2023-10-09T03:44:29.476 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-09T03:44:29.476 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi --verbose --owner scheduled_teuthology@teuthology --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --archive /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454 -- /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/orig.config.yaml 2023-10-09T03:44:29.532 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T03:44:29.613 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7417454' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 5305 sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: a340fb5463aa3715e93e7472d5f60dfdcf184ab3 targets: smithi029.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi177.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-10-07_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-09T03:44:29.613 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa; will attempt to use it 2023-10-09T03:44:29.614 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks 2023-10-09T03:44:29.614 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-09T03:44:29.616 INFO:teuthology.task.internal:Checking packages... 2023-10-09T03:44:29.641 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0' 2023-10-09T03:44:29.641 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:44:29.641 INFO:teuthology.packaging:ref: None 2023-10-09T03:44:29.641 INFO:teuthology.packaging:tag: None 2023-10-09T03:44:29.641 INFO:teuthology.packaging:branch: reef 2023-10-09T03:44:29.641 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:44:29.642 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-09T03:44:29.863 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-610.g397de1b0 2023-10-09T03:44:29.863 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-09T03:44:29.869 INFO:teuthology.task.internal:no buildpackages task found 2023-10-09T03:44:29.869 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-09T03:44:29.888 INFO:teuthology.task.internal:Saving configuration 2023-10-09T03:44:29.912 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-09T03:44:29.918 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T03:44:29.941 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi029.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:39:03.451333', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T03:44:29.966 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:39:03.450024', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T03:44:29.967 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-09T03:44:29.972 INFO:teuthology.task.internal:roles: ubuntu@smithi029.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-09T03:44:29.972 INFO:teuthology.task.internal:roles: ubuntu@smithi177.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-09T03:44:29.972 INFO:teuthology.run_tasks:Running task console_log... 2023-10-09T03:44:30.040 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f784aa32af0>, signals=[15]) 2023-10-09T03:44:30.040 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-09T03:44:30.045 INFO:teuthology.task.internal:Opening connections... 2023-10-09T03:44:30.046 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi029.front.sepia.ceph.com 2023-10-09T03:44:30.047 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:44:30.115 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi177.front.sepia.ceph.com 2023-10-09T03:44:30.116 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:44:30.182 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-09T03:44:30.188 DEBUG:teuthology.orchestra.run.smithi029:> uname -m 2023-10-09T03:44:30.210 INFO:teuthology.orchestra.run.smithi029.stdout:x86_64 2023-10-09T03:44:30.210 DEBUG:teuthology.orchestra.run.smithi029:> cat /etc/os-release 2023-10-09T03:44:30.270 INFO:teuthology.orchestra.run.smithi029.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T03:44:30.270 INFO:teuthology.orchestra.run.smithi029.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:ID="rhel" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:ID_LIKE="fedora" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:VERSION_ID="8.6" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:PLATFORM_ID="platform:el8" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:ANSI_COLOR="0;31" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:44:30.271 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T03:44:30.272 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T03:44:30.272 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T03:44:30.272 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T03:44:30.272 INFO:teuthology.lock.ops:Updating smithi029.front.sepia.ceph.com on lock server 2023-10-09T03:44:30.290 DEBUG:teuthology.orchestra.run.smithi177:> uname -m 2023-10-09T03:44:30.311 INFO:teuthology.orchestra.run.smithi177.stdout:x86_64 2023-10-09T03:44:30.312 DEBUG:teuthology.orchestra.run.smithi177:> cat /etc/os-release 2023-10-09T03:44:30.371 INFO:teuthology.orchestra.run.smithi177.stdout:NAME="Red Hat Enterprise Linux" 2023-10-09T03:44:30.371 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION="8.6 (Ootpa)" 2023-10-09T03:44:30.371 INFO:teuthology.orchestra.run.smithi177.stdout:ID="rhel" 2023-10-09T03:44:30.371 INFO:teuthology.orchestra.run.smithi177.stdout:ID_LIKE="fedora" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION_ID="8.6" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:PLATFORM_ID="platform:el8" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:ANSI_COLOR="0;31" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:HOME_URL="https://www.redhat.com/" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-09T03:44:30.372 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-09T03:44:30.373 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-09T03:44:30.373 INFO:teuthology.lock.ops:Updating smithi177.front.sepia.ceph.com on lock server 2023-10-09T03:44:30.391 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-09T03:44:30.399 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-09T03:44:30.404 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-09T03:44:30.404 DEBUG:teuthology.orchestra.run.smithi029:> test '!' -e /home/ubuntu/cephtest 2023-10-09T03:44:30.407 DEBUG:teuthology.orchestra.run.smithi177:> test '!' -e /home/ubuntu/cephtest 2023-10-09T03:44:30.430 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-09T03:44:30.436 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-09T03:44:30.436 DEBUG:teuthology.orchestra.run.smithi029:> test -z $(ls -A /var/lib/ceph) 2023-10-09T03:44:30.467 DEBUG:teuthology.orchestra.run.smithi177:> test -z $(ls -A /var/lib/ceph) 2023-10-09T03:44:30.505 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-09T03:44:30.555 INFO:teuthology.run_tasks:Running task kernel... 2023-10-09T03:44:30.569 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-09T03:44:30.569 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-09T03:44:30.569 DEBUG:teuthology.orchestra.run.smithi029:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T03:44:30.570 DEBUG:teuthology.orchestra.run.smithi177:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-09T03:44:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:44:30.588 DEBUG:teuthology.orchestra.run.smithi177:> uname -r 2023-10-09T03:44:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:44:30.589 DEBUG:teuthology.orchestra.run.smithi029:> uname -r 2023-10-09T03:44:30.646 INFO:teuthology.orchestra.run.smithi177.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:30.647 INFO:teuthology.task.kernel:Running kernel on smithi177: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:30.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum install -y kernel 2023-10-09T03:44:30.649 INFO:teuthology.orchestra.run.smithi029.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:30.649 INFO:teuthology.task.kernel:Running kernel on smithi029: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:30.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum install -y kernel 2023-10-09T03:44:31.826 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:31.826 INFO:teuthology.orchestra.run.smithi177.stdout:Unable to read consumer identity 2023-10-09T03:44:32.275 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:32.276 INFO:teuthology.orchestra.run.smithi029.stdout:Unable to read consumer identity 2023-10-09T03:44:33.228 INFO:teuthology.orchestra.run.smithi177.stdout:Copr repo for python3-asyncssh owned by ceph 18 kB/s | 3.5 kB 00:00 2023-10-09T03:44:34.029 INFO:teuthology.orchestra.run.smithi177.stdout:Extra Packages for Enterprise Linux 20 MB/s | 16 MB 00:00 2023-10-09T03:44:34.964 INFO:teuthology.orchestra.run.smithi029.stdout:Copr repo for python3-asyncssh owned by ceph 25 kB/s | 3.5 kB 00:00 2023-10-09T03:44:36.082 INFO:teuthology.orchestra.run.smithi029.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-10-09T03:44:37.571 INFO:teuthology.orchestra.run.smithi177.stdout:lab-extras 388 kB/s | 24 kB 00:00 2023-10-09T03:44:38.240 INFO:teuthology.orchestra.run.smithi177.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T03:44:38.291 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:44:38.292 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:44:38.292 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:44:38.452 DEBUG:teuthology.orchestra.run.smithi177:> echo no | sudo yum reinstall kernel || true 2023-10-09T03:44:38.782 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:38.802 INFO:teuthology.orchestra.run.smithi177.stdout:Unable to read consumer identity 2023-10-09T03:44:39.051 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 03:44:37 AM UTC. 2023-10-09T03:44:39.094 INFO:teuthology.orchestra.run.smithi177.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:44:39.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error: No packages marked for reinstall. 2023-10-09T03:44:39.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum reinstall -y kernel || true 2023-10-09T03:44:39.517 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:39.518 INFO:teuthology.orchestra.run.smithi177.stdout:Unable to read consumer identity 2023-10-09T03:44:39.779 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 03:44:37 AM UTC. 2023-10-09T03:44:40.325 INFO:teuthology.orchestra.run.smithi029.stdout:lab-extras 456 kB/s | 24 kB 00:00 2023-10-09T03:44:40.326 INFO:teuthology.orchestra.run.smithi177.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:44:40.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error: No packages marked for reinstall. 2023-10-09T03:44:40.327 DEBUG:teuthology.orchestra.run.smithi177:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T03:44:40.408 INFO:teuthology.orchestra.run.smithi177.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:40.408 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:40.408 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T03:44:40.408 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-09T03:44:40.409 DEBUG:teuthology.orchestra.run.smithi177:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T03:44:40.440 INFO:teuthology.orchestra.run.smithi177.stdout:ttyS1 2023-10-09T03:44:40.455 DEBUG:teuthology.parallel:result is None 2023-10-09T03:44:40.539 INFO:teuthology.orchestra.run.smithi029.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-09T03:44:40.631 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:44:40.632 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:44:40.632 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:44:40.804 DEBUG:teuthology.orchestra.run.smithi029:> echo no | sudo yum reinstall kernel || true 2023-10-09T03:44:41.245 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:41.245 INFO:teuthology.orchestra.run.smithi029.stdout:Unable to read consumer identity 2023-10-09T03:44:41.525 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:02 ago on Mon 09 Oct 2023 03:44:39 AM UTC. 2023-10-09T03:44:41.580 INFO:teuthology.orchestra.run.smithi029.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:44:41.634 INFO:teuthology.orchestra.run.smithi029.stderr:Error: No packages marked for reinstall. 2023-10-09T03:44:41.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum reinstall -y kernel || true 2023-10-09T03:44:42.031 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:44:42.031 INFO:teuthology.orchestra.run.smithi029.stdout:Unable to read consumer identity 2023-10-09T03:44:42.306 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:03 ago on Mon 09 Oct 2023 03:44:39 AM UTC. 2023-10-09T03:44:42.349 INFO:teuthology.orchestra.run.smithi029.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-09T03:44:42.382 INFO:teuthology.orchestra.run.smithi029.stderr:Error: No packages marked for reinstall. 2023-10-09T03:44:42.435 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q kernel | sort -rV | head -n 1 2023-10-09T03:44:42.517 INFO:teuthology.orchestra.run.smithi029.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:42.518 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-09T03:44:42.518 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-09T03:44:42.518 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-09T03:44:42.518 DEBUG:teuthology.orchestra.run.smithi029:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-09T03:44:42.551 INFO:teuthology.orchestra.run.smithi029.stdout:ttyS1 2023-10-09T03:44:42.564 DEBUG:teuthology.parallel:result is None 2023-10-09T03:44:42.565 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-09T03:44:42.571 INFO:teuthology.task.internal:Creating test directory... 2023-10-09T03:44:42.572 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T03:44:42.606 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-09T03:44:42.626 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-09T03:44:42.633 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-09T03:44:42.639 INFO:teuthology.task.internal:Creating archive directory... 2023-10-09T03:44:42.639 DEBUG:teuthology.orchestra.run.smithi029:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T03:44:42.663 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-09T03:44:42.690 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-09T03:44:42.697 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-09T03:44:42.697 DEBUG:teuthology.orchestra.run.smithi029:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T03:44:42.721 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-09T03:44:42.762 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:44:42.772 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:44:42.773 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:44:42.784 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-09T03:44:42.785 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-09T03:44:42.792 INFO:teuthology.task.internal:Configuring sudo... 2023-10-09T03:44:42.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T03:44:42.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-09T03:44:42.845 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-09T03:44:42.853 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-09T03:44:42.853 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T03:44:42.883 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-09T03:44:42.902 DEBUG:teuthology.orchestra.run.smithi029:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:44:42.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:44:43.037 DEBUG:teuthology.orchestra.run.smithi029:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:44:43.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:44:43.163 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T03:44:43.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T03:44:43.270 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:44:43.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-09T03:44:43.374 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:44:43.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-09T03:44:43.504 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T03:44:43.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-09T03:44:43.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo service rsyslog restart 2023-10-09T03:44:43.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo service rsyslog restart 2023-10-09T03:44:43.643 INFO:teuthology.orchestra.run.smithi029.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T03:44:43.679 INFO:teuthology.orchestra.run.smithi177.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T03:44:44.037 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-09T03:44:44.045 INFO:teuthology.task.internal:Starting timer... 2023-10-09T03:44:44.045 INFO:teuthology.run_tasks:Running task pcp... 2023-10-09T03:44:44.066 INFO:teuthology.run_tasks:Running task selinux... 2023-10-09T03:44:44.075 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-09T03:44:44.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo service auditd rotate 2023-10-09T03:44:44.132 INFO:teuthology.orchestra.run.smithi029.stdout:Rotating logs: 2023-10-09T03:44:44.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo service auditd rotate 2023-10-09T03:44:44.202 INFO:teuthology.orchestra.run.smithi177.stdout:Rotating logs: 2023-10-09T03:44:44.204 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-09T03:44:44.204 DEBUG:teuthology.orchestra.run.smithi029:> /usr/sbin/getenforce 2023-10-09T03:44:44.227 INFO:teuthology.orchestra.run.smithi029.stdout:Permissive 2023-10-09T03:44:44.227 DEBUG:teuthology.orchestra.run.smithi177:> /usr/sbin/getenforce 2023-10-09T03:44:44.246 INFO:teuthology.orchestra.run.smithi177.stdout:Permissive 2023-10-09T03:44:44.246 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi029.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi177.front.sepia.ceph.com': 'permissive'} 2023-10-09T03:44:44.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T03:44:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:44:44.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T03:44:44.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T03:44:44.323 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-09T03:44:44.324 DEBUG:teuthology.orchestra.run.smithi029:> sudo /usr/sbin/setenforce permissive 2023-10-09T03:44:44.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo /usr/sbin/setenforce permissive 2023-10-09T03:44:44.517 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-09T03:44:44.527 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-09T03:44:44.534 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-09T03:44:44.709 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-09T03:44:44.725 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-09T03:44:44.726 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi029.front.sepia.ceph.com,smithi177.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-09T03:53:54.007 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi029.front.sepia.ceph.com'), Remote(name='ubuntu@smithi177.front.sepia.ceph.com')] 2023-10-09T03:53:54.008 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2023-10-09T03:53:54.009 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:53:54.078 DEBUG:teuthology.orchestra.run.smithi029:> true 2023-10-09T03:53:54.149 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi029.front.sepia.ceph.com' 2023-10-09T03:53:54.149 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2023-10-09T03:53:54.150 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-09T03:53:54.219 DEBUG:teuthology.orchestra.run.smithi177:> true 2023-10-09T03:53:54.288 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi177.front.sepia.ceph.com' 2023-10-09T03:53:54.289 INFO:teuthology.run_tasks:Running task clock... 2023-10-09T03:53:54.301 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-09T03:53:54.301 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T03:53:54.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T03:53:54.304 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-09T03:53:54.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T03:53:54.335 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T03:53:54.351 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T03:53:54.361 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-09T03:53:54.376 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-09T03:53:54.422 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: ntpd: command not found 2023-10-09T03:53:54.434 INFO:teuthology.orchestra.run.smithi029.stdout:506 Cannot talk to daemon 2023-10-09T03:53:54.447 INFO:teuthology.orchestra.run.smithi177.stderr:sudo: ntpd: command not found 2023-10-09T03:53:54.449 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T03:53:54.459 INFO:teuthology.orchestra.run.smithi177.stdout:506 Cannot talk to daemon 2023-10-09T03:53:54.463 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T03:53:54.473 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-09T03:53:54.486 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-09T03:53:54.537 INFO:teuthology.orchestra.run.smithi029.stderr:bash: ntpq: command not found 2023-10-09T03:53:54.541 INFO:teuthology.orchestra.run.smithi029.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T03:53:54.541 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================== 2023-10-09T03:53:54.541 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.541 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.541 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.542 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.562 INFO:teuthology.orchestra.run.smithi177.stderr:bash: ntpq: command not found 2023-10-09T03:53:54.565 INFO:teuthology.orchestra.run.smithi177.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T03:53:54.566 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================== 2023-10-09T03:53:54.566 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.566 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.566 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.566 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T03:53:54.566 INFO:teuthology.run_tasks:Running task pexec... 2023-10-09T03:53:54.576 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-09T03:53:54.576 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi029.front.sepia.ceph.com 2023-10-09T03:53:54.576 DEBUG:teuthology.orchestra.run.smithi029:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T03:53:54.577 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi177.front.sepia.ceph.com 2023-10-09T03:53:54.577 DEBUG:teuthology.orchestra.run.smithi177:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-09T03:53:54.974 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:53:54.979 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:53:55.829 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:49 ago on Mon 09 Oct 2023 03:52:06 AM UTC. 2023-10-09T03:53:55.865 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:30 ago on Mon 09 Oct 2023 03:52:25 AM UTC. 2023-10-09T03:53:57.588 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:53:57.588 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:53:57.589 INFO:teuthology.orchestra.run.smithi177.stdout: Package Architecture Version Repository Size 2023-10-09T03:53:57.589 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:53:57.589 INFO:teuthology.orchestra.run.smithi177.stdout:Resetting modules: 2023-10-09T03:53:57.589 INFO:teuthology.orchestra.run.smithi177.stdout: container-tools 2023-10-09T03:53:57.589 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:53:57.590 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:53:57.590 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:53:57.590 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:53:57.878 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout:Resetting modules: 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout: container-tools 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:53:57.880 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:53:58.557 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:52 ago on Mon 09 Oct 2023 03:52:06 AM UTC. 2023-10-09T03:53:58.901 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:33 ago on Mon 09 Oct 2023 03:52:25 AM UTC. 2023-10-09T03:54:01.212 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:54:01.673 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:54:01.924 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:54:02.412 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:54:02.569 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:56 ago on Mon 09 Oct 2023 03:52:06 AM UTC. 2023-10-09T03:54:03.287 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:37 ago on Mon 09 Oct 2023 03:52:25 AM UTC. 2023-10-09T03:54:04.258 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:54:04.350 INFO:teuthology.orchestra.run.smithi177.stdout:========================================================================================================================== 2023-10-09T03:54:04.350 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:54:04.350 INFO:teuthology.orchestra.run.smithi177.stdout:========================================================================================================================== 2023-10-09T03:54:04.350 INFO:teuthology.orchestra.run.smithi177.stdout:Installing group/module packages: 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout:Downgrading: 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T03:54:04.351 INFO:teuthology.orchestra.run.smithi177.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout:Installing module profiles: 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout: container-tools/common 2023-10-09T03:54:04.352 INFO:teuthology.orchestra.run.smithi177.stdout:Enabling module streams: 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout: container-tools 3.0 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout:========================================================================================================================== 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout:Install 7 Packages 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout:Downgrade 12 Packages 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:54:04.353 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 51 M 2023-10-09T03:54:04.354 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:54:04.534 INFO:teuthology.orchestra.run.smithi177.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 285 kB/s | 51 kB 00:00 2023-10-09T03:54:04.668 INFO:teuthology.orchestra.run.smithi177.stdout:(2/19): criu-3.15-1.module+el8.6.0+14874+644362 3.7 MB/s | 511 kB 00:00 2023-10-09T03:54:04.693 INFO:teuthology.orchestra.run.smithi177.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 156 kB/s | 53 kB 00:00 2023-10-09T03:54:04.710 INFO:teuthology.orchestra.run.smithi177.stdout:(4/19): containers-common-1.2.4-1.module+el8.6. 255 kB/s | 91 kB 00:00 2023-10-09T03:54:04.735 INFO:teuthology.orchestra.run.smithi177.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.0 MB/s | 69 kB 00:00 2023-10-09T03:54:04.760 INFO:teuthology.orchestra.run.smithi177.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.4 MB/s | 72 kB 00:00 2023-10-09T03:54:04.811 INFO:teuthology.orchestra.run.smithi177.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.1 MB/s | 56 kB 00:00 2023-10-09T03:54:04.869 INFO:teuthology.orchestra.run.smithi177.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 16 MB/s | 2.9 MB 00:00 2023-10-09T03:54:04.895 INFO:teuthology.orchestra.run.smithi177.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 612 kB/s | 51 kB 00:00 2023-10-09T03:54:04.945 INFO:teuthology.orchestra.run.smithi177.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-10-09T03:54:04.987 INFO:teuthology.orchestra.run.smithi177.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 978 kB/s | 41 kB 00:00 2023-10-09T03:54:05.103 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:54:05.162 INFO:teuthology.orchestra.run.smithi177.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 26 MB/s | 11 MB 00:00 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout:========================================================================================================================== 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout:========================================================================================================================== 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout:Installing group/module packages: 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-09T03:54:05.204 INFO:teuthology.orchestra.run.smithi029.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout:Downgrading: 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-09T03:54:05.205 INFO:teuthology.orchestra.run.smithi029.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-09T03:54:05.206 INFO:teuthology.orchestra.run.smithi029.stdout:Installing module profiles: 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout: container-tools/common 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling module streams: 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout: container-tools 3.0 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout:========================================================================================================================== 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout:Install 7 Packages 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout:Downgrade 12 Packages 2023-10-09T03:54:05.207 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:54:05.208 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 51 M 2023-10-09T03:54:05.208 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:54:05.212 INFO:teuthology.orchestra.run.smithi177.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-10-09T03:54:05.311 INFO:teuthology.orchestra.run.smithi029.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 513 kB/s | 53 kB 00:00 2023-10-09T03:54:05.337 INFO:teuthology.orchestra.run.smithi029.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 399 kB/s | 51 kB 00:00 2023-10-09T03:54:05.371 INFO:teuthology.orchestra.run.smithi029.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 560 kB/s | 91 kB 00:00 2023-10-09T03:54:05.371 INFO:teuthology.orchestra.run.smithi177.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 31 MB/s | 6.4 MB 00:00 2023-10-09T03:54:05.433 INFO:teuthology.orchestra.run.smithi029.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 717 kB/s | 69 kB 00:00 2023-10-09T03:54:05.480 INFO:teuthology.orchestra.run.smithi029.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 3.0 MB/s | 511 kB 00:00 2023-10-09T03:54:05.506 INFO:teuthology.orchestra.run.smithi029.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.0 MB/s | 72 kB 00:00 2023-10-09T03:54:05.548 INFO:teuthology.orchestra.run.smithi029.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.3 MB/s | 56 kB 00:00 2023-10-09T03:54:05.579 INFO:teuthology.orchestra.run.smithi177.stdout:(15/19): containernetworking-plugins-0.9.1-1.mo 27 MB/s | 19 MB 00:00 2023-10-09T03:54:05.613 INFO:teuthology.orchestra.run.smithi177.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 4.4 MB/s | 1.1 MB 00:00 2023-10-09T03:54:05.630 INFO:teuthology.orchestra.run.smithi177.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 3.6 MB/s | 185 kB 00:00 2023-10-09T03:54:05.640 INFO:teuthology.orchestra.run.smithi029.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 11 MB/s | 2.9 MB 00:00 2023-10-09T03:54:05.647 INFO:teuthology.orchestra.run.smithi177.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.5 MB/s | 51 kB 00:00 2023-10-09T03:54:05.666 INFO:teuthology.orchestra.run.smithi029.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 435 kB/s | 51 kB 00:00 2023-10-09T03:54:05.716 INFO:teuthology.orchestra.run.smithi029.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-10-09T03:54:05.731 INFO:teuthology.orchestra.run.smithi177.stdout:(19/19): buildah-1.19.9-3.module+el8.6.0+14874+ 13 MB/s | 6.8 MB 00:00 2023-10-09T03:54:05.731 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:54:05.731 INFO:teuthology.orchestra.run.smithi177.stdout:Total 37 MB/s | 51 MB 00:01 2023-10-09T03:54:05.766 INFO:teuthology.orchestra.run.smithi029.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 811 kB/s | 41 kB 00:00 2023-10-09T03:54:05.883 INFO:teuthology.orchestra.run.smithi029.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 19 MB/s | 2.2 MB 00:00 2023-10-09T03:54:06.149 INFO:teuthology.orchestra.run.smithi029.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 16 MB/s | 11 MB 00:00 2023-10-09T03:54:06.210 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:54:06.253 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:54:06.253 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:54:06.385 INFO:teuthology.orchestra.run.smithi029.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 13 MB/s | 6.4 MB 00:00 2023-10-09T03:54:06.460 INFO:teuthology.orchestra.run.smithi029.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 14 MB/s | 1.1 MB 00:00 2023-10-09T03:54:06.594 INFO:teuthology.orchestra.run.smithi029.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 15 MB/s | 6.8 MB 00:00 2023-10-09T03:54:06.621 INFO:teuthology.orchestra.run.smithi029.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.1 MB/s | 185 kB 00:00 2023-10-09T03:54:06.655 INFO:teuthology.orchestra.run.smithi029.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 836 kB/s | 51 kB 00:00 2023-10-09T03:54:06.693 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:54:06.696 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:54:07.047 INFO:teuthology.orchestra.run.smithi029.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 13 MB/s | 19 MB 00:01 2023-10-09T03:54:07.048 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:54:07.048 INFO:teuthology.orchestra.run.smithi029.stdout:Total 27 MB/s | 51 MB 00:01 2023-10-09T03:54:07.548 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:54:07.594 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:54:07.594 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:54:07.889 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:54:07.946 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T03:54:08.060 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:54:08.064 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:54:08.115 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:54:08.126 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:54:08.142 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:54:08.340 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:54:09.281 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:54:09.330 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-09T03:54:09.536 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:54:09.545 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-09T03:54:09.598 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:54:09.802 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:54:25.301 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:54:27.082 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T03:54:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T03:54:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T03:54:27.710 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T03:54:27.876 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-09T03:54:27.937 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T03:54:27.938 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T03:54:27.938 INFO:teuthology.orchestra.run.smithi177.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T03:54:27.938 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:54:28.693 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T03:54:28.911 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T03:54:29.762 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-09T03:54:30.005 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-09T03:54:30.229 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-09T03:54:30.338 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T03:54:30.458 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-09T03:54:30.655 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T03:54:30.691 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-09T03:54:30.691 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-09T03:54:30.691 INFO:teuthology.orchestra.run.smithi029.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-09T03:54:30.691 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:54:31.028 INFO:teuthology.orchestra.run.smithi177.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T03:54:31.285 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T03:54:31.567 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-09T03:54:31.902 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-09T03:54:32.002 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T03:54:32.716 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T03:54:32.944 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T03:54:33.102 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T03:54:33.269 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T03:54:33.425 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T03:54:33.426 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:54:33.739 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:54:33.742 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-09T03:54:33.896 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T03:54:33.897 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:54:34.016 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-09T03:54:34.046 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:54:34.203 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:54:34.355 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T03:54:34.416 INFO:teuthology.orchestra.run.smithi029.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-09T03:54:34.496 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T03:54:34.655 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T03:54:34.705 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-09T03:54:34.787 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T03:54:34.937 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T03:54:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T03:54:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:54:35.236 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:54:35.474 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-09T03:54:35.758 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T03:54:36.144 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T03:54:36.145 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T03:54:36.146 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T03:54:36.300 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-09T03:54:36.554 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-09T03:54:36.764 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-09T03:54:37.025 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-09T03:54:37.150 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-09T03:54:37.150 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:54:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-09T03:54:37.567 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T03:54:37.567 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:31 ago on Mon 09 Oct 2023 03:52:06 AM UTC. 2023-10-09T03:54:37.682 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-09T03:54:37.682 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:54:37.868 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-09T03:54:38.009 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:54:38.205 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-09T03:54:38.417 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-09T03:54:38.599 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-09T03:54:38.766 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-09T03:54:38.908 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-09T03:54:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-09T03:54:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:54:39.325 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:54:39.889 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-09T03:54:40.292 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-09T03:54:40.292 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-09T03:54:40.292 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-09T03:54:40.292 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-09T03:54:40.292 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-09T03:54:40.292 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-09T03:54:40.293 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-09T03:54:40.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-09T03:54:40.295 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-09T03:54:41.224 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout:Downgraded: 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T03:54:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:54:41.426 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:54:41.625 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:41.852 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-09T03:54:41.852 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:16 ago on Mon 09 Oct 2023 03:52:25 AM UTC. 2023-10-09T03:54:44.586 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:54:44.798 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:54:44.798 INFO:teuthology.orchestra.run.smithi029.stdout:Downgraded: 2023-10-09T03:54:44.798 INFO:teuthology.orchestra.run.smithi029.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.798 INFO:teuthology.orchestra.run.smithi029.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.799 INFO:teuthology.orchestra.run.smithi029.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:54:44.800 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:54:45.040 DEBUG:teuthology.parallel:result is None 2023-10-09T03:54:45.040 INFO:teuthology.run_tasks:Running task install... 2023-10-09T03:54:45.050 DEBUG:teuthology.task.install:project ceph 2023-10-09T03:54:45.050 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}} 2023-10-09T03:54:45.050 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T03:54:45.050 INFO:teuthology.task.install:Using flavor: default 2023-10-09T03:54:45.056 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-09T03:54:45.056 INFO:teuthology.task.install:extra packages: [] 2023-10-09T03:54:45.056 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T03:54:45.057 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:54:45.057 INFO:teuthology.packaging:ref: None 2023-10-09T03:54:45.057 INFO:teuthology.packaging:tag: None 2023-10-09T03:54:45.057 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:54:45.057 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:54:45.057 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:54:45.059 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0', 'tag': None, 'wait_for_package': False} 2023-10-09T03:54:45.059 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:54:45.059 INFO:teuthology.packaging:ref: None 2023-10-09T03:54:45.059 INFO:teuthology.packaging:tag: None 2023-10-09T03:54:45.059 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:54:45.060 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:54:45.060 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:54:45.341 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T03:54:45.341 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T03:54:45.426 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/ 2023-10-09T03:54:45.427 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1303.gb2903cb0 2023-10-09T03:54:45.459 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T03:54:45.459 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T03:54:45.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T03:54:45.492 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T03:54:45.492 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:54:45.492 INFO:teuthology.packaging:ref: None 2023-10-09T03:54:45.492 INFO:teuthology.packaging:tag: None 2023-10-09T03:54:45.492 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:54:45.492 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:54:45.493 DEBUG:teuthology.orchestra.run.smithi177:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T03:54:45.535 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/b2903cb0842b1846a00474d4d51d49d1dc33bce1/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-09T03:54:45.535 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T03:54:45.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-09T03:54:45.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T03:54:45.567 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-09T03:54:45.567 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:54:45.567 INFO:teuthology.packaging:ref: None 2023-10-09T03:54:45.567 INFO:teuthology.packaging:tag: None 2023-10-09T03:54:45.567 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:54:45.567 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:54:45.567 DEBUG:teuthology.orchestra.run.smithi029:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-09T03:54:45.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-09T03:54:45.648 DEBUG:teuthology.orchestra.run.smithi177:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T03:54:45.716 INFO:teuthology.orchestra.run.smithi177.stdout:check_obsoletes = 1 2023-10-09T03:54:45.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum clean all 2023-10-09T03:54:45.721 DEBUG:teuthology.orchestra.run.smithi029:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-09T03:54:45.790 INFO:teuthology.orchestra.run.smithi029.stdout:check_obsoletes = 1 2023-10-09T03:54:45.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum clean all 2023-10-09T03:54:46.092 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:54:46.187 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:54:47.021 INFO:teuthology.orchestra.run.smithi177.stdout:44 files removed 2023-10-09T03:54:47.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-radosgw 2023-10-09T03:54:47.226 INFO:teuthology.orchestra.run.smithi029.stdout:44 files removed 2023-10-09T03:54:47.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-radosgw 2023-10-09T03:54:47.402 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:54:47.633 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:54:48.802 INFO:teuthology.orchestra.run.smithi177.stdout:ceph packages for x86_64 318 kB/s | 78 kB 00:00 2023-10-09T03:54:49.036 INFO:teuthology.orchestra.run.smithi029.stdout:ceph packages for x86_64 262 kB/s | 78 kB 00:00 2023-10-09T03:54:49.094 INFO:teuthology.orchestra.run.smithi177.stdout:ceph noarch packages 35 kB/s | 9.1 kB 00:00 2023-10-09T03:54:49.387 INFO:teuthology.orchestra.run.smithi177.stdout:ceph source packages 5.3 kB/s | 1.4 kB 00:00 2023-10-09T03:54:49.413 INFO:teuthology.orchestra.run.smithi029.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2023-10-09T03:54:49.537 INFO:teuthology.orchestra.run.smithi177.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-10-09T03:54:49.764 INFO:teuthology.orchestra.run.smithi029.stdout:ceph source packages 4.4 kB/s | 1.4 kB 00:00 2023-10-09T03:54:49.915 INFO:teuthology.orchestra.run.smithi029.stdout:Copr repo for python3-asyncssh owned by ceph 27 kB/s | 3.5 kB 00:00 2023-10-09T03:54:50.313 INFO:teuthology.orchestra.run.smithi177.stdout:Extra Packages for Enterprise Linux 21 MB/s | 16 MB 00:00 2023-10-09T03:54:53.771 INFO:teuthology.orchestra.run.smithi177.stdout:lab-extras 446 kB/s | 24 kB 00:00 2023-10-09T03:54:53.938 INFO:teuthology.orchestra.run.smithi029.stdout:Extra Packages for Enterprise Linux 4.0 MB/s | 16 MB 00:03 2023-10-09T03:54:54.923 INFO:teuthology.orchestra.run.smithi177.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 44 MB/s | 47 MB 00:01 2023-10-09T03:54:57.662 INFO:teuthology.orchestra.run.smithi029.stdout:lab-extras 387 kB/s | 24 kB 00:00 2023-10-09T03:54:59.324 INFO:teuthology.orchestra.run.smithi029.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 30 MB/s | 47 MB 00:01 2023-10-09T03:55:03.066 INFO:teuthology.orchestra.run.smithi177.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 53 MB/s | 53 MB 00:01 2023-10-09T03:55:08.454 INFO:teuthology.orchestra.run.smithi029.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 33 MB/s | 53 MB 00:01 2023-10-09T03:55:14.257 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout:==================================================================================================== 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout:==================================================================================================== 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout:Upgrading: 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:55:14.260 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T03:55:14.261 INFO:teuthology.orchestra.run.smithi177.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:55:14.262 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:55:14.263 INFO:teuthology.orchestra.run.smithi177.stdout:==================================================================================================== 2023-10-09T03:55:14.263 INFO:teuthology.orchestra.run.smithi177.stdout:Install 21 Packages 2023-10-09T03:55:14.263 INFO:teuthology.orchestra.run.smithi177.stdout:Upgrade 2 Packages 2023-10-09T03:55:14.263 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:55:14.263 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 63 M 2023-10-09T03:55:14.263 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:55:15.707 INFO:teuthology.orchestra.run.smithi177.stdout:(1/23): ceph-common-17.2.6-1303.gb2903cb0.el8.x 17 MB/s | 23 MB 00:01 2023-10-09T03:55:15.749 INFO:teuthology.orchestra.run.smithi177.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 590 kB/s | 24 kB 00:00 2023-10-09T03:55:15.834 INFO:teuthology.orchestra.run.smithi177.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 9.3 MB/s | 797 kB 00:00 2023-10-09T03:55:15.891 INFO:teuthology.orchestra.run.smithi177.stdout:(4/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 3.3 MB/s | 5.2 MB 00:01 2023-10-09T03:55:15.917 INFO:teuthology.orchestra.run.smithi177.stdout:(5/23): libradosstriper1-17.2.6-1303.gb2903cb0. 5.9 MB/s | 509 kB 00:00 2023-10-09T03:55:15.959 INFO:teuthology.orchestra.run.smithi177.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 1.1 MB/s | 45 kB 00:00 2023-10-09T03:55:16.013 INFO:teuthology.orchestra.run.smithi177.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 2.3 MB/s | 115 kB 00:00 2023-10-09T03:55:16.069 INFO:teuthology.orchestra.run.smithi177.stdout:(8/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 3.2 MB/s | 190 kB 00:00 2023-10-09T03:55:16.126 INFO:teuthology.orchestra.run.smithi177.stdout:(9/23): python3-rados-17.2.6-1303.gb2903cb0.el8 6.2 MB/s | 370 kB 00:00 2023-10-09T03:55:16.201 INFO:teuthology.orchestra.run.smithi177.stdout:(10/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 4.7 MB/s | 358 kB 00:00 2023-10-09T03:55:16.260 INFO:teuthology.orchestra.run.smithi177.stdout:(11/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 1.8 MB/s | 106 kB 00:00 2023-10-09T03:55:16.500 INFO:teuthology.orchestra.run.smithi177.stdout:(12/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_ 9.1 MB/s | 5.6 MB 00:00 2023-10-09T03:55:16.836 INFO:teuthology.orchestra.run.smithi177.stdout:(13/23): gperftools-libs-2.7-9.el8.x86_64.rpm 532 kB/s | 306 kB 00:00 2023-10-09T03:55:16.852 INFO:teuthology.orchestra.run.smithi177.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 167 kB/s | 59 kB 00:00 2023-10-09T03:55:16.920 INFO:teuthology.orchestra.run.smithi177.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 900 kB/s | 75 kB 00:00 2023-10-09T03:55:17.095 INFO:teuthology.orchestra.run.smithi177.stdout:(16/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.4 MB/s | 259 kB 00:00 2023-10-09T03:55:17.220 INFO:teuthology.orchestra.run.smithi177.stdout:(17/23): python3-prettytable-0.7.2-14.el8.noarc 352 kB/s | 44 kB 00:00 2023-10-09T03:55:17.262 INFO:teuthology.orchestra.run.smithi177.stdout:(18/23): thrift-0.13.0-2.el8.x86_64.rpm 4.2 MB/s | 1.7 MB 00:00 2023-10-09T03:55:17.387 INFO:teuthology.orchestra.run.smithi177.stdout:(19/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 375 kB/s | 47 kB 00:00 2023-10-09T03:55:17.412 INFO:teuthology.orchestra.run.smithi177.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 1.8 MB/s | 354 kB 00:00 2023-10-09T03:55:17.729 INFO:teuthology.orchestra.run.smithi177.stdout:(21/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 11 MB/s | 3.6 MB 00:00 2023-10-09T03:55:17.930 INFO:teuthology.orchestra.run.smithi177.stdout:(22/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 6.4 MB/s | 3.5 MB 00:00 2023-10-09T03:55:18.413 INFO:teuthology.orchestra.run.smithi177.stdout:(23/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 4.1 MB/s | 17 MB 00:04 2023-10-09T03:55:18.418 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:55:18.418 INFO:teuthology.orchestra.run.smithi177.stdout:Total 15 MB/s | 63 MB 00:04 2023-10-09T03:55:18.466 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:55:18.511 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:55:18.512 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:55:18.854 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:55:18.858 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:55:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:55:20.069 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:55:20.454 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:55:20.549 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:55:20.551 INFO:teuthology.orchestra.run.smithi029.stdout:==================================================================================================== 2023-10-09T03:55:20.551 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout:==================================================================================================== 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-radosgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout:Upgrading: 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout: librados2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout: librbd1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.6 M 2023-10-09T03:55:20.552 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-base x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.2 M 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 23 M 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-selinux x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 24 k 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 797 k 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-09T03:55:20.553 INFO:teuthology.orchestra.run.smithi029.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-09T03:55:20.554 INFO:teuthology.orchestra.run.smithi029.stdout: libradosstriper1 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 509 k 2023-10-09T03:55:20.554 INFO:teuthology.orchestra.run.smithi029.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-09T03:55:20.554 INFO:teuthology.orchestra.run.smithi029.stdout: librgw2 x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 5.6 M 2023-10-09T03:55:20.554 INFO:teuthology.orchestra.run.smithi029.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-09T03:55:20.554 INFO:teuthology.orchestra.run.smithi029.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-09T03:55:20.554 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-argparse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 45 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-common x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 115 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cephfs x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 190 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rados x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 370 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 358 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rgw x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 106 k 2023-10-09T03:55:20.555 INFO:teuthology.orchestra.run.smithi029.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-10-09T03:55:20.556 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:55:20.556 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:55:20.556 INFO:teuthology.orchestra.run.smithi029.stdout:==================================================================================================== 2023-10-09T03:55:20.556 INFO:teuthology.orchestra.run.smithi029.stdout:Install 21 Packages 2023-10-09T03:55:20.556 INFO:teuthology.orchestra.run.smithi029.stdout:Upgrade 2 Packages 2023-10-09T03:55:20.556 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:55:20.557 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 63 M 2023-10-09T03:55:20.557 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:55:20.602 INFO:teuthology.orchestra.run.smithi177.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:55:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:55:20.850 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:55:21.076 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T03:55:21.244 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:55:21.622 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:55:21.676 INFO:teuthology.orchestra.run.smithi029.stdout:(1/23): ceph-base-17.2.6-1303.gb2903cb0.el8.x86 4.9 MB/s | 5.2 MB 00:01 2023-10-09T03:55:21.734 INFO:teuthology.orchestra.run.smithi029.stdout:(2/23): ceph-selinux-17.2.6-1303.gb2903cb0.el8. 416 kB/s | 24 kB 00:00 2023-10-09T03:55:21.827 INFO:teuthology.orchestra.run.smithi177.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:55:21.877 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:55:22.059 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-10-09T03:55:22.101 INFO:teuthology.orchestra.run.smithi029.stdout:(3/23): libcephfs2-17.2.6-1303.gb2903cb0.el8.x8 2.1 MB/s | 797 kB 00:00 2023-10-09T03:55:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:55:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:55:22.551 INFO:teuthology.orchestra.run.smithi029.stdout:(4/23): ceph-common-17.2.6-1303.gb2903cb0.el8.x 12 MB/s | 23 MB 00:01 2023-10-09T03:55:22.585 INFO:teuthology.orchestra.run.smithi029.stdout:(5/23): libradosstriper1-17.2.6-1303.gb2903cb0. 1.0 MB/s | 509 kB 00:00 2023-10-09T03:55:22.660 INFO:teuthology.orchestra.run.smithi029.stdout:(6/23): python3-ceph-argparse-17.2.6-1303.gb290 597 kB/s | 45 kB 00:00 2023-10-09T03:55:22.819 INFO:teuthology.orchestra.run.smithi029.stdout:(7/23): python3-ceph-common-17.2.6-1303.gb2903c 728 kB/s | 115 kB 00:00 2023-10-09T03:55:22.970 INFO:teuthology.orchestra.run.smithi029.stdout:(8/23): librgw2-17.2.6-1303.gb2903cb0.el8.x86_6 13 MB/s | 5.6 MB 00:00 2023-10-09T03:55:23.004 INFO:teuthology.orchestra.run.smithi029.stdout:(9/23): python3-cephfs-17.2.6-1303.gb2903cb0.el 1.0 MB/s | 190 kB 00:00 2023-10-09T03:55:23.038 INFO:teuthology.orchestra.run.smithi029.stdout:(10/23): python3-rados-17.2.6-1303.gb2903cb0.el 5.4 MB/s | 370 kB 00:00 2023-10-09T03:55:23.080 INFO:teuthology.orchestra.run.smithi029.stdout:(11/23): python3-rgw-17.2.6-1303.gb2903cb0.el8. 2.5 MB/s | 106 kB 00:00 2023-10-09T03:55:23.133 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T03:55:23.293 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:55:23.365 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:55:23.414 INFO:teuthology.orchestra.run.smithi029.stdout:(12/23): ceph-radosgw-17.2.6-1303.gb2903cb0.el8 6.0 MB/s | 17 MB 00:02 2023-10-09T03:55:23.448 INFO:teuthology.orchestra.run.smithi029.stdout:(13/23): python3-rbd-17.2.6-1303.gb2903cb0.el8. 806 kB/s | 358 kB 00:00 2023-10-09T03:55:23.479 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T03:55:23.629 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:55:23.828 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:55:23.873 INFO:teuthology.orchestra.run.smithi029.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 129 kB/s | 59 kB 00:00 2023-10-09T03:55:23.949 INFO:teuthology.orchestra.run.smithi029.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 352 kB/s | 306 kB 00:00 2023-10-09T03:55:23.976 INFO:teuthology.orchestra.run.smithi029.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 142 kB/s | 75 kB 00:00 2023-10-09T03:55:24.043 INFO:teuthology.orchestra.run.smithi029.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.7 MB/s | 259 kB 00:00 2023-10-09T03:55:24.053 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T03:55:24.069 INFO:teuthology.orchestra.run.smithi029.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 523 kB/s | 44 kB 00:00 2023-10-09T03:55:24.103 INFO:teuthology.orchestra.run.smithi029.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2023-10-09T03:55:24.128 INFO:teuthology.orchestra.run.smithi029.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 793 kB/s | 47 kB 00:00 2023-10-09T03:55:24.218 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:55:24.278 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:55:24.394 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T03:55:24.396 INFO:teuthology.orchestra.run.smithi029.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 3.3 MB/s | 1.7 MB 00:00 2023-10-09T03:55:24.577 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T03:55:24.783 INFO:teuthology.orchestra.run.smithi029.stdout:(22/23): librbd1-17.2.6-1303.gb2903cb0.el8.x86_ 5.5 MB/s | 3.6 MB 00:00 2023-10-09T03:55:24.842 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T03:55:25.019 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T03:55:25.655 INFO:teuthology.orchestra.run.smithi029.stdout:(23/23): librados2-17.2.6-1303.gb2903cb0.el8.x8 2.2 MB/s | 3.5 MB 00:01 2023-10-09T03:55:25.660 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:55:25.660 INFO:teuthology.orchestra.run.smithi029.stdout:Total 12 MB/s | 63 MB 00:05 2023-10-09T03:55:25.708 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:55:25.758 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:55:25.758 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:55:26.129 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:55:26.133 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:55:27.255 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:55:27.469 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:55:27.887 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-09T03:55:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: Upgrading : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:55:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:55:28.432 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:55:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-09T03:55:28.809 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:55:28.851 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:55:29.035 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:55:29.280 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:55:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: Upgrading : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:55:29.577 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/25 2023-10-09T03:55:29.599 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:55:29.819 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:55:29.847 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-10-09T03:55:29.900 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:55:30.010 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:55:30.051 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:55:30.243 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-10-09T03:55:31.552 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-10-09T03:55:31.759 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:55:31.824 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:55:31.954 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 11/25 2023-10-09T03:55:32.129 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:55:32.405 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:55:32.618 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 14/25 2023-10-09T03:55:32.776 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:55:32.849 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 15/25 2023-10-09T03:55:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-09T03:55:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-09T03:55:33.507 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-09T03:55:33.719 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 19/25 2023-10-09T03:55:39.442 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:55:41.893 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:55:42.534 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 20/25 2023-10-09T03:55:42.807 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:55:42.898 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 21/25 2023-10-09T03:55:43.108 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:55:48.740 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:55:48.741 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /sys 2023-10-09T03:55:48.741 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /proc 2023-10-09T03:55:48.741 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /dev 2023-10-09T03:55:48.741 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /run 2023-10-09T03:55:48.742 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /mnt 2023-10-09T03:55:48.742 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /var/tmp 2023-10-09T03:55:48.742 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /home 2023-10-09T03:55:48.742 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /tmp 2023-10-09T03:55:48.742 INFO:teuthology.orchestra.run.smithi177.stdout:skipping the directory /dev 2023-10-09T03:55:48.742 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:55:50.424 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:55:50.499 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:55:50.499 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:55:50.691 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:55:50.691 INFO:teuthology.orchestra.run.smithi177.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:55:51.251 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:55:51.252 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T03:55:51.252 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:55:51.252 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:55:51.252 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T03:55:51.252 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:55:51.253 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T03:55:51.253 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T03:55:51.253 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T03:55:51.253 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T03:55:51.253 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:55:51.253 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T03:55:51.254 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:55:51.254 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:55:51.254 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T03:55:51.254 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T03:55:51.254 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T03:55:51.254 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T03:55:51.255 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T03:55:52.613 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:55:52.614 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:00:50 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:55:58.252 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:55:58.444 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout:Upgraded: 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T03:55:58.445 INFO:teuthology.orchestra.run.smithi177.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T03:55:58.446 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.447 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.447 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:55:58.447 INFO:teuthology.orchestra.run.smithi177.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T03:55:58.447 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:55:58.447 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:55:58.623 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-test 2023-10-09T03:55:58.969 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:56:00.305 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:00:58 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:01.664 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:56:01.665 INFO:teuthology.orchestra.run.smithi177.stdout:============================================================================================ 2023-10-09T03:56:01.665 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:56:01.665 INFO:teuthology.orchestra.run.smithi177.stdout:============================================================================================ 2023-10-09T03:56:01.665 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:56:01.665 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout:============================================================================================ 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout:Install 6 Packages 2023-10-09T03:56:01.666 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:01.667 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 53 M 2023-10-09T03:56:01.667 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 233 M 2023-10-09T03:56:01.667 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:56:01.700 INFO:teuthology.orchestra.run.smithi177.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.1 MB/s | 70 kB 00:00 2023-10-09T03:56:01.859 INFO:teuthology.orchestra.run.smithi177.stdout:(2/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 915 kB/s | 175 kB 00:00 2023-10-09T03:56:01.884 INFO:teuthology.orchestra.run.smithi177.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.8 MB/s | 323 kB 00:00 2023-10-09T03:56:01.959 INFO:teuthology.orchestra.run.smithi177.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 2.8 MB/s | 202 kB 00:00 2023-10-09T03:56:01.993 INFO:teuthology.orchestra.run.smithi177.stdout:(5/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.4 MB/s | 187 kB 00:00 2023-10-09T03:56:04.276 INFO:teuthology.orchestra.run.smithi177.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 20 MB/s | 52 MB 00:02 2023-10-09T03:56:04.278 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:56:04.278 INFO:teuthology.orchestra.run.smithi177.stdout:Total 20 MB/s | 53 MB 00:02 2023-10-09T03:56:04.315 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:56:04.336 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:56:04.336 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:56:04.378 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:56:04.378 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /sys 2023-10-09T03:56:04.378 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /proc 2023-10-09T03:56:04.378 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /dev 2023-10-09T03:56:04.378 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /run 2023-10-09T03:56:04.379 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /mnt 2023-10-09T03:56:04.379 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /var/tmp 2023-10-09T03:56:04.379 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /home 2023-10-09T03:56:04.379 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /tmp 2023-10-09T03:56:04.379 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /dev 2023-10-09T03:56:04.379 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:04.609 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:56:04.613 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:56:05.070 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:56:05.178 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:56:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:56:05.465 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T03:56:05.603 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T03:56:05.793 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T03:56:05.953 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:56:06.171 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:56:06.235 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 23/25 2023-10-09T03:56:06.235 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:56:06.508 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-09T03:56:06.508 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:56:07.140 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:56:07.140 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/25 2023-10-09T03:56:07.140 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/25 2023-10-09T03:56:07.140 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/25 2023-10-09T03:56:07.140 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_6 6/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 7/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8. 8/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x8 9/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 10/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 11/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 12/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 13/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-10-09T03:56:07.141 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 22/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-09T03:56:07.142 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 24/25 2023-10-09T03:56:08.603 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-09T03:56:08.603 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:01 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:56:10.848 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:56:11.074 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:56:11.307 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:56:11.307 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T03:56:11.307 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T03:56:11.307 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T03:56:11.308 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T03:56:11.308 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T03:56:12.390 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T03:56:12.390 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:10 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:13.606 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:56:13.813 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:13.813 INFO:teuthology.orchestra.run.smithi029.stdout:Upgraded: 2023-10-09T03:56:13.813 INFO:teuthology.orchestra.run.smithi029.stdout: librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-radosgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-selinux-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-09T03:56:13.814 INFO:teuthology.orchestra.run.smithi029.stdout: libradosstriper1-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: librgw2-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-argparse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-common-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:13.815 INFO:teuthology.orchestra.run.smithi029.stdout: thrift-0.13.0-2.el8.x86_64 2023-10-09T03:56:13.816 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:13.816 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:56:13.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-test 2023-10-09T03:56:14.364 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:56:15.888 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:08 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:56:17.426 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:56:17.427 INFO:teuthology.orchestra.run.smithi029.stdout:============================================================================================ 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout:============================================================================================ 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-test x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 52 M 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: libcephsqlite x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 175 k 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-09T03:56:17.428 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:17.429 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:56:17.429 INFO:teuthology.orchestra.run.smithi029.stdout:============================================================================================ 2023-10-09T03:56:17.429 INFO:teuthology.orchestra.run.smithi029.stdout:Install 6 Packages 2023-10-09T03:56:17.429 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:17.429 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 53 M 2023-10-09T03:56:17.429 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 233 M 2023-10-09T03:56:17.430 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:56:17.478 INFO:teuthology.orchestra.run.smithi029.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.4 MB/s | 70 kB 00:00 2023-10-09T03:56:17.662 INFO:teuthology.orchestra.run.smithi029.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.8 MB/s | 323 kB 00:00 2023-10-09T03:56:17.738 INFO:teuthology.orchestra.run.smithi029.stdout:(3/6): libcephsqlite-17.2.6-1303.gb2903cb0.el8. 568 kB/s | 175 kB 00:00 2023-10-09T03:56:17.814 INFO:teuthology.orchestra.run.smithi029.stdout:(4/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.2 MB/s | 187 kB 00:00 2023-10-09T03:56:17.914 INFO:teuthology.orchestra.run.smithi029.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 1.1 MB/s | 202 kB 00:00 2023-10-09T03:56:18.123 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:56:18.309 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:18.309 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:56:18.309 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:18.309 INFO:teuthology.orchestra.run.smithi177.stdout: jq-1.6-3.el8.x86_64 2023-10-09T03:56:18.309 INFO:teuthology.orchestra.run.smithi177.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:18.309 INFO:teuthology.orchestra.run.smithi177.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T03:56:18.310 INFO:teuthology.orchestra.run.smithi177.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T03:56:18.310 INFO:teuthology.orchestra.run.smithi177.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T03:56:18.310 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:18.310 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:56:18.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph 2023-10-09T03:56:18.886 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:56:20.331 INFO:teuthology.orchestra.run.smithi029.stdout:(6/6): ceph-test-17.2.6-1303.gb2903cb0.el8.x86_ 18 MB/s | 52 MB 00:02 2023-10-09T03:56:20.333 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:56:20.333 INFO:teuthology.orchestra.run.smithi029.stdout:Total 18 MB/s | 53 MB 00:02 2023-10-09T03:56:20.372 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:56:20.396 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:56:20.396 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:56:20.445 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:18 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:20.686 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:56:20.690 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:56:21.149 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:56:21.295 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:56:21.383 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-09T03:56:21.647 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-09T03:56:21.799 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================= 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================= 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T03:56:21.803 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T03:56:21.804 INFO:teuthology.orchestra.run.smithi177.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T03:56:21.805 INFO:teuthology.orchestra.run.smithi177.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================= 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout:Install 32 Packages 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:21.806 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 28 M 2023-10-09T03:56:21.807 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 110 M 2023-10-09T03:56:21.807 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:56:21.821 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-09T03:56:22.003 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-09T03:56:22.160 INFO:teuthology.orchestra.run.smithi177.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 35 kB/s | 6.4 kB 00:00 2023-10-09T03:56:22.212 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:56:22.301 INFO:teuthology.orchestra.run.smithi177.stdout:(2/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 7.1 MB/s | 2.3 MB 00:00 2023-10-09T03:56:22.594 INFO:teuthology.orchestra.run.smithi177.stdout:(3/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 2.6 MB/s | 1.6 MB 00:00 2023-10-09T03:56:22.685 INFO:teuthology.orchestra.run.smithi177.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 2.5 MB/s | 238 kB 00:00 2023-10-09T03:56:22.828 INFO:teuthology.orchestra.run.smithi177.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 307 kB/s | 44 kB 00:00 2023-10-09T03:56:22.953 INFO:teuthology.orchestra.run.smithi177.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.4 MB/s | 185 kB 00:00 2023-10-09T03:56:23.019 INFO:teuthology.orchestra.run.smithi177.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.5 MB/s | 173 kB 00:00 2023-10-09T03:56:23.187 INFO:teuthology.orchestra.run.smithi177.stdout:(8/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 19 MB/s | 17 MB 00:00 2023-10-09T03:56:23.212 INFO:teuthology.orchestra.run.smithi177.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 2.0 MB/s | 384 kB 00:00 2023-10-09T03:56:23.262 INFO:teuthology.orchestra.run.smithi177.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 351 kB/s | 18 kB 00:00 2023-10-09T03:56:23.288 INFO:teuthology.orchestra.run.smithi177.stdout:(11/32): python3-jaraco-6.2-6.el8.noarch.rpm 110 kB/s | 11 kB 00:00 2023-10-09T03:56:23.321 INFO:teuthology.orchestra.run.smithi177.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 832 kB/s | 49 kB 00:00 2023-10-09T03:56:23.405 INFO:teuthology.orchestra.run.smithi177.stdout:(13/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.3 MB/s | 283 kB 00:00 2023-10-09T03:56:23.430 INFO:teuthology.orchestra.run.smithi177.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 413 kB/s | 59 kB 00:00 2023-10-09T03:56:23.455 INFO:teuthology.orchestra.run.smithi177.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 311 kB/s | 16 kB 00:00 2023-10-09T03:56:23.489 INFO:teuthology.orchestra.run.smithi177.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 319 kB/s | 19 kB 00:00 2023-10-09T03:56:23.688 INFO:teuthology.orchestra.run.smithi177.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 103 kB/s | 24 kB 00:00 2023-10-09T03:56:23.849 INFO:teuthology.orchestra.run.smithi177.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 81 kB/s | 29 kB 00:00 2023-10-09T03:56:23.961 INFO:teuthology.orchestra.run.smithi177.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 101 kB/s | 27 kB 00:00 2023-10-09T03:56:24.274 INFO:teuthology.orchestra.run.smithi177.stdout:(20/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86 1.9 MB/s | 3.9 MB 00:02 2023-10-09T03:56:24.470 INFO:teuthology.orchestra.run.smithi177.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 411 kB/s | 255 kB 00:00 2023-10-09T03:56:24.500 INFO:teuthology.orchestra.run.smithi177.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 466 kB/s | 251 kB 00:00 2023-10-09T03:56:24.526 INFO:teuthology.orchestra.run.smithi177.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 420 kB/s | 23 kB 00:00 2023-10-09T03:56:24.617 INFO:teuthology.orchestra.run.smithi177.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 249 kB/s | 85 kB 00:00 2023-10-09T03:56:24.651 INFO:teuthology.orchestra.run.smithi177.stdout:(25/32): python3-markupsafe-0.23-19.el8.x86_64. 325 kB/s | 39 kB 00:00 2023-10-09T03:56:24.684 INFO:teuthology.orchestra.run.smithi177.stdout:(26/32): python3-pytz-2017.2-9.el8.noarch.rpm 294 kB/s | 54 kB 00:00 2023-10-09T03:56:24.801 INFO:teuthology.orchestra.run.smithi177.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 853 kB/s | 157 kB 00:00 2023-10-09T03:56:24.827 INFO:teuthology.orchestra.run.smithi177.stdout:(28/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.6 MB/s | 457 kB 00:00 2023-10-09T03:56:24.852 INFO:teuthology.orchestra.run.smithi177.stdout:(29/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 619 kB/s | 103 kB 00:00 2023-10-09T03:56:24.960 INFO:teuthology.orchestra.run.smithi177.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 686 kB/s | 109 kB 00:00 2023-10-09T03:56:24.994 INFO:teuthology.orchestra.run.smithi177.stdout:(31/32): python3-cryptography-3.2.1-5.el8.x86_6 3.3 MB/s | 559 kB 00:00 2023-10-09T03:56:25.069 INFO:teuthology.orchestra.run.smithi177.stdout:(32/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.1 MB/s | 238 kB 00:00 2023-10-09T03:56:25.070 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:56:25.071 INFO:teuthology.orchestra.run.smithi177.stdout:Total 8.7 MB/s | 28 MB 00:03 2023-10-09T03:56:25.163 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:56:25.200 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:56:25.200 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:56:25.404 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:56:25.408 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:56:26.479 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:56:26.729 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T03:56:26.944 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T03:56:27.142 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-10-09T03:56:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-10-09T03:56:27.515 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/6 2023-10-09T03:56:27.759 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:56:27.759 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-10-09T03:56:27.931 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-10-09T03:56:27.991 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 6/6 2023-10-09T03:56:27.991 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/6 2023-10-09T03:56:27.991 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/6 2023-10-09T03:56:27.991 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-09T03:56:27.991 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-09T03:56:27.991 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-09T03:56:28.099 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-10-09T03:56:28.398 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-10-09T03:56:28.602 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-10-09T03:56:28.808 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-10-09T03:56:29.047 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-10-09T03:56:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-09T03:56:29.110 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:22 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:56:29.214 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-10-09T03:56:29.430 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T03:56:29.588 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T03:56:29.819 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T03:56:29.990 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T03:56:30.142 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T03:56:30.279 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T03:56:30.516 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T03:56:30.784 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T03:56:30.953 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T03:56:31.190 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T03:56:31.426 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T03:56:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T03:56:31.939 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T03:56:32.294 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T03:56:32.637 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T03:56:32.824 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:56:34.303 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:56:34.508 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:56:34.908 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:56:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:56:35.289 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:56:35.447 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:56:35.465 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:56:35.587 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:56:35.777 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:56:35.935 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:56:35.935 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T03:56:35.936 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T03:56:35.937 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-10-09T03:56:35.938 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-test-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: jq-1.6-3.el8.x86_64 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: libcephsqlite-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-09T03:56:35.978 INFO:teuthology.orchestra.run.smithi029.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-09T03:56:35.979 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:35.979 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:56:36.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph 2023-10-09T03:56:36.487 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:56:37.470 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-10-09T03:56:37.471 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:35 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:38.124 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:31 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:56:39.657 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:56:39.660 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================= 2023-10-09T03:56:39.660 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:56:39.660 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================= 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: ceph x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 6.4 k 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mds x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 2.3 M 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 1.6 M 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 238 k 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mon x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.9 M 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-osd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 17 M 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-09T03:56:39.661 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-09T03:56:39.662 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:39.663 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:56:39.664 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================= 2023-10-09T03:56:39.664 INFO:teuthology.orchestra.run.smithi029.stdout:Install 32 Packages 2023-10-09T03:56:39.664 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:56:39.664 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 28 M 2023-10-09T03:56:39.664 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 110 M 2023-10-09T03:56:39.664 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:56:40.047 INFO:teuthology.orchestra.run.smithi029.stdout:(1/32): ceph-17.2.6-1303.gb2903cb0.el8.x86_64.r 29 kB/s | 6.4 kB 00:00 2023-10-09T03:56:40.239 INFO:teuthology.orchestra.run.smithi029.stdout:(2/32): ceph-mds-17.2.6-1303.gb2903cb0.el8.x86_ 5.6 MB/s | 2.3 MB 00:00 2023-10-09T03:56:40.708 INFO:teuthology.orchestra.run.smithi029.stdout:(3/32): ceph-mgr-17.2.6-1303.gb2903cb0.el8.x86_ 1.8 MB/s | 1.6 MB 00:00 2023-10-09T03:56:40.842 INFO:teuthology.orchestra.run.smithi029.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1303.gb290 1.7 MB/s | 238 kB 00:00 2023-10-09T03:56:41.209 INFO:teuthology.orchestra.run.smithi029.stdout:(5/32): ceph-osd-17.2.6-1303.gb2903cb0.el8.x86_ 17 MB/s | 17 MB 00:00 2023-10-09T03:56:41.235 INFO:teuthology.orchestra.run.smithi029.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 111 kB/s | 44 kB 00:00 2023-10-09T03:56:41.344 INFO:teuthology.orchestra.run.smithi029.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.6 MB/s | 173 kB 00:00 2023-10-09T03:56:41.470 INFO:teuthology.orchestra.run.smithi029.stdout:(8/32): ceph-mon-17.2.6-1303.gb2903cb0.el8.x86_ 2.8 MB/s | 3.9 MB 00:01 2023-10-09T03:56:41.504 INFO:teuthology.orchestra.run.smithi029.stdout:(9/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 632 kB/s | 185 kB 00:00 2023-10-09T03:56:41.537 INFO:teuthology.orchestra.run.smithi029.stdout:(10/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.9 MB/s | 384 kB 00:00 2023-10-09T03:56:41.564 INFO:teuthology.orchestra.run.smithi029.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 324 kB/s | 18 kB 00:00 2023-10-09T03:56:41.597 INFO:teuthology.orchestra.run.smithi029.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 837 kB/s | 49 kB 00:00 2023-10-09T03:56:41.622 INFO:teuthology.orchestra.run.smithi029.stdout:(13/32): python3-jaraco-6.2-6.el8.noarch.rpm 73 kB/s | 11 kB 00:00 2023-10-09T03:56:41.647 INFO:teuthology.orchestra.run.smithi029.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 695 kB/s | 59 kB 00:00 2023-10-09T03:56:41.688 INFO:teuthology.orchestra.run.smithi029.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.3 MB/s | 283 kB 00:00 2023-10-09T03:56:41.708 INFO:teuthology.orchestra.run.smithi029.stdout:(16/32): python3-portend-2.6-1.el8.noarch.rpm 185 kB/s | 16 kB 00:00 2023-10-09T03:56:41.737 INFO:teuthology.orchestra.run.smithi029.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 222 kB/s | 19 kB 00:00 2023-10-09T03:56:41.760 INFO:teuthology.orchestra.run.smithi029.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 308 kB/s | 24 kB 00:00 2023-10-09T03:56:41.786 INFO:teuthology.orchestra.run.smithi029.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 376 kB/s | 29 kB 00:00 2023-10-09T03:56:41.810 INFO:teuthology.orchestra.run.smithi029.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 354 kB/s | 27 kB 00:00 2023-10-09T03:56:41.895 INFO:teuthology.orchestra.run.smithi029.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.9 MB/s | 251 kB 00:00 2023-10-09T03:56:41.920 INFO:teuthology.orchestra.run.smithi029.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 784 kB/s | 85 kB 00:00 2023-10-09T03:56:41.945 INFO:teuthology.orchestra.run.smithi029.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 453 kB/s | 23 kB 00:00 2023-10-09T03:56:41.979 INFO:teuthology.orchestra.run.smithi029.stdout:(24/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.1 MB/s | 255 kB 00:00 2023-10-09T03:56:42.038 INFO:teuthology.orchestra.run.smithi029.stdout:(25/32): python3-pytz-2017.2-9.el8.noarch.rpm 460 kB/s | 54 kB 00:00 2023-10-09T03:56:42.102 INFO:teuthology.orchestra.run.smithi029.stdout:(26/32): python3-markupsafe-0.23-19.el8.x86_64. 262 kB/s | 39 kB 00:00 2023-10-09T03:56:42.148 INFO:teuthology.orchestra.run.smithi029.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 938 kB/s | 157 kB 00:00 2023-10-09T03:56:42.190 INFO:teuthology.orchestra.run.smithi029.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.2 MB/s | 103 kB 00:00 2023-10-09T03:56:42.232 INFO:teuthology.orchestra.run.smithi029.stdout:(29/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.3 MB/s | 457 kB 00:00 2023-10-09T03:56:42.282 INFO:teuthology.orchestra.run.smithi029.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 811 kB/s | 109 kB 00:00 2023-10-09T03:56:42.324 INFO:teuthology.orchestra.run.smithi029.stdout:(31/32): python3-cryptography-3.2.1-5.el8.x86_6 4.1 MB/s | 559 kB 00:00 2023-10-09T03:56:42.358 INFO:teuthology.orchestra.run.smithi029.stdout:(32/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.8 MB/s | 238 kB 00:00 2023-10-09T03:56:42.362 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:56:42.363 INFO:teuthology.orchestra.run.smithi029.stdout:Total 10 MB/s | 28 MB 00:02 2023-10-09T03:56:42.459 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:56:42.499 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:56:42.499 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:56:42.719 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:56:42.723 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:56:42.886 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:56:43.072 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:43.072 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:56:43.072 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:43.072 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:43.072 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T03:56:43.073 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T03:56:43.074 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T03:56:43.074 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T03:56:43.074 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T03:56:43.074 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T03:56:43.074 INFO:teuthology.orchestra.run.smithi177.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T03:56:43.074 INFO:teuthology.orchestra.run.smithi177.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T03:56:43.075 INFO:teuthology.orchestra.run.smithi177.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T03:56:43.076 INFO:teuthology.orchestra.run.smithi177.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T03:56:43.077 INFO:teuthology.orchestra.run.smithi177.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T03:56:43.077 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:43.077 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:56:43.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-base 2023-10-09T03:56:43.576 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:56:43.872 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:56:44.163 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-09T03:56:44.424 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-09T03:56:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-10-09T03:56:45.079 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-10-09T03:56:45.094 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:43 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:45.414 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-10-09T03:56:45.584 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-10-09T03:56:45.778 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-10-09T03:56:46.113 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-10-09T03:56:46.331 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-10-09T03:56:46.418 INFO:teuthology.orchestra.run.smithi177.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:56:46.526 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:56:46.527 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:56:46.527 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:56:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-10-09T03:56:46.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install cephadm 2023-10-09T03:56:46.866 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-10-09T03:56:47.013 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:56:47.051 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-10-09T03:56:47.294 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-09T03:56:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-09T03:56:47.688 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-09T03:56:47.855 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-09T03:56:48.062 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-09T03:56:48.198 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-09T03:56:48.441 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-09T03:56:48.556 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:46 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:48.722 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-09T03:56:48.931 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-09T03:56:49.202 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-09T03:56:49.511 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-09T03:56:49.746 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-09T03:56:49.994 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:56:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:49.996 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 80 k 2023-10-09T03:56:49.996 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 364 k 2023-10-09T03:56:49.996 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:56:50.127 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-09T03:56:50.250 INFO:teuthology.orchestra.run.smithi177.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 316 kB/s | 80 kB 00:00 2023-10-09T03:56:50.250 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:56:50.250 INFO:teuthology.orchestra.run.smithi177.stdout:Total 315 kB/s | 80 kB 00:00 2023-10-09T03:56:50.251 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:56:50.259 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:56:50.259 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:56:50.260 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:56:50.264 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:56:50.304 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:56:50.437 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-09T03:56:50.838 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 27/32 2023-10-09T03:56:51.024 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:56:51.968 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:56:52.141 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:56:52.310 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:56:52.597 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 28/32 2023-10-09T03:56:52.824 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:56:53.257 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 29/32 2023-10-09T03:56:53.400 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:56:53.410 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:56:53.411 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:51 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:56:53.629 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 30/32 2023-10-09T03:56:53.824 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:56:53.844 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 31/32 2023-10-09T03:56:53.982 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 32/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 3/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 4/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 5/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8. 6/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-09T03:56:54.346 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-09T03:56:54.347 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-10-09T03:56:54.348 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-10-09T03:56:54.349 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-10-09T03:56:55.968 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-10-09T03:56:55.968 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:48 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:56:58.969 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:56:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:59.150 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:56:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:56:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:56:59.150 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:56:59.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T03:56:59.649 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:01.178 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:59 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:02.532 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repo Size 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:57:02.533 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:57:02.534 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:57:02.534 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:02.534 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 161 k 2023-10-09T03:57:02.534 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 490 k 2023-10-09T03:57:02.534 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:57:02.825 INFO:teuthology.orchestra.run.smithi177.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 557 kB/s | 161 kB 00:00 2023-10-09T03:57:02.848 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:02.849 INFO:teuthology.orchestra.run.smithi177.stdout:Total 555 kB/s | 161 kB 00:00 2023-10-09T03:57:02.849 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:57:02.849 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:57:02.849 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:57:02.887 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:57:02.891 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:57:03.003 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:57:03.194 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:57:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:57:04.595 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:57:04.595 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:02 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:05.458 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:57:05.664 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:05.664 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mds-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-modules-core-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mon-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-osd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-09T03:57:05.665 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-portend-2.6-1.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-09T03:57:05.666 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:05.667 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:57:05.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-base 2023-10-09T03:57:06.193 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:08.077 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:01 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:09.499 INFO:teuthology.orchestra.run.smithi029.stdout:Package ceph-base-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:57:09.598 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:57:09.615 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:57:09.617 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:57:09.617 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:57:09.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install cephadm 2023-10-09T03:57:09.780 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:09.780 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:57:09.780 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:09.780 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:09.780 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:57:09.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-mgr 2023-10-09T03:57:10.114 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:10.263 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:11.201 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:04 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:11.298 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:09 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:12.620 INFO:teuthology.orchestra.run.smithi177.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:57:12.712 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout: cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 80 k 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:57:12.713 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:57:12.714 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:57:12.714 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:12.714 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 80 k 2023-10-09T03:57:12.714 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 364 k 2023-10-09T03:57:12.714 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:57:12.727 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:57:12.729 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:57:12.729 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:57:12.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T03:57:13.011 INFO:teuthology.orchestra.run.smithi029.stdout:cephadm-17.2.6-1303.gb2903cb0.el8.noarch.rpm 271 kB/s | 80 kB 00:00 2023-10-09T03:57:13.011 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:13.011 INFO:teuthology.orchestra.run.smithi029.stdout:Total 270 kB/s | 80 kB 00:00 2023-10-09T03:57:13.012 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:57:13.021 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:57:13.021 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:57:13.023 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:57:13.027 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:57:13.069 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:57:13.194 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:14.382 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:12 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:14.972 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:57:15.148 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:57:15.324 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:57:15.738 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout:=================================================================================================== 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout:=================================================================================================== 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T03:57:15.739 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout:=================================================================================================== 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout:Install 6 Packages 2023-10-09T03:57:15.740 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:15.741 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 2.0 M 2023-10-09T03:57:15.741 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 8.0 M 2023-10-09T03:57:15.741 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:57:15.952 INFO:teuthology.orchestra.run.smithi177.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1303.gb29 164 kB/s | 23 kB 00:00 2023-10-09T03:57:16.061 INFO:teuthology.orchestra.run.smithi177.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1303.gb290 58 kB/s | 14 kB 00:00 2023-10-09T03:57:16.169 INFO:teuthology.orchestra.run.smithi177.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 155 kB/s | 34 kB 00:00 2023-10-09T03:57:16.237 INFO:teuthology.orchestra.run.smithi177.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 644 kB/s | 43 kB 00:00 2023-10-09T03:57:16.270 INFO:teuthology.orchestra.run.smithi177.stdout:(5/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 3.7 MB/s | 1.7 MB 00:00 2023-10-09T03:57:16.362 INFO:teuthology.orchestra.run.smithi177.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 655 kB/s | 196 kB 00:00 2023-10-09T03:57:16.365 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:16.365 INFO:teuthology.orchestra.run.smithi177.stdout:Total 3.2 MB/s | 2.0 MB 00:00 2023-10-09T03:57:16.376 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:57:16.385 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:57:16.385 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:57:16.441 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:57:16.445 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:57:16.518 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/1 2023-10-09T03:57:16.518 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:09 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:16.773 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:57:16.933 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T03:57:17.129 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T03:57:17.326 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T03:57:17.446 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T03:57:17.765 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T03:57:17.960 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:57:18.204 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:57:18.204 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T03:57:18.204 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T03:57:18.205 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T03:57:18.205 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T03:57:18.205 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T03:57:19.265 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T03:57:19.265 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:17 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:19.989 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:57:20.187 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:20.187 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:57:20.188 INFO:teuthology.orchestra.run.smithi029.stdout: cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:20.188 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:20.188 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:57:20.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-immutable-object-cache 2023-10-09T03:57:20.767 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:21.874 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:14 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:23.330 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:57:23.379 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:57:23.380 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:57:23.380 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2023-10-09T03:57:23.380 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:57:23.380 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:57:23.380 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 161 k 2023-10-09T03:57:23.380 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 161 k 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 490 k 2023-10-09T03:57:23.381 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T03:57:23.512 INFO:teuthology.orchestra.run.smithi177.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T03:57:23.513 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:23.513 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:57:23.681 INFO:teuthology.orchestra.run.smithi029.stdout:ceph-immutable-object-cache-17.2.6-1303.gb2903c 538 kB/s | 161 kB 00:00 2023-10-09T03:57:23.682 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:23.682 INFO:teuthology.orchestra.run.smithi029.stdout:Total 536 kB/s | 161 kB 00:00 2023-10-09T03:57:23.682 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:57:23.694 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:57:23.694 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:57:23.752 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:57:23.756 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:57:23.875 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:57:23.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T03:57:24.096 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:57:24.306 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:24.465 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:57:25.581 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0. 1/1 2023-10-09T03:57:25.581 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:18 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:25.850 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:23 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:27.272 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:57:27.273 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================================= 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================================= 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T03:57:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T03:57:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T03:57:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:27.275 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:57:27.275 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================================= 2023-10-09T03:57:27.275 INFO:teuthology.orchestra.run.smithi177.stdout:Install 8 Packages 2023-10-09T03:57:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:27.276 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 35 M 2023-10-09T03:57:27.276 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 210 M 2023-10-09T03:57:27.276 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:57:27.475 INFO:teuthology.orchestra.run.smithi177.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 19 MB/s | 3.7 MB 00:00 2023-10-09T03:57:27.592 INFO:teuthology.orchestra.run.smithi177.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.9 MB/s | 225 kB 00:00 2023-10-09T03:57:27.793 INFO:teuthology.orchestra.run.smithi177.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 27 MB/s | 14 MB 00:00 2023-10-09T03:57:27.893 INFO:teuthology.orchestra.run.smithi177.stdout:(4/8): ceph-mgr-diskprediction-local-17.2.6-130 12 MB/s | 7.4 MB 00:00 2023-10-09T03:57:28.001 INFO:teuthology.orchestra.run.smithi177.stdout:(5/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.9 MB 00:00 2023-10-09T03:57:28.085 INFO:teuthology.orchestra.run.smithi177.stdout:(6/8): openblas-0.3.15-3.el8.x86_64.rpm 16 MB/s | 4.7 MB 00:00 2023-10-09T03:57:28.119 INFO:teuthology.orchestra.run.smithi177.stdout:(7/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.8 MB/s | 644 kB 00:00 2023-10-09T03:57:28.194 INFO:teuthology.orchestra.run.smithi177.stdout:(8/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 891 kB/s | 171 kB 00:00 2023-10-09T03:57:28.195 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:28.195 INFO:teuthology.orchestra.run.smithi177.stdout:Total 38 MB/s | 35 MB 00:00 2023-10-09T03:57:28.432 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:57:28.450 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:57:28.450 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:57:28.743 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:57:28.747 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:57:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:57:29.454 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:57:29.591 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:57:29.737 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:57:29.862 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:57:30.058 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:30.058 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:57:30.059 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-immutable-object-cache-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:57:30.059 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:30.059 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:57:30.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr 2023-10-09T03:57:30.350 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:57:30.487 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:57:30.570 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:31.083 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:57:31.270 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:57:31.956 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:57:32.151 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:25 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:32.256 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T03:57:32.470 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:57:33.589 INFO:teuthology.orchestra.run.smithi029.stdout:Package ceph-mgr-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:57:33.712 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:57:33.713 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:57:33.713 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:57:33.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-dashboard 2023-10-09T03:57:34.216 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:34.234 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:57:35.460 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T03:57:35.603 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:57:35.610 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:28 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:35.892 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:57:35.892 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T03:57:35.892 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-10-09T03:57:35.892 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-10-09T03:57:35.893 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T03:57:35.893 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-10-09T03:57:35.893 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-10-09T03:57:35.893 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-10-09T03:57:36.971 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-10-09T03:57:36.971 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:34 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:37.132 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:57:37.133 INFO:teuthology.orchestra.run.smithi029.stdout:=================================================================================================== 2023-10-09T03:57:37.133 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:57:37.133 INFO:teuthology.orchestra.run.smithi029.stdout:=================================================================================================== 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 1.7 M 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 23 k 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 14 k 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout:=================================================================================================== 2023-10-09T03:57:37.134 INFO:teuthology.orchestra.run.smithi029.stdout:Install 6 Packages 2023-10-09T03:57:37.135 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:37.135 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 2.0 M 2023-10-09T03:57:37.135 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 8.0 M 2023-10-09T03:57:37.135 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:57:37.342 INFO:teuthology.orchestra.run.smithi029.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1303.gb29 151 kB/s | 23 kB 00:00 2023-10-09T03:57:37.368 INFO:teuthology.orchestra.run.smithi029.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1303.gb290 81 kB/s | 14 kB 00:00 2023-10-09T03:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1303.gb2903cb0 5.0 MB/s | 1.7 MB 00:00 2023-10-09T03:57:37.736 INFO:teuthology.orchestra.run.smithi029.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 204 kB/s | 43 kB 00:00 2023-10-09T03:57:37.811 INFO:teuthology.orchestra.run.smithi029.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 72 kB/s | 34 kB 00:00 2023-10-09T03:57:38.160 INFO:teuthology.orchestra.run.smithi029.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 248 kB/s | 196 kB 00:00 2023-10-09T03:57:38.163 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:38.164 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.9 MB/s | 2.0 MB 00:01 2023-10-09T03:57:38.177 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:57:38.188 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:57:38.188 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:57:38.254 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:57:38.259 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:57:38.577 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:57:38.821 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-09T03:57:39.039 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-09T03:57:39.239 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-09T03:57:39.352 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 4/6 2023-10-09T03:57:39.757 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 5/6 2023-10-09T03:57:39.962 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:57:40.251 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 6/6 2023-10-09T03:57:40.251 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8. 1/6 2023-10-09T03:57:40.252 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarc 2/6 2023-10-09T03:57:40.252 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.n 3/6 2023-10-09T03:57:40.252 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-09T03:57:40.252 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-09T03:57:41.345 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-09T03:57:41.345 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:34 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:42.818 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:57:43.002 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:43.002 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:57:43.002 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:43.002 INFO:teuthology.orchestra.run.smithi177.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:57:43.002 INFO:teuthology.orchestra.run.smithi177.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:57:43.002 INFO:teuthology.orchestra.run.smithi177.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T03:57:43.003 INFO:teuthology.orchestra.run.smithi177.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T03:57:43.003 INFO:teuthology.orchestra.run.smithi177.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T03:57:43.003 INFO:teuthology.orchestra.run.smithi177.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T03:57:43.003 INFO:teuthology.orchestra.run.smithi177.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T03:57:43.003 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:43.003 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:57:43.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-mgr-rook 2023-10-09T03:57:43.502 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:45.012 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:43 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:46.401 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:57:46.402 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================== 2023-10-09T03:57:46.402 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================== 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T03:57:46.403 INFO:teuthology.orchestra.run.smithi177.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout:======================================================================================================== 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout:Install 13 Packages 2023-10-09T03:57:46.404 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:46.405 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 1.9 M 2023-10-09T03:57:46.405 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 25 M 2023-10-09T03:57:46.405 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:57:46.690 INFO:teuthology.orchestra.run.smithi177.stdout:(1/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 190 kB/s | 47 kB 00:00 2023-10-09T03:57:47.498 INFO:teuthology.orchestra.run.smithi177.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 15 kB/s | 16 kB 00:01 2023-10-09T03:57:47.524 INFO:teuthology.orchestra.run.smithi177.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 30 kB/s | 33 kB 00:01 2023-10-09T03:57:47.558 INFO:teuthology.orchestra.run.smithi177.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 94 kB/s | 81 kB 00:00 2023-10-09T03:57:47.633 INFO:teuthology.orchestra.run.smithi177.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 620 kB/s | 64 kB 00:00 2023-10-09T03:57:47.658 INFO:teuthology.orchestra.run.smithi177.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 643 kB/s | 61 kB 00:00 2023-10-09T03:57:47.700 INFO:teuthology.orchestra.run.smithi177.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 5.8 MB/s | 1.2 MB 00:00 2023-10-09T03:57:47.867 INFO:teuthology.orchestra.run.smithi177.stdout:(8/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 811 kB/s | 126 kB 00:00 2023-10-09T03:57:47.892 INFO:teuthology.orchestra.run.smithi177.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 425 kB/s | 110 kB 00:00 2023-10-09T03:57:47.917 INFO:teuthology.orchestra.run.smithi177.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 103 kB/s | 27 kB 00:00 2023-10-09T03:57:48.005 INFO:teuthology.orchestra.run.smithi177.stdout:(11/13): python3-jsonpointer-1.10-11.el8.noarch 145 kB/s | 20 kB 00:00 2023-10-09T03:57:48.051 INFO:teuthology.orchestra.run.smithi177.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 269 kB/s | 43 kB 00:00 2023-10-09T03:57:48.076 INFO:teuthology.orchestra.run.smithi177.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 977 kB/s | 155 kB 00:00 2023-10-09T03:57:48.077 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:48.078 INFO:teuthology.orchestra.run.smithi177.stdout:Total 1.2 MB/s | 1.9 MB 00:01 2023-10-09T03:57:48.144 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:57:48.160 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:57:48.160 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:57:48.174 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:57:48.232 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:57:48.236 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:57:48.376 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-grafana-dashboards-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-dashboard-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-prometheus-alerts-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:48.377 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:57:48.709 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:57:48.909 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T03:57:49.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-09T03:57:49.128 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T03:57:49.352 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T03:57:49.400 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-10-09T03:57:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-10-09T03:57:50.041 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-10-09T03:57:50.204 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-10-09T03:57:50.442 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:43 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:57:50.450 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T03:57:50.599 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T03:57:50.790 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T03:57:51.861 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T03:57:51.972 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================================= 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================================= 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 7.4 M 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-09T03:57:51.973 INFO:teuthology.orchestra.run.smithi029.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================================= 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout:Install 8 Packages 2023-10-09T03:57:51.974 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:57:51.975 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 35 M 2023-10-09T03:57:51.975 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 210 M 2023-10-09T03:57:51.975 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:57:52.084 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T03:57:52.217 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:57:52.276 INFO:teuthology.orchestra.run.smithi029.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 12 MB/s | 3.7 MB 00:00 2023-10-09T03:57:52.410 INFO:teuthology.orchestra.run.smithi029.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.6 MB/s | 225 kB 00:00 2023-10-09T03:57:52.457 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:57:52.457 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T03:57:52.457 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T03:57:52.457 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T03:57:52.457 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T03:57:52.457 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T03:57:52.458 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-10-09T03:57:52.761 INFO:teuthology.orchestra.run.smithi029.stdout:(3/8): ceph-mgr-diskprediction-local-17.2.6-130 9.4 MB/s | 7.4 MB 00:00 2023-10-09T03:57:53.011 INFO:teuthology.orchestra.run.smithi029.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 13 MB/s | 14 MB 00:01 2023-10-09T03:57:53.166 INFO:teuthology.orchestra.run.smithi029.stdout:(5/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 6.4 MB/s | 4.9 MB 00:00 2023-10-09T03:57:53.229 INFO:teuthology.orchestra.run.smithi029.stdout:(6/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.0 MB/s | 644 kB 00:00 2023-10-09T03:57:53.396 INFO:teuthology.orchestra.run.smithi029.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 7.4 MB/s | 4.7 MB 00:00 2023-10-09T03:57:53.421 INFO:teuthology.orchestra.run.smithi029.stdout:(8/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 673 kB/s | 171 kB 00:00 2023-10-09T03:57:53.422 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:57:53.422 INFO:teuthology.orchestra.run.smithi029.stdout:Total 24 MB/s | 35 MB 00:01 2023-10-09T03:57:53.546 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-10-09T03:57:53.546 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:51 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:53.671 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:57:53.690 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:57:53.690 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:57:54.000 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:57:54.004 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:57:54.569 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:57:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:57:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-09T03:57:55.009 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:57:55.663 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-09T03:57:55.884 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:57:56.521 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-09T03:57:56.716 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:57:57.443 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-09T03:57:57.782 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:57:57.833 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-09T03:57:57.965 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:57.965 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:57:57.965 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:57:57.965 INFO:teuthology.orchestra.run.smithi177.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T03:57:57.965 INFO:teuthology.orchestra.run.smithi177.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T03:57:57.966 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:57:57.967 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:57:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:57:58.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T03:57:58.466 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:57:59.423 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:02:57 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:57:59.915 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-09T03:58:00.780 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:00.781 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================================== 2023-10-09T03:58:00.781 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================================== 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================================== 2023-10-09T03:58:00.782 INFO:teuthology.orchestra.run.smithi177.stdout:Install 5 Packages 2023-10-09T03:58:00.783 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:00.783 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 6.0 M 2023-10-09T03:58:00.783 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 26 M 2023-10-09T03:58:00.783 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:58:01.167 INFO:teuthology.orchestra.run.smithi177.stdout:(1/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.7 MB/s | 476 kB 00:00 2023-10-09T03:58:01.192 INFO:teuthology.orchestra.run.smithi177.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 204 kB/s | 60 kB 00:00 2023-10-09T03:58:01.218 INFO:teuthology.orchestra.run.smithi177.stdout:(3/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 379 kB/s | 121 kB 00:00 2023-10-09T03:58:01.281 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-09T03:58:01.309 INFO:teuthology.orchestra.run.smithi177.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.7 MB/s | 538 kB 00:00 2023-10-09T03:58:01.401 INFO:teuthology.orchestra.run.smithi177.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 23 MB/s | 4.8 MB 00:00 2023-10-09T03:58:01.403 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:01.403 INFO:teuthology.orchestra.run.smithi177.stdout:Total 9.6 MB/s | 6.0 MB 00:00 2023-10-09T03:58:01.456 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:58:01.457 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:58:01.467 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:58:01.467 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:58:01.546 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:58:01.549 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 8/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb 1/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-10-09T03:58:01.778 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-10-09T03:58:02.499 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:58:02.853 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T03:58:02.868 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-10-09T03:58:02.869 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:02:55 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:03.066 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T03:58:03.402 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:58:03.683 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T03:58:03.864 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:58:04.088 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:58:04.088 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T03:58:04.089 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T03:58:04.089 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:58:04.089 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T03:58:05.137 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T03:58:05.138 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:03 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:06.796 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:58:06.999 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:07.000 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:58:07.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-rook 2023-10-09T03:58:07.537 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:07.999 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:58:08.181 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:08.181 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:58:08.181 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:58:08.182 INFO:teuthology.orchestra.run.smithi177.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T03:58:08.182 INFO:teuthology.orchestra.run.smithi177.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T03:58:08.182 INFO:teuthology.orchestra.run.smithi177.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T03:58:08.182 INFO:teuthology.orchestra.run.smithi177.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T03:58:08.182 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:08.182 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:08.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install ceph-fuse 2023-10-09T03:58:08.626 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:01 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:08.696 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:10.174 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:58:10.174 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================== 2023-10-09T03:58:10.174 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:58:10.174 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================== 2023-10-09T03:58:10.174 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-rook noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 47 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-09T03:58:10.175 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================================== 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout:Install 13 Packages 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 1.9 M 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 25 M 2023-10-09T03:58:10.176 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:58:10.566 INFO:teuthology.orchestra.run.smithi029.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 66 kB/s | 16 kB 00:00 2023-10-09T03:58:10.592 INFO:teuthology.orchestra.run.smithi029.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 120 kB/s | 33 kB 00:00 2023-10-09T03:58:10.625 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:08 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:10.759 INFO:teuthology.orchestra.run.smithi029.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 422 kB/s | 81 kB 00:00 2023-10-09T03:58:10.844 INFO:teuthology.orchestra.run.smithi029.stdout:(4/13): ceph-mgr-rook-17.2.6-1303.gb2903cb0.el8 90 kB/s | 47 kB 00:00 2023-10-09T03:58:10.870 INFO:teuthology.orchestra.run.smithi029.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 590 kB/s | 64 kB 00:00 2023-10-09T03:58:10.936 INFO:teuthology.orchestra.run.smithi029.stdout:(6/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.4 MB/s | 1.2 MB 00:00 2023-10-09T03:58:11.053 INFO:teuthology.orchestra.run.smithi029.stdout:(7/13): python3-pyasn1-modules-0.3.7-6.el8.noar 598 kB/s | 110 kB 00:00 2023-10-09T03:58:11.120 INFO:teuthology.orchestra.run.smithi029.stdout:(8/13): python3-websocket-client-0.56.0-5.el8.n 221 kB/s | 61 kB 00:00 2023-10-09T03:58:11.146 INFO:teuthology.orchestra.run.smithi029.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 127 kB/s | 27 kB 00:00 2023-10-09T03:58:11.248 INFO:teuthology.orchestra.run.smithi029.stdout:(10/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 649 kB/s | 126 kB 00:00 2023-10-09T03:58:11.282 INFO:teuthology.orchestra.run.smithi029.stdout:(11/13): python3-requests-oauthlib-1.0.0-1.el8. 336 kB/s | 43 kB 00:00 2023-10-09T03:58:11.307 INFO:teuthology.orchestra.run.smithi029.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 107 kB/s | 20 kB 00:00 2023-10-09T03:58:11.332 INFO:teuthology.orchestra.run.smithi029.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 1.8 MB/s | 155 kB 00:00 2023-10-09T03:58:11.336 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:11.336 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.7 MB/s | 1.9 MB 00:01 2023-10-09T03:58:11.405 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:58:11.422 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:58:11.423 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:58:11.501 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:58:11.505 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:58:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:58:12.045 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repository Size 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-fuse 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T03:58:12.046 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout:Install 2 Packages 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 959 k 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 3.0 M 2023-10-09T03:58:12.047 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:58:12.123 INFO:teuthology.orchestra.run.smithi177.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2023-10-09T03:58:12.225 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-09T03:58:12.478 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-09T03:58:12.673 INFO:teuthology.orchestra.run.smithi177.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 1.4 MB/s | 876 kB 00:00 2023-10-09T03:58:12.674 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:12.674 INFO:teuthology.orchestra.run.smithi177.stdout:Total 1.5 MB/s | 959 kB 00:00 2023-10-09T03:58:12.686 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:58:12.707 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:58:12.707 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:58:12.781 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:58:12.782 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-09T03:58:12.785 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:58:12.954 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:58:13.046 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-10-09T03:58:13.199 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T03:58:13.249 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-10-09T03:58:13.395 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:58:13.465 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-10-09T03:58:13.647 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-10-09T03:58:13.747 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:58:13.747 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T03:58:13.849 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-09T03:58:14.057 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-09T03:58:14.250 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-09T03:58:14.805 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T03:58:14.805 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:12 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:15.422 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-09T03:58:15.666 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-09T03:58:15.873 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 13/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 1/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-09T03:58:16.125 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-09T03:58:16.126 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-10-09T03:58:16.126 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-10-09T03:58:16.126 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-10-09T03:58:16.126 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-10-09T03:58:16.126 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-10-09T03:58:17.271 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-10-09T03:58:17.272 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:10 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:19.087 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:58:19.267 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:19.267 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:58:19.267 INFO:teuthology.orchestra.run.smithi177.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T03:58:19.268 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:19.268 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:19.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install librados-devel 2023-10-09T03:58:19.784 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:21.066 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:19 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:21.180 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:58:21.380 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-rook-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-09T03:58:21.381 INFO:teuthology.orchestra.run.smithi029.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:21.382 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:58:21.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-cephadm 2023-10-09T03:58:21.928 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repo Size 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 130 k 2023-10-09T03:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 464 k 2023-10-09T03:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:58:22.797 INFO:teuthology.orchestra.run.smithi177.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 437 kB/s | 130 kB 00:00 2023-10-09T03:58:22.798 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:22.798 INFO:teuthology.orchestra.run.smithi177.stdout:Total 435 kB/s | 130 kB 00:00 2023-10-09T03:58:22.798 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:58:22.803 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:58:22.804 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:58:22.858 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:58:22.862 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:58:22.961 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:15 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:22.979 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:58:23.134 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:23.356 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:24.402 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:24.402 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:22 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:24.487 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:58:24.488 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================================== 2023-10-09T03:58:24.488 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:58:24.488 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================================== 2023-10-09T03:58:24.488 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1303.gb2903cb0.el8 ceph-noarch 121 k 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================================== 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout:Install 5 Packages 2023-10-09T03:58:24.489 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:24.490 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 6.0 M 2023-10-09T03:58:24.490 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 26 M 2023-10-09T03:58:24.490 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:58:24.852 INFO:teuthology.orchestra.run.smithi029.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1303.gb2903cb0.e 580 kB/s | 121 kB 00:00 2023-10-09T03:58:24.878 INFO:teuthology.orchestra.run.smithi029.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 257 kB/s | 60 kB 00:00 2023-10-09T03:58:24.945 INFO:teuthology.orchestra.run.smithi029.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 5.7 MB/s | 538 kB 00:00 2023-10-09T03:58:24.970 INFO:teuthology.orchestra.run.smithi029.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.4 MB/s | 476 kB 00:00 2023-10-09T03:58:25.104 INFO:teuthology.orchestra.run.smithi029.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 21 MB/s | 4.8 MB 00:00 2023-10-09T03:58:25.105 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:25.106 INFO:teuthology.orchestra.run.smithi029.stdout:Total 9.7 MB/s | 6.0 MB 00:00 2023-10-09T03:58:25.161 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:58:25.173 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:58:25.174 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:58:25.257 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:58:25.261 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:58:26.256 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:58:26.642 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-10-09T03:58:26.958 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-10-09T03:58:27.065 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:58:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:27.248 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:58:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:58:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:27.249 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:27.313 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:58:27.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install libcephfs2 2023-10-09T03:58:27.625 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-10-09T03:58:27.738 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:27.821 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:58:28.070 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 5/5 2023-10-09T03:58:28.070 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 1/5 2023-10-09T03:58:28.070 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-10-09T03:58:28.070 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-10-09T03:58:28.071 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-10-09T03:58:28.597 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:26 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:29.164 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-10-09T03:58:29.164 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:22 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:29.839 INFO:teuthology.orchestra.run.smithi177.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:29.947 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:29.949 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:29.949 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:30.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install libcephfs-devel 2023-10-09T03:58:30.414 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:31.280 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:29 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:31.997 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-cephadm-2:17.2.6-1303.gb2903cb0.el8.noarch 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-09T03:58:32.202 INFO:teuthology.orchestra.run.smithi029.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-10-09T03:58:32.203 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:32.203 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:58:32.716 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:32.717 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repo Size 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 25 k 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 109 k 2023-10-09T03:58:32.718 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:58:33.076 INFO:teuthology.orchestra.run.smithi177.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 69 kB/s | 25 kB 00:00 2023-10-09T03:58:33.077 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:33.077 INFO:teuthology.orchestra.run.smithi177.stdout:Total 69 kB/s | 25 kB 00:00 2023-10-09T03:58:33.077 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:58:33.081 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:58:33.081 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:58:33.083 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:58:33.087 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:58:33.141 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:58:33.348 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:33.568 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:34.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-fuse 2023-10-09T03:58:34.518 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:34.564 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:34.564 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:32 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:35.407 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:28 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:36.895 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:58:36.895 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-fuse 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout: x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 876 k 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout:Install 2 Packages 2023-10-09T03:58:36.896 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:36.897 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 959 k 2023-10-09T03:58:36.897 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 3.0 M 2023-10-09T03:58:36.897 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:58:36.982 INFO:teuthology.orchestra.run.smithi029.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 972 kB/s | 83 kB 00:00 2023-10-09T03:58:37.030 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:58:37.183 INFO:teuthology.orchestra.run.smithi029.stdout:(2/2): ceph-fuse-17.2.6-1303.gb2903cb0.el8.x86_ 3.0 MB/s | 876 kB 00:00 2023-10-09T03:58:37.184 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:37.184 INFO:teuthology.orchestra.run.smithi029.stdout:Total 3.3 MB/s | 959 kB 00:00 2023-10-09T03:58:37.195 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:58:37.212 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:37.212 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:58:37.212 INFO:teuthology.orchestra.run.smithi177.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:58:37.212 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:37.212 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:37.217 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:58:37.217 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:58:37.297 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:58:37.301 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:58:37.354 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install librados2 2023-10-09T03:58:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:58:37.706 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:37.734 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-09T03:58:37.894 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:58:38.294 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2/2 2023-10-09T03:58:38.294 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/2 2023-10-09T03:58:38.570 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:36 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:39.446 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-09T03:58:39.446 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:32 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:39.823 INFO:teuthology.orchestra.run.smithi177.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:39.929 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:39.930 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:39.931 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:40.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install librbd1 2023-10-09T03:58:40.407 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:41.227 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:39 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:41.892 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:58:42.091 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:42.091 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:58:42.091 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-09T03:58:42.091 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:42.091 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:58:42.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librados-devel 2023-10-09T03:58:42.477 INFO:teuthology.orchestra.run.smithi177.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:42.585 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:42.586 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:42.586 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:42.620 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:42.706 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install python3-rados 2023-10-09T03:58:43.056 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:43.471 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:36 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:43.894 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:41 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:45.007 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout: librados-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 130 k 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:58:45.008 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:45.009 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 130 k 2023-10-09T03:58:45.009 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 464 k 2023-10-09T03:58:45.009 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:58:45.135 INFO:teuthology.orchestra.run.smithi177.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:45.244 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:45.245 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:45.245 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:45.274 INFO:teuthology.orchestra.run.smithi029.stdout:librados-devel-17.2.6-1303.gb2903cb0.el8.x86_64 490 kB/s | 130 kB 00:00 2023-10-09T03:58:45.274 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:45.275 INFO:teuthology.orchestra.run.smithi029.stdout:Total 488 kB/s | 130 kB 00:00 2023-10-09T03:58:45.275 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:58:45.281 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:58:45.281 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:58:45.341 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:58:45.345 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:58:45.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install python3-rgw 2023-10-09T03:58:45.462 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:58:45.634 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:45.721 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:45.919 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:46.567 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:44 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:47.018 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:47.018 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:40 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:47.822 INFO:teuthology.orchestra.run.smithi177.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:47.930 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:47.931 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:47.931 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:48.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install python3-cephfs 2023-10-09T03:58:48.404 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:49.233 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:47 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:49.476 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:58:49.675 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:49.676 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:58:49.676 INFO:teuthology.orchestra.run.smithi029.stdout: librados-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:58:49.676 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:49.676 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:58:49.822 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install libcephfs2 2023-10-09T03:58:50.199 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:50.500 INFO:teuthology.orchestra.run.smithi177.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:50.607 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:50.608 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:50.608 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:50.727 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install python3-rbd 2023-10-09T03:58:51.040 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:44 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:51.080 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:51.881 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:49 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:52.441 INFO:teuthology.orchestra.run.smithi029.stdout:Package libcephfs2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:52.558 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:58:52.559 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:58:52.559 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:58:52.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install libcephfs-devel 2023-10-09T03:58:53.060 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:53.126 INFO:teuthology.orchestra.run.smithi177.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:58:53.235 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:53.236 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2023-10-09T03:58:53.236 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:58:53.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install rbd-fuse 2023-10-09T03:58:53.707 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:58:53.947 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:46 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:54.537 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:52 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:55.421 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs-devel x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 25 k 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:55.422 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:58:55.423 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:58:55.423 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:58:55.423 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:58:55.423 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 25 k 2023-10-09T03:58:55.423 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 109 k 2023-10-09T03:58:55.423 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:58:55.575 INFO:teuthology.orchestra.run.smithi029.stdout:libcephfs-devel-17.2.6-1303.gb2903cb0.el8.x86_6 164 kB/s | 25 kB 00:00 2023-10-09T03:58:55.575 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:55.575 INFO:teuthology.orchestra.run.smithi029.stdout:Total 163 kB/s | 25 kB 00:00 2023-10-09T03:58:55.576 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:58:55.580 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:58:55.580 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:58:55.582 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:58:55.586 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:58:55.636 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:58:55.892 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout: Package Architecture Version Repository Size 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:58:55.893 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:58:55.894 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:58:55.894 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 90 k 2023-10-09T03:58:55.894 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 264 k 2023-10-09T03:58:55.894 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:58:56.024 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:56.079 INFO:teuthology.orchestra.run.smithi177.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 488 kB/s | 90 kB 00:00 2023-10-09T03:58:56.079 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:58:56.080 INFO:teuthology.orchestra.run.smithi177.stdout:Total 484 kB/s | 90 kB 00:00 2023-10-09T03:58:56.080 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:58:56.089 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:58:56.090 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:58:56.144 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:58:56.148 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:58:56.251 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:56.252 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:58:56.436 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:56.666 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:57.288 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:57.288 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:50 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:58:57.709 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:58:57.709 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:55 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:58:59.895 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:59:00.090 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:59:00.100 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:00.100 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:59:00.100 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs-devel-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:00.100 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:00.100 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:00.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librados2 2023-10-09T03:59:00.272 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:00.272 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:59:00.272 INFO:teuthology.orchestra.run.smithi177.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:00.272 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:00.273 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:59:00.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install rbd-mirror 2023-10-09T03:59:00.615 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:00.774 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:01.454 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:54 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:01.573 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:03:59 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:59:02.828 INFO:teuthology.orchestra.run.smithi029.stdout:Package librados2-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:59:02.940 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout: Package Arch Version Repo Size 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:59:02.941 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:59:02.942 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:59:02.942 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:02.942 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 3.5 M 2023-10-09T03:59:02.942 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 16 M 2023-10-09T03:59:02.942 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:59:02.948 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:02.950 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:59:02.950 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:03.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librbd1 2023-10-09T03:59:03.356 INFO:teuthology.orchestra.run.smithi177.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 8.3 MB/s | 3.5 MB 00:00 2023-10-09T03:59:03.357 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:59:03.357 INFO:teuthology.orchestra.run.smithi177.stdout:Total 8.3 MB/s | 3.5 MB 00:00 2023-10-09T03:59:03.358 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:59:03.369 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:59:03.369 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:59:03.435 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:59:03.438 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:59:03.449 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:03.874 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:59:04.196 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:04.279 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:03:57 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:04.559 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:05.631 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:05.632 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:04:03 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:59:05.659 INFO:teuthology.orchestra.run.smithi029.stdout:Package librbd1-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:59:05.778 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:05.779 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:59:05.779 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:05.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rados 2023-10-09T03:59:06.287 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:07.170 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:00 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:07.994 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:59:08.175 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:08.312 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:59:08.312 INFO:teuthology.orchestra.run.smithi177.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:08.312 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:08.312 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:59:08.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum -y install rbd-nbd 2023-10-09T03:59:08.525 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rados-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:59:08.644 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:08.646 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:59:08.646 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:08.674 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:08.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rgw 2023-10-09T03:59:09.151 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:09.490 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:04:07 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:59:09.992 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:02 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:10.853 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2023-10-09T03:59:10.854 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:59:10.854 INFO:teuthology.orchestra.run.smithi177.stdout: Package Architecture Version Repository Size 2023-10-09T03:59:10.854 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:59:10.854 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2023-10-09T03:59:10.854 INFO:teuthology.orchestra.run.smithi177.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T03:59:10.854 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:10.855 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2023-10-09T03:59:10.855 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2023-10-09T03:59:10.855 INFO:teuthology.orchestra.run.smithi177.stdout:Install 1 Package 2023-10-09T03:59:10.855 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:10.855 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 173 k 2023-10-09T03:59:10.855 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 552 k 2023-10-09T03:59:10.856 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2023-10-09T03:59:11.152 INFO:teuthology.orchestra.run.smithi177.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 583 kB/s | 173 kB 00:00 2023-10-09T03:59:11.152 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:59:11.152 INFO:teuthology.orchestra.run.smithi177.stdout:Total 581 kB/s | 173 kB 00:00 2023-10-09T03:59:11.153 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2023-10-09T03:59:11.163 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2023-10-09T03:59:11.164 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2023-10-09T03:59:11.217 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2023-10-09T03:59:11.221 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2023-10-09T03:59:11.340 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2023-10-09T03:59:11.399 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rgw-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:59:11.492 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:11.518 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:11.519 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:59:11.519 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:11.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-cephfs 2023-10-09T03:59:11.722 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:12.038 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:12.763 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:12.764 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:04:10 ago on Mon 09 Oct 2023 03:55:02 AM UTC. 2023-10-09T03:59:12.907 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:05 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:14.318 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-cephfs-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:59:14.436 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:14.437 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:59:14.438 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:14.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rbd 2023-10-09T03:59:14.952 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:15.154 INFO:teuthology.orchestra.run.smithi177.stdout:Installed products updated. 2023-10-09T03:59:15.333 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:15.334 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2023-10-09T03:59:15.334 INFO:teuthology.orchestra.run.smithi177.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:15.334 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T03:59:15.334 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2023-10-09T03:59:15.481 DEBUG:teuthology.parallel:result is None 2023-10-09T03:59:15.813 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:08 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:17.209 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 is already installed. 2023-10-09T03:59:17.328 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:17.329 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2023-10-09T03:59:17.329 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:17.465 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-fuse 2023-10-09T03:59:17.840 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:18.729 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:11 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:20.278 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-fuse x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 90 k 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:20.279 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:59:20.280 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:20.280 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:59:20.280 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:20.281 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 90 k 2023-10-09T03:59:20.281 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 264 k 2023-10-09T03:59:20.281 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:59:20.448 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-fuse-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 539 kB/s | 90 kB 00:00 2023-10-09T03:59:20.448 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:59:20.448 INFO:teuthology.orchestra.run.smithi029.stdout:Total 535 kB/s | 90 kB 00:00 2023-10-09T03:59:20.449 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:59:20.459 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:59:20.459 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:59:20.518 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:59:20.522 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:59:20.637 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:59:20.780 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:22.128 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:22.128 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:15 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:24.804 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:59:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:25.005 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:59:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-fuse-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:25.005 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:25.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-mirror 2023-10-09T03:59:25.528 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:26.397 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:19 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:27.927 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:27.927 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:27.927 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-mirror x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 3.5 M 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:59:27.928 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:27.929 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 3.5 M 2023-10-09T03:59:27.929 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 16 M 2023-10-09T03:59:27.929 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:59:28.646 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-mirror-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 4.8 MB/s | 3.5 MB 00:00 2023-10-09T03:59:28.647 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:59:28.647 INFO:teuthology.orchestra.run.smithi029.stdout:Total 4.8 MB/s | 3.5 MB 00:00 2023-10-09T03:59:28.648 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:59:28.661 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:59:28.661 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:59:28.735 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:59:28.739 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:59:29.215 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:59:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:29.827 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:30.939 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:30.939 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:23 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:34.081 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:59:34.281 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:34.281 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:59:34.281 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-mirror-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:34.281 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:34.281 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:34.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-nbd 2023-10-09T03:59:34.806 INFO:teuthology.orchestra.run.smithi029.stdout:Updating Subscription Management repositories. 2023-10-09T03:59:35.695 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:28 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:37.210 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2023-10-09T03:59:37.210 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:37.210 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-nbd x86_64 2:17.2.6-1303.gb2903cb0.el8 ceph 173 k 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2023-10-09T03:59:37.211 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:37.212 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 173 k 2023-10-09T03:59:37.212 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 552 k 2023-10-09T03:59:37.212 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2023-10-09T03:59:37.418 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-nbd-17.2.6-1303.gb2903cb0.el8.x86_64.rpm 840 kB/s | 173 kB 00:00 2023-10-09T03:59:37.418 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T03:59:37.418 INFO:teuthology.orchestra.run.smithi029.stdout:Total 835 kB/s | 173 kB 00:00 2023-10-09T03:59:37.419 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2023-10-09T03:59:37.430 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2023-10-09T03:59:37.430 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2023-10-09T03:59:37.489 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2023-10-09T03:59:37.493 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2023-10-09T03:59:37.610 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2023-10-09T03:59:37.803 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:38.052 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:39.190 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 1/1 2023-10-09T03:59:39.190 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:04:32 ago on Mon 09 Oct 2023 03:55:07 AM UTC. 2023-10-09T03:59:43.348 INFO:teuthology.orchestra.run.smithi029.stdout:Installed products updated. 2023-10-09T03:59:43.550 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:43.550 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2023-10-09T03:59:43.550 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-nbd-2:17.2.6-1303.gb2903cb0.el8.x86_64 2023-10-09T03:59:43.551 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T03:59:43.551 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2023-10-09T03:59:43.711 DEBUG:teuthology.parallel:result is None 2023-10-09T03:59:43.712 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:59:43.712 INFO:teuthology.packaging:ref: None 2023-10-09T03:59:43.712 INFO:teuthology.packaging:tag: None 2023-10-09T03:59:43.712 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:59:43.712 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:59:43.713 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:59:44.077 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T03:59:44.101 INFO:teuthology.orchestra.run.smithi029.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T03:59:44.101 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T03:59:44.102 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T03:59:44.102 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-09T03:59:44.102 INFO:teuthology.packaging:ref: None 2023-10-09T03:59:44.102 INFO:teuthology.packaging:tag: None 2023-10-09T03:59:44.102 INFO:teuthology.packaging:branch: quincy 2023-10-09T03:59:44.102 INFO:teuthology.packaging:sha1: b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T03:59:44.102 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-10-09T03:59:44.498 DEBUG:teuthology.orchestra.run.smithi177:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-09T03:59:44.526 INFO:teuthology.orchestra.run.smithi177.stdout:17.2.6-1303.gb2903cb0.el8 2023-10-09T03:59:44.526 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1303.gb2903cb0.el8 2023-10-09T03:59:44.526 INFO:teuthology.task.install:The correct ceph version 17.2.6-1303.gb2903cb0 is installed. 2023-10-09T03:59:44.526 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-09T03:59:44.526 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T03:59:44.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T03:59:44.559 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T03:59:44.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-09T03:59:44.598 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-09T03:59:44.598 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T03:59:44.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T03:59:44.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T03:59:44.701 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T03:59:44.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/usr/bin/daemon-helper 2023-10-09T03:59:44.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-09T03:59:44.801 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-09T03:59:44.801 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T03:59:44.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T03:59:44.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T03:59:44.902 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T03:59:44.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-09T03:59:44.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-09T03:59:44.999 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-09T03:59:45.000 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T03:59:45.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T03:59:45.030 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T03:59:45.100 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T03:59:45.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/usr/bin/stdin-killer 2023-10-09T03:59:45.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-09T03:59:45.196 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-09T03:59:45.282 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'} 2023-10-09T03:59:45.282 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-10-09T03:59:45.283 INFO:tasks.cephadm:Cluster fsid is 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T03:59:45.283 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-09T03:59:45.283 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.29', 'mon.c': '[v2:172.21.15.29:3301,v1:172.21.15.29:6790]', 'mon.b': '172.21.15.177'} 2023-10-09T03:59:45.283 INFO:tasks.cephadm:First mon is mon.a on smithi029 2023-10-09T03:59:45.283 INFO:tasks.cephadm:First mgr is y 2023-10-09T03:59:45.283 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-09T03:59:45.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo hostname $(hostname -s) 2023-10-09T03:59:45.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo hostname $(hostname -s) 2023-10-09T03:59:45.345 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-10-09T03:59:45.345 DEBUG:teuthology.orchestra.run.smithi029:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:45.492 INFO:teuthology.orchestra.run.smithi029.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 03:59 /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:45.492 DEBUG:teuthology.orchestra.run.smithi177:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:45.629 INFO:teuthology.orchestra.run.smithi177.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366887 Oct 9 03:59 /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:45.630 DEBUG:teuthology.orchestra.run.smithi029:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:45.652 DEBUG:teuthology.orchestra.run.smithi177:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-09T03:59:45.681 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-10-09T03:59:45.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T03:59:45.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-10-09T03:59:46.624 INFO:teuthology.orchestra.run.smithi177.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T03:59:47.249 INFO:teuthology.orchestra.run.smithi029.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout: "repo_digests": [ 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:00:29.140 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:00:36.381 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:00:36.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph_version": "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)", 2023-10-09T04:00:36.381 INFO:teuthology.orchestra.run.smithi029.stdout: "image_id": "5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf", 2023-10-09T04:00:36.381 INFO:teuthology.orchestra.run.smithi029.stdout: "repo_digests": [ 2023-10-09T04:00:36.382 INFO:teuthology.orchestra.run.smithi029.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:67105d45415a7f6878d82dd27e27caaa40ebf0ca6488030995660f1cad5fce9d" 2023-10-09T04:00:36.382 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:00:36.382 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:00:36.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo mkdir -p /etc/ceph 2023-10-09T04:00:36.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo mkdir -p /etc/ceph 2023-10-09T04:00:36.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 777 /etc/ceph 2023-10-09T04:00:36.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 777 /etc/ceph 2023-10-09T04:00:36.546 INFO:tasks.cephadm:Writing seed config... 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-09T04:00:36.547 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-09T04:00:36.548 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-09T04:00:36.548 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-09T04:00:36.548 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:00:36.548 DEBUG:teuthology.orchestra.run.smithi029:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-09T04:00:36.569 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 486585fa-6658-11ee-8db5-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-09T04:00:36.569 DEBUG:teuthology.orchestra.run.smithi029:mon.a> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-09T04:00:36.612 DEBUG:teuthology.orchestra.run.smithi029:mgr.y> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-09T04:00:36.641 INFO:journalctl@ceph.mon.a.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:00:36.642 INFO:tasks.cephadm:Bootstrapping... 2023-10-09T04:00:36.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.29 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:00:36.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:00:36.855 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2023-10-09T04:00:36.855 INFO:teuthology.orchestra.run.smithi029.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '486585fa-6658-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.29', '--skip-admin-label'] 2023-10-09T04:00:36.920 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 3.0.1 2023-10-09T04:00:36.920 INFO:teuthology.orchestra.run.smithi029.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-10-09T04:00:36.921 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying podman|docker is present... 2023-10-09T04:00:36.982 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 3.0.1 2023-10-09T04:00:36.982 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying lvm2 is present... 2023-10-09T04:00:36.982 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying time synchronization is in place... 2023-10-09T04:00:36.989 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T04:00:36.989 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T04:00:36.995 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T04:00:36.995 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2023-10-09T04:00:37.001 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout enabled 2023-10-09T04:00:37.008 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout active 2023-10-09T04:00:37.008 INFO:teuthology.orchestra.run.smithi029.stdout:Unit chronyd.service is enabled and running 2023-10-09T04:00:37.008 INFO:teuthology.orchestra.run.smithi029.stdout:Repeating the final host check... 2023-10-09T04:00:37.072 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 3.0.1 2023-10-09T04:00:37.072 INFO:teuthology.orchestra.run.smithi029.stdout:podman (/bin/podman) version 3.0.1 is present 2023-10-09T04:00:37.073 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl is present 2023-10-09T04:00:37.073 INFO:teuthology.orchestra.run.smithi029.stdout:lvcreate is present 2023-10-09T04:00:37.078 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-10-09T04:00:37.079 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-10-09T04:00:37.084 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-10-09T04:00:37.084 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2023-10-09T04:00:37.090 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout enabled 2023-10-09T04:00:37.096 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout active 2023-10-09T04:00:37.096 INFO:teuthology.orchestra.run.smithi029.stdout:Unit chronyd.service is enabled and running 2023-10-09T04:00:37.096 INFO:teuthology.orchestra.run.smithi029.stdout:Host looks OK 2023-10-09T04:00:37.097 INFO:teuthology.orchestra.run.smithi029.stdout:Cluster fsid: 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:37.097 INFO:teuthology.orchestra.run.smithi029.stdout:Acquiring lock 140129002998752 on /run/cephadm/486585fa-6658-11ee-8db5-212e2dc638e7.lock 2023-10-09T04:00:37.097 INFO:teuthology.orchestra.run.smithi029.stdout:Lock 140129002998752 acquired on /run/cephadm/486585fa-6658-11ee-8db5-212e2dc638e7.lock 2023-10-09T04:00:37.097 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying IP 172.21.15.29 port 3300 ... 2023-10-09T04:00:37.097 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying IP 172.21.15.29 port 6789 ... 2023-10-09T04:00:37.098 INFO:teuthology.orchestra.run.smithi029.stdout:Base mon IP(s) is [172.21.15.29:3300, 172.21.15.29:6789], mon addrv is [v2:172.21.15.29:3300,v1:172.21.15.29:6789] 2023-10-09T04:00:37.101 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-10-09T04:00:37.101 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.29 metric 100 2023-10-09T04:00:37.104 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-10-09T04:00:37.104 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-10-09T04:00:37.104 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1799sec hoplimit 64 pref medium 2023-10-09T04:00:37.106 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-09T04:00:37.106 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-10-09T04:00:37.107 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T04:00:37.107 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-10-09T04:00:37.107 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:125a/64 scope link 2023-10-09T04:00:37.107 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-10-09T04:00:37.108 INFO:teuthology.orchestra.run.smithi029.stdout:Mon IP `172.21.15.29` is in CIDR network `172.21.0.0/20` 2023-10-09T04:00:37.108 INFO:teuthology.orchestra.run.smithi029.stdout:Mon IP `172.21.15.29` is in CIDR network `172.21.0.0/20` 2023-10-09T04:00:37.108 INFO:teuthology.orchestra.run.smithi029.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-09T04:00:37.108 INFO:teuthology.orchestra.run.smithi029.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-09T04:00:37.108 INFO:teuthology.orchestra.run.smithi029.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T04:00:37.530 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T04:00:37.531 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-10-09T04:00:37.531 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Getting image source signatures 2023-10-09T04:00:37.531 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying blob sha256:1b55aded3a89dccf5f05a82d583393d22b3476851fe870b413c77ca909f5f760 2023-10-09T04:00:37.531 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying blob sha256:be9ed931e0f930233626f1c4d225977fbcc33e056e1a8fc67c7f63553d9455a2 2023-10-09T04:00:37.531 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying config sha256:5517396c50d4b07181c79c353f571b455dbdd80614d068cc9a6e42eb27b7bccf 2023-10-09T04:00:37.531 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Writing manifest to image destination 2023-10-09T04:00:37.532 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Storing signatures 2023-10-09T04:00:38.715 INFO:teuthology.orchestra.run.smithi029.stdout:ceph: stdout ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T04:00:38.715 INFO:teuthology.orchestra.run.smithi029.stdout:Ceph version: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable) 2023-10-09T04:00:38.715 INFO:teuthology.orchestra.run.smithi029.stdout:Extracting ceph user uid/gid from container image... 2023-10-09T04:00:39.756 INFO:teuthology.orchestra.run.smithi029.stdout:stat: stdout 167 167 2023-10-09T04:00:39.756 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial keys... 2023-10-09T04:00:40.794 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQDoeiNlqfvtBhAAploXgaH59ItE3lxFl59Ehw== 2023-10-09T04:00:41.771 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQDpeiNl0lvXBxAA9wMtINXk2xXDju09yeUDYw== 2023-10-09T04:00:42.827 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQDqeiNlNppgCBAA01ynSxP3Qryxcf11GolSIg== 2023-10-09T04:00:42.828 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial monmap... 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:monmaptool for a [v2:172.21.15.29:3300,v1:172.21.15.29:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:setting min_mon_release = octopus 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: set fsid to 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:00:43.812 INFO:teuthology.orchestra.run.smithi029.stdout:Creating mon... 2023-10-09T04:00:44.909 INFO:teuthology.orchestra.run.smithi029.stdout:create mon.a on 2023-10-09T04:00:45.021 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-09T04:00:45.113 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-09T04:00:45.214 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-486585fa-6658-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-486585fa-6658-11ee-8db5-212e2dc638e7.target. 2023-10-09T04:00:45.214 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-486585fa-6658-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-486585fa-6658-11ee-8db5-212e2dc638e7.target. 2023-10-09T04:00:45.341 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T04:00:45.341 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to reset failed state of unit ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-09T04:00:45.432 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-486585fa-6658-11ee-8db5-212e2dc638e7.target.wants/ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-486585fa-6658-11ee-8db5-212e2dc638e7@.service. 2023-10-09T04:00:45.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:45 smithi029 systemd[1]: Starting Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:00:46.025 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T04:00:46.025 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2023-10-09T04:00:46.032 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T04:00:46.032 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2023-10-09T04:00:46.032 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2023-10-09T04:00:46.033 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T04:00:46.033 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mon to start... 2023-10-09T04:00:46.033 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mon... 2023-10-09T04:00:46.277 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:46 smithi029 bash[100032]: 4d6976c473e929775d45aec3def3bec850c375091ecdedb3962aca230df7b21d 2023-10-09T04:00:46.277 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:46 smithi029 systemd[1]: Started Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:00:46.277 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:46 smithi029 ceph-mon[100098]: mkfs 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:46.277 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:46 smithi029 ceph-mon[100098]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:00:47.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:47 smithi029 ceph-mon[100098]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:00:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:47 smithi029 ceph-mon[100098]: monmap e1: 1 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0]} removed_ranks: {} 2023-10-09T04:00:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:47 smithi029 ceph-mon[100098]: fsmap 2023-10-09T04:00:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:47 smithi029 ceph-mon[100098]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T04:00:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:47 smithi029 ceph-mon[100098]: mgrmap e1: no daemons active 2023-10-09T04:00:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:47 smithi029 ceph-mon[100098]: from='client.? 172.21.15.29:0/1699198689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-09T04:00:47.471 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout cluster: 2023-10-09T04:00:47.471 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout id: 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout services: 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.607453s) 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:47.472 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout data: 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout pgs: 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:mon is available 2023-10-09T04:00:47.473 INFO:teuthology.orchestra.run.smithi029.stdout:Assimilating anything we can from ceph.conf... 2023-10-09T04:00:48.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:48 smithi029 ceph-mon[100098]: from='client.? 172.21.15.29:0/1071902631' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T04:00:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:48 smithi029 ceph-mon[100098]: from='client.? 172.21.15.29:0/1071902631' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-09T04:00:48.662 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout fsid = 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.29:3300,v1:172.21.15.29:6789] 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T04:00:48.663 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:48.664 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T04:00:48.664 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T04:00:48.664 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T04:00:48.664 INFO:teuthology.orchestra.run.smithi029.stdout:Generating new minimal ceph.conf... 2023-10-09T04:00:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:49 smithi029 ceph-mon[100098]: from='client.? 172.21.15.29:0/1584780694' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:00:49.945 INFO:teuthology.orchestra.run.smithi029.stdout:Restarting the monitor... 2023-10-09T04:00:50.302 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:49 smithi029 systemd[1]: Stopping Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:00:50.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:50 smithi029 conmon[100073]: 2023-10-09T04:00:50.192+0000 7ff04064a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:00:50.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:50 smithi029 conmon[100073]: 2023-10-09T04:00:50.192+0000 7ff04064a700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-09T04:00:51.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:50 smithi029 bash[100655]: 4d6976c473e929775d45aec3def3bec850c375091ecdedb3962aca230df7b21d 2023-10-09T04:00:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:50 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T04:00:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:50 smithi029 systemd[1]: Stopped Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:00:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:50 smithi029 systemd[1]: Starting Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:00:51.498 INFO:teuthology.orchestra.run.smithi029.stdout:Setting mon public_network to 172.21.0.0/20 2023-10-09T04:00:51.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 bash[100978]: 5d294109024f4fa65129e3df83107a7f046f2279dc3f346e6bcbc496bcf26123 2023-10-09T04:00:51.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:00:51.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T04:00:51.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: pidfile_write: ignore empty --pid-file 2023-10-09T04:00:51.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 systemd[1]: Started Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:00:51.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: load: jerasure load: lrc 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: RocksDB version: 6.15.5 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Compile date Oct 8 2023 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: DB SUMMARY 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: DB Session ID: 9N5WF0AH6LNQ09IW2Z2L 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83589 ; 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:00:51.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.env: 0x55b3828e8a80 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.fs: Posix File System 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.info_log: 0x55b3838e85a0 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.statistics: (nil) 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.use_fsync: 0 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:00:51.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.db_log_dir: 2023-10-09T04:00:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.write_buffer_manager: 0x55b38465a960 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:00:51.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.unordered_write: 0 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:00:51.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.row_cache: None 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.wal_filter: None 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.preserve_deletes: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:00:51.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:00:51.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_open_files: -1 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:00:51.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Compression algorithms supported: 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kZSTD supported: 0 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:00:51.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.merge_operator: 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_filter: None 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b38393afb8) 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:00:51.682 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_type: 0 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_index_type: 0 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_shortening: 1 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: hash_index_allow_collision: 1 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: checksum: 1 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: no_block_cache: 0 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache: 0x55b38396d090 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_options: 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: capacity : 536870912 2023-10-09T04:00:51.683 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_shard_bits : 4 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: strict_capacity_limit : 0 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_compressed: (nil) 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: persistent_cache: (nil) 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size: 4096 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size_deviation: 10 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_restart_interval: 16 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_block_restart_interval: 1 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_block_size: 4096 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: partition_filters: 0 2023-10-09T04:00:51.684 INFO:journalctl@ceph.mon.a.smithi029.stdout: use_delta_encoding: 1 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: whole_key_filtering: 1 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: verify_compression: 0 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: format_version: 4 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_index_compression: 1 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_align: 0 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression: NoCompression 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:00:51.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.num_levels: 7 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:00:51.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:00:51.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:00:51.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:00:51.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:00:51.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.ttl: 2592000 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824051502313, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-10-09T04:00:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824051503530, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80136, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78404, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72612, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696824051, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "7d007fbd-c8a8-4b8e-9007-3978688ecf5f", "db_session_id": "9N5WF0AH6LNQ09IW2Z2L"}} 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824051504037, "job": 1, "event": "recovery_finished"} 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55b383901180 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: DB pointer 0x55b3839d6000 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** DB Stats ** 2023-10-09T04:00:51.692 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:51.693 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:00:51.694 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 15.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 15.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.695 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 97.1 0.00 0.00 1 0.001 0 0 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:00:51.696 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 15.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: starting mon.a rank 0 at public addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] at bind addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???) e1 preinit fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:00:51.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).mds e1 new map 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).mds e1 print_map 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout: e1 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout: legacy client fscid: -1 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout: No filesystems configured 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:00:51.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-09T04:00:51.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mon.a is new leader, mons a in quorum (ranks 0) 2023-10-09T04:00:51.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: monmap e1: 1 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0]} removed_ranks: {} 2023-10-09T04:00:51.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: fsmap 2023-10-09T04:00:51.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: osdmap e1: 0 total, 0 up, 0 in 2023-10-09T04:00:51.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:51 smithi029 ceph-mon[101043]: mgrmap e1: no daemons active 2023-10-09T04:00:52.737 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote config to /etc/ceph/ceph.conf 2023-10-09T04:00:52.737 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:00:52.738 INFO:teuthology.orchestra.run.smithi029.stdout:Creating mgr... 2023-10-09T04:00:52.738 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying port 9283 ... 2023-10-09T04:00:52.871 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T04:00:52.872 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to reset failed state of unit ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-09T04:00:52.965 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-486585fa-6658-11ee-8db5-212e2dc638e7.target.wants/ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-486585fa-6658-11ee-8db5-212e2dc638e7@.service. 2023-10-09T04:00:53.083 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:52 smithi029 systemd[1]: Starting Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:00:53.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:53 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4287495911' entity='client.admin' 2023-10-09T04:00:53.599 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T04:00:53.599 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2023-10-09T04:00:53.606 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T04:00:53.606 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2023-10-09T04:00:53.607 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2023-10-09T04:00:53.607 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to enable service . firewalld.service is not available 2023-10-09T04:00:53.614 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T04:00:53.615 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2023-10-09T04:00:53.623 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T04:00:53.623 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2023-10-09T04:00:53.623 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2023-10-09T04:00:53.624 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-09T04:00:53.624 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr to start... 2023-10-09T04:00:53.624 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr... 2023-10-09T04:00:53.848 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:53 smithi029 bash[101425]: 308b3b05d6d1a4abf0bb3b0b704d13b5b7f2b20d2fcefeef9adb96ba731184fe 2023-10-09T04:00:53.848 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:53 smithi029 systemd[1]: Started Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:00:53.848 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:53 smithi029 conmon[101466]: 2023-10-09T04:00:53.728+0000 7ff4e85981c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:00:53.848 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:53 smithi029 conmon[101466]: 2023-10-09T04:00:53.826+0000 7ff4e85981c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:00:54.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:53 smithi029 conmon[101466]: 2023-10-09T04:00:53.934+0000 7ff4e85981c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:00:54.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:54 smithi029 conmon[101466]: 2023-10-09T04:00:54.043+0000 7ff4e85981c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:00:54.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:54 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/430776517' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:00:54.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:54 smithi029 conmon[101466]: 2023-10-09T04:00:54.253+0000 7ff4e85981c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:00:54.910 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:54.916 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:00:54.916 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "486585fa-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:00:54.917 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:00:54.918 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:00:54.919 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:00:54.920 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:00:46.051258+0000", 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:00:54.921 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:00:54.922 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (1/15)... 2023-10-09T04:00:55.005 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:54 smithi029 conmon[101466]: 2023-10-09T04:00:54.748+0000 7ff4e85981c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:00:55.005 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:54 smithi029 conmon[101466]: 2023-10-09T04:00:54.852+0000 7ff4e85981c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:00:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:55 smithi029 conmon[101466]: 2023-10-09T04:00:55.004+0000 7ff4e85981c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:00:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:55 smithi029 conmon[101466]: 2023-10-09T04:00:55.116+0000 7ff4e85981c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:00:55.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:55 smithi029 conmon[101466]: 2023-10-09T04:00:55.618+0000 7ff4e85981c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:00:56.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:56 smithi029 conmon[101466]: 2023-10-09T04:00:56.303+0000 7ff4e85981c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:00:56.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:56 smithi029 conmon[101466]: 2023-10-09T04:00:56.492+0000 7ff4e85981c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:00:57.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:00:57 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2461054190' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:00:57.742 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:57 smithi029 conmon[101466]: 2023-10-09T04:00:57.470+0000 7ff4e85981c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:00:57.742 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:57 smithi029 conmon[101466]: 2023-10-09T04:00:57.578+0000 7ff4e85981c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:00:58.137 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:57 smithi029 conmon[101466]: 2023-10-09T04:00:57.799+0000 7ff4e85981c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "486585fa-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:00:58.179 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:00:58.180 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:00:58.181 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:00:58.184 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.184 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:00:58.185 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:00:58.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:00:58.187 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:00:46.051258+0000", 2023-10-09T04:00:58.187 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:00:58.187 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:00:58.187 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:00:58.187 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:00:58.187 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (2/15)... 2023-10-09T04:00:58.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:58 smithi029 conmon[101466]: 2023-10-09T04:00:58.136+0000 7ff4e85981c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:00:58.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:58 smithi029 conmon[101466]: 2023-10-09T04:00:58.241+0000 7ff4e85981c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:00:58.884 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:58 smithi029 conmon[101466]: 2023-10-09T04:00:58.567+0000 7ff4e85981c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:00:59.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:58 smithi029 conmon[101466]: 2023-10-09T04:00:58.883+0000 7ff4e85981c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:01:00.090 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:59 smithi029 conmon[101466]: 2023-10-09T04:00:59.698+0000 7ff4e85981c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:01:00.090 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:00:59 smithi029 conmon[101466]: 2023-10-09T04:00:59.855+0000 7ff4e85981c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:01:00.376 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:00 smithi029 conmon[101466]: 2023-10-09T04:01:00.089+0000 7ff4e85981c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:01:00.376 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:00 smithi029 conmon[101466]: 2023-10-09T04:01:00.229+0000 7ff4e85981c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:00.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:00 smithi029 conmon[101466]: 2023-10-09T04:01:00.505+0000 7ff4e85981c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:01:01.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:00 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/314528520' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:01:01.481 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "486585fa-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:01:01.482 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:01:01.483 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:01:01.484 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:01:01.485 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:01:01.485 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:01:01.485 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:01:01.485 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.485 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:01:01.485 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:01:01.486 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:00:46.051258+0000", 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:01:01.487 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (3/15)... 2023-10-09T04:01:01.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:01 smithi029 conmon[101466]: 2023-10-09T04:01:01.548+0000 7ff4e85981c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:01:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: Activating manager daemon y 2023-10-09T04:01:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: mgrmap e2: y(active, starting, since 0.00293173s) 2023-10-09T04:01:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:01:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:01:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:01:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: Manager daemon y is now available 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' 2023-10-09T04:01:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:02 smithi029 ceph-mon[101043]: from='mgr.14100 172.21.15.29:0/3991537279' entity='mgr.y' 2023-10-09T04:01:03.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:03 smithi029 ceph-mon[101043]: mgrmap e3: y(active, since 1.00731s) 2023-10-09T04:01:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:04 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4001735819' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-09T04:01:05.153 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:01:05.153 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "486585fa-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:01:05.154 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-10-09T04:01:05.155 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-10-09T04:01:05.156 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-10-09T04:01:05.156 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-10-09T04:01:05.156 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-10-09T04:01:05.156 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-10-09T04:01:05.159 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-10-09T04:01:05.160 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2023-10-09T04:00:46.051258+0000", 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-10-09T04:01:05.161 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:01:05.162 INFO:teuthology.orchestra.run.smithi029.stdout:mgr is available 2023-10-09T04:01:05.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:05 smithi029 ceph-mon[101043]: mgrmap e4: y(active, since 3s) 2023-10-09T04:01:06.668 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [global] 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout fsid = 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [mgr] 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-10-09T04:01:06.669 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2023-10-09T04:01:06.670 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [osd] 2023-10-09T04:01:06.670 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-10-09T04:01:06.670 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-10-09T04:01:06.670 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling cephadm module... 2023-10-09T04:01:06.849 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:06 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2701449335' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-09T04:01:07.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:07 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/400635328' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-09T04:01:07.917 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:07 smithi029 conmon[101466]: ignoring --setuser ceph since I am not root 2023-10-09T04:01:07.917 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:07 smithi029 conmon[101466]: ignoring --setgroup ceph since I am not root 2023-10-09T04:01:07.917 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:07 smithi029 conmon[101466]: 2023-10-09T04:01:07.819+0000 7fc1bb88f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:01:08.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:07 smithi029 conmon[101466]: 2023-10-09T04:01:07.916+0000 7fc1bb88f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:08.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:08 smithi029 conmon[101466]: 2023-10-09T04:01:08.023+0000 7fc1bb88f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:01:08.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:08 smithi029 conmon[101466]: 2023-10-09T04:01:08.131+0000 7fc1bb88f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:01:08.562 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:08 smithi029 conmon[101466]: 2023-10-09T04:01:08.338+0000 7fc1bb88f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:01:08.854 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:08 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/400635328' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-09T04:01:08.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:08 smithi029 ceph-mon[101043]: mgrmap e5: y(active, since 6s) 2023-10-09T04:01:08.855 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:08 smithi029 conmon[101466]: 2023-10-09T04:01:08.743+0000 7fc1bb88f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:01:09.118 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:08 smithi029 conmon[101466]: 2023-10-09T04:01:08.853+0000 7fc1bb88f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:01:09.118 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:09 smithi029 conmon[101466]: 2023-10-09T04:01:09.003+0000 7fc1bb88f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:01:09.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:09 smithi029 conmon[101466]: 2023-10-09T04:01:09.117+0000 7fc1bb88f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:01:09.875 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for the mgr to restart... 2023-10-09T04:01:09.876 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr epoch 5... 2023-10-09T04:01:09.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:09 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1031110826' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T04:01:09.881 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:09 smithi029 conmon[101466]: 2023-10-09T04:01:09.633+0000 7fc1bb88f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:01:10.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:10 smithi029 conmon[101466]: 2023-10-09T04:01:10.334+0000 7fc1bb88f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:01:10.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:10 smithi029 conmon[101466]: 2023-10-09T04:01:10.525+0000 7fc1bb88f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:01:11.801 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:11 smithi029 conmon[101466]: 2023-10-09T04:01:11.477+0000 7fc1bb88f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:11.801 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:11 smithi029 conmon[101466]: 2023-10-09T04:01:11.586+0000 7fc1bb88f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:01:12.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:11 smithi029 conmon[101466]: 2023-10-09T04:01:11.800+0000 7fc1bb88f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:12.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:12 smithi029 conmon[101466]: 2023-10-09T04:01:12.142+0000 7fc1bb88f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:01:12.425 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:12 smithi029 conmon[101466]: 2023-10-09T04:01:12.245+0000 7fc1bb88f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:01:12.894 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:12 smithi029 conmon[101466]: 2023-10-09T04:01:12.574+0000 7fc1bb88f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:01:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:12 smithi029 conmon[101466]: 2023-10-09T04:01:12.893+0000 7fc1bb88f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:01:14.111 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:13 smithi029 conmon[101466]: 2023-10-09T04:01:13.716+0000 7fc1bb88f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:01:14.111 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:13 smithi029 conmon[101466]: 2023-10-09T04:01:13.873+0000 7fc1bb88f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:01:14.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:14 smithi029 conmon[101466]: 2023-10-09T04:01:14.109+0000 7fc1bb88f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:01:14.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:14 smithi029 conmon[101466]: 2023-10-09T04:01:14.245+0000 7fc1bb88f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:14.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:14 smithi029 conmon[101466]: 2023-10-09T04:01:14.512+0000 7fc1bb88f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:01:15.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:15 smithi029 conmon[101466]: 2023-10-09T04:01:15.578+0000 7fc1bb88f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:01:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: Active manager daemon y restarted 2023-10-09T04:01:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: Activating manager daemon y 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: osdmap e2: 0 total, 0 up, 0 in 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: mgrmap e6: y(active, starting, since 0.00325001s) 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: Manager daemon y is now available 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:15.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:15 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:16.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:16 smithi029 ceph-mon[101043]: Found migration_current of "None". Setting to last migration. 2023-10-09T04:01:16.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:16 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:01:16.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:16 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:01:16.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:16 smithi029 ceph-mon[101043]: mgrmap e7: y(active, since 1.00505s) 2023-10-09T04:01:17.290 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:01:17.291 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-10-09T04:01:17.291 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T04:01:17.291 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:01:17.291 INFO:teuthology.orchestra.run.smithi029.stdout:mgr epoch 5 is available 2023-10-09T04:01:17.292 INFO:teuthology.orchestra.run.smithi029.stdout:Setting orchestrator backend to cephadm... 2023-10-09T04:01:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:17 smithi029 ceph-mon[101043]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T04:01:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:17 smithi029 ceph-mon[101043]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T04:01:18.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:18 smithi029 ceph-mon[101043]: [09/Oct/2023:04:01:17] ENGINE Bus STARTING 2023-10-09T04:01:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:18 smithi029 ceph-mon[101043]: mgrmap e8: y(active, since 2s) 2023-10-09T04:01:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:18 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:18 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:18 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:18 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:19.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:19 smithi029 ceph-mon[101043]: [09/Oct/2023:04:01:17] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:01:19.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:19 smithi029 ceph-mon[101043]: [09/Oct/2023:04:01:17] ENGINE Bus STARTED 2023-10-09T04:01:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:19 smithi029 ceph-mon[101043]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:20.216 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout value unchanged 2023-10-09T04:01:20.216 INFO:teuthology.orchestra.run.smithi029.stdout:Generating ssh key... 2023-10-09T04:01:20.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:20 smithi029 ceph-mon[101043]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:21.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: Generating public/private rsa key pair. 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: Your identification has been saved in /tmp/tmpk_omz8cz/key. 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: Your public key has been saved in /tmp/tmpk_omz8cz/key.pub. 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: The key fingerprint is: 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: SHA256:1I5oUqRF4qEVFDc3FBf52PsUUwqVlej68fBnPEfxXpY ceph-486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: The key's randomart image is: 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: +---[RSA 3072]----+ 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | .B+*.=.oo .ooo| 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | + B o +. .....| 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | . o . . .+.. o | 2023-10-09T04:01:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | . o o. o.+. | 2023-10-09T04:01:21.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | . o S . .. o+| 2023-10-09T04:01:21.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | o ..o.E=| 2023-10-09T04:01:21.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | .o==.| 2023-10-09T04:01:21.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | ..+*| 2023-10-09T04:01:21.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: | .+| 2023-10-09T04:01:21.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:21 smithi029 conmon[101466]: +----[SHA256]-----+ 2023-10-09T04:01:22.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:22 smithi029 ceph-mon[101043]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:22 smithi029 ceph-mon[101043]: Generating ssh key... 2023-10-09T04:01:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:22 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:22 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:23.490 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkcr8R6C/EV25Lw8OvnLbLtRWeihCwDGNm/vBx+TZ3bH3o+U3E4Zm9ga8sSM6ztDkosozoldnhUrQyuS371Wr85ANI2MPknAQV4Z4f/Ynd/K8aApi5/bxd/nY6SBnaVKog3SH1QXQaquOdpsW34a6m2W7+Ct1D2pGq5zi/wKe+eUypDyz5HJQV6sBxjlImwryE48Mq5+lyFiiwDYaAw82zaCdghgX5MH2Rd+c21ns7lqebCzdwuVv5HYNIuNc86WZQg/lgcmm2I9OlJPHnGZIaI1YMUtmwJ4R1huR3zZoUtdkPjwJyk29BrvRmLQWsXRZ5ojVDd3/wwrkI0EitM0JZRRjXdeju92cZFRYccjiYQPfFpnYt+rrxHJ32SY1qljgGx4wgoM375sJwez+rHE9S6BOa58OTwvfH5WI4eD0D/C3tMo6tI9Spe43hwThWw9zd3stfWxAgJpLhEOM7MP70xPYtvy3vwe2y3M6mClH17BD4PmntiIukch1LSEUUru8= ceph-486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:23.491 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-09T04:01:23.491 INFO:teuthology.orchestra.run.smithi029.stdout:Adding key to root@localhost authorized_keys... 2023-10-09T04:01:23.492 INFO:teuthology.orchestra.run.smithi029.stdout:Adding host smithi029... 2023-10-09T04:01:23.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:23 smithi029 ceph-mon[101043]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:24.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:24 smithi029 ceph-mon[101043]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi029", "addr": "172.21.15.29", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:26.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:25 smithi029 ceph-mon[101043]: Deploying cephadm binary to smithi029 2023-10-09T04:01:26.940 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Added host 'smithi029' with addr '172.21.15.29' 2023-10-09T04:01:26.940 INFO:teuthology.orchestra.run.smithi029.stdout:Deploying unmanaged mon service... 2023-10-09T04:01:27.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:27 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:27.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:27 smithi029 ceph-mon[101043]: Added host smithi029 2023-10-09T04:01:27.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:27 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:28.856 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-10-09T04:01:28.856 INFO:teuthology.orchestra.run.smithi029.stdout:Deploying unmanaged mgr service... 2023-10-09T04:01:29.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:28 smithi029 ceph-mon[101043]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:29.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:28 smithi029 ceph-mon[101043]: Saving service mon spec with placement count:5 2023-10-09T04:01:29.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:28 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:30.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:30 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:30.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:30 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:30.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:30 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:30.805 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-10-09T04:01:31.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:31 smithi029 ceph-mon[101043]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:31 smithi029 ceph-mon[101043]: Saving service mgr spec with placement count:2 2023-10-09T04:01:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:31 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:32 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2774937991' entity='client.admin' 2023-10-09T04:01:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:32 smithi029 ceph-mon[101043]: from='mgr.14120 172.21.15.29:0/1789617387' entity='mgr.y' 2023-10-09T04:01:33.956 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling the dashboard module... 2023-10-09T04:01:34.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:34 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2033964311' entity='client.admin' 2023-10-09T04:01:35.365 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:35 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/145497801' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-09T04:01:35.366 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: ignoring --setuser ceph since I am not root 2023-10-09T04:01:35.366 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: ignoring --setgroup ceph since I am not root 2023-10-09T04:01:35.366 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: 2023-10-09T04:01:35.269+0000 7f25743e41c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:01:35.366 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: 2023-10-09T04:01:35.364+0000 7f25743e41c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:35.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: 2023-10-09T04:01:35.470+0000 7f25743e41c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:01:35.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: 2023-10-09T04:01:35.578+0000 7f25743e41c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:01:36.044 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:35 smithi029 conmon[101466]: 2023-10-09T04:01:35.779+0000 7f25743e41c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:01:36.298 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:36 smithi029 conmon[101466]: 2023-10-09T04:01:36.188+0000 7f25743e41c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:01:36.299 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:36 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/145497801' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-09T04:01:36.299 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:36 smithi029 ceph-mon[101043]: mgrmap e9: y(active, since 19s) 2023-10-09T04:01:36.564 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:36 smithi029 conmon[101466]: 2023-10-09T04:01:36.297+0000 7f25743e41c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:01:36.564 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:36 smithi029 conmon[101466]: 2023-10-09T04:01:36.450+0000 7f25743e41c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:01:36.565 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:36 smithi029 conmon[101466]: 2023-10-09T04:01:36.563+0000 7f25743e41c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:01:37.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:37 smithi029 conmon[101466]: 2023-10-09T04:01:37.076+0000 7f25743e41c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:01:37.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for the mgr to restart... 2023-10-09T04:01:37.350 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr epoch 9... 2023-10-09T04:01:37.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:37 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4064697831' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-09T04:01:37.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:37 smithi029 conmon[101466]: 2023-10-09T04:01:37.773+0000 7f25743e41c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:01:38.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:37 smithi029 conmon[101466]: 2023-10-09T04:01:37.965+0000 7f25743e41c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:01:39.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:38 smithi029 conmon[101466]: 2023-10-09T04:01:38.916+0000 7f25743e41c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:39.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:39 smithi029 conmon[101466]: 2023-10-09T04:01:39.025+0000 7f25743e41c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:01:39.581 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:39 smithi029 conmon[101466]: 2023-10-09T04:01:39.239+0000 7f25743e41c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:01:39.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:39 smithi029 conmon[101466]: 2023-10-09T04:01:39.581+0000 7f25743e41c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:01:39.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:39 smithi029 conmon[101466]: 2023-10-09T04:01:39.684+0000 7f25743e41c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:01:40.335 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:40 smithi029 conmon[101466]: 2023-10-09T04:01:40.012+0000 7f25743e41c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:01:41.581 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:40 smithi029 conmon[101466]: 2023-10-09T04:01:40.334+0000 7f25743e41c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:01:41.582 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:41 smithi029 conmon[101466]: 2023-10-09T04:01:41.183+0000 7f25743e41c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:01:41.582 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:41 smithi029 conmon[101466]: 2023-10-09T04:01:41.339+0000 7f25743e41c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:01:41.582 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:41 smithi029 conmon[101466]: 2023-10-09T04:01:41.581+0000 7f25743e41c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:01:41.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:41 smithi029 conmon[101466]: 2023-10-09T04:01:41.718+0000 7f25743e41c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:01:42.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:41 smithi029 conmon[101466]: 2023-10-09T04:01:41.989+0000 7f25743e41c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:01:43.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:01:43 smithi029 conmon[101466]: 2023-10-09T04:01:43.051+0000 7f25743e41c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:01:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: Active manager daemon y restarted 2023-10-09T04:01:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: Activating manager daemon y 2023-10-09T04:01:43.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: osdmap e3: 0 total, 0 up, 0 in 2023-10-09T04:01:43.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: mgrmap e10: y(active, starting, since 0.00309198s) 2023-10-09T04:01:43.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:01:43.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:01:43.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:01:43.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:01:43.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:01:43.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: Manager daemon y is now available 2023-10-09T04:01:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:01:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:01:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:44 smithi029 ceph-mon[101043]: mgrmap e11: y(active, since 1.00582s) 2023-10-09T04:01:44.729 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2023-10-09T04:01:44.730 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-10-09T04:01:44.730 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "initialized": true 2023-10-09T04:01:44.730 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2023-10-09T04:01:44.730 INFO:teuthology.orchestra.run.smithi029.stdout:mgr epoch 9 is available 2023-10-09T04:01:44.730 INFO:teuthology.orchestra.run.smithi029.stdout:Generating a dashboard self-signed certificate... 2023-10-09T04:01:45.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:45 smithi029 ceph-mon[101043]: [09/Oct/2023:04:01:43] ENGINE Bus STARTING 2023-10-09T04:01:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:45 smithi029 ceph-mon[101043]: [09/Oct/2023:04:01:43] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:01:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:45 smithi029 ceph-mon[101043]: [09/Oct/2023:04:01:43] ENGINE Bus STARTED 2023-10-09T04:01:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:45 smithi029 ceph-mon[101043]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-09T04:01:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:45 smithi029 ceph-mon[101043]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-09T04:01:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:46.312 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-10-09T04:01:46.312 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial admin user... 2023-10-09T04:01:46.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:46.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:46 smithi029 ceph-mon[101043]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:46.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:46.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:46.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:46 smithi029 ceph-mon[101043]: mgrmap e12: y(active, since 2s) 2023-10-09T04:01:48.214 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$swLk20amlusP.keNYwzaoe6KoQxGgi.2Xcj4V1zRqlTe.4uIbDoXO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696824107, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-09T04:01:48.214 INFO:teuthology.orchestra.run.smithi029.stdout:Fetching dashboard port number... 2023-10-09T04:01:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:48 smithi029 ceph-mon[101043]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:01:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:48 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:49.646 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 8443 2023-10-09T04:01:49.657 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-10-09T04:01:49.657 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2023-10-09T04:01:49.664 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-10-09T04:01:49.664 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2023-10-09T04:01:49.664 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2023-10-09T04:01:49.665 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-09T04:01:49.666 INFO:teuthology.orchestra.run.smithi029.stdout:Ceph Dashboard is now available at: 2023-10-09T04:01:49.666 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:49.666 INFO:teuthology.orchestra.run.smithi029.stdout: URL: https://smithi029:8443/ 2023-10-09T04:01:49.666 INFO:teuthology.orchestra.run.smithi029.stdout: User: admin 2023-10-09T04:01:49.667 INFO:teuthology.orchestra.run.smithi029.stdout: Password: 8y4wrcgdm0 2023-10-09T04:01:49.667 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:49.667 INFO:teuthology.orchestra.run.smithi029.stdout:Saving cluster configuration to /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config directory 2023-10-09T04:01:49.667 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling autotune for osd_memory_target 2023-10-09T04:01:49.893 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:49 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4262621417' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-09T04:01:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:50 smithi029 ceph-mon[101043]: mgrmap e13: y(active, since 6s) 2023-10-09T04:01:52.812 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-10-09T04:01:52.812 INFO:teuthology.orchestra.run.smithi029.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-09T04:01:52.812 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.813 INFO:teuthology.orchestra.run.smithi029.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-09T04:01:52.813 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.813 INFO:teuthology.orchestra.run.smithi029.stdout:Or, if you are only running a single cluster on this host: 2023-10-09T04:01:52.813 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout:Please consider enabling telemetry to help improve Ceph: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: ceph telemetry on 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout:For more information see: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-10-09T04:01:52.814 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:01:52.815 INFO:teuthology.orchestra.run.smithi029.stdout:Bootstrap complete. 2023-10-09T04:01:52.850 INFO:tasks.cephadm:Fetching config... 2023-10-09T04:01:52.851 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:01:52.851 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-09T04:01:52.868 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-09T04:01:52.868 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:01:52.868 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-09T04:01:52.927 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-09T04:01:52.927 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:01:52.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-09T04:01:52.997 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-09T04:01:52.998 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:01:52.998 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-09T04:01:53.054 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-09T04:01:53.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkcr8R6C/EV25Lw8OvnLbLtRWeihCwDGNm/vBx+TZ3bH3o+U3E4Zm9ga8sSM6ztDkosozoldnhUrQyuS371Wr85ANI2MPknAQV4Z4f/Ynd/K8aApi5/bxd/nY6SBnaVKog3SH1QXQaquOdpsW34a6m2W7+Ct1D2pGq5zi/wKe+eUypDyz5HJQV6sBxjlImwryE48Mq5+lyFiiwDYaAw82zaCdghgX5MH2Rd+c21ns7lqebCzdwuVv5HYNIuNc86WZQg/lgcmm2I9OlJPHnGZIaI1YMUtmwJ4R1huR3zZoUtdkPjwJyk29BrvRmLQWsXRZ5ojVDd3/wwrkI0EitM0JZRRjXdeju92cZFRYccjiYQPfFpnYt+rrxHJ32SY1qljgGx4wgoM375sJwez+rHE9S6BOa58OTwvfH5WI4eD0D/C3tMo6tI9Spe43hwThWw9zd3stfWxAgJpLhEOM7MP70xPYtvy3vwe2y3M6mClH17BD4PmntiIukch1LSEUUru8= ceph-486585fa-6658-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T04:01:53.133 INFO:teuthology.orchestra.run.smithi029.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkcr8R6C/EV25Lw8OvnLbLtRWeihCwDGNm/vBx+TZ3bH3o+U3E4Zm9ga8sSM6ztDkosozoldnhUrQyuS371Wr85ANI2MPknAQV4Z4f/Ynd/K8aApi5/bxd/nY6SBnaVKog3SH1QXQaquOdpsW34a6m2W7+Ct1D2pGq5zi/wKe+eUypDyz5HJQV6sBxjlImwryE48Mq5+lyFiiwDYaAw82zaCdghgX5MH2Rd+c21ns7lqebCzdwuVv5HYNIuNc86WZQg/lgcmm2I9OlJPHnGZIaI1YMUtmwJ4R1huR3zZoUtdkPjwJyk29BrvRmLQWsXRZ5ojVDd3/wwrkI0EitM0JZRRjXdeju92cZFRYccjiYQPfFpnYt+rrxHJ32SY1qljgGx4wgoM375sJwez+rHE9S6BOa58OTwvfH5WI4eD0D/C3tMo6tI9Spe43hwThWw9zd3stfWxAgJpLhEOM7MP70xPYtvy3vwe2y3M6mClH17BD4PmntiIukch1LSEUUru8= ceph-486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:53.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkcr8R6C/EV25Lw8OvnLbLtRWeihCwDGNm/vBx+TZ3bH3o+U3E4Zm9ga8sSM6ztDkosozoldnhUrQyuS371Wr85ANI2MPknAQV4Z4f/Ynd/K8aApi5/bxd/nY6SBnaVKog3SH1QXQaquOdpsW34a6m2W7+Ct1D2pGq5zi/wKe+eUypDyz5HJQV6sBxjlImwryE48Mq5+lyFiiwDYaAw82zaCdghgX5MH2Rd+c21ns7lqebCzdwuVv5HYNIuNc86WZQg/lgcmm2I9OlJPHnGZIaI1YMUtmwJ4R1huR3zZoUtdkPjwJyk29BrvRmLQWsXRZ5ojVDd3/wwrkI0EitM0JZRRjXdeju92cZFRYccjiYQPfFpnYt+rrxHJ32SY1qljgGx4wgoM375sJwez+rHE9S6BOa58OTwvfH5WI4eD0D/C3tMo6tI9Spe43hwThWw9zd3stfWxAgJpLhEOM7MP70xPYtvy3vwe2y3M6mClH17BD4PmntiIukch1LSEUUru8= ceph-486585fa-6658-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-09T04:01:53.192 INFO:teuthology.orchestra.run.smithi177.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkcr8R6C/EV25Lw8OvnLbLtRWeihCwDGNm/vBx+TZ3bH3o+U3E4Zm9ga8sSM6ztDkosozoldnhUrQyuS371Wr85ANI2MPknAQV4Z4f/Ynd/K8aApi5/bxd/nY6SBnaVKog3SH1QXQaquOdpsW34a6m2W7+Ct1D2pGq5zi/wKe+eUypDyz5HJQV6sBxjlImwryE48Mq5+lyFiiwDYaAw82zaCdghgX5MH2Rd+c21ns7lqebCzdwuVv5HYNIuNc86WZQg/lgcmm2I9OlJPHnGZIaI1YMUtmwJ4R1huR3zZoUtdkPjwJyk29BrvRmLQWsXRZ5ojVDd3/wwrkI0EitM0JZRRjXdeju92cZFRYccjiYQPfFpnYt+rrxHJ32SY1qljgGx4wgoM375sJwez+rHE9S6BOa58OTwvfH5WI4eD0D/C3tMo6tI9Spe43hwThWw9zd3stfWxAgJpLhEOM7MP70xPYtvy3vwe2y3M6mClH17BD4PmntiIukch1LSEUUru8= ceph-486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:01:53.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-09T04:01:53.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:53 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/329879284' entity='client.admin' 2023-10-09T04:01:53.561 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:01:56.559 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-09T04:01:56.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-09T04:01:57.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:56 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3041495891' entity='client.admin' 2023-10-09T04:01:57.315 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:01:58.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:58.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:58.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi029", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:01:58.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:58.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:01:58.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:01:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:01:59.777 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi177 2023-10-09T04:01:59.777 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T04:01:59.777 DEBUG:teuthology.orchestra.run.smithi177:> dd of=/etc/ceph/ceph.conf 2023-10-09T04:01:59.797 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T04:01:59.797 DEBUG:teuthology.orchestra.run.smithi177:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:01:59.857 INFO:tasks.cephadm:Adding host smithi177 to orchestrator... 2023-10-09T04:01:59.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi177 2023-10-09T04:02:00.220 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:02:00.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:00 smithi029 ceph-mon[101043]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:00.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:00 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:00.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:00 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:00.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:00 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:00.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:00 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:00.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:00 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:02:01.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:01 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:01.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:01 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:01.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:01 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:02:01.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:01 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:01.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:01 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:01.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:01 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:02 smithi029 ceph-mon[101043]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi177", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:03.900 INFO:teuthology.orchestra.run.smithi029.stdout:Added host 'smithi177' with addr '172.21.15.177' 2023-10-09T04:02:04.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:04 smithi029 ceph-mon[101043]: Deploying cephadm binary to smithi177 2023-10-09T04:02:04.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:04 smithi029 ceph-mon[101043]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:04.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-09T04:02:04.934 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:02:05.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:05 smithi029 ceph-mon[101043]: Added host smithi177 2023-10-09T04:02:05.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:05 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:05.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:05 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:06.241 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:06 smithi029 ceph-mon[101043]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:06.241 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:06 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:06.624 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:02:06.624 INFO:teuthology.orchestra.run.smithi029.stdout:[{"addr": "172.21.15.29", "hostname": "smithi029", "labels": [], "status": ""}, {"addr": "172.21.15.177", "hostname": "smithi177", "labels": [], "status": ""}] 2023-10-09T04:02:07.342 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-09T04:02:07.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-09T04:02:07.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:07 smithi029 ceph-mon[101043]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:07 smithi029 ceph-mon[101043]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:07.696 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:02:09.750 INFO:teuthology.orchestra.run.smithi029.stderr:adjusted tunables profile to default 2023-10-09T04:02:10.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:02:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/108051490' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T04:02:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:10.460 INFO:tasks.cephadm:Adding mon.a on smithi029 2023-10-09T04:02:10.461 INFO:tasks.cephadm:Adding mon.c on smithi029 2023-10-09T04:02:10.461 INFO:tasks.cephadm:Adding mon.b on smithi177 2023-10-09T04:02:10.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b' 2023-10-09T04:02:10.829 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:11.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:10 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:10 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:02:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:10 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/108051490' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T04:02:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:10 smithi029 ceph-mon[101043]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:12.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:11 smithi029 ceph-mon[101043]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:12.519 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled mon update... 2023-10-09T04:02:13.108 DEBUG:teuthology.orchestra.run.smithi029:mon.c> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-09T04:02:13.111 DEBUG:teuthology.orchestra.run.smithi177:mon.b> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-09T04:02:13.113 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:02:13.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:02:13.143 INFO:journalctl@ceph.mon.b.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:02:13.148 INFO:journalctl@ceph.mon.c.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:02:13.577 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b;count:3 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:13.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: Deploying daemon mon.b on smithi177 2023-10-09T04:02:13.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:13 smithi029 ceph-mon[101043]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:16.117 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T04:02:16.117 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:00:43.172716Z","created":"2023-10-09T04:00:43.172716Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-09T04:02:16.118 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2023-10-09T04:02:16.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:16 smithi029 ceph-mon[101043]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:16 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/2048644156' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:16.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:16 smithi177 systemd[1]: Starting Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:17.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:02:17.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T04:02:17.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pidfile_write: ignore empty --pid-file 2023-10-09T04:02:17.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: load: jerasure load: lrc 2023-10-09T04:02:17.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: RocksDB version: 6.15.5 2023-10-09T04:02:17.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Compile date Oct 8 2023 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: DB SUMMARY 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: DB Session ID: RSC13260Z6CF5FTGUKT0 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:02:17.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.env: 0x563bf4b20a80 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.fs: Posix File System 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.info_log: 0x563bf66d25a0 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.statistics: (nil) 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.use_fsync: 0 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:02:17.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.db_log_dir: 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:02:17.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.write_buffer_manager: 0x563bf7444960 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:02:17.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.unordered_write: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.row_cache: None 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.wal_filter: None 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.preserve_deletes: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:02:17.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:02:17.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_open_files: -1 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:02:17.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Compression algorithms supported: 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kZSTD supported: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:02:17.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.merge_operator: 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_filter: None 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563bf6724fb8) 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:02:17.537 INFO:journalctl@ceph.mon.b.smithi177.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: index_type: 0 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: data_block_index_type: 0 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: index_shortening: 1 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: hash_index_allow_collision: 1 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: checksum: 1 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: no_block_cache: 0 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache: 0x563bf6757090 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache_options: 2023-10-09T04:02:17.538 INFO:journalctl@ceph.mon.b.smithi177.stdout: capacity : 536870912 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: num_shard_bits : 4 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: strict_capacity_limit : 0 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache_compressed: (nil) 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: persistent_cache: (nil) 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_size: 4096 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_size_deviation: 10 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_restart_interval: 16 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: index_block_restart_interval: 1 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: metadata_block_size: 4096 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: partition_filters: 0 2023-10-09T04:02:17.539 INFO:journalctl@ceph.mon.b.smithi177.stdout: use_delta_encoding: 1 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: whole_key_filtering: 1 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: verify_compression: 0 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: format_version: 4 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: enable_index_compression: 1 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_align: 0 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression: NoCompression 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:02:17.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.num_levels: 7 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:02:17.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:02:17.543 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:02:17.544 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:02:17.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:02:17.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.ttl: 2592000 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:02:17.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824137107891, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824137108154, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696824137, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a48596dd-d9f3-4dbf-a9b0-e1b5bb0abe2f", "db_session_id": "RSC13260Z6CF5FTGUKT0"}} 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824137108478, "job": 1, "event": "recovery_finished"} 2023-10-09T04:02:17.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563bf66eb180 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: DB pointer 0x563bf67c0000 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** DB Stats ** 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:02:17.549 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:17.550 INFO:journalctl@ceph.mon.b.smithi177.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.551 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:17.552 INFO:journalctl@ceph.mon.b.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-09T04:02:17.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: using public_addr v2:172.21.15.177:0/0 -> [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: starting mon.b rank -1 at public addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] at bind addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(???) e0 preinit fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 bash[106005]: f19800b5161693708d00ecf019d8edf8491f3fc2e39448428cef4ee2678bc212 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 systemd[1]: Started Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).mds e1 new map 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).mds e1 print_map 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: e1 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: legacy client fscid: -1 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:02:17.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: No filesystems configured 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Added host smithi177 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.555 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:17.556 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/108051490' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:02:17.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/108051490' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b;count:3 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:17.558 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:17.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: Deploying daemon mon.b on smithi177 2023-10-09T04:02:17.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:17.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:17.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/2048644156' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:17.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T04:02:17.559 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:17 smithi177 ceph-mon[106070]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-10-09T04:02:17.652 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:02:17.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:02:17.987 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:02:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: mon.a calling monitor election 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: mon.b calling monitor election 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:02:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} removed_ranks: {} 2023-10-09T04:02:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: fsmap 2023-10-09T04:02:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: mgrmap e13: y(active, since 39s) 2023-10-09T04:02:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: overall HEALTH_OK 2023-10-09T04:02:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:22.660 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T04:02:22.660 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":2,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:02:17.129280Z","created":"2023-10-09T04:00:43.172716Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:3300","nonce":0},{"type":"v1","addr":"172.21.15.177:6789","nonce":0}]},"addr":"172.21.15.177:6789/0","public_addr":"172.21.15.177:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-09T04:02:22.662 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 2 2023-10-09T04:02:22.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:22.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:22.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: mon.a calling monitor election 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: mon.b calling monitor election 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} removed_ranks: {} 2023-10-09T04:02:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: fsmap 2023-10-09T04:02:22.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:22.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: mgrmap e13: y(active, since 39s) 2023-10-09T04:02:22.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: overall HEALTH_OK 2023-10-09T04:02:22.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:22.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:23.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:23.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:23.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:23.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: Deploying daemon mon.c on smithi029 2023-10-09T04:02:23.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/4156879611' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:23.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:23.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:23.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:23.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: Deploying daemon mon.c on smithi029 2023-10-09T04:02:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/4156879611' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:24.206 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-09T04:02:24.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-09T04:02:24.553 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:02:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:24 smithi029 systemd[1]: Starting Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:02:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable), process ceph-mon, pid 7 2023-10-09T04:02:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pidfile_write: ignore empty --pid-file 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: load: jerasure load: lrc 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: RocksDB version: 6.15.5 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Compile date Oct 8 2023 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: DB SUMMARY 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: DB Session ID: KTBYLWNXAP8CQBQQ16YL 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-10-09T04:02:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.env: 0x55aae90dea80 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.fs: Posix File System 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.info_log: 0x55aaea7a25a0 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.statistics: (nil) 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.use_fsync: 0 2023-10-09T04:02:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.db_log_dir: 2023-10-09T04:02:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.write_buffer_manager: 0x55aaeb514930 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:02:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.unordered_write: 0 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.row_cache: None 2023-10-09T04:02:25.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.wal_filter: None 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.preserve_deletes: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:02:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:02:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_open_files: -1 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Compression algorithms supported: 2023-10-09T04:02:25.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kZSTD supported: 0 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:02:25.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 bash[119066]: c8851494e8fd5b2bfa8ff6afa1915574d9118a787c60627e7612bfccb7e86d37 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.merge_operator: 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_filter: None 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55aaea7f4fb8) 2023-10-09T04:02:25.931 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_type: 0 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_index_type: 0 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_shortening: 1 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: hash_index_allow_collision: 1 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: checksum: 1 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: no_block_cache: 0 2023-10-09T04:02:25.932 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache: 0x55aaea827090 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_options: 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: capacity : 536870912 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_shard_bits : 4 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: strict_capacity_limit : 0 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_compressed: (nil) 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: persistent_cache: (nil) 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size: 4096 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size_deviation: 10 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_restart_interval: 16 2023-10-09T04:02:25.933 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_block_restart_interval: 1 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_block_size: 4096 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: partition_filters: 0 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: use_delta_encoding: 1 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: whole_key_filtering: 1 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: verify_compression: 0 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: format_version: 4 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_index_compression: 1 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_align: 0 2023-10-09T04:02:25.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression: NoCompression 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.num_levels: 7 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:02:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:02:25.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:02:25.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:02:25.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:02:25.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:02:25.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:02:25.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.arena_block_size: 4194304 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:02:25.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:02:25.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.ttl: 2592000 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:02:25.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824145460163, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824145460561, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696824145, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a33e7a1c-39b8-4df7-9c56-991955364e33", "db_session_id": "KTBYLWNXAP8CQBQQ16YL"}} 2023-10-09T04:02:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824145461032, "job": 1, "event": "recovery_finished"} 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55aaea7bb180 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: DB pointer 0x55aaea890000 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** DB Stats ** 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:02:25.943 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:25.944 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:02:25.945 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:02:25.946 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:02:25.947 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: using public_addrv [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: starting mon.c rank -1 at public addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] at bind addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(???) e0 preinit fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 systemd[1]: Started Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).mds e1 new map 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).mds e1 print_map 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout: e1 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:02:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout: legacy client fscid: -1 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout: No filesystems configured 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:25.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Added host smithi177 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.950 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/108051490' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/108051490' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-09T04:02:25.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi177:172.21.15.177=b;count:3 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Deploying daemon mon.b on smithi177 2023-10-09T04:02:25.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/2048644156' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.a calling monitor election 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.b calling monitor election 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} removed_ranks: {} 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: fsmap 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mgrmap e13: y(active, since 39s) 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: overall HEALTH_OK 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.954 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: Deploying daemon mon.c on smithi029 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/4156879611' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-09T04:02:25.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:25 smithi029 ceph-mon[119131]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: mon.a calling monitor election 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: mon.b calling monitor election 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: mon.c calling monitor election 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: fsmap 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: mgrmap e13: y(active, since 47s) 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: overall HEALTH_OK 2023-10-09T04:02:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: mon.a calling monitor election 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: mon.b calling monitor election 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: mon.c calling monitor election 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: fsmap 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: mgrmap e13: y(active, since 47s) 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: overall HEALTH_OK 2023-10-09T04:02:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:02:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: mon.a calling monitor election 2023-10-09T04:02:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: mon.b calling monitor election 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: mon.c calling monitor election 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:02:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: fsmap 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: osdmap e4: 0 total, 0 up, 0 in 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: mgrmap e13: y(active, since 47s) 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: overall HEALTH_OK 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[101043]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[119131]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:31 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:31 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:31 smithi177 ceph-mon[106070]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:31 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:02:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:31 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:31 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:32.232 INFO:teuthology.orchestra.run.smithi177.stdout: 2023-10-09T04:02:32.232 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":3,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","modified":"2023-10-09T04:02:25.484827Z","created":"2023-10-09T04:00:43.172716Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:3300","nonce":0},{"type":"v1","addr":"172.21.15.177:6789","nonce":0}]},"addr":"172.21.15.177:6789/0","public_addr":"172.21.15.177:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3301","nonce":0},{"type":"v1","addr":"172.21.15.29:6790","nonce":0}]},"addr":"172.21.15.29:6790/0","public_addr":"172.21.15.29:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-09T04:02:32.234 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 3 2023-10-09T04:02:32.845 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-09T04:02:32.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-09T04:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/612486717' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:33.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:33.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/612486717' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:02:33.031 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/612486717' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:33.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:33.963 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:02:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: Reconfiguring daemon mon.a on smithi029 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:34.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: Reconfiguring daemon mon.a on smithi029 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:34.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: Reconfiguring mon.a (unknown last config time)... 2023-10-09T04:02:34.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: Reconfiguring daemon mon.a on smithi029 2023-10-09T04:02:34.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:35.231 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:35 smithi029 ceph-mon[101043]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:02:35.231 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:35 smithi029 ceph-mon[101043]: Reconfiguring daemon mon.c on smithi029 2023-10-09T04:02:35.231 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:35 smithi029 ceph-mon[119131]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:02:35.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:35 smithi029 ceph-mon[119131]: Reconfiguring daemon mon.c on smithi029 2023-10-09T04:02:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:35 smithi177 ceph-mon[106070]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:02:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:35 smithi177 ceph-mon[106070]: Reconfiguring daemon mon.c on smithi029 2023-10-09T04:02:35.834 INFO:teuthology.orchestra.run.smithi029.stdout:# minimal ceph.conf for 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:35.834 INFO:teuthology.orchestra.run.smithi029.stdout:[global] 2023-10-09T04:02:35.834 INFO:teuthology.orchestra.run.smithi029.stdout: fsid = 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:02:35.834 INFO:teuthology.orchestra.run.smithi029.stdout: mon_host = [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] 2023-10-09T04:02:36.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:36 smithi029 ceph-mon[119131]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:36 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1975752622' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:36 smithi029 ceph-mon[101043]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:36 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1975752622' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:36 smithi177 ceph-mon[106070]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:36 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1975752622' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:36.753 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-09T04:02:36.753 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:02:36.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T04:02:36.783 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:02:36.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:36.851 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T04:02:36.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.conf 2023-10-09T04:02:36.881 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T04:02:36.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:02:36.950 INFO:tasks.cephadm:Adding mgr.y on smithi029 2023-10-09T04:02:36.950 INFO:tasks.cephadm:Adding mgr.x on smithi177 2023-10-09T04:02:36.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi029=y;smithi177=x' 2023-10-09T04:02:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:02:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: Reconfiguring daemon mon.b on smithi177 2023-10-09T04:02:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:37 smithi177 ceph-mon[106070]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: Reconfiguring daemon mon.b on smithi177 2023-10-09T04:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[101043]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: Reconfiguring daemon mon.b on smithi177 2023-10-09T04:02:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:37 smithi029 ceph-mon[119131]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:37.945 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:02:39.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:38 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.150 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:38 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.150 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:38 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:39.150 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:38 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:39.150 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:38 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:39.150 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:38 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:38 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:39.504 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled mgr update... 2023-10-09T04:02:40.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:02:40.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:40.157 DEBUG:teuthology.orchestra.run.smithi177:mgr.x> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-09T04:02:40.159 INFO:tasks.cephadm:Deploying OSDs... 2023-10-09T04:02:40.160 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:02:40.160 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T04:02:40.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:40.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:40.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:02:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:40.181 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T04:02:40.181 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_1 2023-10-09T04:02:40.190 INFO:journalctl@ceph.mgr.x.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:02:40.242 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T04:02:40.242 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:40.242 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 6h/6d Inode: 393364 Links: 1 2023-10-09T04:02:40.243 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:40.243 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:40.243 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2023-10-09 04:02:24.815033702 +0000 2023-10-09T04:02:40.243 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2023-10-09 03:53:24.340583446 +0000 2023-10-09T04:02:40.243 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2023-10-09 03:53:24.340583446 +0000 2023-10-09T04:02:40.243 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: - 2023-10-09T04:02:40.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T04:02:40.313 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2023-10-09T04:02:40.314 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2023-10-09T04:02:40.314 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000135588 s, 3.8 MB/s 2023-10-09T04:02:40.315 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T04:02:40.376 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_2 2023-10-09T04:02:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T04:02:40.436 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:40.436 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 6h/6d Inode: 393428 Links: 1 2023-10-09T04:02:40.436 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:40.436 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:40.436 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2023-10-09 04:02:24.815033702 +0000 2023-10-09T04:02:40.437 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2023-10-09 03:53:24.963573471 +0000 2023-10-09T04:02:40.437 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2023-10-09 03:53:24.963573471 +0000 2023-10-09T04:02:40.437 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: - 2023-10-09T04:02:40.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T04:02:40.505 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2023-10-09T04:02:40.505 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2023-10-09T04:02:40.506 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.00012606 s, 4.1 MB/s 2023-10-09T04:02:40.507 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T04:02:40.568 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_3 2023-10-09T04:02:40.629 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T04:02:40.629 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:40.629 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 6h/6d Inode: 388025 Links: 1 2023-10-09T04:02:40.629 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:40.629 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:40.630 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2023-10-09 04:02:24.816033687 +0000 2023-10-09T04:02:40.630 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2023-10-09 03:53:25.507564761 +0000 2023-10-09T04:02:40.630 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2023-10-09 03:53:25.507564761 +0000 2023-10-09T04:02:40.630 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: - 2023-10-09T04:02:40.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T04:02:40.700 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2023-10-09T04:02:40.700 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2023-10-09T04:02:40.700 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000127209 s, 4.0 MB/s 2023-10-09T04:02:40.701 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T04:02:40.762 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_4 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 6h/6d Inode: 395422 Links: 1 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2023-10-09 04:02:24.816033687 +0000 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2023-10-09 03:53:26.044556163 +0000 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2023-10-09 03:53:26.044556163 +0000 2023-10-09T04:02:40.823 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: - 2023-10-09T04:02:40.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T04:02:40.895 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2023-10-09T04:02:40.895 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2023-10-09T04:02:40.895 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000128648 s, 4.0 MB/s 2023-10-09T04:02:40.896 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T04:02:40.957 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T04:02:40.957 DEBUG:teuthology.orchestra.run.smithi177:> dd if=/scratch_devs of=/dev/stdout 2023-10-09T04:02:40.975 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-09T04:02:40.975 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_1 2023-10-09T04:02:41.033 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-09T04:02:41.033 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:41.033 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 6h/6d Inode: 386951 Links: 1 2023-10-09T04:02:41.034 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:41.034 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:41.034 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2023-10-09 04:02:16.412015847 +0000 2023-10-09T04:02:41.034 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2023-10-09 03:53:00.507766193 +0000 2023-10-09T04:02:41.034 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2023-10-09 03:53:00.507766193 +0000 2023-10-09T04:02:41.034 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2023-10-09T04:02:41.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-09T04:02:41.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:40 smithi177 ceph-mon[106070]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi177=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:41.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:40 smithi177 ceph-mon[106070]: Saving service mgr spec with placement smithi029=y;smithi177=x;count:2 2023-10-09T04:02:41.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:40 smithi177 ceph-mon[106070]: Deploying daemon mgr.x on smithi177 2023-10-09T04:02:41.144 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2023-10-09T04:02:41.145 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2023-10-09T04:02:41.145 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000240234 s, 2.1 MB/s 2023-10-09T04:02:41.146 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-09T04:02:41.163 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_2 2023-10-09T04:02:41.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:40 smithi029 ceph-mon[101043]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi177=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:41.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:40 smithi029 ceph-mon[101043]: Saving service mgr spec with placement smithi029=y;smithi177=x;count:2 2023-10-09T04:02:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:40 smithi029 ceph-mon[101043]: Deploying daemon mgr.x on smithi177 2023-10-09T04:02:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:40 smithi029 ceph-mon[119131]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi177=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:40 smithi029 ceph-mon[119131]: Saving service mgr spec with placement smithi029=y;smithi177=x;count:2 2023-10-09T04:02:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:40 smithi029 ceph-mon[119131]: Deploying daemon mgr.x on smithi177 2023-10-09T04:02:41.222 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-09T04:02:41.222 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:41.222 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 6h/6d Inode: 392822 Links: 1 2023-10-09T04:02:41.223 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:41.223 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:41.223 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2023-10-09 04:02:41.093349032 +0000 2023-10-09T04:02:41.223 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2023-10-09 03:53:01.012752527 +0000 2023-10-09T04:02:41.223 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2023-10-09 03:53:01.012752527 +0000 2023-10-09T04:02:41.223 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2023-10-09T04:02:41.223 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-09T04:02:41.291 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2023-10-09T04:02:41.291 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2023-10-09T04:02:41.291 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000217421 s, 2.4 MB/s 2023-10-09T04:02:41.292 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-09T04:02:41.351 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_3 2023-10-09T04:02:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-09T04:02:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 6h/6d Inode: 392877 Links: 1 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2023-10-09 04:02:41.224345493 +0000 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2023-10-09 03:53:01.507739132 +0000 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2023-10-09 03:53:01.507739132 +0000 2023-10-09T04:02:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2023-10-09T04:02:41.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-09T04:02:41.474 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:41 smithi177 systemd[1]: Starting Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:02:41.476 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2023-10-09T04:02:41.476 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2023-10-09T04:02:41.477 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000228948 s, 2.2 MB/s 2023-10-09T04:02:41.478 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-09T04:02:41.535 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_4 2023-10-09T04:02:41.593 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 6h/6d Inode: 393780 Links: 1 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2023-10-09 04:02:41.225345466 +0000 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2023-10-09 03:53:02.014725413 +0000 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2023-10-09 03:53:02.014725413 +0000 2023-10-09T04:02:41.594 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: - 2023-10-09T04:02:41.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-09T04:02:41.659 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2023-10-09T04:02:41.659 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2023-10-09T04:02:41.659 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000212594 s, 2.4 MB/s 2023-10-09T04:02:41.660 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-09T04:02:41.719 INFO:tasks.cephadm:Deploying osd.0 on smithi029 with /dev/vg_nvme/lv_4... 2023-10-09T04:02:41.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T04:02:42.086 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:02:42.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:41 smithi029 ceph-mon[101043]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:41 smithi029 ceph-mon[119131]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:41 smithi177 ceph-mon[106070]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:41 smithi177 ceph-mgr[109303]: -- 172.21.15.177:0/3624494476 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56510b94a9c0 con 0x56510b923000 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:41 smithi177 bash[109153]: caddb7c690a30bb509b9ab07c89b2724b490eb04561a6d53eb7b286abf6d9914 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:41 smithi177 systemd[1]: Started Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.019+0000 7f1efbc161c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.096+0000 7f1efbc161c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:02:42.187 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.186+0000 7f1efbc161c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:02:42.468 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.284+0000 7f1efbc161c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:02:42.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.467+0000 7f1efbc161c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:02:43.143 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:42 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:42 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:42 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:42 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:42 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.814+0000 7f1efbc161c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:42 smithi177 conmon[109257]: 2023-10-09T04:02:42.905+0000 7f1efbc161c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:02:43.144 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:43 smithi177 conmon[109257]: 2023-10-09T04:02:43.041+0000 7f1efbc161c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:02:43.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:43.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:43.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:42 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:43.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:43 smithi177 conmon[109257]: 2023-10-09T04:02:43.143+0000 7f1efbc161c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:02:43.997 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T04:02:43.997 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T04:02:43.997 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2023-10-09T04:02:43.997 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2023-10-09T04:02:43.997 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00914915 s, 1.1 GB/s 2023-10-09T04:02:43.997 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2023-10-09T04:02:44.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:43 smithi177 conmon[109257]: 2023-10-09T04:02:43.559+0000 7f1efbc161c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:02:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:02:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:44.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:44 smithi177 conmon[109257]: 2023-10-09T04:02:44.123+0000 7f1efbc161c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:02:44.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:44 smithi177 conmon[109257]: 2023-10-09T04:02:44.290+0000 7f1efbc161c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:02:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:02:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:02:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:45.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi029:vg_nvme/lv_4 2023-10-09T04:02:45.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:45 smithi177 ceph-mon[106070]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:02:45.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:45 smithi177 ceph-mon[106070]: Reconfiguring daemon mgr.y on smithi029 2023-10-09T04:02:45.299 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:45 smithi177 conmon[109257]: 2023-10-09T04:02:45.036+0000 7f1efbc161c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:02:45.299 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:45 smithi177 conmon[109257]: 2023-10-09T04:02:45.127+0000 7f1efbc161c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:02:45.418 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:02:45.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:45 smithi029 ceph-mon[101043]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:02:45.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:45 smithi029 ceph-mon[101043]: Reconfiguring daemon mgr.y on smithi029 2023-10-09T04:02:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:45 smithi029 ceph-mon[119131]: Reconfiguring mgr.y (unknown last config time)... 2023-10-09T04:02:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:45 smithi029 ceph-mon[119131]: Reconfiguring daemon mgr.y on smithi029 2023-10-09T04:02:45.567 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:45 smithi177 conmon[109257]: 2023-10-09T04:02:45.299+0000 7f1efbc161c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:02:45.936 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:45 smithi177 conmon[109257]: 2023-10-09T04:02:45.566+0000 7f1efbc161c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:02:45.936 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:45 smithi177 conmon[109257]: 2023-10-09T04:02:45.651+0000 7f1efbc161c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:02:45.936 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:45 smithi177 conmon[109257]: 2023-10-09T04:02:45.936+0000 7f1efbc161c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:02:46.188 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:46.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:46.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:46.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:46.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:02:46.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:46.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:02:46.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:02:46.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:46 smithi177 conmon[109257]: 2023-10-09T04:02:46.188+0000 7f1efbc161c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:02:47.194 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:46 smithi177 conmon[109257]: 2023-10-09T04:02:46.881+0000 7f1efbc161c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:02:47.194 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:47 smithi177 conmon[109257]: 2023-10-09T04:02:47.009+0000 7f1efbc161c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:02:47.514 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:47 smithi177 conmon[109257]: 2023-10-09T04:02:47.193+0000 7f1efbc161c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:02:47.514 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:47 smithi177 conmon[109257]: 2023-10-09T04:02:47.304+0000 7f1efbc161c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:02:47.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:47 smithi177 conmon[109257]: 2023-10-09T04:02:47.513+0000 7f1efbc161c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:47 smithi177 ceph-mon[106070]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:02:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:02:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[101043]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[119131]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:02:47.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:02:47.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:48.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:02:48 smithi177 conmon[109257]: 2023-10-09T04:02:48.358+0000 7f1efbc161c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:02:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:48 smithi177 ceph-mon[106070]: from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:48 smithi177 ceph-mon[106070]: Standby manager daemon x started 2023-10-09T04:02:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:48 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:02:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:48 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:02:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:48 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:02:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:48 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:02:48.915 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[101043]: from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[101043]: Standby manager daemon x started 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[119131]: from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:02:48.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[119131]: Standby manager daemon x started 2023-10-09T04:02:48.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:02:48.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:02:48.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:02:48.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:48 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/850159063' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:02:49.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:49 smithi029 ceph-mon[101043]: mgrmap e14: y(active, since 65s), standbys: x 2023-10-09T04:02:49.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:49 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:02:49.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:49 smithi029 ceph-mon[101043]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:49.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:49 smithi029 ceph-mon[119131]: mgrmap e14: y(active, since 65s), standbys: x 2023-10-09T04:02:49.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:49 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:02:49.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:49 smithi029 ceph-mon[119131]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:49 smithi177 ceph-mon[106070]: mgrmap e14: y(active, since 65s), standbys: x 2023-10-09T04:02:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:49 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:02:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:49 smithi177 ceph-mon[106070]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1097971137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]: dispatch 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]: dispatch 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]': finished 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[101043]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3437890342' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:02:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1097971137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]: dispatch 2023-10-09T04:02:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]: dispatch 2023-10-09T04:02:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]': finished 2023-10-09T04:02:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[119131]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:02:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:02:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:50 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3437890342' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:02:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:50 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1097971137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]: dispatch 2023-10-09T04:02:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:50 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]: dispatch 2023-10-09T04:02:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:50 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3e888dc-7381-4dc9-9e9e-ea492336810d"}]': finished 2023-10-09T04:02:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:50 smithi177 ceph-mon[106070]: osdmap e5: 1 total, 0 up, 1 in 2023-10-09T04:02:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:50 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:02:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:50 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3437890342' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:02:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:51 smithi029 ceph-mon[101043]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:51 smithi029 ceph-mon[119131]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:51 smithi177 ceph-mon[106070]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:54.270 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:54 smithi029 ceph-mon[101043]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:54.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:54 smithi029 ceph-mon[119131]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:54 smithi177 ceph-mon[106070]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:56.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:56 smithi029 ceph-mon[101043]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:56.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:56 smithi029 ceph-mon[119131]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:56 smithi177 ceph-mon[106070]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:57.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:57 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:02:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:57 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:57 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:02:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:57 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:57 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:02:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:57 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:02:58.305 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:58 smithi029 ceph-mon[101043]: Deploying daemon osd.0 on smithi029 2023-10-09T04:02:58.305 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:02:58 smithi029 ceph-mon[101043]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:58.305 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:58 smithi029 ceph-mon[119131]: Deploying daemon osd.0 on smithi029 2023-10-09T04:02:58.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:02:58 smithi029 ceph-mon[119131]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:02:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:58 smithi177 ceph-mon[106070]: Deploying daemon osd.0 on smithi029 2023-10-09T04:02:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:02:58 smithi177 ceph-mon[106070]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:00.394 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:00 smithi029 ceph-mon[101043]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:00.395 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:00 smithi029 ceph-mon[119131]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:00 smithi177 ceph-mon[106070]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:02.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[119131]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[101043]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:02 smithi177 ceph-mon[106070]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[101043]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[119131]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:04 smithi177 ceph-mon[106070]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:04.950 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 0 on host 'smithi029' 2023-10-09T04:03:05.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[101043]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[119131]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:05 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:05 smithi177 ceph-mon[106070]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:03:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:05 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:05 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:05 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:05.657 DEBUG:teuthology.orchestra.run.smithi029:osd.0> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-09T04:03:05.660 INFO:tasks.cephadm:Deploying osd.1 on smithi029 with /dev/vg_nvme/lv_3... 2023-10-09T04:03:05.660 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T04:03:05.692 INFO:journalctl@ceph.osd.0.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:03:06.285 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:03:06 smithi029 conmon[125444]: 2023-10-09T04:03:06.155+0000 7fafc1de4700 -1 osd.0 0 waiting for initial osdmap 2023-10-09T04:03:06.285 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:03:06 smithi029 conmon[125444]: 2023-10-09T04:03:06.157+0000 7fafbbbdd700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:03:06.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[101043]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[101043]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[101043]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[101043]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[119131]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[119131]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[119131]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:06.286 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:06 smithi029 ceph-mon[119131]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:06 smithi177 ceph-mon[106070]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:06 smithi177 ceph-mon[106070]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:03:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:06 smithi177 ceph-mon[106070]: osdmap e6: 1 total, 0 up, 1 in 2023-10-09T04:03:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:06 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:06 smithi177 ceph-mon[106070]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:06.757 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[101043]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[101043]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:03:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[101043]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[119131]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[119131]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:03:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:07 smithi029 ceph-mon[119131]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:07 smithi177 ceph-mon[106070]: from='osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:07 smithi177 ceph-mon[106070]: osdmap e7: 1 total, 0 up, 1 in 2023-10-09T04:03:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:07 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:07 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:07 smithi177 ceph-mon[106070]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-09T04:03:08.300 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:03:08.300 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:03:08.300 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[101043]: osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938] boot 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[101043]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[119131]: osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938] boot 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[119131]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:03:08.301 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:08 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:03:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:08 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:03:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:08 smithi177 ceph-mon[106070]: osd.0 [v2:172.21.15.29:6802/1054408938,v1:172.21.15.29:6803/1054408938] boot 2023-10-09T04:03:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:08 smithi177 ceph-mon[106070]: osdmap e8: 1 total, 1 up, 1 in 2023-10-09T04:03:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:03:09.172 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T04:03:09.172 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T04:03:09.172 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2023-10-09T04:03:09.172 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2023-10-09T04:03:09.172 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00801378 s, 1.3 GB/s 2023-10-09T04:03:09.172 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi029 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi029 to 16149M 2023-10-09T04:03:09.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi029 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi029 to 16149M 2023-10-09T04:03:09.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:09.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:09.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:09.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:09 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:09.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi029:vg_nvme/lv_3 2023-10-09T04:03:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: osdmap e9: 1 total, 1 up, 1 in 2023-10-09T04:03:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi029 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi029 to 16149M 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:09 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:10.175 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:12.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[101043]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[119131]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:12 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:12 smithi177 ceph-mon[106070]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:12 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:12 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:12 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:13.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:13 smithi029 ceph-mon[101043]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:13.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:13 smithi029 ceph-mon[119131]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:13 smithi177 ceph-mon[106070]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:14 smithi029 ceph-mon[101043]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:14.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:14 smithi029 ceph-mon[119131]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:14 smithi177 ceph-mon[106070]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:15.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4199724058' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]: dispatch 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]: dispatch 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]': finished 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[101043]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2116081934' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4199724058' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]: dispatch 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]: dispatch 2023-10-09T04:03:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]': finished 2023-10-09T04:03:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[119131]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:03:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:15 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2116081934' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:15 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4199724058' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]: dispatch 2023-10-09T04:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:15 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]: dispatch 2023-10-09T04:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:15 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12e83e56-ffd0-4eb6-bb51-5612a7768792"}]': finished 2023-10-09T04:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:15 smithi177 ceph-mon[106070]: osdmap e10: 2 total, 1 up, 2 in 2023-10-09T04:03:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:15 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:15 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2116081934' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:16.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:16 smithi029 ceph-mon[101043]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:16 smithi029 ceph-mon[119131]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:16 smithi177 ceph-mon[106070]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:17 smithi029 ceph-mon[101043]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:17.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:17 smithi029 ceph-mon[119131]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:17 smithi177 ceph-mon[106070]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:20 smithi029 ceph-mon[101043]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:20.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:20 smithi029 ceph-mon[119131]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:20.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:20 smithi177 ceph-mon[106070]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:21 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:03:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:21 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:21 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:03:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:21 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:21 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:03:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:21 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:22.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:22 smithi029 ceph-mon[119131]: Deploying daemon osd.1 on smithi029 2023-10-09T04:03:22.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:22 smithi029 ceph-mon[119131]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:22 smithi029 ceph-mon[101043]: Deploying daemon osd.1 on smithi029 2023-10-09T04:03:22.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:22 smithi029 ceph-mon[101043]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:22 smithi177 ceph-mon[106070]: Deploying daemon osd.1 on smithi029 2023-10-09T04:03:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:22 smithi177 ceph-mon[106070]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:24 smithi029 ceph-mon[101043]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:24 smithi029 ceph-mon[119131]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:24 smithi177 ceph-mon[106070]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:26.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:26 smithi029 ceph-mon[101043]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:26 smithi029 ceph-mon[119131]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:26 smithi177 ceph-mon[106070]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[101043]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:27.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:27.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:27 smithi029 ceph-mon[119131]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:27 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:27 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:27 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:27 smithi177 ceph-mon[106070]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:29.581 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 1 on host 'smithi029' 2023-10-09T04:03:29.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:29.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:29.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[101043]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:29.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:29.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:29.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:29 smithi029 ceph-mon[119131]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:29 smithi177 ceph-mon[106070]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:30.264 DEBUG:teuthology.orchestra.run.smithi029:osd.1> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-09T04:03:30.266 INFO:tasks.cephadm:Deploying osd.2 on smithi029 with /dev/vg_nvme/lv_2... 2023-10-09T04:03:30.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T04:03:30.296 INFO:journalctl@ceph.osd.1.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:03:30.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[101043]: from='osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:03:30.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[101043]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[119131]: from='osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[119131]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:30.568 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:30.569 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:30 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:30 smithi177 ceph-mon[106070]: from='osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:03:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:30 smithi177 ceph-mon[106070]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:03:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:30 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:30.820 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:31.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[101043]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:03:31.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[101043]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:03:31.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[101043]: from='osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[101043]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[101043]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[119131]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[119131]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[119131]: from='osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:31.631 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[119131]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:31.632 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:31 smithi029 ceph-mon[119131]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:31.632 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:03:31 smithi029 conmon[134427]: 2023-10-09T04:03:31.374+0000 7fc8d9c3e700 -1 osd.1 0 waiting for initial osdmap 2023-10-09T04:03:31.632 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:03:31 smithi029 conmon[134427]: 2023-10-09T04:03:31.377+0000 7fc8d2a35700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:03:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:31 smithi177 ceph-mon[106070]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:03:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:31 smithi177 ceph-mon[106070]: osdmap e11: 2 total, 1 up, 2 in 2023-10-09T04:03:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:31 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:31 smithi177 ceph-mon[106070]: from='osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:31 smithi177 ceph-mon[106070]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:31 smithi177 ceph-mon[106070]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-09T04:03:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[101043]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[101043]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:03:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[119131]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[119131]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:03:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:32 smithi177 ceph-mon[106070]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:32 smithi177 ceph-mon[106070]: osdmap e12: 2 total, 1 up, 2 in 2023-10-09T04:03:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:33.522 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T04:03:33.523 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T04:03:33.523 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2023-10-09T04:03:33.523 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2023-10-09T04:03:33.523 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0085022 s, 1.2 GB/s 2023-10-09T04:03:33.523 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2023-10-09T04:03:33.555 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:03:33.555 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:03:33.555 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[101043]: osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275] boot 2023-10-09T04:03:33.555 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[101043]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[101043]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[119131]: osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275] boot 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[119131]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:33.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:33 smithi029 ceph-mon[119131]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:33 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:03:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:33 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:03:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:33 smithi177 ceph-mon[106070]: osd.1 [v2:172.21.15.29:6810/3631252275,v1:172.21.15.29:6811/3631252275] boot 2023-10-09T04:03:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:33 smithi177 ceph-mon[106070]: osdmap e13: 2 total, 2 up, 2 in 2023-10-09T04:03:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:33 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:03:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:33 smithi177 ceph-mon[106070]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:34.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi029:vg_nvme/lv_2 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi029 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi029 to 8074M 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi029 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi029 to 8074M 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: osdmap e14: 2 total, 2 up, 2 in 2023-10-09T04:03:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi029 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi029 to 8074M 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:34.781 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:35 smithi029 ceph-mon[101043]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:35.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:35 smithi029 ceph-mon[119131]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:35.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:35 smithi177 ceph-mon[106070]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[101043]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[101043]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[119131]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:37.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:37 smithi029 ceph-mon[119131]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:37 smithi177 ceph-mon[106070]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:03:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:03:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:03:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:37 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:37 smithi177 ceph-mon[106070]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:39 smithi177 ceph-mon[106070]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:39 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4056840710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]: dispatch 2023-10-09T04:03:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:39 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]: dispatch 2023-10-09T04:03:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:39 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]': finished 2023-10-09T04:03:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:39 smithi177 ceph-mon[106070]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:03:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[101043]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4056840710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]: dispatch 2023-10-09T04:03:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]: dispatch 2023-10-09T04:03:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]': finished 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[101043]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[119131]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4056840710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]: dispatch 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]: dispatch 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6620b1f9-fce6-4509-8c69-741afdb0332d"}]': finished 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[119131]: osdmap e15: 3 total, 2 up, 3 in 2023-10-09T04:03:39.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:40 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2525949510' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:40.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:40 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2525949510' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:40 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2525949510' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:03:41.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:41 smithi177 ceph-mon[106070]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:41.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:41 smithi029 ceph-mon[101043]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:41 smithi029 ceph-mon[119131]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:44 smithi029 ceph-mon[101043]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:44 smithi029 ceph-mon[119131]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:44 smithi177 ceph-mon[106070]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:46.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:46 smithi029 ceph-mon[101043]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:46.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:03:46.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:46 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:46.412 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:46 smithi029 ceph-mon[119131]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:46.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:03:46.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:46 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:46 smithi177 ceph-mon[106070]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:03:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:46 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:47.366 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:47 smithi029 ceph-mon[101043]: Deploying daemon osd.2 on smithi029 2023-10-09T04:03:47.366 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:47 smithi029 ceph-mon[119131]: Deploying daemon osd.2 on smithi029 2023-10-09T04:03:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:47 smithi177 ceph-mon[106070]: Deploying daemon osd.2 on smithi029 2023-10-09T04:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:48 smithi029 ceph-mon[101043]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:48 smithi029 ceph-mon[119131]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:48 smithi177 ceph-mon[106070]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:50 smithi029 ceph-mon[101043]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:50.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:50 smithi029 ceph-mon[119131]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:50 smithi177 ceph-mon[106070]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[101043]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:51 smithi029 ceph-mon[119131]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:51 smithi177 ceph-mon[106070]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:54.234 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 2 on host 'smithi029' 2023-10-09T04:03:54.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[101043]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:53 smithi029 ceph-mon[119131]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:53 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:53 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:53 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:03:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:53 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:03:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:53 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:53 smithi177 ceph-mon[106070]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:54.876 DEBUG:teuthology.orchestra.run.smithi029:osd.2> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T04:03:54.878 INFO:tasks.cephadm:Deploying osd.3 on smithi029 with /dev/vg_nvme/lv_1... 2023-10-09T04:03:54.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T04:03:54.910 INFO:journalctl@ceph.osd.2.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:03:55.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[101043]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[119131]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:55.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:54 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:54 smithi177 ceph-mon[106070]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:03:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:54 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:54 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:03:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:54 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:03:55.567 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:03:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:55 smithi177 ceph-mon[106070]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:03:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:55 smithi177 ceph-mon[106070]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:03:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:55 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:55 smithi177 ceph-mon[106070]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:55 smithi177 ceph-mon[106070]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:56.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[101043]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:03:56.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[101043]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:03:56.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:56.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[101043]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:56.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[101043]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:56.298 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:03:55 smithi029 conmon[144116]: 2023-10-09T04:03:55.976+0000 7fb959823700 -1 osd.2 0 waiting for initial osdmap 2023-10-09T04:03:56.298 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:03:55 smithi029 conmon[144116]: 2023-10-09T04:03:55.979+0000 7fb95261a700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:03:56.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[119131]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:03:56.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[119131]: osdmap e16: 3 total, 2 up, 3 in 2023-10-09T04:03:56.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:56.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[119131]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:03:56.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:55 smithi029 ceph-mon[119131]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-09T04:03:57.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[119131]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:57.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[119131]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:03:57.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:57.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[101043]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[101043]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:03:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:56 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:56 smithi177 ceph-mon[106070]: from='osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:03:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:56 smithi177 ceph-mon[106070]: osdmap e17: 3 total, 2 up, 3 in 2023-10-09T04:03:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:56 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:56 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:58.227 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T04:03:58.227 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T04:03:58.227 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2023-10-09T04:03:58.227 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2023-10-09T04:03:58.227 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114499 s, 916 MB/s 2023-10-09T04:03:58.227 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2023-10-09T04:03:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:03:58.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566] boot 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: pgmap v75: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566] boot 2023-10-09T04:03:58.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:03:58.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:58.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: pgmap v75: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:03:58.270 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:57 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:03:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:03:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:03:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: osd.2 [v2:172.21.15.29:6818/1532034566,v1:172.21.15.29:6819/1532034566] boot 2023-10-09T04:03:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: osdmap e18: 3 total, 3 up, 3 in 2023-10-09T04:03:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:03:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: pgmap v75: 0 pgs: ; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:03:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:57 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-09T04:03:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:58 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:03:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:58 smithi177 ceph-mon[106070]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:03:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:58 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:03:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:58 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:03:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:58 smithi029 ceph-mon[119131]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:03:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:58 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:03:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-09T04:03:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:58 smithi029 ceph-mon[101043]: osdmap e19: 3 total, 3 up, 3 in 2023-10-09T04:03:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:58 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:03:59.489 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi029:vg_nvme/lv_1 2023-10-09T04:03:59.897 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:04:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi029 2023-10-09T04:04:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:04:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi029 to 5383M 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:03:59 smithi177 ceph-mon[106070]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi029 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi029 to 5383M 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[101043]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi029 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:04:00.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: osdmap e20: 3 total, 3 up, 3 in 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi029 to 5383M 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:00.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:03:59 smithi029 ceph-mon[119131]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:00.326 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150743]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T04:04:00.326 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150743]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:00.326 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150743]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:00.326 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150743]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:00.921 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150798]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T04:04:00.921 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150798]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:00.921 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150798]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:00.921 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:04:00 smithi029 sudo[150798]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:01 smithi177 ceph-mon[106070]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:04:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:01 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:01.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:01 smithi029 ceph-mon[101043]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:04:01.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:01 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:01.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:01 smithi029 ceph-mon[119131]: osdmap e21: 3 total, 3 up, 3 in 2023-10-09T04:04:01.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:01 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:01.281 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150829]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T04:04:01.282 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150829]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:01.282 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150829]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:01.282 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150829]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:01.920 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150906]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T04:04:01.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150906]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:01.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150906]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:01.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:04:01 smithi029 sudo[150906]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:02 smithi177 ceph-mon[106070]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:02.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[101043]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:02.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:02.368 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:02.368 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:02.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[119131]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:02.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:02.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:02.369 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:02.369 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151022]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T04:04:02.369 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151022]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:02.369 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151022]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:02.369 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151022]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:02.670 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151152]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T04:04:02.671 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151152]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:02.671 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151152]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:02.671 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151152]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:03.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151207]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T04:04:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151207]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151207]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:02 smithi029 sudo[151207]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:03.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:03 smithi029 sudo[151236]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T04:04:03.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:03 smithi029 sudo[151236]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:03.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:03 smithi029 sudo[151236]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:03.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:03 smithi029 sudo[151236]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:03.906 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110181]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T04:04:03.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110181]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:03.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110181]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:03.907 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110181]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:03 smithi029 ceph-mon[101043]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:03 smithi029 ceph-mon[101043]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:03 smithi029 ceph-mon[119131]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:03 smithi029 ceph-mon[119131]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110184]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T04:04:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110184]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110184]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 sudo[110184]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 ceph-mon[106070]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:03 smithi177 ceph-mon[106070]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151331]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-10-09T04:04:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151331]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151331]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151331]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1064682242' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "266c755e-25de-4dd5-a2a8-1b67b5fc8e3a"}]: dispatch 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1064682242' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "266c755e-25de-4dd5-a2a8-1b67b5fc8e3a"}]': finished 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:04:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151335]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151335]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151335]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 sudo[151335]: pam_unix(sudo:session): session closed for user root 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T04:04:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1064682242' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "266c755e-25de-4dd5-a2a8-1b67b5fc8e3a"}]: dispatch 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1064682242' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "266c755e-25de-4dd5-a2a8-1b67b5fc8e3a"}]': finished 2023-10-09T04:04:05.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:04:05.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: mgrmap e15: y(active, since 2m), standbys: x 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:04:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:04:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:04:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1064682242' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "266c755e-25de-4dd5-a2a8-1b67b5fc8e3a"}]: dispatch 2023-10-09T04:04:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-09T04:04:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1064682242' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "266c755e-25de-4dd5-a2a8-1b67b5fc8e3a"}]': finished 2023-10-09T04:04:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: osdmap e22: 4 total, 3 up, 4 in 2023-10-09T04:04:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:06.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:05 smithi029 ceph-mon[101043]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:05 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/499803728' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:06.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:05 smithi029 ceph-mon[119131]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:06.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:05 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/499803728' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:05 smithi177 ceph-mon[106070]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:05 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/499803728' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:07 smithi029 ceph-mon[101043]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:07.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:07 smithi029 ceph-mon[119131]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:07 smithi177 ceph-mon[106070]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:10.220 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:10 smithi029 ceph-mon[101043]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-09T04:04:10.221 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:10 smithi029 ceph-mon[119131]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-09T04:04:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:10 smithi177 ceph-mon[106070]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-10-09T04:04:12.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:12 smithi029 ceph-mon[101043]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-09T04:04:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:12 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:04:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:12 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:12 smithi029 ceph-mon[119131]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-09T04:04:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:12 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:04:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:12 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:12 smithi177 ceph-mon[106070]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-10-09T04:04:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:12 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:04:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:12 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:13.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:13 smithi029 ceph-mon[101043]: Deploying daemon osd.3 on smithi029 2023-10-09T04:04:13.372 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:13 smithi029 ceph-mon[119131]: Deploying daemon osd.3 on smithi029 2023-10-09T04:04:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:13 smithi177 ceph-mon[106070]: Deploying daemon osd.3 on smithi029 2023-10-09T04:04:14.287 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:14 smithi029 ceph-mon[101043]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T04:04:14.287 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:14 smithi029 ceph-mon[119131]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T04:04:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:14 smithi177 ceph-mon[106070]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-10-09T04:04:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:15 smithi029 ceph-mon[101043]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 76 KiB/s wr, 5 op/s 2023-10-09T04:04:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:15 smithi029 ceph-mon[119131]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 76 KiB/s wr, 5 op/s 2023-10-09T04:04:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:15 smithi177 ceph-mon[106070]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 76 KiB/s wr, 5 op/s 2023-10-09T04:04:17.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[101043]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:17 smithi029 ceph-mon[119131]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:17.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:17.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:17 smithi177 ceph-mon[106070]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:04:19.772 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 3 on host 'smithi029' 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:04:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[101043]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:04:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:19 smithi029 ceph-mon[119131]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:04:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2023-10-09T04:04:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:04:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:19 smithi177 ceph-mon[106070]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:04:20.458 DEBUG:teuthology.orchestra.run.smithi029:osd.3> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-09T04:04:20.460 INFO:tasks.cephadm:Deploying osd.4 on smithi177 with /dev/vg_nvme/lv_4... 2023-10-09T04:04:20.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-09T04:04:20.490 INFO:journalctl@ceph.osd.3.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:04:20.801 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:04:21.011 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:04:21.012 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:20 smithi177 ceph-mon[106070]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:04:21.101 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[101043]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:04:21.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: osdmap e23: 4 total, 3 up, 4 in 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:04:21.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:20 smithi029 ceph-mon[119131]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:04:21.420 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:04:21 smithi029 conmon[153884]: 2023-10-09T04:04:21.099+0000 7fec09db6700 -1 osd.3 0 waiting for initial osdmap 2023-10-09T04:04:21.421 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:04:21 smithi029 conmon[153884]: 2023-10-09T04:04:21.103+0000 7fec053b2700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:04:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[101043]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[101043]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:04:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[101043]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[119131]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[119131]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[119131]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:22 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:22 smithi177 ceph-mon[106070]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:22 smithi177 ceph-mon[106070]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2023-10-09T04:04:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:22 smithi177 ceph-mon[106070]: osdmap e24: 4 total, 3 up, 4 in 2023-10-09T04:04:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:22 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:22.533 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-09T04:04:22.533 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-09T04:04:22.534 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2023-10-09T04:04:22.534 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2023-10-09T04:04:22.534 INFO:teuthology.orchestra.run.smithi177.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111007 s, 945 MB/s 2023-10-09T04:04:22.534 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 2023-10-09T04:04:22.534 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2023-10-09T04:04:23.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi177:vg_nvme/lv_4 2023-10-09T04:04:23.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[101043]: osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544] boot 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[101043]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[119131]: osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544] boot 2023-10-09T04:04:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[119131]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:04:23.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:23 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:23.461 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:04:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:23 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:04:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:23 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:04:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:23 smithi177 ceph-mon[106070]: osd.3 [v2:172.21.15.29:6826/3716476544,v1:172.21.15.29:6827/3716476544] boot 2023-10-09T04:04:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:23 smithi177 ceph-mon[106070]: osdmap e25: 4 total, 4 up, 4 in 2023-10-09T04:04:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:04:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:24.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:24 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: osdmap e26: 4 total, 4 up, 4 in 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:24.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:24.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:24.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:24.520 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:24 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:25 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi029 2023-10-09T04:04:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:25 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi029 to 4037M 2023-10-09T04:04:25.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:25 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi029 2023-10-09T04:04:25.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:25 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi029 to 4037M 2023-10-09T04:04:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:25 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi029 2023-10-09T04:04:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:25 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi029 to 4037M 2023-10-09T04:04:26.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[101043]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[101043]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[119131]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[119131]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:26 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:26 smithi177 ceph-mon[106070]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:26 smithi177 ceph-mon[106070]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:26 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:26 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:26 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[101043]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/186895106' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]: dispatch 2023-10-09T04:04:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]: dispatch 2023-10-09T04:04:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]': finished 2023-10-09T04:04:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[101043]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:04:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[119131]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/186895106' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]: dispatch 2023-10-09T04:04:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]: dispatch 2023-10-09T04:04:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]': finished 2023-10-09T04:04:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[119131]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:04:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:27 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:27 smithi177 ceph-mon[106070]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:27 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/186895106' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]: dispatch 2023-10-09T04:04:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:27 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]: dispatch 2023-10-09T04:04:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:27 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4eed67c8-385c-41ab-9afc-a65ad85104d0"}]': finished 2023-10-09T04:04:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:27 smithi177 ceph-mon[106070]: osdmap e27: 5 total, 4 up, 5 in 2023-10-09T04:04:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:27 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:28.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:28 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/398322468' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:28 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/398322468' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:28 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/398322468' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:29 smithi029 ceph-mon[101043]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:29 smithi029 ceph-mon[119131]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:29 smithi177 ceph-mon[106070]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:32 smithi177 ceph-mon[106070]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:32.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:32 smithi029 ceph-mon[101043]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:32.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:32 smithi029 ceph-mon[119131]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:34 smithi029 ceph-mon[101043]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:04:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:34 smithi029 ceph-mon[119131]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:04:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:34.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:34 smithi177 ceph-mon[106070]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:34.442 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:04:34.442 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:35 smithi177 ceph-mon[106070]: Deploying daemon osd.4 on smithi177 2023-10-09T04:04:35.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:35 smithi029 ceph-mon[101043]: Deploying daemon osd.4 on smithi177 2023-10-09T04:04:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:35 smithi029 ceph-mon[119131]: Deploying daemon osd.4 on smithi177 2023-10-09T04:04:36.411 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:36 smithi177 ceph-mon[106070]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:36.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:36 smithi029 ceph-mon[101043]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:36 smithi029 ceph-mon[119131]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:37 smithi177 ceph-mon[106070]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:37 smithi029 ceph-mon[101043]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:37 smithi029 ceph-mon[119131]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:39.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:39.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:39.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:39 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:39.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:39 smithi177 ceph-mon[106070]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:39.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[101043]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:39 smithi029 ceph-mon[119131]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:41.467 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 4 on host 'smithi177' 2023-10-09T04:04:41.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:41 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:41.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:41 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:04:41.989 DEBUG:teuthology.orchestra.run.smithi177:osd.4> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T04:04:41.991 INFO:tasks.cephadm:Deploying osd.5 on smithi177 with /dev/vg_nvme/lv_3... 2023-10-09T04:04:41.992 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-09T04:04:42.022 INFO:journalctl@ceph.osd.4.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:04:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:42 smithi029 ceph-mon[101043]: from='osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:04:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:42 smithi029 ceph-mon[101043]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:04:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:42 smithi029 ceph-mon[119131]: from='osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:04:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:42 smithi029 ceph-mon[119131]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:04:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:42 smithi177 ceph-mon[106070]: from='osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:04:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:42 smithi177 ceph-mon[106070]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:04:43.044 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:04:43.777 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:04:43 smithi177 conmon[114075]: 2023-10-09T04:04:43.568+0000 7fb7e2ce4700 -1 osd.4 0 waiting for initial osdmap 2023-10-09T04:04:43.777 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:04:43 smithi177 conmon[114075]: 2023-10-09T04:04:43.573+0000 7fb7deae1700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:04:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:43 smithi177 ceph-mon[106070]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:04:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:43 smithi177 ceph-mon[106070]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:04:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:43 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:43 smithi177 ceph-mon[106070]: from='osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:04:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:43 smithi177 ceph-mon[106070]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:04:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:43 smithi177 ceph-mon[106070]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[101043]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[101043]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[101043]: from='osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[101043]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[101043]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:43.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[119131]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:04:43.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[119131]: osdmap e28: 5 total, 4 up, 5 in 2023-10-09T04:04:43.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:43.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[119131]: from='osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:04:43.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[119131]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:04:43.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:43 smithi029 ceph-mon[119131]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-09T04:04:44.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:44 smithi177 ceph-mon[106070]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:04:44.726 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:44 smithi177 ceph-mon[106070]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:04:44.726 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:44.726 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[119131]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[119131]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[101043]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[101043]: osdmap e29: 5 total, 4 up, 5 in 2023-10-09T04:04:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:44.931 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-09T04:04:44.931 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-09T04:04:44.931 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2023-10-09T04:04:44.932 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2023-10-09T04:04:44.932 INFO:teuthology.orchestra.run.smithi177.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0120864 s, 868 MB/s 2023-10-09T04:04:44.932 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2023-10-09T04:04:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:04:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:04:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139] boot 2023-10-09T04:04:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:45 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi177:vg_nvme/lv_3 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139] boot 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: osd.4 [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139] boot 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: osdmap e30: 5 total, 5 up, 5 in 2023-10-09T04:04:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:04:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:45 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:04:46.171 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:04:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:46 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi177 2023-10-09T04:04:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:46 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi177 to 17168M 2023-10-09T04:04:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:46 smithi177 ceph-mon[106070]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:04:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:46 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi177 2023-10-09T04:04:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:46 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi177 to 17168M 2023-10-09T04:04:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:46 smithi029 ceph-mon[101043]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:04:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:46 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi177 2023-10-09T04:04:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:46 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi177 to 17168M 2023-10-09T04:04:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:46 smithi029 ceph-mon[119131]: osdmap e31: 5 total, 5 up, 5 in 2023-10-09T04:04:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:47 smithi177 ceph-mon[106070]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:04:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:47 smithi177 ceph-mon[106070]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:47.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:47 smithi029 ceph-mon[101043]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:04:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:47 smithi029 ceph-mon[101043]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:47 smithi029 ceph-mon[119131]: osdmap e32: 5 total, 5 up, 5 in 2023-10-09T04:04:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:47 smithi029 ceph-mon[119131]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:48 smithi177 ceph-mon[106070]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T04:04:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:48 smithi177 ceph-mon[106070]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:48 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:48 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:48 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[101043]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T04:04:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[101043]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[119131]: osdmap e33: 5 total, 5 up, 5 in 2023-10-09T04:04:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[119131]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:04:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:04:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:04:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:48 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:49 smithi177 ceph-mon[106070]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T04:04:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:49 smithi177 ceph-mon[106070]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-10-09T04:04:49.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:49 smithi029 ceph-mon[101043]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T04:04:49.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:49 smithi029 ceph-mon[101043]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-10-09T04:04:49.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:49 smithi029 ceph-mon[119131]: osdmap e34: 5 total, 5 up, 5 in 2023-10-09T04:04:49.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:49 smithi029 ceph-mon[119131]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-10-09T04:04:50.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/4042548817' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]: dispatch 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]: dispatch 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]': finished 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[101043]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/4042548817' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]: dispatch 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]: dispatch 2023-10-09T04:04:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]': finished 2023-10-09T04:04:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[119131]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:04:50.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:50 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:04:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:50 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/4042548817' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]: dispatch 2023-10-09T04:04:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:50 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]: dispatch 2023-10-09T04:04:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:50 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d1c822a8-9ffa-4586-9773-007ad79e1b5f"}]': finished 2023-10-09T04:04:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:50 smithi177 ceph-mon[106070]: osdmap e35: 6 total, 5 up, 6 in 2023-10-09T04:04:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:50 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:04:51.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:51 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/3345921327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:51 smithi029 ceph-mon[101043]: pgmap v119: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T04:04:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:51 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/3345921327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:51 smithi029 ceph-mon[119131]: pgmap v119: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T04:04:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:51 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/3345921327' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:04:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:51 smithi177 ceph-mon[106070]: pgmap v119: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T04:04:54.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:54 smithi029 ceph-mon[101043]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:04:54.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:54 smithi029 ceph-mon[119131]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:04:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:54 smithi177 ceph-mon[106070]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:04:56.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:56 smithi029 ceph-mon[119131]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:56 smithi029 ceph-mon[101043]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:56 smithi177 ceph-mon[106070]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:57.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:57 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:04:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:57 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:57 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:04:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:57 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:57.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:57 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:04:57.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:57 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:04:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:58 smithi029 ceph-mon[101043]: Deploying daemon osd.5 on smithi177 2023-10-09T04:04:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:04:58 smithi029 ceph-mon[101043]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:58.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:58 smithi029 ceph-mon[119131]: Deploying daemon osd.5 on smithi177 2023-10-09T04:04:58.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:04:58 smithi029 ceph-mon[119131]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:04:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:58 smithi177 ceph-mon[106070]: Deploying daemon osd.5 on smithi177 2023-10-09T04:04:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:04:58 smithi177 ceph-mon[106070]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:00 smithi029 ceph-mon[101043]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:00.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:00 smithi029 ceph-mon[119131]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:00 smithi177 ceph-mon[106070]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:02 smithi177 ceph-mon[106070]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:02.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:02.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:02 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[101043]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[119131]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:02.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:02.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:02 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:03.784 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 5 on host 'smithi177' 2023-10-09T04:05:04.392 DEBUG:teuthology.orchestra.run.smithi177:osd.5> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-09T04:05:04.394 INFO:tasks.cephadm:Deploying osd.6 on smithi177 with /dev/vg_nvme/lv_2... 2023-10-09T04:05:04.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.424 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:04.424 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:04.429 INFO:journalctl@ceph.osd.5.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:05:04.947 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:05:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:05 smithi029 ceph-mon[101043]: from='osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:05:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:05 smithi029 ceph-mon[101043]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:05:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:05 smithi029 ceph-mon[119131]: from='osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:05:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:05 smithi029 ceph-mon[119131]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:05:05.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:05 smithi177 ceph-mon[106070]: from='osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:05:05.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:05 smithi177 ceph-mon[106070]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[101043]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[101043]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[101043]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[101043]: from='osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[101043]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[119131]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[119131]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:05:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[119131]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T04:05:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[119131]: from='osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:06 smithi029 ceph-mon[119131]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:06.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:06 smithi177 ceph-mon[106070]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:06.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:06 smithi177 ceph-mon[106070]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-09T04:05:06.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:06 smithi177 ceph-mon[106070]: osdmap e36: 6 total, 5 up, 6 in 2023-10-09T04:05:06.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:06 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:06.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:06 smithi177 ceph-mon[106070]: from='osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:06.516 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:06 smithi177 ceph-mon[106070]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:06.516 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 04:05:06 smithi177 conmon[123039]: 2023-10-09T04:05:06.156+0000 7f634c50f700 -1 osd.5 0 waiting for initial osdmap 2023-10-09T04:05:06.517 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 04:05:06 smithi177 conmon[123039]: 2023-10-09T04:05:06.161+0000 7f634930e700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:05:07.022 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-09T04:05:07.022 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-09T04:05:07.022 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2023-10-09T04:05:07.022 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2023-10-09T04:05:07.022 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112307 s, 934 MB/s 2023-10-09T04:05:07.023 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946] boot 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:05:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946] boot 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:05:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:07 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: osdmap e37: 6 total, 5 up, 6 in 2023-10-09T04:05:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: osd.5 [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946] boot 2023-10-09T04:05:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: osdmap e38: 6 total, 6 up, 6 in 2023-10-09T04:05:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:07 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:05:07.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi177:vg_nvme/lv_2 2023-10-09T04:05:08.283 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:05:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:05:08.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:05:08.465 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:08.466 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:08.467 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:08.468 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.468 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:08.468 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:08.468 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:08 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:05:08.468 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: osdmap e39: 6 total, 6 up, 6 in 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:08.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:08 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:09.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:09 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi177 2023-10-09T04:05:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:09 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi177 to 8584M 2023-10-09T04:05:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:09 smithi029 ceph-mon[101043]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T04:05:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:09 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi177 2023-10-09T04:05:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:09 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi177 to 8584M 2023-10-09T04:05:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:09 smithi029 ceph-mon[119131]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T04:05:09.438 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:09 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi177 2023-10-09T04:05:09.438 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:09 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi177 to 8584M 2023-10-09T04:05:09.438 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:09 smithi177 ceph-mon[106070]: osdmap e40: 6 total, 6 up, 6 in 2023-10-09T04:05:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[101043]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[101043]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:05:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[119131]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[119131]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:10 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:10 smithi177 ceph-mon[106070]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:10 smithi177 ceph-mon[106070]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:05:10.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:10 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:10.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:10 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:10.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:10 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:11 smithi029 ceph-mon[101043]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:11 smithi029 ceph-mon[101043]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T04:05:11.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:11 smithi029 ceph-mon[119131]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:11.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:11 smithi029 ceph-mon[119131]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T04:05:11.453 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:11 smithi177 ceph-mon[106070]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:11.454 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:11 smithi177 ceph-mon[106070]: osdmap e41: 6 total, 6 up, 6 in 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/282992871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]: dispatch 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]: dispatch 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]': finished 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T04:05:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/282992871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]: dispatch 2023-10-09T04:05:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]: dispatch 2023-10-09T04:05:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]': finished 2023-10-09T04:05:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T04:05:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:12 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: osdmap e42: 6 total, 6 up, 6 in 2023-10-09T04:05:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/282992871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]: dispatch 2023-10-09T04:05:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]: dispatch 2023-10-09T04:05:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "41a89acf-f32f-41bf-9eb6-5a6edeea9043"}]': finished 2023-10-09T04:05:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: osdmap e43: 7 total, 6 up, 7 in 2023-10-09T04:05:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:12 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:13 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/861039156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:13 smithi177 ceph-mon[106070]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T04:05:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:13 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/861039156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:13 smithi029 ceph-mon[101043]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T04:05:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:13 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/861039156' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:13 smithi029 ceph-mon[119131]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 127 KiB/s, 0 objects/s recovering 2023-10-09T04:05:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:14 smithi177 ceph-mon[106070]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:05:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:14 smithi177 ceph-mon[106070]: Cluster is now healthy 2023-10-09T04:05:14.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:14 smithi029 ceph-mon[101043]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:05:14.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:14 smithi029 ceph-mon[101043]: Cluster is now healthy 2023-10-09T04:05:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:14 smithi029 ceph-mon[119131]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T04:05:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:14 smithi029 ceph-mon[119131]: Cluster is now healthy 2023-10-09T04:05:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:15 smithi177 ceph-mon[106070]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:05:15.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:15 smithi029 ceph-mon[101043]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:05:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:15 smithi029 ceph-mon[119131]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:05:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:17 smithi029 ceph-mon[101043]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-10-09T04:05:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:17 smithi029 ceph-mon[119131]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-10-09T04:05:18.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:17 smithi177 ceph-mon[106070]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-10-09T04:05:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:18 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:05:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:18 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:18 smithi029 ceph-mon[101043]: Deploying daemon osd.6 on smithi177 2023-10-09T04:05:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:18 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:05:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:18 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:18 smithi029 ceph-mon[119131]: Deploying daemon osd.6 on smithi177 2023-10-09T04:05:18.967 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:18 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:05:18.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:18 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:18.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:18 smithi177 ceph-mon[106070]: Deploying daemon osd.6 on smithi177 2023-10-09T04:05:19.661 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:19 smithi177 ceph-mon[106070]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:05:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:19 smithi029 ceph-mon[101043]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:05:19.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:19 smithi029 ceph-mon[119131]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:05:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:22 smithi029 ceph-mon[101043]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:05:22.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:22 smithi029 ceph-mon[119131]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:05:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:22 smithi177 ceph-mon[106070]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:05:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:23 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:23 smithi177 ceph-mon[106070]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-10-09T04:05:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[101043]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-10-09T04:05:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:23 smithi029 ceph-mon[119131]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-10-09T04:05:25.871 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 6 on host 'smithi177' 2023-10-09T04:05:26.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:26.059 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:26.060 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.060 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:26.060 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.060 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:26.060 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:25 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:25 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:26.447 DEBUG:teuthology.orchestra.run.smithi177:osd.6> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-09T04:05:26.449 INFO:tasks.cephadm:Deploying osd.7 on smithi177 with /dev/vg_nvme/lv_1... 2023-10-09T04:05:26.449 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-09T04:05:26.479 INFO:journalctl@ceph.osd.6.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:05:27.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:26 smithi177 ceph-mon[106070]: from='osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:05:27.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:26 smithi177 ceph-mon[106070]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:05:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:26 smithi029 ceph-mon[101043]: from='osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:05:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:26 smithi029 ceph-mon[101043]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:05:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:26 smithi029 ceph-mon[119131]: from='osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:05:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:26 smithi029 ceph-mon[119131]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-09T04:05:27.385 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:05:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:27 smithi177 ceph-mon[106070]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:05:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:27 smithi177 ceph-mon[106070]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T04:05:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:27 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:27 smithi177 ceph-mon[106070]: from='osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:27 smithi177 ceph-mon[106070]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:27 smithi177 ceph-mon[106070]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:28.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 04:05:27 smithi177 conmon[131831]: 2023-10-09T04:05:27.885+0000 7fb2c3073700 -1 osd.6 0 waiting for initial osdmap 2023-10-09T04:05:28.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 04:05:27 smithi177 conmon[131831]: 2023-10-09T04:05:27.894+0000 7fb2bde6e700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:05:28.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[101043]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[101043]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[101043]: from='osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[101043]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[101043]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[119131]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[119131]: osdmap e44: 7 total, 6 up, 7 in 2023-10-09T04:05:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[119131]: from='osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[119131]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:27 smithi029 ceph-mon[119131]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[101043]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[101043]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[119131]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[119131]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T04:05:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:28 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:28 smithi177 ceph-mon[106070]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:28 smithi177 ceph-mon[106070]: osdmap e45: 7 total, 6 up, 7 in 2023-10-09T04:05:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:28 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:28 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:29.349 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-09T04:05:29.349 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-09T04:05:29.349 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2023-10-09T04:05:29.349 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2023-10-09T04:05:29.349 INFO:teuthology.orchestra.run.smithi177.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0109028 s, 962 MB/s 2023-10-09T04:05:29.349 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2023-10-09T04:05:29.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi177:vg_nvme/lv_1 2023-10-09T04:05:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:05:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:05:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886] boot 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:29 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886] boot 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:05:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: osd.6 [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886] boot 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: osdmap e46: 7 total, 7 up, 7 in 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:29 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:30.298 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:05:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:30 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi177 2023-10-09T04:05:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:30 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi177 to 5722M 2023-10-09T04:05:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:30 smithi029 ceph-mon[101043]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T04:05:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:30 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi177 2023-10-09T04:05:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:30 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi177 to 5722M 2023-10-09T04:05:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:30 smithi029 ceph-mon[119131]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T04:05:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:30 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi177 2023-10-09T04:05:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:30 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi177 to 5722M 2023-10-09T04:05:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:30 smithi177 ceph-mon[106070]: osdmap e47: 7 total, 7 up, 7 in 2023-10-09T04:05:32.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:31 smithi029 ceph-mon[101043]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T04:05:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:31 smithi029 ceph-mon[101043]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:31 smithi029 ceph-mon[101043]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T04:05:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:31 smithi029 ceph-mon[119131]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T04:05:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:31 smithi029 ceph-mon[119131]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:31 smithi029 ceph-mon[119131]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T04:05:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:31 smithi177 ceph-mon[106070]: osdmap e48: 7 total, 7 up, 7 in 2023-10-09T04:05:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:31 smithi177 ceph-mon[106070]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:31 smithi177 ceph-mon[106070]: osdmap e49: 7 total, 7 up, 7 in 2023-10-09T04:05:33.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[101043]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[101043]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[119131]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:32 smithi029 ceph-mon[119131]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T04:05:33.269 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:32 smithi177 ceph-mon[106070]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:05:33.270 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-09T04:05:33.270 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-09T04:05:33.270 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:32 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:33.270 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:32 smithi177 ceph-mon[106070]: osdmap e50: 7 total, 7 up, 7 in 2023-10-09T04:05:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:33 smithi029 ceph-mon[101043]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:33 smithi029 ceph-mon[119131]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:33 smithi177 ceph-mon[106070]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/1408661297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]: dispatch 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]: dispatch 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[101043]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]': finished 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[101043]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/3698646769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/1408661297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]: dispatch 2023-10-09T04:05:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]: dispatch 2023-10-09T04:05:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[119131]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]': finished 2023-10-09T04:05:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[119131]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T04:05:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:34 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/3698646769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:34 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/1408661297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]: dispatch 2023-10-09T04:05:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:34 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]: dispatch 2023-10-09T04:05:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:34 smithi177 ceph-mon[106070]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf54f5e9-1efd-44de-968c-59accfa44021"}]': finished 2023-10-09T04:05:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:34 smithi177 ceph-mon[106070]: osdmap e51: 8 total, 7 up, 8 in 2023-10-09T04:05:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:34 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:34 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/3698646769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-09T04:05:36.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:35 smithi029 ceph-mon[101043]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2023-10-09T04:05:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:35 smithi029 ceph-mon[119131]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2023-10-09T04:05:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:35 smithi177 ceph-mon[106070]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 137 KiB/s, 0 objects/s recovering 2023-10-09T04:05:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:37 smithi029 ceph-mon[101043]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:05:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:37 smithi029 ceph-mon[119131]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:05:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:37 smithi177 ceph-mon[106070]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-10-09T04:05:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:40 smithi029 ceph-mon[101043]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-10-09T04:05:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:40 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:05:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:40 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:40 smithi029 ceph-mon[119131]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-10-09T04:05:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:40 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:05:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:40 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:40 smithi177 ceph-mon[106070]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-10-09T04:05:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:40 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:05:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:40 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:41 smithi177 ceph-mon[106070]: Deploying daemon osd.7 on smithi177 2023-10-09T04:05:41.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:41 smithi029 ceph-mon[101043]: Deploying daemon osd.7 on smithi177 2023-10-09T04:05:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:41 smithi029 ceph-mon[119131]: Deploying daemon osd.7 on smithi177 2023-10-09T04:05:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:42 smithi177 ceph-mon[106070]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2023-10-09T04:05:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:42 smithi029 ceph-mon[101043]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2023-10-09T04:05:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:42 smithi029 ceph-mon[119131]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2023-10-09T04:05:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:43 smithi177 ceph-mon[106070]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:05:43.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:43 smithi029 ceph-mon[101043]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:05:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:43 smithi029 ceph-mon[119131]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-10-09T04:05:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:44 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:45.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:44 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:44 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:45 smithi177 ceph-mon[106070]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:46.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:45 smithi029 ceph-mon[101043]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:45 smithi029 ceph-mon[119131]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:47.774 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 7 on host 'smithi177' 2023-10-09T04:05:47.981 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:47.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:47 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:48.384 DEBUG:teuthology.orchestra.run.smithi177:osd.7> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-09T04:05:48.386 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-09T04:05:48.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T04:05:48.418 INFO:journalctl@ceph.osd.7.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:05:48.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:47 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:05:48.751 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:48 smithi177 ceph-mon[106070]: from='osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:05:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:48 smithi177 ceph-mon[106070]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:05:49.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:48 smithi029 ceph-mon[101043]: from='osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:05:49.299 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:48 smithi029 ceph-mon[101043]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:05:49.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:48 smithi029 ceph-mon[119131]: from='osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:05:49.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:48 smithi029 ceph-mon[119131]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-09T04:05:50.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[101043]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[101043]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[101043]: from='osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[101043]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[101043]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[119131]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[119131]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[119131]: from='osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:50.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[119131]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:50.204 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:49 smithi029 ceph-mon[119131]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:49 smithi177 ceph-mon[106070]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-09T04:05:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:49 smithi177 ceph-mon[106070]: osdmap e52: 8 total, 7 up, 8 in 2023-10-09T04:05:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:49 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:49 smithi177 ceph-mon[106070]: from='osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:49 smithi177 ceph-mon[106070]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:05:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:49 smithi177 ceph-mon[106070]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:50.278 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 04:05:49 smithi177 conmon[140729]: 2023-10-09T04:05:49.955+0000 7fe2be426700 -1 osd.7 0 waiting for initial osdmap 2023-10-09T04:05:50.278 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 04:05:49 smithi177 conmon[140729]: 2023-10-09T04:05:49.962+0000 7fe2b721d700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:05:50.586 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:05:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: purged_snaps scrub starts 2023-10-09T04:05:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: purged_snaps scrub ok 2023-10-09T04:05:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T04:05:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:50 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1314381853' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:05:51.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: purged_snaps scrub starts 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: purged_snaps scrub ok 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1314381853' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:05:51.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: purged_snaps scrub starts 2023-10-09T04:05:51.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: purged_snaps scrub ok 2023-10-09T04:05:51.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2023-10-09T04:05:51.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: osdmap e53: 8 total, 7 up, 8 in 2023-10-09T04:05:51.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:51.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:51.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:50 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1314381853' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:05:51.422 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1696824328,"num_in_osds":8,"osd_in_since":1696824334,"num_remapped_pgs":0} 2023-10-09T04:05:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577] boot 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:52.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:52.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:51 smithi177 ceph-mon[106070]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:05:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577] boot 2023-10-09T04:05:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[101043]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: osd.7 [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577] boot 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: osdmap e54: 8 total, 8 up, 8 in 2023-10-09T04:05:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:05:52.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:05:52.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:05:52.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:05:52.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:51 smithi029 ceph-mon[119131]: osdmap e55: 8 total, 8 up, 8 in 2023-10-09T04:05:52.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-09T04:05:52.798 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:52 smithi177 ceph-mon[106070]: Detected new or changed devices on smithi177 2023-10-09T04:05:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:52 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi177 to 4292M 2023-10-09T04:05:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:52 smithi177 ceph-mon[106070]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:05:53.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:52 smithi029 ceph-mon[101043]: Detected new or changed devices on smithi177 2023-10-09T04:05:53.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:52 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi177 to 4292M 2023-10-09T04:05:53.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:52 smithi029 ceph-mon[101043]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:05:53.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:52 smithi029 ceph-mon[119131]: Detected new or changed devices on smithi177 2023-10-09T04:05:53.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:52 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi177 to 4292M 2023-10-09T04:05:53.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:52 smithi029 ceph-mon[119131]: osdmap e56: 8 total, 8 up, 8 in 2023-10-09T04:05:54.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:53 smithi029 ceph-mon[101043]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:05:54.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:53 smithi029 ceph-mon[101043]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:05:54.144 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:53 smithi029 ceph-mon[119131]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:05:54.144 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:53 smithi029 ceph-mon[119131]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:53 smithi177 ceph-mon[106070]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:05:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:53 smithi177 ceph-mon[106070]: osdmap e57: 8 total, 8 up, 8 in 2023-10-09T04:05:54.549 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:05:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:54 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1530266723' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:05:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:54 smithi177 ceph-mon[106070]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:05:55.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:54 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1530266723' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:05:55.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:54 smithi029 ceph-mon[101043]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:05:55.351 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:54 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1530266723' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-09T04:05:55.351 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:54 smithi029 ceph-mon[119131]: osdmap e58: 8 total, 8 up, 8 in 2023-10-09T04:05:55.368 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824350,"num_in_osds":8,"osd_in_since":1696824334,"num_remapped_pgs":0} 2023-10-09T04:05:55.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:05:55.735 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:05:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:55 smithi177 ceph-mon[106070]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:05:56.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:55 smithi029 ceph-mon[101043]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:05:56.331 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:55 smithi029 ceph-mon[119131]: pgmap v174: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:05:57.483 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:05:57.483 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":58,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:05:54.570232+0000","last_up_change":"2023-10-09T04:05:50.951832+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6803","nonce":1054408938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6805","nonce":1054408938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6809","nonce":1054408938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6807","nonce":1054408938}]},"public_addr":"172.21.15.29:6803/1054408938","cluster_addr":"172.21.15.29:6805/1054408938","heartbeat_back_addr":"172.21.15.29:6809/1054408938","heartbeat_front_addr":"172.21.15.29:6807/1054408938","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6811","nonce":3631252275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6813","nonce":3631252275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6817","nonce":3631252275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6815","nonce":3631252275}]},"public_addr":"172.21.15.29:6811/3631252275","cluster_addr":"172.21.15.29:6813/3631252275","heartbeat_back_addr":"172.21.15.29:6817/3631252275","heartbeat_front_addr":"172.21.15.29:6815/3631252275","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6819","nonce":1532034566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6821","nonce":1532034566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6825","nonce":1532034566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6823","nonce":1532034566}]},"public_addr":"172.21.15.29:6819/1532034566","cluster_addr":"172.21.15.29:6821/1532034566","heartbeat_back_addr":"172.21.15.29:6825/1532034566","heartbeat_front_addr":"172.21.15.29:6823/1532034566","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6827","nonce":3716476544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6829","nonce":3716476544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6833","nonce":3716476544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6831","nonce":3716476544}]},"public_addr":"172.21.15.29:6827/3716476544","cluster_addr":"172.21.15.29:6829/3716476544","heartbeat_back_addr":"172.21.15.29:6833/3716476544","heartbeat_front_addr":"172.21.15.29:6831/3716476544","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:05:57.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:57 smithi029 ceph-mon[101043]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-10-09T04:05:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:05:57 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1920477001' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:05:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:57 smithi029 ceph-mon[119131]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-10-09T04:05:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:05:57 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1920477001' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:05:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:57 smithi177 ceph-mon[106070]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-10-09T04:05:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:05:57 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1920477001' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:05:58.250 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-10-09T04:03:57.603026+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-10-09T04:05:58.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-10-09T04:05:58.606 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:00.317 INFO:teuthology.orchestra.run.smithi029.stdout:pg_num: 1 2023-10-09T04:06:00.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:00 smithi029 ceph-mon[101043]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-10-09T04:06:00.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:00 smithi029 ceph-mon[119131]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-10-09T04:06:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:00 smithi177 ceph-mon[106070]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-10-09T04:06:01.089 INFO:tasks.cephadm:Adding prometheus.a on smithi177 2023-10-09T04:06:01.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi177=a' 2023-10-09T04:06:01.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:01 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4068505284' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T04:06:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:01 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4068505284' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T04:06:01.446 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:01 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4068505284' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-10-09T04:06:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:02 smithi177 ceph-mon[106070]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:06:02.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:02 smithi029 ceph-mon[101043]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:06:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:02 smithi029 ceph-mon[119131]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:06:03.262 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled prometheus update... 2023-10-09T04:06:03.451 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:03 smithi177 ceph-mon[106070]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-10-09T04:06:03.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:03 smithi029 ceph-mon[101043]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-10-09T04:06:03.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:03 smithi029 ceph-mon[119131]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-10-09T04:06:03.924 DEBUG:teuthology.orchestra.run.smithi177:prometheus.a> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-09T04:06:03.926 INFO:tasks.cephadm:Adding node-exporter.a on smithi029 2023-10-09T04:06:03.926 INFO:tasks.cephadm:Adding node-exporter.b on smithi177 2023-10-09T04:06:03.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi029=a;smithi177=b' 2023-10-09T04:06:03.958 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:06:04.294 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:04.525 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: Saving service prometheus spec with placement smithi177=a;count:1 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:04.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:04 smithi177 ceph-mon[106070]: Deploying daemon prometheus.a on smithi177 2023-10-09T04:06:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: Saving service prometheus spec with placement smithi177=a;count:1 2023-10-09T04:06:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[101043]: Deploying daemon prometheus.a on smithi177 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: Saving service prometheus spec with placement smithi177=a;count:1 2023-10-09T04:06:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:04.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:04.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:04.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:04.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:04.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:04 smithi029 ceph-mon[119131]: Deploying daemon prometheus.a on smithi177 2023-10-09T04:06:05.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:05 smithi177 ceph-mon[106070]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T04:06:05.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:05 smithi029 ceph-mon[101043]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T04:06:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:05 smithi029 ceph-mon[119131]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-09T04:06:06.198 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled node-exporter update... 2023-10-09T04:06:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:07 smithi177 ceph-mon[106070]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi177=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:07 smithi177 ceph-mon[106070]: Saving service node-exporter spec with placement smithi029=a;smithi177=b;count:2 2023-10-09T04:06:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:07 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:07 smithi177 ceph-mon[106070]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[101043]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi177=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[101043]: Saving service node-exporter spec with placement smithi029=a;smithi177=b;count:2 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[101043]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[119131]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi177=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[119131]: Saving service node-exporter spec with placement smithi029=a;smithi177=b;count:2 2023-10-09T04:06:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:07.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:07 smithi029 ceph-mon[119131]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-09T04:06:07.698 DEBUG:teuthology.orchestra.run.smithi029:node-exporter.a> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-09T04:06:07.700 DEBUG:teuthology.orchestra.run.smithi177:node-exporter.b> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-09T04:06:07.702 INFO:tasks.cephadm:Adding alertmanager.a on smithi029 2023-10-09T04:06:07.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi029=a' 2023-10-09T04:06:07.735 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:06:07.738 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:06:10.157 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:10 smithi177 ceph-mon[106070]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:10.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:10 smithi029 ceph-mon[101043]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:10 smithi029 ceph-mon[119131]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:12.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:12 smithi177 ceph-mon[106070]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:12 smithi029 ceph-mon[101043]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:12.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:12 smithi029 ceph-mon[119131]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:13 smithi177 ceph-mon[106070]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:13 smithi029 ceph-mon[101043]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:13 smithi029 ceph-mon[119131]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:14.867 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled alertmanager update... 2023-10-09T04:06:15.583 DEBUG:teuthology.orchestra.run.smithi029:alertmanager.a> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-09T04:06:15.588 INFO:tasks.cephadm:Adding grafana.a on smithi177 2023-10-09T04:06:15.588 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi177=a' 2023-10-09T04:06:15.622 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:06:16.147 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:15 smithi177 ceph-mon[106070]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:16.147 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:15 smithi177 ceph-mon[106070]: Saving service alertmanager spec with placement smithi029=a;count:1 2023-10-09T04:06:16.147 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:15 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:16.147 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:15 smithi177 ceph-mon[106070]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:16.148 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[101043]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[101043]: Saving service alertmanager spec with placement smithi029=a;count:1 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[101043]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[119131]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[119131]: Saving service alertmanager spec with placement smithi029=a;count:1 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:15 smithi029 ceph-mon[119131]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:16.434 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:16 smithi177 systemd[1]: Starting Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:06:17.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.453Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:06:17.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.453Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:06:17.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.453Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi177 (none))" 2023-10-09T04:06:17.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.453Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:06:17.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.453Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:06:17.578 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.454Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.454Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.456Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.456Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.458Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.458Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.848µs 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.458Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.458Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.458Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.14µs wal_replay_duration=257.805µs wbl_replay_duration=120ns total_replay_duration=294.045µs 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.459Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.459Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:06:17.579 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.459Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:06:17.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 bash[148996]: d772191b46e074c5efe0022f1185441946d6484ff278dd5351c5639266aed147 2023-10-09T04:06:17.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 systemd[1]: Started Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:06:17.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.475Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.114041ms db_storage=706ns remote_storage=1.465µs web_handler=334ns query_engine=679ns scrape=185.996µs scrape_sd=20.415µs notify=680ns notify_sd=1.731µs rules=15.55958ms tracing=6.241µs 2023-10-09T04:06:17.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.475Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:06:17.580 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:06:17 smithi177 conmon[149043]: ts=2023-10-09T04:06:17.475Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[101043]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[119131]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:17 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:06:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:17 smithi177 ceph-mon[106070]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:17 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-09T04:06:18.116 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled grafana update... 2023-10-09T04:06:18.684 DEBUG:teuthology.orchestra.run.smithi177:grafana.a> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-09T04:06:18.689 INFO:tasks.cephadm:Setting up client nodes... 2023-10-09T04:06:18.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T04:06:18.718 INFO:journalctl@ceph.grafana.a.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:06:18.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:18 smithi029 conmon[101466]: ignoring --setuser ceph since I am not root 2023-10-09T04:06:18.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:18 smithi029 conmon[101466]: ignoring --setgroup ceph since I am not root 2023-10-09T04:06:18.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:18 smithi029 ceph-mgr[101490]: -- 172.21.15.29:0/163533910 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dff7a969c0 con 0x55dff7a6e800 2023-10-09T04:06:18.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:18 smithi029 conmon[101466]: 2023-10-09T04:06:18.745+0000 7fcc6eb7f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:06:18.884 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:18 smithi177 conmon[109257]: ignoring --setuser ceph since I am not root 2023-10-09T04:06:18.885 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:18 smithi177 conmon[109257]: ignoring --setgroup ceph since I am not root 2023-10-09T04:06:18.885 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:18 smithi177 conmon[109257]: 2023-10-09T04:06:18.721+0000 7fbac2f161c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:06:18.885 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:18 smithi177 conmon[109257]: 2023-10-09T04:06:18.798+0000 7fbac2f161c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:06:18.885 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:18 smithi177 conmon[109257]: 2023-10-09T04:06:18.884+0000 7fbac2f161c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:06:19.073 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:19.105 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:18 smithi029 conmon[101466]: 2023-10-09T04:06:18.842+0000 7fcc6eb7f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:06:19.105 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:18 smithi029 conmon[101466]: 2023-10-09T04:06:18.970+0000 7fcc6eb7f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:06:19.105 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:19 smithi029 conmon[101466]: 2023-10-09T04:06:19.104+0000 7fcc6eb7f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:06:19.140 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:18 smithi177 conmon[109257]: 2023-10-09T04:06:18.975+0000 7fbac2f161c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:06:19.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:19 smithi177 ceph-mon[106070]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:19.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:19 smithi177 ceph-mon[106070]: Saving service grafana spec with placement smithi177=a;count:1 2023-10-09T04:06:19.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:19.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:19.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:19 smithi177 ceph-mon[106070]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:06:19.141 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:19 smithi177 ceph-mon[106070]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:19 smithi029 conmon[101466]: 2023-10-09T04:06:19.317+0000 7fcc6eb7f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[101043]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[101043]: Saving service grafana spec with placement smithi177=a;count:1 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[101043]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:06:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[101043]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T04:06:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[119131]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi177=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:06:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[119131]: Saving service grafana spec with placement smithi177=a;count:1 2023-10-09T04:06:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' 2023-10-09T04:06:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[119131]: from='mgr.14152 172.21.15.29:0/3336967384' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-09T04:06:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:19 smithi029 ceph-mon[119131]: mgrmap e16: y(active, since 4m), standbys: x 2023-10-09T04:06:19.446 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:19 smithi177 conmon[109257]: 2023-10-09T04:06:19.140+0000 7fbac2f161c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:06:19.750 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:19 smithi177 conmon[109257]: 2023-10-09T04:06:19.445+0000 7fbac2f161c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:06:19.750 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:19 smithi177 conmon[109257]: 2023-10-09T04:06:19.532+0000 7fbac2f161c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:06:19.750 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:19 smithi177 conmon[109257]: 2023-10-09T04:06:19.658+0000 7fbac2f161c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:06:19.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:19 smithi029 conmon[101466]: 2023-10-09T04:06:19.734+0000 7fcc6eb7f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:06:19.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:19 smithi029 conmon[101466]: 2023-10-09T04:06:19.842+0000 7fcc6eb7f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:06:20.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:19 smithi177 conmon[109257]: 2023-10-09T04:06:19.749+0000 7fbac2f161c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:06:20.105 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:19 smithi029 conmon[101466]: 2023-10-09T04:06:19.991+0000 7fcc6eb7f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:06:20.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:20 smithi029 conmon[101466]: 2023-10-09T04:06:20.104+0000 7fcc6eb7f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:06:20.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:20 smithi177 conmon[109257]: 2023-10-09T04:06:20.163+0000 7fbac2f161c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:06:20.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:20 smithi029 conmon[101466]: 2023-10-09T04:06:20.648+0000 7fcc6eb7f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout:[client.0] 2023-10-09T04:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: key = AQA8fCNlHEkKORAAzR7/cgpRkGd1ItCmMUM7RA== 2023-10-09T04:06:21.008 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:20 smithi177 conmon[109257]: 2023-10-09T04:06:20.720+0000 7fbac2f161c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:06:21.009 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:20 smithi177 conmon[109257]: 2023-10-09T04:06:20.887+0000 7fbac2f161c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:06:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:21 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/119479989' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:21 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:21 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:06:21.345 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:21 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/119479989' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:21.346 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:21 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:21.346 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:21 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:06:21.346 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:21 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/119479989' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:21.347 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:21 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:21.347 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:21 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:06:21.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:21 smithi029 conmon[101466]: 2023-10-09T04:06:21.344+0000 7fcc6eb7f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:06:21.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:21 smithi029 conmon[101466]: 2023-10-09T04:06:21.535+0000 7fcc6eb7f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:06:21.744 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T04:06:21.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-09T04:06:21.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-09T04:06:21.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-09T04:06:21.910 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:21 smithi177 conmon[109257]: 2023-10-09T04:06:21.643+0000 7fbac2f161c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:06:21.910 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:21 smithi177 conmon[109257]: 2023-10-09T04:06:21.732+0000 7fbac2f161c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:06:21.910 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:21 smithi177 conmon[109257]: 2023-10-09T04:06:21.909+0000 7fbac2f161c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:06:22.159 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:06:22.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:22 smithi177 conmon[109257]: 2023-10-09T04:06:22.237+0000 7fbac2f161c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:06:22.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:22 smithi177 conmon[109257]: 2023-10-09T04:06:22.326+0000 7fbac2f161c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:06:22.828 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:22 smithi029 conmon[101466]: 2023-10-09T04:06:22.498+0000 7fcc6eb7f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:06:22.828 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:22 smithi029 conmon[101466]: 2023-10-09T04:06:22.608+0000 7fcc6eb7f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:06:22.902 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:22 smithi177 conmon[109257]: 2023-10-09T04:06:22.636+0000 7fbac2f161c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:06:23.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:22 smithi029 conmon[101466]: 2023-10-09T04:06:22.827+0000 7fcc6eb7f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:06:23.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:22 smithi177 conmon[109257]: 2023-10-09T04:06:22.901+0000 7fbac2f161c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:06:23.628 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:23 smithi029 conmon[101466]: 2023-10-09T04:06:23.175+0000 7fcc6eb7f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:06:23.628 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:23 smithi029 conmon[101466]: 2023-10-09T04:06:23.280+0000 7fcc6eb7f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:06:23.835 INFO:teuthology.orchestra.run.smithi177.stdout:[client.1] 2023-10-09T04:06:23.835 INFO:teuthology.orchestra.run.smithi177.stdout: key = AQA/fCNl6P98MRAAG2UcfT53JA7FHUWj8UxPVg== 2023-10-09T04:06:23.882 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:23 smithi029 conmon[101466]: 2023-10-09T04:06:23.627+0000 7fcc6eb7f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:06:23.979 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:23 smithi177 ceph-mon[106070]: from='client.? 172.21.15.177:0/3093536895' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:23.979 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:23 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:23.979 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:23 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:06:23.979 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:23 smithi177 conmon[109257]: 2023-10-09T04:06:23.647+0000 7fbac2f161c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:06:23.979 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:23 smithi177 conmon[109257]: 2023-10-09T04:06:23.780+0000 7fbac2f161c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:06:24.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:23 smithi029 conmon[101466]: 2023-10-09T04:06:23.958+0000 7fcc6eb7f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:06:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:23 smithi029 ceph-mon[101043]: from='client.? 172.21.15.177:0/3093536895' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:23 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:23 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:06:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:23 smithi029 ceph-mon[119131]: from='client.? 172.21.15.177:0/3093536895' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:23 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-09T04:06:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:23 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-09T04:06:24.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:23 smithi177 conmon[109257]: 2023-10-09T04:06:23.978+0000 7fbac2f161c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:06:24.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:24 smithi177 conmon[109257]: 2023-10-09T04:06:24.099+0000 7fbac2f161c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:06:24.390 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2023-10-09T04:06:24.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-09T04:06:24.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-09T04:06:24.432 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-09T04:06:24.432 DEBUG:teuthology.orchestra.run.smithi029:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-09T04:06:24.769 INFO:teuthology.misc.health.smithi029.stdout: 2023-10-09T04:06:24.770 INFO:teuthology.misc.health.smithi029.stdout:{"epoch":58,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:05:54.570232+0000","last_up_change":"2023-10-09T04:05:50.951832+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6803","nonce":1054408938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6805","nonce":1054408938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6809","nonce":1054408938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6807","nonce":1054408938}]},"public_addr":"172.21.15.29:6803/1054408938","cluster_addr":"172.21.15.29:6805/1054408938","heartbeat_back_addr":"172.21.15.29:6809/1054408938","heartbeat_front_addr":"172.21.15.29:6807/1054408938","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6811","nonce":3631252275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6813","nonce":3631252275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6817","nonce":3631252275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6815","nonce":3631252275}]},"public_addr":"172.21.15.29:6811/3631252275","cluster_addr":"172.21.15.29:6813/3631252275","heartbeat_back_addr":"172.21.15.29:6817/3631252275","heartbeat_front_addr":"172.21.15.29:6815/3631252275","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6819","nonce":1532034566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6821","nonce":1532034566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6825","nonce":1532034566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6823","nonce":1532034566}]},"public_addr":"172.21.15.29:6819/1532034566","cluster_addr":"172.21.15.29:6821/1532034566","heartbeat_back_addr":"172.21.15.29:6825/1532034566","heartbeat_front_addr":"172.21.15.29:6823/1532034566","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6827","nonce":3716476544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6829","nonce":3716476544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6833","nonce":3716476544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6831","nonce":3716476544}]},"public_addr":"172.21.15.29:6827/3716476544","cluster_addr":"172.21.15.29:6829/3716476544","heartbeat_back_addr":"172.21.15.29:6833/3716476544","heartbeat_front_addr":"172.21.15.29:6831/3716476544","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:06:24.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:24 smithi177 conmon[109257]: 2023-10-09T04:06:24.316+0000 7fbac2f161c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:06:24.783 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-09T04:06:24.783 INFO:tasks.cephadm:Creating RBD pool 2023-10-09T04:06:24.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-09T04:06:25.143 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:25.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:24 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1217014425' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:24 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1217014425' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:25.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:24 smithi029 conmon[101466]: 2023-10-09T04:06:24.798+0000 7fcc6eb7f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:06:25.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:24 smithi029 conmon[101466]: 2023-10-09T04:06:24.963+0000 7fcc6eb7f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:06:25.180 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:24 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1217014425' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:25.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: 2023-10-09T04:06:25.179+0000 7fbac2f161c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:06:25.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: [09/Oct/2023:04:06:25] ENGINE Bus STARTING 2023-10-09T04:06:25.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: CherryPy Checker: 2023-10-09T04:06:25.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: The Application mounted at '' has an empty config. 2023-10-09T04:06:25.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: 2023-10-09T04:06:25.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: [09/Oct/2023:04:06:25] ENGINE Serving on http://:::9283 2023-10-09T04:06:25.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:25 smithi177 conmon[109257]: [09/Oct/2023:04:06:25] ENGINE Bus STARTED 2023-10-09T04:06:25.580 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:25 smithi029 conmon[101466]: 2023-10-09T04:06:25.227+0000 7fcc6eb7f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:06:25.580 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:25 smithi029 conmon[101466]: 2023-10-09T04:06:25.367+0000 7fcc6eb7f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:06:25.884 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:25 smithi029 conmon[101466]: 2023-10-09T04:06:25.654+0000 7fcc6eb7f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:06:26.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[101043]: Standby manager daemon x restarted 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[101043]: Standby manager daemon x started 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[119131]: Standby manager daemon x restarted 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[119131]: Standby manager daemon x started 2023-10-09T04:06:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:06:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:06:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:06:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:25 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:06:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:25 smithi177 ceph-mon[106070]: Standby manager daemon x restarted 2023-10-09T04:06:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:25 smithi177 ceph-mon[106070]: Standby manager daemon x started 2023-10-09T04:06:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:25 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:06:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:25 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:06:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:25 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:06:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:25 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/80686897' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:06:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: mgrmap e17: y(active, since 4m), standbys: x 2023-10-09T04:06:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: Active manager daemon y restarted 2023-10-09T04:06:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: Activating manager daemon y 2023-10-09T04:06:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:06:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: mgrmap e18: y(active, starting, since 0.00661857s), standbys: x 2023-10-09T04:06:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:06:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: Manager daemon y is now available 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:06:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: mgrmap e17: y(active, since 4m), standbys: x 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: Active manager daemon y restarted 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: Activating manager daemon y 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: mgrmap e18: y(active, starting, since 0.00661857s), standbys: x 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:06:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:06:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: Manager daemon y is now available 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: 2023-10-09T04:06:26.724+0000 7fcc6eb7f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:06:26.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: [09/Oct/2023:04:06:26] ENGINE Bus STARTING 2023-10-09T04:06:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: mgrmap e17: y(active, since 4m), standbys: x 2023-10-09T04:06:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: Active manager daemon y restarted 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: Activating manager daemon y 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: osdmap e59: 8 total, 8 up, 8 in 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: mgrmap e18: y(active, starting, since 0.00661857s), standbys: x 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:06:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: Manager daemon y is now available 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:06:27.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:06:27.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: CherryPy Checker: 2023-10-09T04:06:27.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: The Application mounted at '' has an empty config. 2023-10-09T04:06:27.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: 2023-10-09T04:06:27.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: [09/Oct/2023:04:06:26] ENGINE Serving on http://:::9283 2023-10-09T04:06:27.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:26 smithi029 conmon[101466]: [09/Oct/2023:04:06:26] ENGINE Bus STARTED 2023-10-09T04:06:27.903 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'rbd' created 2023-10-09T04:06:28.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:27 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3924429446' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:06:28.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:27 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:06:28.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:27 smithi029 ceph-mon[101043]: mgrmap e19: y(active, since 1.01303s), standbys: x 2023-10-09T04:06:28.153 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:27 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3924429446' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:06:28.153 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:27 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:06:28.153 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:27 smithi029 ceph-mon[119131]: mgrmap e19: y(active, since 1.01303s), standbys: x 2023-10-09T04:06:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:27 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3924429446' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:06:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:27 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-09T04:06:28.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:27 smithi177 ceph-mon[106070]: mgrmap e19: y(active, since 1.01303s), standbys: x 2023-10-09T04:06:28.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: [09/Oct/2023:04:06:28] ENGINE Bus STARTING 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: [09/Oct/2023:04:06:28] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: [09/Oct/2023:04:06:28] ENGINE Bus STARTED 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: [09/Oct/2023:04:06:28] ENGINE Bus STARTING 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: [09/Oct/2023:04:06:28] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: [09/Oct/2023:04:06:28] ENGINE Bus STARTED 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:28 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-09T04:06:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: osdmap e60: 8 total, 8 up, 8 in 2023-10-09T04:06:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: [09/Oct/2023:04:06:28] ENGINE Bus STARTING 2023-10-09T04:06:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: [09/Oct/2023:04:06:28] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:06:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: [09/Oct/2023:04:06:28] ENGINE Bus STARTED 2023-10-09T04:06:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:28 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:29.365 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:06:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi177 to 4036M 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[119131]: mgrmap e20: y(active, since 3s), standbys: x 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi177 to 4036M 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:06:30.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:29 smithi029 ceph-mon[101043]: mgrmap e20: y(active, since 3s), standbys: x 2023-10-09T04:06:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: osdmap e61: 8 total, 8 up, 8 in 2023-10-09T04:06:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:06:30.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi177 to 4036M 2023-10-09T04:06:30.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:30.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:06:30.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:29 smithi177 ceph-mon[106070]: mgrmap e20: y(active, since 3s), standbys: x 2023-10-09T04:06:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:06:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:31.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:06:31.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:06:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:06:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:31.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:31.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:06:31.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:30 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:06:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: osdmap e62: 8 total, 8 up, 8 in 2023-10-09T04:06:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:06:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:06:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:06:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:30 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:06:31.921 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-09T04:06:32.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3238104832' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[101043]: Deploying daemon alertmanager.a on smithi029 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:06:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3238104832' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:31 smithi029 ceph-mon[119131]: Deploying daemon alertmanager.a on smithi029 2023-10-09T04:06:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:06:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:06:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:06:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:06:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3238104832' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:31 smithi177 ceph-mon[106070]: Deploying daemon alertmanager.a on smithi029 2023-10-09T04:06:32.835 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:06:32.835 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:06:32.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:06:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:32 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3238104832' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:06:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:32 smithi029 ceph-mon[101043]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:06:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:32 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3238104832' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:06:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:32 smithi029 ceph-mon[119131]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:06:33.274 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:32 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3238104832' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:06:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:32 smithi177 ceph-mon[106070]: osdmap e63: 8 total, 8 up, 8 in 2023-10-09T04:06:33.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:32 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:06:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:06:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:33 smithi029 ceph-mon[101043]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:33 smithi029 ceph-mon[101043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:06:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:33 smithi029 ceph-mon[101043]: Cluster is now healthy 2023-10-09T04:06:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:33 smithi029 ceph-mon[119131]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:33 smithi029 ceph-mon[119131]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:06:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:33 smithi029 ceph-mon[119131]: Cluster is now healthy 2023-10-09T04:06:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:33 smithi177 ceph-mon[106070]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:33 smithi177 ceph-mon[106070]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:06:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:33 smithi177 ceph-mon[106070]: Cluster is now healthy 2023-10-09T04:06:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:35 smithi177 ceph-mon[106070]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:35 smithi029 ceph-mon[101043]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:35 smithi029 ceph-mon[119131]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:38.794 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:06:38.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:38 smithi029 ceph-mon[101043]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:38 smithi029 ceph-mon[119131]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:38 smithi177 ceph-mon[106070]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:39.865 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:39 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1389356686' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:06:39.865 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:39 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1389356686' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:06:39.886 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":20,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":3407458368},{"type":"v1","addr":"172.21.15.29:6801","nonce":3407458368}]},"active_addr":"172.21.15.29:6801/3407458368","active_change":"2023-10-09T04:06:26.739515+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2321967128}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1371585021}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1787157051}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3506139723}]}]} 2023-10-09T04:06:39.890 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:06:39.891 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:06:39.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:06:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:39 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1389356686' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:06:40.734 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:40.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:40 smithi029 ceph-mon[101043]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:40.848 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:40 smithi029 ceph-mon[119131]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:40.849 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:40 smithi029 systemd[1]: Starting Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:06:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:40 smithi177 ceph-mon[106070]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:42.552 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.323Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.323Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.323Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.324Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.353Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.353Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.356Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[165950]: ts=2023-10-09T04:06:42.356Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 bash[165866]: d4e0409956ad046932bc9dd1940d6d6fea5beb9e519c8d39c121a457360ec7eb 2023-10-09T04:06:42.553 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:42 smithi029 systemd[1]: Started Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:06:42.554 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[101466]: [09/Oct/2023:04:06:42] ENGINE Bus STOPPING 2023-10-09T04:06:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[101466]: [09/Oct/2023:04:06:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:06:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[101466]: [09/Oct/2023:04:06:42] ENGINE Bus STOPPED 2023-10-09T04:06:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[101466]: [09/Oct/2023:04:06:42] ENGINE Bus STARTING 2023-10-09T04:06:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[101466]: [09/Oct/2023:04:06:42] ENGINE Serving on http://:::9283 2023-10-09T04:06:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:06:42 smithi029 conmon[101466]: [09/Oct/2023:04:06:42] ENGINE Bus STARTED 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:06:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:06:42.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:42 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:06:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:06:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:06:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:43.048 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:06:43.049 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":63,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:06:31.909033+0000","last_up_change":"2023-10-09T04:05:50.951832+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6803","nonce":1054408938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6805","nonce":1054408938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6809","nonce":1054408938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6807","nonce":1054408938}]},"public_addr":"172.21.15.29:6803/1054408938","cluster_addr":"172.21.15.29:6805/1054408938","heartbeat_back_addr":"172.21.15.29:6809/1054408938","heartbeat_front_addr":"172.21.15.29:6807/1054408938","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6811","nonce":3631252275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6813","nonce":3631252275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6817","nonce":3631252275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6815","nonce":3631252275}]},"public_addr":"172.21.15.29:6811/3631252275","cluster_addr":"172.21.15.29:6813/3631252275","heartbeat_back_addr":"172.21.15.29:6817/3631252275","heartbeat_front_addr":"172.21.15.29:6815/3631252275","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6819","nonce":1532034566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6821","nonce":1532034566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6825","nonce":1532034566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6823","nonce":1532034566}]},"public_addr":"172.21.15.29:6819/1532034566","cluster_addr":"172.21.15.29:6821/1532034566","heartbeat_back_addr":"172.21.15.29:6825/1532034566","heartbeat_front_addr":"172.21.15.29:6823/1532034566","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6827","nonce":3716476544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6829","nonce":3716476544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6833","nonce":3716476544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6831","nonce":3716476544}]},"public_addr":"172.21.15.29:6827/3716476544","cluster_addr":"172.21.15.29:6829/3716476544","heartbeat_back_addr":"172.21.15.29:6833/3716476544","heartbeat_front_addr":"172.21.15.29:6831/3716476544","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:06:43.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:43 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:06:43.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:43 smithi029 ceph-mon[101043]: Deploying daemon grafana.a on smithi177 2023-10-09T04:06:43.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:43 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/608107972' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:43.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:43 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:06:43.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:43 smithi029 ceph-mon[119131]: Deploying daemon grafana.a on smithi177 2023-10-09T04:06:43.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:43 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/608107972' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:43.921 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:06:43.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:06:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:43 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-09T04:06:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:43 smithi177 ceph-mon[106070]: Deploying daemon grafana.a on smithi177 2023-10-09T04:06:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:43 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/608107972' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:44.279 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:44.594 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:44 smithi029 conmon[165950]: ts=2023-10-09T04:06:44.324Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000054538s 2023-10-09T04:06:44.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:44 smithi029 ceph-mon[101043]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:44 smithi029 ceph-mon[119131]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:44 smithi177 ceph-mon[106070]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:46.105 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:06:46.106 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":63,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:06:31.909033+0000","last_up_change":"2023-10-09T04:05:50.951832+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6803","nonce":1054408938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6805","nonce":1054408938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6809","nonce":1054408938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6807","nonce":1054408938}]},"public_addr":"172.21.15.29:6803/1054408938","cluster_addr":"172.21.15.29:6805/1054408938","heartbeat_back_addr":"172.21.15.29:6809/1054408938","heartbeat_front_addr":"172.21.15.29:6807/1054408938","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6811","nonce":3631252275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6813","nonce":3631252275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6817","nonce":3631252275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6815","nonce":3631252275}]},"public_addr":"172.21.15.29:6811/3631252275","cluster_addr":"172.21.15.29:6813/3631252275","heartbeat_back_addr":"172.21.15.29:6817/3631252275","heartbeat_front_addr":"172.21.15.29:6815/3631252275","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6819","nonce":1532034566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6821","nonce":1532034566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6825","nonce":1532034566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6823","nonce":1532034566}]},"public_addr":"172.21.15.29:6819/1532034566","cluster_addr":"172.21.15.29:6821/1532034566","heartbeat_back_addr":"172.21.15.29:6825/1532034566","heartbeat_front_addr":"172.21.15.29:6823/1532034566","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6827","nonce":3716476544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6829","nonce":3716476544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6833","nonce":3716476544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6831","nonce":3716476544}]},"public_addr":"172.21.15.29:6827/3716476544","cluster_addr":"172.21.15.29:6829/3716476544","heartbeat_back_addr":"172.21.15.29:6833/3716476544","heartbeat_front_addr":"172.21.15.29:6831/3716476544","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:06:46.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:06:46.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:06:46.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:06:46.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:06:46.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:06:46.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:06:46.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:06:46.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:06:46.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:46 smithi029 ceph-mon[101043]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:46.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:46 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4193961595' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:46 smithi029 ceph-mon[119131]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:46 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4193961595' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:46 smithi177 ceph-mon[106070]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:46 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4193961595' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:06:48.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:47 smithi029 ceph-mon[101043]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:48.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:47 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:47 smithi029 ceph-mon[119131]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:47 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:48.194 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:47 smithi177 ceph-mon[106070]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:47 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:06:48.309 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:48.397 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:48.531 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:48.532 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:48.649 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:49.258 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:49.258 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:49 smithi029 ceph-mon[101043]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:49 smithi029 ceph-mon[119131]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:49 smithi177 ceph-mon[106070]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:52.538 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:06:52 smithi029 conmon[165950]: ts=2023-10-09T04:06:52.327Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002337149s 2023-10-09T04:06:52.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:52 smithi029 ceph-mon[101043]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:52.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:52 smithi029 ceph-mon[119131]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:52 smithi177 ceph-mon[106070]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:53.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:06:52 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:06:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:06:53.735 INFO:teuthology.orchestra.run.smithi029.stdout:197568495632 2023-10-09T04:06:53.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:06:54.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:54 smithi029 ceph-mon[101043]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:54.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:54 smithi029 ceph-mon[119131]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:54 smithi177 ceph-mon[106070]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:56.841 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:56 smithi029 ceph-mon[101043]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:56.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:56 smithi029 ceph-mon[119131]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:56 smithi177 ceph-mon[106070]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:57.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:57 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:57.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:57 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:57.951 INFO:teuthology.orchestra.run.smithi029.stdout:77309411365 2023-10-09T04:06:57.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:06:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:57 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:06:58.743 INFO:teuthology.orchestra.run.smithi029.stdout:163208757271 2023-10-09T04:06:58.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:06:58.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:06:58 smithi029 ceph-mon[101043]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:58.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:06:58 smithi029 ceph-mon[119131]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:06:58 smithi177 ceph-mon[106070]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:06:59.907 INFO:teuthology.orchestra.run.smithi029.stdout:34359738416 2023-10-09T04:06:59.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:06:59.962 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:06:59.987 INFO:teuthology.orchestra.run.smithi029.stdout:55834574891 2023-10-09T04:06:59.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:07:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:00 smithi177 ceph-mon[106070]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:00.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:00 smithi029 ceph-mon[101043]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:00 smithi029 ceph-mon[119131]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:01.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 systemd[1]: Starting Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:01.841 INFO:teuthology.orchestra.run.smithi029.stdout:231928234000 2023-10-09T04:07:01.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:07:01.902 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 bash[152175]: 3d5a415c7f4fa886850c4452ddfb01020cf1db82aa986091d58d0c65a97be922 2023-10-09T04:07:01.902 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 systemd[1]: Started Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:01.902 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.752813005Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T04:07:01.902 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753013523Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T04:07:01.902 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753031017Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T04:07:01.902 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753040188Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753045863Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753051362Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753057206Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753062645Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753068408Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753073745Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753078819Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753083956Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753089308Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=settings t=2023-10-09T04:07:01.753094619Z level=info msg="App mode production" 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=sqlstore t=2023-10-09T04:07:01.753158442Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T04:07:01.903 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=sqlstore t=2023-10-09T04:07:01.753175206Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T04:07:01.904 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.75402776Z level=info msg="Starting DB migrations" 2023-10-09T04:07:01.904 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.75410392Z level=info msg="Executing migration" id="create migration_log table" 2023-10-09T04:07:01.904 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.818899102Z level=info msg="Executing migration" id="create user table" 2023-10-09T04:07:01.904 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.860472731Z level=info msg="Executing migration" id="add unique index user.login" 2023-10-09T04:07:01.904 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.902413235Z level=info msg="Executing migration" id="add unique index user.email" 2023-10-09T04:07:02.107 INFO:teuthology.orchestra.run.smithi029.stdout:128849018908 2023-10-09T04:07:02.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.944877801Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.969898901Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:01 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:01.995023926Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.020736049Z level=info msg="Executing migration" id="create user table v2" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.045714371Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.087580909Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.146213481Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-10-09T04:07:02.171 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.171352435Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-10-09T04:07:02.439 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.213457404Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-10-09T04:07:02.439 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.238918177Z level=info msg="Executing migration" id="Update user table charset" 2023-10-09T04:07:02.440 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.280562196Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-10-09T04:07:02.440 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.322530366Z level=info msg="Executing migration" id="Add missing user data" 2023-10-09T04:07:02.440 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.347645235Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-10-09T04:07:02.440 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.372931837Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-10-09T04:07:02.440 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.414753271Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-10-09T04:07:02.440 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.439966318Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-10-09T04:07:02.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:02 smithi177 ceph-mon[106070]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:02.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:02 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:02 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:02 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:02 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:02 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.700 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.465295676Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-10-09T04:07:02.700 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.49061434Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-10-09T04:07:02.700 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.532533941Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-10-09T04:07:02.701 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.582534551Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-10-09T04:07:02.701 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.599439987Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-10-09T04:07:02.701 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.633144883Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-10-09T04:07:02.701 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.666657845Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-10-09T04:07:02.701 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.683455619Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-10-09T04:07:02.701 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.700237615Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-10-09T04:07:02.866 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[101043]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:02.866 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.866 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.866 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[119131]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.867 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:02 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:02.868 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:02 smithi029 systemd[1]: Starting Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:02.952 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.742467069Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.775899253Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.792828221Z level=info msg="Executing migration" id="create temp_user v2" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.826394019Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.843233811Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.876770619Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.893582799Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.918851542Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.935826004Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-10-09T04:07:02.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.952659517Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-10-09T04:07:03.145 INFO:teuthology.orchestra.run.smithi029.stdout:107374182434 2023-10-09T04:07:03.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:07:03.179 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:03.221 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:02 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:02.994598991Z level=info msg="Executing migration" id="create star table" 2023-10-09T04:07:03.221 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.011568158Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-10-09T04:07:03.221 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.028276051Z level=info msg="Executing migration" id="create org table v1" 2023-10-09T04:07:03.221 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.045262738Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-10-09T04:07:03.222 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.087458283Z level=info msg="Executing migration" id="create org_user table v1" 2023-10-09T04:07:03.222 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.104255772Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-10-09T04:07:03.222 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.129302553Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-10-09T04:07:03.222 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.154605014Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-10-09T04:07:03.222 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.179706798Z level=info msg="Executing migration" id="Update org table charset" 2023-10-09T04:07:03.222 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.221738575Z level=info msg="Executing migration" id="Update org_user table charset" 2023-10-09T04:07:03.243 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:03.443 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:03.481 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.272089427Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-10-09T04:07:03.481 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.347366001Z level=info msg="Executing migration" id="create dashboard table" 2023-10-09T04:07:03.481 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.389243624Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-10-09T04:07:03.482 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.43115073Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-10-09T04:07:03.482 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.456364768Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-10-09T04:07:03.482 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.481735082Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-10-09T04:07:03.625 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:03.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:03 smithi029 ceph-mon[101043]: Deploying daemon node-exporter.a on smithi029 2023-10-09T04:07:03.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:03 smithi029 ceph-mon[119131]: Deploying daemon node-exporter.a on smithi029 2023-10-09T04:07:03.741 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:03 smithi177 ceph-mon[106070]: Deploying daemon node-exporter.a on smithi029 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.515374107Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.532043903Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.548899534Z level=info msg="Executing migration" id="create dashboard v2" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.56573377Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.582746262Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.599475454Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.633128126Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-10-09T04:07:03.742 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.675098766Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-10-09T04:07:03.743 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.700063931Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-10-09T04:07:03.743 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.742031991Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.784144856Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.809215201Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.834385649Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.859714927Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.884749028Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.910054557Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.952014897Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:03 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:03.976954645Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-10-09T04:07:04.002 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.002239105Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-10-09T04:07:04.270 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.027682598Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.052702148Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.119576546Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.144832384Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.169972277Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.195189575Z level=info msg="Executing migration" id="Update dashboard title length" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.245802132Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-09T04:07:04.271 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.271031221Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-10-09T04:07:04.527 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.313039121Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-09T04:07:04.527 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.329922085Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-10-09T04:07:04.527 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.363567998Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-09T04:07:04.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.380333314Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-09T04:07:04.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.397147426Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-10-09T04:07:04.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.414040911Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-10-09T04:07:04.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.430864251Z level=info msg="Executing migration" id="Add check_sum column" 2023-10-09T04:07:04.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.447735878Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-10-09T04:07:04.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.464717826Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-10-09T04:07:04.529 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.48141173Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-10-09T04:07:04.529 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.498289679Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-10-09T04:07:04.529 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.515125575Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-10-09T04:07:04.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:04 smithi177 ceph-mon[106070]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:04.809 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.557293858Z level=info msg="Executing migration" id="create data_source table" 2023-10-09T04:07:04.810 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.574136765Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-10-09T04:07:04.810 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.590915211Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-10-09T04:07:04.810 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.607888087Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-10-09T04:07:04.810 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.641309408Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-10-09T04:07:04.810 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.674942657Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-10-09T04:07:04.811 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.691752972Z level=info msg="Executing migration" id="create data_source table v2" 2023-10-09T04:07:04.811 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.708702131Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-10-09T04:07:04.811 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.742156064Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-10-09T04:07:04.811 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.759080294Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-10-09T04:07:04.811 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.775971756Z level=info msg="Executing migration" id="Add column with_credentials" 2023-10-09T04:07:04.812 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.792794942Z level=info msg="Executing migration" id="Add secure json data column" 2023-10-09T04:07:04.812 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.809598646Z level=info msg="Executing migration" id="Update data_source table charset" 2023-10-09T04:07:04.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:04 smithi029 ceph-mon[101043]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:04.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:04 smithi029 ceph-mon[119131]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:05.086 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.884973927Z level=info msg="Executing migration" id="Update initial version to 1" 2023-10-09T04:07:05.086 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.918550694Z level=info msg="Executing migration" id="Add read_only data column" 2023-10-09T04:07:05.086 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.935365556Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-10-09T04:07:05.086 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.952202183Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-10-09T04:07:05.087 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.969149958Z level=info msg="Executing migration" id="Add uid column" 2023-10-09T04:07:05.087 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:04 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:04.986085429Z level=info msg="Executing migration" id="Update uid value" 2023-10-09T04:07:05.087 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.002779677Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-10-09T04:07:05.087 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.044727606Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-10-09T04:07:05.087 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.086534385Z level=info msg="Executing migration" id="create api_key table" 2023-10-09T04:07:05.170 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:04 smithi029 bash[171842]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T04:07:05.355 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.128752999Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-10-09T04:07:05.355 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.154064452Z level=info msg="Executing migration" id="add index api_key.key" 2023-10-09T04:07:05.355 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.179295224Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-10-09T04:07:05.355 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.221275671Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-10-09T04:07:05.355 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.246475434Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-10-09T04:07:05.356 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.271770875Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-10-09T04:07:05.356 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.31360321Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-10-09T04:07:05.356 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.355396858Z level=info msg="Executing migration" id="create api_key table v2" 2023-10-09T04:07:05.491 INFO:teuthology.orchestra.run.smithi029.stdout:197568495634 2023-10-09T04:07:05.615 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:05 smithi029 bash[171842]: Getting image source signatures 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.405866291Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.447786715Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.473058533Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.498090377Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.523343539Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.548703931Z level=info msg="Executing migration" id="Update api_key table charset" 2023-10-09T04:07:05.616 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.573693613Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-10-09T04:07:05.883 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:05 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2740884692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.657798376Z level=info msg="Executing migration" id="Add service account foreign key" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.699431074Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.716279927Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.758104353Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.783389519Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.808476504Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.833767673Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.858881354Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-10-09T04:07:05.884 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.884039404Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-09T04:07:05.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:05 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2740884692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:07:05.889 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:05 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2740884692' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:07:05.890 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:05 smithi029 bash[171842]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T04:07:05.890 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:05 smithi029 bash[171842]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T04:07:05.890 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:05 smithi029 bash[171842]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T04:07:06.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.934366943Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-09T04:07:06.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.959577674Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-10-09T04:07:06.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:05 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:05.984760523Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-10-09T04:07:06.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.009989397Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-10-09T04:07:06.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.035134036Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-10-09T04:07:06.152 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.060137743Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-09T04:07:06.153 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.085475288Z level=info msg="Executing migration" id="create quota table v1" 2023-10-09T04:07:06.153 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.127260949Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-09T04:07:06.153 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.152451261Z level=info msg="Executing migration" id="Update quota table charset" 2023-10-09T04:07:06.412 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.194533099Z level=info msg="Executing migration" id="create plugin_setting table" 2023-10-09T04:07:06.412 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.236389783Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-09T04:07:06.412 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.278292743Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-10-09T04:07:06.412 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.303453946Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-10-09T04:07:06.412 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.328621786Z level=info msg="Executing migration" id="create session table" 2023-10-09T04:07:06.413 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.370572107Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-10-09T04:07:06.413 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.412677364Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-10-09T04:07:06.616 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:06 smithi029 bash[171842]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T04:07:06.672 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:06 smithi177 ceph-mon[106070]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:06.672 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.454582102Z level=info msg="Executing migration" id="create playlist table v2" 2023-10-09T04:07:06.672 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.479771137Z level=info msg="Executing migration" id="create playlist item table v2" 2023-10-09T04:07:06.672 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.505022019Z level=info msg="Executing migration" id="Update playlist table charset" 2023-10-09T04:07:06.673 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.530036979Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-10-09T04:07:06.673 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.563711587Z level=info msg="Executing migration" id="drop preferences table v2" 2023-10-09T04:07:06.673 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.605464824Z level=info msg="Executing migration" id="drop preferences table v3" 2023-10-09T04:07:06.673 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.647414232Z level=info msg="Executing migration" id="create preferences table v3" 2023-10-09T04:07:06.673 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.672644937Z level=info msg="Executing migration" id="Update preferences table charset" 2023-10-09T04:07:06.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:06 smithi029 ceph-mon[101043]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:06 smithi029 ceph-mon[119131]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.714678996Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.74825258Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.798443734Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.831967243Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.86549482Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.898973836Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-10-09T04:07:06.932 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.932447019Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-10-09T04:07:07.193 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:06 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:06.991248674Z level=info msg="Executing migration" id="create alert table v1" 2023-10-09T04:07:07.193 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.016354785Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-10-09T04:07:07.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.041458513Z level=info msg="Executing migration" id="add index alert state" 2023-10-09T04:07:07.194 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.074929528Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-10-09T04:07:07.195 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.100036863Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-10-09T04:07:07.195 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.142016008Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-09T04:07:07.195 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.192074587Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-09T04:07:07.421 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:07 smithi029 bash[171842]: Writing manifest to image destination 2023-10-09T04:07:07.421 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:07 smithi029 bash[171842]: Storing signatures 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.23423406Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.259383923Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.284704908Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.326594898Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.351784315Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.39353203Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.435432782Z level=info msg="Executing migration" id="Add column is_default" 2023-10-09T04:07:07.477 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.477306313Z level=info msg="Executing migration" id="Add column frequency" 2023-10-09T04:07:07.762 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.519221774Z level=info msg="Executing migration" id="Add column send_reminder" 2023-10-09T04:07:07.762 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.544329693Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-10-09T04:07:07.762 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.569481379Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-10-09T04:07:07.762 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.594744955Z level=info msg="Executing migration" id="Update alert table charset" 2023-10-09T04:07:07.762 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.63672343Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-10-09T04:07:07.762 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.678499022Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-10-09T04:07:07.763 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.720394169Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-09T04:07:07.763 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.762364418Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-10-09T04:07:07.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495632 got 197568495634 for osd.6 2023-10-09T04:07:07.891 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.814735113Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.839929587Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.865113796Z level=info msg="Executing migration" id="Add for to alert table" 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.90708553Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.973885107Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:07 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:07.999105219Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-10-09T04:07:08.024 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.024230426Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-10-09T04:07:08.184 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:08.184 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:08.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.057999548Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-10-09T04:07:08.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.08317887Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-10-09T04:07:08.277 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.108216262Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-10-09T04:07:08.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.150124369Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-10-09T04:07:08.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.175347803Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-10-09T04:07:08.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.200381781Z level=info msg="Executing migration" id="create annotation table v5" 2023-10-09T04:07:08.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.24232879Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-10-09T04:07:08.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.267476787Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-10-09T04:07:08.475 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:08.577 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.317847591Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-10-09T04:07:08.577 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.343116536Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-10-09T04:07:08.577 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.385037644Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-10-09T04:07:08.577 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.426945466Z level=info msg="Executing migration" id="Update annotation table charset" 2023-10-09T04:07:08.577 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.452003363Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-10-09T04:07:08.578 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.47707695Z level=info msg="Executing migration" id="Drop category_id index" 2023-10-09T04:07:08.578 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.502266217Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-10-09T04:07:08.578 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.527421441Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-10-09T04:07:08.578 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.552671068Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-09T04:07:08.578 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.577818047Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-09T04:07:08.845 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:08 smithi177 ceph-mon[106070]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:08.845 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.611377265Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-09T04:07:08.846 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.653267805Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-10-09T04:07:08.846 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.72853457Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-09T04:07:08.846 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.778695162Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-10-09T04:07:08.846 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.812251857Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-10-09T04:07:08.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:08 smithi029 ceph-mon[101043]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:08 smithi029 ceph-mon[119131]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:09.130 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.845684561Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-10-09T04:07:09.130 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.879397518Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-10-09T04:07:09.130 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.929436974Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-10-09T04:07:09.131 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:08 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:08.963136782Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-10-09T04:07:09.131 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.013184843Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-10-09T04:07:09.131 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.038517797Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-10-09T04:07:09.131 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.063546832Z level=info msg="Executing migration" id="Add epoch_end column" 2023-10-09T04:07:09.131 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.088754927Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-10-09T04:07:09.131 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.130758153Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-10-09T04:07:09.407 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.181381895Z level=info msg="Executing migration" id="Move region to single row" 2023-10-09T04:07:09.407 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.223114306Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-10-09T04:07:09.407 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.248556091Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-09T04:07:09.408 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.273743769Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-09T04:07:09.408 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.315547764Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-09T04:07:09.408 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.340641343Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-09T04:07:09.408 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.365961603Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-10-09T04:07:09.408 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.407741131Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-10-09T04:07:09.668 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:09 smithi177 ceph-mon[106070]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:09.668 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.475747252Z level=info msg="Executing migration" id="create test_data table" 2023-10-09T04:07:09.668 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.525139958Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-10-09T04:07:09.668 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.550332395Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-10-09T04:07:09.668 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.575625865Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-09T04:07:09.668 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.600831054Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-10-09T04:07:09.669 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.642723984Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-10-09T04:07:09.669 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.667810146Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-10-09T04:07:09.728 INFO:teuthology.orchestra.run.smithi029.stdout:55834574893 2023-10-09T04:07:09.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:09 smithi029 ceph-mon[101043]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:09.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:09 smithi029 ceph-mon[119131]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:09.852 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.849Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.849Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.850Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:07:09.853 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:07:09.854 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:07:09.855 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:07:09.855 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:07:09.855 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:07:09.855 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.718343785Z level=info msg="Executing migration" id="create team table" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.760126528Z level=info msg="Executing migration" id="add index team.org_id" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.785331673Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.810662632Z level=info msg="Executing migration" id="create team member table" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.835767845Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.860973524Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.886107654Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-10-09T04:07:09.953 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.911215675Z level=info msg="Executing migration" id="Add column email to team table" 2023-10-09T04:07:09.954 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.95326758Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-10-09T04:07:10.156 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:07:10.157 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:07:10.157 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:07:10.157 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:07:10.157 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:07:10.157 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:07:10.158 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:07:10.158 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:07:10.158 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:07:10.158 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:07:10.158 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:07:10.159 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:07:10.160 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:07:10.160 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:07:10.160 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.851Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:07:10.160 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.852Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:07:10.160 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:09 smithi029 conmon[172521]: ts=2023-10-09T04:07:09.852Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:07:10.245 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:09 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:09.995335518Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-10-09T04:07:10.245 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.020362503Z level=info msg="Executing migration" id="create dashboard acl table" 2023-10-09T04:07:10.245 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.045588678Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-10-09T04:07:10.246 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.087420989Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-09T04:07:10.246 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.112671098Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-09T04:07:10.246 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.13797777Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-10-09T04:07:10.246 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.179907037Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-10-09T04:07:10.246 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.245597149Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-10-09T04:07:10.420 INFO:teuthology.orchestra.run.smithi029.stdout:163208757274 2023-10-09T04:07:10.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.280180754Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-10-09T04:07:10.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.305579962Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-10-09T04:07:10.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.330757089Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-10-09T04:07:10.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.355938388Z level=info msg="Executing migration" id="create tag table" 2023-10-09T04:07:10.507 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.389671033Z level=info msg="Executing migration" id="add index tag.key_value" 2023-10-09T04:07:10.508 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.431466915Z level=info msg="Executing migration" id="create login attempt table" 2023-10-09T04:07:10.508 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.448419616Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-10-09T04:07:10.508 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.465225415Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-10-09T04:07:10.508 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.481972096Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-09T04:07:10.508 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.507301878Z level=info msg="Executing migration" id="create login_attempt v2" 2023-10-09T04:07:10.767 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:10 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3717253324' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:07:10.768 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:10 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2918187693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.54939125Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.566099768Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.58298874Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.599986082Z level=info msg="Executing migration" id="create user auth table" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.616784426Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.650202592Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.667161578Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.700642362Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-10-09T04:07:10.768 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.717406428Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-10-09T04:07:10.769 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.751033913Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-10-09T04:07:10.769 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.767890711Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-10-09T04:07:10.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:10 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3717253324' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:07:10.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:10 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2918187693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:07:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:10 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3717253324' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:07:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:10 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2918187693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:07:11.027 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.810157934Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-10-09T04:07:11.027 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.843680368Z level=info msg="Executing migration" id="create server_lock table" 2023-10-09T04:07:11.027 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.860514083Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-10-09T04:07:11.027 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.877470155Z level=info msg="Executing migration" id="create user auth token table" 2023-10-09T04:07:11.027 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.894227231Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-10-09T04:07:11.028 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.911118151Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-10-09T04:07:11.028 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.928005856Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-10-09T04:07:11.028 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.944727695Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-10-09T04:07:11.028 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:10 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:10.970003831Z level=info msg="Executing migration" id="create cache_data table" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.040416523Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.087295759Z level=info msg="Executing migration" id="create short_url table v1" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.112611361Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.137577314Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.162729117Z level=info msg="Executing migration" id="delete alert_definition table" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.204804193Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.229870177Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-10-09T04:07:11.305 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.255074993Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-10-09T04:07:11.306 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.280258021Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-10-09T04:07:11.306 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.305226212Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-10-09T04:07:11.573 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.330776768Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-10-09T04:07:11.573 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.355992071Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-10-09T04:07:11.573 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.406392982Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-10-09T04:07:11.574 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.456487698Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-10-09T04:07:11.574 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.50680161Z level=info msg="Executing migration" id="drop alert_definition table" 2023-10-09T04:07:11.574 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.540348116Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-10-09T04:07:11.574 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.573789064Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-10-09T04:07:11.832 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.623927737Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-09T04:07:11.833 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.674241918Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-09T04:07:11.833 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.707826379Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-09T04:07:11.833 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.749535311Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-10-09T04:07:11.833 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.808088571Z level=info msg="Executing migration" id="create alert_instance table" 2023-10-09T04:07:11.833 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.833120104Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-09T04:07:12.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.866944839Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-09T04:07:12.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.892076525Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-10-09T04:07:12.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.917188635Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-09T04:07:12.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:11 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:11.959241886Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-10-09T04:07:12.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.001004754Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-10-09T04:07:12.093 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.026172342Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-10-09T04:07:12.094 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.068055442Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-09T04:07:12.094 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.093264381Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-10-09T04:07:12.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.118790324Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-10-09T04:07:12.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.160522147Z level=info msg="Executing migration" id="create alert_rule table" 2023-10-09T04:07:12.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.185821888Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-10-09T04:07:12.353 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.227695158Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-10-09T04:07:12.354 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.252890934Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-09T04:07:12.354 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.278142723Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-10-09T04:07:12.354 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.303191529Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-10-09T04:07:12.354 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.328503553Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-10-09T04:07:12.354 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.353715093Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-10-09T04:07:12.540 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574893 for osd.1 2023-10-09T04:07:12.541 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:12.621 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:12 smithi177 ceph-mon[106070]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:12.621 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.37882309Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.420797627Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.462718083Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.487803839Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.55471207Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.580034528Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-10-09T04:07:12.622 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.621683156Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-10-09T04:07:12.631 INFO:teuthology.orchestra.run.smithi029.stdout:77309411368 2023-10-09T04:07:12.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:12 smithi029 ceph-mon[101043]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:12.715 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:12.715 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:12 smithi029 ceph-mon[119131]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:12.715 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:12.716 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:12 smithi029 bash[171842]: 15156916bd109a8c5bc71eb23637984df313af7a5770caf814e289a9b0457088 2023-10-09T04:07:12.716 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:07:12 smithi029 systemd[1]: Started Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:12.873 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.664002448Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.697407668Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.739615918Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.781655021Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.823441769Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.848552688Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-10-09T04:07:12.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.873741939Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-10-09T04:07:13.142 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.898891183Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-10-09T04:07:13.142 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.924386436Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-10-09T04:07:13.143 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:12 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:12.96601577Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-10-09T04:07:13.143 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.007936125Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-10-09T04:07:13.143 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.049821759Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-10-09T04:07:13.143 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.091981375Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-10-09T04:07:13.143 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.142152418Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-09T04:07:13.394 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:13 smithi177 systemd[1]: Starting Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:13.394 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.184395847Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-10-09T04:07:13.394 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.226327649Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-10-09T04:07:13.395 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.251644036Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-10-09T04:07:13.395 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.319073149Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-10-09T04:07:13.395 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.352139663Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-10-09T04:07:13.395 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.377212571Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-10-09T04:07:13.395 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.394072789Z level=info msg="Executing migration" id="create provenance_type table" 2023-10-09T04:07:13.654 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:13 smithi177 bash[152643]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-10-09T04:07:13.654 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:13 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.654 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:13 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.654 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:13 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/569438982' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:07:13.654 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:13 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.436138846Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.478079628Z level=info msg="Executing migration" id="create alert_image table" 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.520145153Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.545140357Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.570341419Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.612447714Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.65416082Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T04:07:13.655 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.654328306Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-10-09T04:07:13.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/569438982' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/569438982' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:07:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:13 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:13.812 INFO:teuthology.orchestra.run.smithi029.stdout:34359738418 2023-10-09T04:07:13.914 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.704819604Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-10-09T04:07:13.914 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.746729895Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-10-09T04:07:13.914 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.771809281Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-10-09T04:07:13.914 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.771959014Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-10-09T04:07:13.914 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.77209458Z level=info msg="alerts found to migrate" alerts=0 2023-10-09T04:07:13.914 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.797101699Z level=info msg="Executing migration" id="create library_element table v1" 2023-10-09T04:07:13.915 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.821995114Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-10-09T04:07:13.915 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.847425936Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-10-09T04:07:13.915 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.889050124Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-10-09T04:07:13.915 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.914494482Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-10-09T04:07:14.232 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:14 smithi177 bash[152643]: Getting image source signatures 2023-10-09T04:07:14.232 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.96484819Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-10-09T04:07:14.233 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:13 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:13.98992548Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-10-09T04:07:14.233 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.031949529Z level=info msg="Executing migration" id="create data_keys table" 2023-10-09T04:07:14.233 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.107185349Z level=info msg="Executing migration" id="create secrets table" 2023-10-09T04:07:14.233 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.13232843Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-10-09T04:07:14.233 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.157810531Z level=info msg="Executing migration" id="add name column into data_keys" 2023-10-09T04:07:14.233 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.232863223Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-10-09T04:07:14.527 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:14 smithi177 bash[152643]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-10-09T04:07:14.527 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:14 smithi177 bash[152643]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-10-09T04:07:14.527 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:14 smithi177 bash[152643]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-10-09T04:07:14.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.317328555Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-10-09T04:07:14.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.400644518Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-10-09T04:07:14.528 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.459141794Z level=info msg="Executing migration" id="create kv_store table v1" 2023-10-09T04:07:14.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:14 smithi177 ceph-mon[106070]: Deploying daemon node-exporter.b on smithi177 2023-10-09T04:07:14.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:14 smithi177 ceph-mon[106070]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:14.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:14 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3494373224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.542789163Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.639523985Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.685196145Z level=info msg="Executing migration" id="create permission table" 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.718872135Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.752401381Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.785959906Z level=info msg="Executing migration" id="create role table" 2023-10-09T04:07:14.820 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.819868007Z level=info msg="Executing migration" id="add column display_name" 2023-10-09T04:07:14.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:14 smithi029 ceph-mon[101043]: Deploying daemon node-exporter.b on smithi177 2023-10-09T04:07:14.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:14 smithi029 ceph-mon[101043]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:14.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:14 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3494373224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:07:14.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:14 smithi029 ceph-mon[119131]: Deploying daemon node-exporter.b on smithi177 2023-10-09T04:07:14.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:14 smithi029 ceph-mon[119131]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:14.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:14 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3494373224' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:07:15.113 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:14 smithi177 bash[152643]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-10-09T04:07:15.113 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:15 smithi177 bash[152643]: Writing manifest to image destination 2023-10-09T04:07:15.113 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:15 smithi177 bash[152643]: Storing signatures 2023-10-09T04:07:15.113 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.878468391Z level=info msg="Executing migration" id="add column group_name" 2023-10-09T04:07:15.114 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.939621251Z level=info msg="Executing migration" id="add index role.org_id" 2023-10-09T04:07:15.114 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:14 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:14.995804849Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-10-09T04:07:15.114 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.057989914Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-10-09T04:07:15.114 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.113475286Z level=info msg="Executing migration" id="create team role table" 2023-10-09T04:07:15.219 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411368 for osd.2 2023-10-09T04:07:15.219 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:15.240 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757271 got 163208757274 for osd.5 2023-10-09T04:07:15.240 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:15.307 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2023-10-09T04:07:15.307 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:15.375 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.172386294Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-10-09T04:07:15.375 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.223598074Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-10-09T04:07:15.375 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.291159229Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-10-09T04:07:15.375 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.341559207Z level=info msg="Executing migration" id="create user role table" 2023-10-09T04:07:15.375 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.375371811Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-10-09T04:07:15.659 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.419331148Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-10-09T04:07:15.659 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.464872185Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-10-09T04:07:15.659 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.541575353Z level=info msg="Executing migration" id="create builtin role table" 2023-10-09T04:07:15.659 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.575202031Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-10-09T04:07:15.659 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.659216958Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-10-09T04:07:15.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:15 smithi029 ceph-mon[101043]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:15 smithi029 ceph-mon[119131]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:15.927 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:15 smithi177 ceph-mon[106070]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:15.928 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.70997884Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-10-09T04:07:15.928 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.751891743Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-10-09T04:07:15.928 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.793886411Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-10-09T04:07:15.928 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.852666075Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-10-09T04:07:15.928 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.928128266Z level=info msg="Executing migration" id="add unique index role.uid" 2023-10-09T04:07:15.987 INFO:teuthology.orchestra.run.smithi029.stdout:128849018911 2023-10-09T04:07:16.205 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:15 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:15.989404112Z level=info msg="Executing migration" id="create seed assignment table" 2023-10-09T04:07:16.205 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.054751296Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-10-09T04:07:16.205 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.130521081Z level=info msg="Executing migration" id="add column hidden to role table" 2023-10-09T04:07:16.205 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.205917084Z level=info msg="Executing migration" id="create query_history table v1" 2023-10-09T04:07:16.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.282415436Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-10-09T04:07:16.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.341405353Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-10-09T04:07:16.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.416966009Z level=info msg="Executing migration" id="teams permissions migration" 2023-10-09T04:07:16.476 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.476162883Z level=info msg="Executing migration" id="dashboard permissions" 2023-10-09T04:07:16.728 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:16 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2541502708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:07:16.728 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.543745952Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-10-09T04:07:16.729 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.621128486Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-10-09T04:07:16.729 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.686911361Z level=info msg="Executing migration" id="alerting notification permissions" 2023-10-09T04:07:16.847 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018911 for osd.4 2023-10-09T04:07:16.847 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:17.013 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.728684365Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-10-09T04:07:17.014 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.770995401Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-10-09T04:07:17.014 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.796181203Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-10-09T04:07:17.014 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.821207615Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-10-09T04:07:17.014 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.87982484Z level=info msg="Executing migration" id="create correlation table v1" 2023-10-09T04:07:17.014 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:16.971884734Z level=info msg="Executing migration" id="add index correlations.uid" 2023-10-09T04:07:17.014 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.901Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.902Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.902Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.902Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.902Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.902Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:07:17.015 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:07:17.016 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:07:17.017 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:07:17.018 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:07:17.019 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 conmon[152720]: ts=2023-10-09T04:07:16.903Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 bash[152643]: b7a4d2d246ccd755dd0a1651ba8eb4e826e8783e0f8d627e463ca6b8fe69f877 2023-10-09T04:07:17.020 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:07:16 smithi177 systemd[1]: Started Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:17.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:16 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2541502708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:07:17.069 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:16 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2541502708' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:07:17.205 INFO:teuthology.orchestra.run.smithi029.stdout:107374182437 2023-10-09T04:07:17.274 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.013836249Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-10-09T04:07:17.274 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.072898866Z level=info msg="Executing migration" id="add correlation config column" 2023-10-09T04:07:17.274 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.115094773Z level=info msg="Executing migration" id="create entity_events table" 2023-10-09T04:07:17.275 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.18206133Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-10-09T04:07:17.275 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.223976897Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T04:07:17.275 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.224121415Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-10-09T04:07:17.275 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.249267479Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T04:07:17.275 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.249412342Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T04:07:17.275 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.274551275Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-10-09T04:07:17.525 INFO:teuthology.orchestra.run.smithi029.stdout:231928234003 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.299782727Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.341669559Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.366846499Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.408752559Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.450807277Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.475965239Z level=info msg="Executing migration" id="Drop public config table" 2023-10-09T04:07:17.526 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.526358227Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-10-09T04:07:17.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.559990673Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-10-09T04:07:17.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.585191098Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-10-09T04:07:17.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.610342737Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-10-09T04:07:17.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.635557447Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-10-09T04:07:17.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.677553267Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-10-09T04:07:17.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.694351577Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-10-09T04:07:17.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.71123412Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-10-09T04:07:17.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.753213306Z level=info msg="Executing migration" id="add share column" 2023-10-09T04:07:18.072 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.820620994Z level=info msg="Executing migration" id="create default alerting folders" 2023-10-09T04:07:18.072 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.854180536Z level=info msg="Executing migration" id="create file table" 2023-10-09T04:07:18.072 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:17 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:17.921701126Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-10-09T04:07:18.072 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.005212988Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-10-09T04:07:18.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:18.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.072 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:18.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1284591877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:07:18.073 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:18 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3192974881' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:07:18.349 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.071922754Z level=info msg="Executing migration" id="create file_meta table" 2023-10-09T04:07:18.349 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.155982487Z level=info msg="Executing migration" id="file table idx: path key" 2023-10-09T04:07:18.349 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.223255584Z level=info msg="Executing migration" id="set path collation in file table" 2023-10-09T04:07:18.349 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.256925738Z level=info msg="Executing migration" id="managed permissions migration" 2023-10-09T04:07:18.349 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.298887918Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-10-09T04:07:18.350 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.315680038Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-10-09T04:07:18.350 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.349305456Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1284591877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3192974881' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1284591877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:07:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:18 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3192974881' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:07:18.378 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182434 got 107374182437 for osd.3 2023-10-09T04:07:18.378 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:18.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.38302761Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-10-09T04:07:18.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.416548379Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-10-09T04:07:18.601 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.433489519Z level=info msg="Executing migration" id="update group index for alert rules" 2023-10-09T04:07:18.602 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.450307308Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-10-09T04:07:18.602 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.48389495Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-10-09T04:07:18.602 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.509080749Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-10-09T04:07:18.602 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.534318649Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-10-09T04:07:18.602 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.576283207Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-10-09T04:07:18.602 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.601571196Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-10-09T04:07:18.625 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234000 got 231928234003 for osd.7 2023-10-09T04:07:18.626 DEBUG:teuthology.parallel:result is None 2023-10-09T04:07:18.626 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:07:18.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:07:18.870 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.635137294Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-10-09T04:07:18.870 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.721665604Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-10-09T04:07:18.870 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.804490839Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-10-09T04:07:18.874 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.870160317Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-10-09T04:07:19.137 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.928449919Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-10-09T04:07:19.137 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:18 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:18.995694556Z level=info msg="Executing migration" id="create folder table" 2023-10-09T04:07:19.138 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:19.09032922Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-10-09T04:07:19.138 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:19.13788168Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-10-09T04:07:19.220 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:19.188266168Z level=info msg="Executing migration" id="Update folder title length" 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:19.230217428Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=migrator t=2023-10-09T04:07:19.255340913Z level=info msg="migrations completed" performed=485 skipped=0 duration=17.50125711s 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=sqlstore t=2023-10-09T04:07:19.25598371Z level=info msg="Created default organization" 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=plugin.loader t=2023-10-09T04:07:19.369757578Z level=info msg="Plugin registered" pluginID=input 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=plugin.loader t=2023-10-09T04:07:19.399934535Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=plugin.loader t=2023-10-09T04:07:19.399973211Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T04:07:19.400 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=secrets t=2023-10-09T04:07:19.400027939Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T04:07:19.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=query_data t=2023-10-09T04:07:19.401976448Z level=info msg="Query Service initialization" 2023-10-09T04:07:19.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:19 smithi177 conmon[152217]: logger=live.push_http t=2023-10-09T04:07:19.405033138Z level=info msg="Live Push Gateway initialization" 2023-10-09T04:07:20.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:20 smithi029 ceph-mon[101043]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:20.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:20 smithi029 ceph-mon[119131]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:20.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:20 smithi177 ceph-mon[106070]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:20.310 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=infra.usagestats.collector t=2023-10-09T04:07:20.216825373Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T04:07:20.310 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=provisioning.datasources t=2023-10-09T04:07:20.217224246Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-09T04:07:20.310 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=provisioning.datasources t=2023-10-09T04:07:20.249587295Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T04:07:20.310 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=provisioning.datasources t=2023-10-09T04:07:20.309921394Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-10-09T04:07:20.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=provisioning.alerting t=2023-10-09T04:07:20.358576834Z level=info msg="starting to provision alerting" 2023-10-09T04:07:20.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=provisioning.alerting t=2023-10-09T04:07:20.358621103Z level=info msg="finished to provision alerting" 2023-10-09T04:07:20.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=grafanaStorageLogger t=2023-10-09T04:07:20.358788771Z level=info msg="storage starting" 2023-10-09T04:07:20.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=ngalert.state.manager t=2023-10-09T04:07:20.358863887Z level=info msg="Warming state cache for startup" 2023-10-09T04:07:20.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=http.server t=2023-10-09T04:07:20.360709371Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T04:07:20.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=sqlstore.transactions t=2023-10-09T04:07:20.369751393Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:07:20.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=ngalert.state.manager t=2023-10-09T04:07:20.40865551Z level=info msg="State cache has been initialized" states=0 duration=49.789404ms 2023-10-09T04:07:20.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=ngalert.multiorg.alertmanager t=2023-10-09T04:07:20.40871421Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T04:07:20.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=ticker t=2023-10-09T04:07:20.408743398Z level=info msg=starting first_tick=2023-10-09T04:07:30Z 2023-10-09T04:07:20.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:07:20 smithi177 conmon[152217]: logger=sqlstore.transactions t=2023-10-09T04:07:20.539609569Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:07:21.281 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:07:21.283 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:07:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:21 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:21 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:21 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:21 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:21 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:21 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.324 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":34,"stamp":"2023-10-09T04:07:20.760529+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384968,"kb_used_data":6664,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576248,"statfs":{"total":767960285184,"available":765518077952,"internally_reserved":0,"allocated":6823936,"data_stored":3876565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002303"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.385476+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:06:30.385476+0000","last_peered":"2023-10-09T04:06:30.385476+0000","last_clean":"2023-10-09T04:06:30.385476+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:06:30.385476+0000","last_undegraded":"2023-10-09T04:06:30.385476+0000","last_fullsized":"2023-10-09T04:06:30.385476+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.211882+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:06:30.211882+0000","last_peered":"2023-10-09T04:06:30.211882+0000","last_clean":"2023-10-09T04:06:30.211882+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:06:30.211882+0000","last_undegraded":"2023-10-09T04:06:30.211882+0000","last_fullsized":"2023-10-09T04:06:30.211882+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.906310+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:06:29.906310+0000","last_peered":"2023-10-09T04:06:29.906310+0000","last_clean":"2023-10-09T04:06:29.906310+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:06:29.906310+0000","last_undegraded":"2023-10-09T04:06:29.906310+0000","last_fullsized":"2023-10-09T04:06:29.906310+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.906291+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:06:29.906291+0000","last_peered":"2023-10-09T04:06:29.906291+0000","last_clean":"2023-10-09T04:06:29.906291+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:06:29.906291+0000","last_undegraded":"2023-10-09T04:06:29.906291+0000","last_fullsized":"2023-10-09T04:06:29.906291+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.906310+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:06:29.906310+0000","last_peered":"2023-10-09T04:06:29.906310+0000","last_clean":"2023-10-09T04:06:29.906310+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:06:29.906310+0000","last_undegraded":"2023-10-09T04:06:29.906310+0000","last_fullsized":"2023-10-09T04:06:29.906310+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.385500+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:06:30.385500+0000","last_peered":"2023-10-09T04:06:30.385500+0000","last_clean":"2023-10-09T04:06:30.385500+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:06:30.385500+0000","last_undegraded":"2023-10-09T04:06:30.385500+0000","last_fullsized":"2023-10-09T04:06:30.385500+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.903786+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:06:29.903786+0000","last_peered":"2023-10-09T04:06:29.903786+0000","last_clean":"2023-10-09T04:06:29.903786+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:06:29.903786+0000","last_undegraded":"2023-10-09T04:06:29.903786+0000","last_fullsized":"2023-10-09T04:06:29.903786+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.385459+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:06:30.385459+0000","last_peered":"2023-10-09T04:06:30.385459+0000","last_clean":"2023-10-09T04:06:30.385459+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:06:30.385459+0000","last_undegraded":"2023-10-09T04:06:30.385459+0000","last_fullsized":"2023-10-09T04:06:30.385459+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.211887+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:06:30.211887+0000","last_peered":"2023-10-09T04:06:30.211887+0000","last_clean":"2023-10-09T04:06:30.211887+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:06:30.211887+0000","last_undegraded":"2023-10-09T04:06:30.211887+0000","last_fullsized":"2023-10-09T04:06:30.211887+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.39400000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.43099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":5.6970000000000001,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.36399999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":0.51100000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.47499999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.1280000000000001,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.498}]}]},{"osd":6,"up_from":46,"seq":197568495638,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.59499999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.441},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.55600000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.60599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.626},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.1779999999999999,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.53000000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.67400000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.252,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.224},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.316}]},{"osd":2,"last update":"Mon Oct 9 04:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.47999999999999998,"15min":0.46000000000000002},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42899999999999999,"15min":0.42799999999999999},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.58299999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.27900000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.29899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.61199999999999999,"15min":0.60999999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78300000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53800000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.63,"15min":0.61299999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.7610000000000001,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.627,"15min":0.627},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.52700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.50600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:06:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.34899999999999998,"15min":0.32300000000000001},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.36699999999999999,"15min":0.34000000000000002},"min":{"1min":0.29499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.58899999999999997,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.46500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.432,"15min":0.433},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60699999999999998,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.5},{"interface":"front","average":{"1min":0.432,"5min":0.436,"15min":0.43099999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.63700000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.48599999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48299999999999998,"15min":0.48099999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.61799999999999999,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52500000000000002,"15min":0.52600000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.52000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.65200000000000002,"15min":0.65000000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.58999999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.621,"15min":0.625},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80900000000000005,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.63400000000000001,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85699999999999998,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.53200000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:06:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.629},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.56499999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:06:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.54300000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.38600000000000001,"15min":0.38},"min":{"1min":0.23300000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.65500000000000003,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.371,"5min":0.40000000000000002,"15min":0.40100000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.57699999999999996,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.39600000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.42099999999999999,"15min":0.41499999999999998},"min":{"1min":0.26200000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.65400000000000003,"5min":0.754,"15min":0.754},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64400000000000002,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:06:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48499999999999999,"15min":0.47999999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.48999999999999999,"15min":0.48499999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.45800000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.59299999999999997,"15min":0.59499999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.63500000000000001,"15min":0.63600000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.69099999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.56200000000000006}]},{"osd":7,"last update":"Mon Oct 9 04:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.57399999999999995}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.42099999999999999,"15min":0.42699999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.56399999999999995,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.45200000000000001,"15min":0.46100000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58299999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.33200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.46899999999999997,"15min":0.47599999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.58699999999999997,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.48299999999999998,"15min":0.49299999999999999},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":0.67400000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.375}]},{"osd":2,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.505,"15min":0.51000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65100000000000002,"5min":0.745,"15min":0.745},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.51100000000000001,"15min":0.51900000000000002},"min":{"1min":0.35499999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.64900000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.622,"15min":0.625},"min":{"1min":0.42199999999999999,"5min":0.375,"15min":0.375},"max":{"1min":0.82199999999999995,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.80900000000000005,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.56399999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78500000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.63300000000000001,"15min":0.63800000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79200000000000004,"5min":0.874,"15min":0.874},"last":0.52400000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.499}]},{"osd":7,"last update":"Mon Oct 9 04:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.57299999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65300000000000002,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.65900000000000003,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.51500000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.50900000000000001,"15min":0.51700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.65200000000000002,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.50700000000000001,"15min":0.50600000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.66600000000000004,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.432}]},{"osd":2,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53400000000000003,"15min":0.53200000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.69899999999999995,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53100000000000003,"15min":0.52700000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73399999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.53100000000000003,"15min":0.52000000000000002},"min":{"1min":0.42299999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.45700000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.42799999999999999,"15min":0.41899999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.69499999999999995,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.504}]},{"osd":6,"last update":"Mon Oct 9 04:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.442},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54100000000000004,"15min":0.54600000000000004},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.38800000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42999999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.78700000000000003,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.43099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71299999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.50700000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.373,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72299999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.439}]},{"osd":4,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.41499999999999998,"15min":0.40899999999999997},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.42799999999999999,"15min":0.41999999999999998},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.374}]},{"osd":6,"last update":"Mon Oct 9 04:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.46700000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:06:55 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.47799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:07:22.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:07:22.704 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:22.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:22.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:22 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:23.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:07:22 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:07:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:22 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:23.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:23 smithi029 ceph-mon[101043]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:07:23.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:23 smithi029 ceph-mon[101043]: Reconfiguring daemon alertmanager.a on smithi029 2023-10-09T04:07:23.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:23 smithi029 ceph-mon[119131]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:07:23.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:23 smithi029 ceph-mon[119131]: Reconfiguring daemon alertmanager.a on smithi029 2023-10-09T04:07:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:23 smithi177 ceph-mon[106070]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:07:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:23 smithi177 ceph-mon[106070]: Reconfiguring daemon alertmanager.a on smithi029 2023-10-09T04:07:24.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:24 smithi029 ceph-mon[101043]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:24.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:24 smithi029 ceph-mon[119131]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:24 smithi177 ceph-mon[106070]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:25.365 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:07:25.367 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:07:26.251 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":36,"stamp":"2023-10-09T04:07:24.761208+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2384968,"kb_used_data":6664,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747576248,"statfs":{"total":767960285184,"available":765518077952,"internally_reserved":0,"allocated":6823936,"data_stored":3876565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002292"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.385476+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:06:30.385476+0000","last_peered":"2023-10-09T04:06:30.385476+0000","last_clean":"2023-10-09T04:06:30.385476+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:06:30.385476+0000","last_undegraded":"2023-10-09T04:06:30.385476+0000","last_fullsized":"2023-10-09T04:06:30.385476+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.211882+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:06:30.211882+0000","last_peered":"2023-10-09T04:06:30.211882+0000","last_clean":"2023-10-09T04:06:30.211882+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:06:30.211882+0000","last_undegraded":"2023-10-09T04:06:30.211882+0000","last_fullsized":"2023-10-09T04:06:30.211882+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.906310+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:06:29.906310+0000","last_peered":"2023-10-09T04:06:29.906310+0000","last_clean":"2023-10-09T04:06:29.906310+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:06:29.906310+0000","last_undegraded":"2023-10-09T04:06:29.906310+0000","last_fullsized":"2023-10-09T04:06:29.906310+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.906291+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:06:29.906291+0000","last_peered":"2023-10-09T04:06:29.906291+0000","last_clean":"2023-10-09T04:06:29.906291+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:06:29.906291+0000","last_undegraded":"2023-10-09T04:06:29.906291+0000","last_fullsized":"2023-10-09T04:06:29.906291+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.906310+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:06:29.906310+0000","last_peered":"2023-10-09T04:06:29.906310+0000","last_clean":"2023-10-09T04:06:29.906310+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:06:29.906310+0000","last_undegraded":"2023-10-09T04:06:29.906310+0000","last_fullsized":"2023-10-09T04:06:29.906310+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.385500+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:06:30.385500+0000","last_peered":"2023-10-09T04:06:30.385500+0000","last_clean":"2023-10-09T04:06:30.385500+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:06:30.385500+0000","last_undegraded":"2023-10-09T04:06:30.385500+0000","last_fullsized":"2023-10-09T04:06:30.385500+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:29.903786+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:06:29.903786+0000","last_peered":"2023-10-09T04:06:29.903786+0000","last_clean":"2023-10-09T04:06:29.903786+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:06:29.903786+0000","last_undegraded":"2023-10-09T04:06:29.903786+0000","last_fullsized":"2023-10-09T04:06:29.903786+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.385459+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:06:30.385459+0000","last_peered":"2023-10-09T04:06:30.385459+0000","last_clean":"2023-10-09T04:06:30.385459+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:06:30.385459+0000","last_undegraded":"2023-10-09T04:06:30.385459+0000","last_fullsized":"2023-10-09T04:06:30.385459+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-09T04:06:30.211887+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:06:30.211887+0000","last_peered":"2023-10-09T04:06:30.211887+0000","last_clean":"2023-10-09T04:06:30.211887+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:06:30.211887+0000","last_undegraded":"2023-10-09T04:06:30.211887+0000","last_fullsized":"2023-10-09T04:06:30.211887+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":10.557,"5min":10.557,"15min":10.557},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.5}]},{"osd":1,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":5.6970000000000001,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.44},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.42799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.55000000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":0.46200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.1280000000000001,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.66200000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298480,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446672,"statfs":{"total":95995035648,"available":95689392128,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.51900000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.35599999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.61599999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.1779999999999999,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.66600000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.81100000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.252,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.39200000000000002,"15min":0.37},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.35399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.47999999999999998,"15min":0.46000000000000002},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42899999999999999,"15min":0.42799999999999999},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.58299999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51400000000000001,"15min":0.51200000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.61199999999999999,"15min":0.60999999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78300000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57499999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.63,"15min":0.61299999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.7610000000000001,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.627,"15min":0.627},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.45700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.60099999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298484,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446668,"statfs":{"total":95995035648,"available":95689388032,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:06:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.34899999999999998,"15min":0.32300000000000001},"min":{"1min":0.25,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.36699999999999999,"15min":0.34000000000000002},"min":{"1min":0.29499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.58899999999999997,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.32800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:07:02 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.432,"15min":0.433},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60699999999999998,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.436,"15min":0.43099999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.63700000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48299999999999998,"15min":0.48099999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.61799999999999999,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52500000000000002,"15min":0.52600000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.40699999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.65200000000000002,"15min":0.65000000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.749}]},{"osd":5,"last update":"Mon Oct 9 04:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.621,"15min":0.625},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80900000000000005,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.63400000000000001,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85699999999999998,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:06:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.76600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:06:56 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.625}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.38600000000000001,"15min":0.38},"min":{"1min":0.23300000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.65500000000000003,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.40000000000000002,"15min":0.40100000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.57699999999999996,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.42099999999999999,"15min":0.41499999999999998},"min":{"1min":0.26200000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.65400000000000003,"5min":0.754,"15min":0.754},"last":0.315},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64400000000000002,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 04:06:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48499999999999999,"15min":0.47999999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.48999999999999999,"15min":0.48499999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.39800000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.59299999999999997,"15min":0.59499999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.80200000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:07:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.63500000000000001,"15min":0.63600000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.76300000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.745},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.78800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.84399999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.42099999999999999,"15min":0.42699999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.56399999999999995,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.45200000000000001,"15min":0.46100000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.58299999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.27700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.46899999999999997,"15min":0.47599999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.58699999999999997,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.48299999999999998,"15min":0.49299999999999999},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":0.67400000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.23699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:06:25 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.505,"15min":0.51000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65100000000000002,"5min":0.745,"15min":0.745},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.51100000000000001,"15min":0.51900000000000002},"min":{"1min":0.35499999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.64900000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.622,"15min":0.625},"min":{"1min":0.42199999999999999,"5min":0.375,"15min":0.375},"max":{"1min":0.82199999999999995,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.80900000000000005,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.71799999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78500000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.63300000000000001,"15min":0.63800000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79200000000000004,"5min":0.874,"15min":0.874},"last":0.69399999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.76400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.77500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65300000000000002,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.65900000000000003,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.47599999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.50900000000000001,"15min":0.51700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.65200000000000002,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.50700000000000001,"15min":0.50600000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.66600000000000004,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.45000000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67300000000000004,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53400000000000003,"15min":0.53200000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.69899999999999995,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.43099999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53100000000000003,"15min":0.52700000000000002},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.73399999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.501},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.53100000000000003,"15min":0.52000000000000002},"min":{"1min":0.42299999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.42099999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:07:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.42799999999999999,"15min":0.41899999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.69499999999999995,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.42599999999999999,"15min":0.41799999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.46500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.55400000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757277,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297904,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447248,"statfs":{"total":95995035648,"available":95689981952,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54100000000000004,"15min":0.54600000000000004},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.65500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.38800000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.47999999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.55500000000000005,"15min":0.56100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.78700000000000003,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.43099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71299999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.46000000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.373,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.72299999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.58499999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:07:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.41499999999999998,"15min":0.40899999999999997},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.433},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.42799999999999999,"15min":0.41999999999999998},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:06:55 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.498},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.59799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:07:26.254 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:07:26.254 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:07:26.255 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:07:26.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:07:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:26 smithi029 ceph-mon[101043]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:26 smithi029 ceph-mon[101043]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:26 smithi029 ceph-mon[119131]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:26 smithi029 ceph-mon[119131]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:26 smithi177 ceph-mon[106070]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:26 smithi177 ceph-mon[106070]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:07:27.061 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:27.312 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:27 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:27.601 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:27 smithi029 conmon[165950]: ts=2023-10-09T04:07:27.384Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:07:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:27.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:27 smithi029 bash[175974]: d4e0409956ad046932bc9dd1940d6d6fea5beb9e519c8d39c121a457360ec7eb 2023-10-09T04:07:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:28.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:28 smithi029 ceph-mon[101043]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:28.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:28 smithi029 ceph-mon[119131]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:28 smithi177 ceph-mon[106070]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:29.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:28 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:07:29.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:28 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:29.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:28 smithi029 systemd[1]: Starting Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:29.508 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:07:29.509 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:07:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:29 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1102426084' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:07:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:29 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1102426084' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:07:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:29 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1102426084' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.177Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.177Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.178Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.179Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.208Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.209Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.212Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T04:07:30.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 conmon[176538]: ts=2023-10-09T04:07:30.212Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T04:07:30.480 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:07:30.480 INFO:tasks.cephadm:Setup complete, yielding 2023-10-09T04:07:30.480 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-09T04:07:30.489 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi029.front.sepia.ceph.com 2023-10-09T04:07:30.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-09T04:07:30.857 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:30 smithi029 ceph-mon[101043]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:30 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:30 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:30 smithi029 ceph-mon[119131]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:30 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:30 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 bash[176482]: 6a4c4bc4e5b31060c5c22b8f00913e104f69932c91365fc552c5b1ebc2463d5b 2023-10-09T04:07:30.922 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:30 smithi029 systemd[1]: Started Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:30 smithi177 ceph-mon[106070]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:30 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:30 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:31 smithi177 ceph-mon[106070]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:07:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:31 smithi177 ceph-mon[106070]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:07:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:31 smithi029 ceph-mon[101043]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:07:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:31 smithi029 ceph-mon[101043]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:07:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:31 smithi029 ceph-mon[119131]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:07:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:31 smithi029 ceph-mon[119131]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:07:32.398 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:32 smithi029 conmon[176538]: ts=2023-10-09T04:07:32.180Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000190394s 2023-10-09T04:07:32.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:32 smithi029 ceph-mon[101043]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:32 smithi029 ceph-mon[119131]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:32.801 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:32 smithi177 ceph-mon[106070]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:33.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:07:32 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:07:33.645 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:33 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:07:33.645 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 systemd[1]: Stopping Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:33.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:33 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:07:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:33 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-09T04:07:34.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.645Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.646Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:07:34.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.646Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:07:34.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.646Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:07:34.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:33 smithi177 conmon[149043]: ts=2023-10-09T04:07:33.646Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:07:34.527 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 bash[154262]: d772191b46e074c5efe0022f1185441946d6484ff278dd5351c5639266aed147 2023-10-09T04:07:34.527 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:07:34.528 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 systemd[1]: Stopped Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:34.528 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 systemd[1]: Starting Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:07:34.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[119131]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:34.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[119131]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:07:34.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:07:34.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[119131]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:07:34.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:07:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[101043]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[101043]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:07:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:07:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[101043]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:07:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:34 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:07:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:34 smithi029 conmon[101019]: 2023-10-09T04:07:34.634+0000 7f3716509700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:07:34.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:34 smithi177 ceph-mon[106070]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:34.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:34 smithi177 ceph-mon[106070]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:07:34.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:34 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-09T04:07:34.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:34 smithi177 ceph-mon[106070]: osdmap e64: 8 total, 8 up, 8 in 2023-10-09T04:07:34.910 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:34 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-09T04:07:35.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STOPPING 2023-10-09T04:07:35.277 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.910Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.910Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.910Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi177 (none))" 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.910Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.910Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.912Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.912Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.913Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.913Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.914Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:07:35.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.914Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.389µs 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.914Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.916Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.916Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.916Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.534µs wal_replay_duration=1.819829ms wbl_replay_duration=108ns total_replay_duration=1.859096ms 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.917Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.917Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.917Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 bash[154588]: 51d99d79686facde26f1219241f16a432b447b605378f7988cc5fe051213c58c 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 systemd[1]: Started Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.933Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.926028ms db_storage=1.069µs remote_storage=1.331µs web_handler=487ns query_engine=598ns scrape=304.672µs scrape_sd=24.612µs notify=28.558µs notify_sd=8.504µs rules=15.270721ms tracing=6.648µs 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.933Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:07:35.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:07:34 smithi177 conmon[154629]: ts=2023-10-09T04:07:34.933Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:07:35.538 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:07:35.538 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STOPPED 2023-10-09T04:07:35.538 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STARTING 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Serving on http://:::9283 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STARTED 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STOPPING 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STOPPED 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STARTING 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Serving on http://:::9283 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STARTED 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STOPPING 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STOPPED 2023-10-09T04:07:35.539 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STARTING 2023-10-09T04:07:35.540 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Serving on http://:::9283 2023-10-09T04:07:35.540 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:35 smithi029 conmon[101466]: [09/Oct/2023:04:07:35] ENGINE Bus STARTED 2023-10-09T04:07:35.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-09T04:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: fsmap foo:0 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: Saving service mds.foo spec with placement count:2 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029:9093"}]: dispatch 2023-10-09T04:07:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029:9093"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: fsmap foo:0 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: Saving service mds.foo spec with placement count:2 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029:9093"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029:9093"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:07:35.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:07:35.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:07:35.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:07:35.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:35.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:35 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-09T04:07:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: osdmap e65: 8 total, 8 up, 8 in 2023-10-09T04:07:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: osdmap e66: 8 total, 8 up, 8 in 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: fsmap foo:0 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: Saving service mds.foo spec with placement count:2 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: pgmap v44: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029:9093"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029:9093"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi177.front.sepia.ceph.com:9095"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:07:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:07:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi177.front.sepia.ceph.com:3000"}]: dispatch 2023-10-09T04:07:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:35 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:36.289 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:36.878 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:36 smithi029 ceph-mon[101043]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:07:36.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:36 smithi029 ceph-mon[119131]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:07:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:36 smithi177 ceph-mon[106070]: osdmap e67: 8 total, 8 up, 8 in 2023-10-09T04:07:37.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:37 smithi029 ceph-mon[119131]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:07:37.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:37 smithi029 ceph-mon[119131]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:37.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:37 smithi029 ceph-mon[101043]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:07:37.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:37 smithi029 ceph-mon[101043]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:37 smithi177 ceph-mon[106070]: osdmap e68: 8 total, 8 up, 8 in 2023-10-09T04:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:37 smithi177 ceph-mon[106070]: pgmap v47: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:38 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:38 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:38 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:38 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:38 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-09T04:07:39.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:39.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:39.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:39.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:07:39.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[119131]: Deploying daemon mds.foo.smithi029.mozfhx on smithi029 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:07:39.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:39.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:39 smithi029 ceph-mon[101043]: Deploying daemon mds.foo.smithi029.mozfhx on smithi029 2023-10-09T04:07:39.923 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: pgmap v48: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:07:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:07:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:39 smithi177 ceph-mon[106070]: Deploying daemon mds.foo.smithi029.mozfhx on smithi029 2023-10-09T04:07:40.671 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:07:40 smithi029 conmon[176538]: ts=2023-10-09T04:07:40.182Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002682398s 2023-10-09T04:07:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:42 smithi029 ceph-mon[101043]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:42 smithi029 ceph-mon[119131]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:43.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:07:42 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:07:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:42 smithi177 ceph-mon[106070]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:43.407 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T04:07:43.418 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:07:43.418 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:07:43.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:07:43.801 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:44.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:44 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:44] "GET /metrics HTTP/1.1" 200 208136 "" "Prometheus/2.43.0" 2023-10-09T04:07:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: Deploying daemon mds.foo.smithi177.pbovgs on smithi177 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:boot 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: daemon mds.foo.smithi029.mozfhx assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:07:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: Cluster is now healthy 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: fsmap foo:0 1 up:standby 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:creating} 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[101043]: daemon mds.foo.smithi029.mozfhx is now active in filesystem foo as rank 0 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: Deploying daemon mds.foo.smithi177.pbovgs on smithi177 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:boot 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: daemon mds.foo.smithi029.mozfhx assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:07:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:07:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: Cluster is now healthy 2023-10-09T04:07:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: fsmap foo:0 1 up:standby 2023-10-09T04:07:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:07:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:creating} 2023-10-09T04:07:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:44 smithi029 ceph-mon[119131]: daemon mds.foo.smithi029.mozfhx is now active in filesystem foo as rank 0 2023-10-09T04:07:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: pgmap v50: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: Deploying daemon mds.foo.smithi177.pbovgs on smithi177 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:boot 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: daemon mds.foo.smithi029.mozfhx assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-09T04:07:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: Cluster is now healthy 2023-10-09T04:07:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: fsmap foo:0 1 up:standby 2023-10-09T04:07:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:07:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:creating} 2023-10-09T04:07:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:44 smithi177 ceph-mon[106070]: daemon mds.foo.smithi029.mozfhx is now active in filesystem foo as rank 0 2023-10-09T04:07:45.741 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:07:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:45 smithi177 ceph-mon[106070]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:active 2023-10-09T04:07:45.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:45 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 2023-10-09T04:07:45.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:45 smithi029 ceph-mon[101043]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:active 2023-10-09T04:07:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:45 smithi029 ceph-mon[101043]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 2023-10-09T04:07:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:45 smithi029 ceph-mon[119131]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:active 2023-10-09T04:07:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:45 smithi029 ceph-mon[119131]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 2023-10-09T04:07:46.666 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":20,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":3407458368},{"type":"v1","addr":"172.21.15.29:6801","nonce":3407458368}]},"active_addr":"172.21.15.29:6801/3407458368","active_change":"2023-10-09T04:06:26.739515+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2321967128}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1371585021}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1787157051}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3506139723}]}]} 2023-10-09T04:07:46.671 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:07:46.671 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:07:46.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3480092373' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: mds.? [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] up:boot 2023-10-09T04:07:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3480092373' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: mds.? [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] up:boot 2023-10-09T04:07:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:46 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3480092373' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: mds.? [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] up:boot 2023-10-09T04:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:46 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:07:47.388 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:48.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:47 smithi029 ceph-mon[101043]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:48.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:47 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:47 smithi029 ceph-mon[119131]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:47 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:47 smithi177 ceph-mon[106070]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:07:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:47 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:49.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:48 smithi029 ceph-mon[101043]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:active 2023-10-09T04:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:48 smithi029 ceph-mon[101043]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:48 smithi029 ceph-mon[119131]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:active 2023-10-09T04:07:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:48 smithi029 ceph-mon[119131]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:49.253 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:07:49.254 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":68,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:07:36.643475+0000","last_up_change":"2023-10-09T04:05:50.951832+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:07:33.266053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:07:34.056234+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6803","nonce":1054408938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6805","nonce":1054408938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6809","nonce":1054408938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6807","nonce":1054408938}]},"public_addr":"172.21.15.29:6803/1054408938","cluster_addr":"172.21.15.29:6805/1054408938","heartbeat_back_addr":"172.21.15.29:6809/1054408938","heartbeat_front_addr":"172.21.15.29:6807/1054408938","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6811","nonce":3631252275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6813","nonce":3631252275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6817","nonce":3631252275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6815","nonce":3631252275}]},"public_addr":"172.21.15.29:6811/3631252275","cluster_addr":"172.21.15.29:6813/3631252275","heartbeat_back_addr":"172.21.15.29:6817/3631252275","heartbeat_front_addr":"172.21.15.29:6815/3631252275","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6819","nonce":1532034566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6821","nonce":1532034566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6825","nonce":1532034566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6823","nonce":1532034566}]},"public_addr":"172.21.15.29:6819/1532034566","cluster_addr":"172.21.15.29:6821/1532034566","heartbeat_back_addr":"172.21.15.29:6825/1532034566","heartbeat_front_addr":"172.21.15.29:6823/1532034566","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6827","nonce":3716476544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6829","nonce":3716476544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6833","nonce":3716476544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6831","nonce":3716476544}]},"public_addr":"172.21.15.29:6827/3716476544","cluster_addr":"172.21.15.29:6829/3716476544","heartbeat_back_addr":"172.21.15.29:6833/3716476544","heartbeat_front_addr":"172.21.15.29:6831/3716476544","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:07:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:48 smithi177 ceph-mon[106070]: mds.? [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] up:active 2023-10-09T04:07:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:48 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:50.166 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:07:50.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:07:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:49 smithi177 ceph-mon[106070]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:49 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:49 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:49 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3836470216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:07:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[101043]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3836470216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:07:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[119131]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:50.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:50.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:49 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3836470216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:07:50.641 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: mds.? [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] up:standby 2023-10-09T04:07:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[101043]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: mds.? [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] up:standby 2023-10-09T04:07:51.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:51 smithi029 ceph-mon[119131]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:07:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:07:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:07:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: mds.? [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] up:standby 2023-10-09T04:07:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:51 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:07:52.625 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:07:52.626 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":68,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:07:36.643475+0000","last_up_change":"2023-10-09T04:05:50.951832+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:07:33.266053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:07:34.056234+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6803","nonce":1054408938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6805","nonce":1054408938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6809","nonce":1054408938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1054408938},{"type":"v1","addr":"172.21.15.29:6807","nonce":1054408938}]},"public_addr":"172.21.15.29:6803/1054408938","cluster_addr":"172.21.15.29:6805/1054408938","heartbeat_back_addr":"172.21.15.29:6809/1054408938","heartbeat_front_addr":"172.21.15.29:6807/1054408938","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6811","nonce":3631252275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6813","nonce":3631252275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6817","nonce":3631252275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3631252275},{"type":"v1","addr":"172.21.15.29:6815","nonce":3631252275}]},"public_addr":"172.21.15.29:6811/3631252275","cluster_addr":"172.21.15.29:6813/3631252275","heartbeat_back_addr":"172.21.15.29:6817/3631252275","heartbeat_front_addr":"172.21.15.29:6815/3631252275","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6819","nonce":1532034566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6821","nonce":1532034566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6825","nonce":1532034566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":1532034566},{"type":"v1","addr":"172.21.15.29:6823","nonce":1532034566}]},"public_addr":"172.21.15.29:6819/1532034566","cluster_addr":"172.21.15.29:6821/1532034566","heartbeat_back_addr":"172.21.15.29:6825/1532034566","heartbeat_front_addr":"172.21.15.29:6823/1532034566","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6827","nonce":3716476544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6829","nonce":3716476544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6833","nonce":3716476544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3716476544},{"type":"v1","addr":"172.21.15.29:6831","nonce":3716476544}]},"public_addr":"172.21.15.29:6827/3716476544","cluster_addr":"172.21.15.29:6829/3716476544","heartbeat_back_addr":"172.21.15.29:6833/3716476544","heartbeat_front_addr":"172.21.15.29:6831/3716476544","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:07:52.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:52 smithi029 ceph-mon[101043]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:52.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:52 smithi029 ceph-mon[119131]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:52 smithi177 ceph-mon[106070]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:53.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:07:52 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:07:53.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:07:53.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:07:53.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:07:53.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:07:53.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:07:53.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:07:53.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:07:53.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:07:53.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:53 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1422591974' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:07:53.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:53 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:53.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:53 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:53 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1422591974' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:07:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:53 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:53 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:53 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1422591974' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:07:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:53 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:53 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:54.220 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:54.469 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:54.578 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:54.585 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:07:54 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:07:54] "GET /metrics HTTP/1.1" 200 208136 "" "Prometheus/2.43.0" 2023-10-09T04:07:54.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:54 smithi029 ceph-mon[101043]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:54.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:54 smithi029 ceph-mon[119131]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:54 smithi177 ceph-mon[106070]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:55.296 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:55.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:55 smithi029 ceph-mon[101043]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:55.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:55 smithi029 ceph-mon[119131]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:55 smithi177 ceph-mon[106070]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:56.058 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:56.062 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:56.386 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:56.574 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:07:58.079 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:57 smithi029 ceph-mon[101043]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:58.079 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:57 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:58.079 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:57 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:58.080 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:57 smithi029 ceph-mon[119131]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:58.080 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:57 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:58.080 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:57 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:07:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:57 smithi177 ceph-mon[106070]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:07:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:57 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:07:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:57 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:00.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:07:59 smithi029 ceph-mon[101043]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:07:59 smithi029 ceph-mon[119131]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:08:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:07:59 smithi177 ceph-mon[106070]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-10-09T04:08:02.729 INFO:teuthology.orchestra.run.smithi029.stdout:34359738429 2023-10-09T04:08:02.729 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:08:02.759 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:02 smithi029 ceph-mon[101043]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:02.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:02 smithi029 ceph-mon[119131]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:02 smithi177 ceph-mon[106070]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:03.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:08:02 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:08:04.599 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:04 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:04] "GET /metrics HTTP/1.1" 200 238822 "" "Prometheus/2.43.0" 2023-10-09T04:08:04.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:04 smithi029 ceph-mon[101043]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:04 smithi029 ceph-mon[119131]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:04 smithi177 ceph-mon[106070]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:06 smithi177 ceph-mon[106070]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:06 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:06 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:06 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:06 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:06 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[101043]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[119131]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-09T04:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:08:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:08:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:06 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:08:07.456 INFO:teuthology.orchestra.run.smithi029.stdout:107374182448 2023-10-09T04:08:07.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:08:07.898 INFO:teuthology.orchestra.run.smithi029.stdout:231928234014 2023-10-09T04:08:07.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:08:07.944 INFO:teuthology.orchestra.run.smithi029.stdout:128849018922 2023-10-09T04:08:07.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:08:07.975 INFO:teuthology.orchestra.run.smithi029.stdout:163208757286 2023-10-09T04:08:07.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:08:08.563 INFO:teuthology.orchestra.run.smithi029.stdout:197568495648 2023-10-09T04:08:08.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:08:08.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:08 smithi029 ceph-mon[101043]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:08 smithi029 ceph-mon[119131]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:08 smithi177 ceph-mon[106070]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:09.113 INFO:teuthology.orchestra.run.smithi029.stdout:55834574905 2023-10-09T04:08:09.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:08:09.148 INFO:teuthology.orchestra.run.smithi029.stdout:77309411380 2023-10-09T04:08:09.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:08:09.426 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:09.470 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:09.557 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:09.584 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:09.694 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:10 smithi029 ceph-mon[101043]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:10.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:10 smithi029 ceph-mon[119131]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:10 smithi177 ceph-mon[106070]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:12.044 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:12.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:12 smithi029 ceph-mon[101043]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:12 smithi029 ceph-mon[119131]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:12.978 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:12.978 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:13.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:08:12 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:08:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:12 smithi177 ceph-mon[106070]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:14.586 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:14 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:14] "GET /metrics HTTP/1.1" 200 238809 "" "Prometheus/2.43.0" 2023-10-09T04:08:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:14 smithi029 ceph-mon[101043]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:14 smithi029 ceph-mon[119131]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:14 smithi177 ceph-mon[106070]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:16.115 INFO:teuthology.orchestra.run.smithi029.stdout:231928234016 2023-10-09T04:08:16.149 INFO:teuthology.orchestra.run.smithi029.stdout:34359738432 2023-10-09T04:08:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:16 smithi029 ceph-mon[101043]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:16 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2217737092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:08:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:16 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3901249431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:16 smithi029 ceph-mon[119131]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:16 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2217737092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:16 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3901249431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:08:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:16 smithi177 ceph-mon[106070]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:16 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2217737092' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:08:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:16 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3901249431' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:08:17.859 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738429 got 34359738432 for osd.0 2023-10-09T04:08:17.859 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:18.204 INFO:teuthology.orchestra.run.smithi029.stdout:107374182450 2023-10-09T04:08:18.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:18 smithi029 ceph-mon[101043]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:18.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:18 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3820457089' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:08:18.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:18 smithi029 ceph-mon[119131]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:18.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:18 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3820457089' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:08:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:18 smithi177 ceph-mon[106070]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:18 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3820457089' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:08:19.230 INFO:teuthology.orchestra.run.smithi029.stdout:163208757288 2023-10-09T04:08:19.756 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234014 got 231928234016 for osd.7 2023-10-09T04:08:19.757 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:19.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:19 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3629365940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:08:19.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:19 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4064231598' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:08:19.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:19 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3629365940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:08:19.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:19 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4064231598' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:08:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:19 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3629365940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:08:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:19 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4064231598' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:08:20.421 INFO:teuthology.orchestra.run.smithi029.stdout:77309411382 2023-10-09T04:08:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:20 smithi029 ceph-mon[101043]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:20 smithi029 ceph-mon[119131]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:20 smithi177 ceph-mon[106070]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:21.204 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182448 got 107374182450 for osd.3 2023-10-09T04:08:21.205 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:21.514 INFO:teuthology.orchestra.run.smithi029.stdout:55834574908 2023-10-09T04:08:21.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:21 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2160366647' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:08:21.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:21 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2160366647' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:08:21.631 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757286 got 163208757288 for osd.5 2023-10-09T04:08:21.631 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:21.834 INFO:teuthology.orchestra.run.smithi029.stdout:197568495651 2023-10-09T04:08:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:21 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2160366647' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:08:22.480 INFO:teuthology.orchestra.run.smithi029.stdout:128849018925 2023-10-09T04:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:22 smithi029 ceph-mon[101043]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:22 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2177043853' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:22 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3353394321' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:22 smithi029 ceph-mon[119131]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:22 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2177043853' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:22 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3353394321' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:08:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:22 smithi177 ceph-mon[106070]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:22 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2177043853' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:08:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:22 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3353394321' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:08:23.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:08:22 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:08:23.563 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411380 got 77309411382 for osd.2 2023-10-09T04:08:23.563 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:24.026 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495648 got 197568495651 for osd.6 2023-10-09T04:08:24.185 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:24.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018922 got 128849018925 for osd.4 2023-10-09T04:08:24.187 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:24.188 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574905 got 55834574908 for osd.1 2023-10-09T04:08:24.188 DEBUG:teuthology.parallel:result is None 2023-10-09T04:08:24.188 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:08:24.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:08:24.595 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:24.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:24 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:24] "GET /metrics HTTP/1.1" 200 238809 "" "Prometheus/2.43.0" 2023-10-09T04:08:24.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:24 smithi029 ceph-mon[101043]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:24 smithi029 ceph-mon[119131]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:24 smithi177 ceph-mon[106070]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:26.527 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:26.528 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:08:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:26 smithi029 ceph-mon[101043]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:26 smithi029 ceph-mon[119131]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:26 smithi177 ceph-mon[106070]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:27.330 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":71,"stamp":"2023-10-09T04:08:24.774819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385544,"kb_used_data":7240,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575672,"statfs":{"total":767960285184,"available":765517488128,"internally_reserved":0,"allocated":7413760,"data_stored":4174155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002860"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283292+0000","last_change":"2023-10-09T04:07:34.659224+0000","last_active":"2023-10-09T04:07:36.283292+0000","last_peered":"2023-10-09T04:07:36.283292+0000","last_clean":"2023-10-09T04:07:36.283292+0000","last_became_active":"2023-10-09T04:07:34.658916+0000","last_became_peered":"2023-10-09T04:07:34.658916+0000","last_unstale":"2023-10-09T04:07:36.283292+0000","last_undegraded":"2023-10-09T04:07:36.283292+0000","last_fullsized":"2023-10-09T04:07:36.283292+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.380019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614395+0000","last_change":"2023-10-09T04:07:34.646776+0000","last_active":"2023-10-09T04:07:43.614395+0000","last_peered":"2023-10-09T04:07:43.614395+0000","last_clean":"2023-10-09T04:07:43.614395+0000","last_became_active":"2023-10-09T04:07:34.644325+0000","last_became_peered":"2023-10-09T04:07:34.644325+0000","last_unstale":"2023-10-09T04:07:43.614395+0000","last_undegraded":"2023-10-09T04:07:43.614395+0000","last_fullsized":"2023-10-09T04:07:43.614395+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:23.379485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.653353+0000","last_change":"2023-10-09T04:07:35.653186+0000","last_active":"2023-10-09T04:07:35.653353+0000","last_peered":"2023-10-09T04:07:35.653353+0000","last_clean":"2023-10-09T04:07:35.653353+0000","last_became_active":"2023-10-09T04:07:35.652908+0000","last_became_peered":"2023-10-09T04:07:35.652908+0000","last_unstale":"2023-10-09T04:07:35.653353+0000","last_undegraded":"2023-10-09T04:07:35.653353+0000","last_fullsized":"2023-10-09T04:07:35.653353+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:42.975764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286552+0000","last_change":"2023-10-09T04:07:34.638165+0000","last_active":"2023-10-09T04:07:37.286552+0000","last_peered":"2023-10-09T04:07:37.286552+0000","last_clean":"2023-10-09T04:07:37.286552+0000","last_became_active":"2023-10-09T04:07:34.637823+0000","last_became_peered":"2023-10-09T04:07:34.637823+0000","last_unstale":"2023-10-09T04:07:37.286552+0000","last_undegraded":"2023-10-09T04:07:37.286552+0000","last_fullsized":"2023-10-09T04:07:37.286552+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:13.118351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283619+0000","last_change":"2023-10-09T04:07:34.676759+0000","last_active":"2023-10-09T04:07:36.283619+0000","last_peered":"2023-10-09T04:07:36.283619+0000","last_clean":"2023-10-09T04:07:36.283619+0000","last_became_active":"2023-10-09T04:07:34.676294+0000","last_became_peered":"2023-10-09T04:07:34.676294+0000","last_unstale":"2023-10-09T04:07:36.283619+0000","last_undegraded":"2023-10-09T04:07:36.283619+0000","last_fullsized":"2023-10-09T04:07:36.283619+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:44.387110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614096+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:07:43.614096+0000","last_peered":"2023-10-09T04:07:43.614096+0000","last_clean":"2023-10-09T04:07:43.614096+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:07:43.614096+0000","last_undegraded":"2023-10-09T04:07:43.614096+0000","last_fullsized":"2023-10-09T04:07:43.614096+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283842+0000","last_change":"2023-10-09T04:07:34.659699+0000","last_active":"2023-10-09T04:07:36.283842+0000","last_peered":"2023-10-09T04:07:36.283842+0000","last_clean":"2023-10-09T04:07:36.283842+0000","last_became_active":"2023-10-09T04:07:34.659189+0000","last_became_peered":"2023-10-09T04:07:34.659189+0000","last_unstale":"2023-10-09T04:07:36.283842+0000","last_undegraded":"2023-10-09T04:07:36.283842+0000","last_fullsized":"2023-10-09T04:07:36.283842+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:44.119093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614137+0000","last_change":"2023-10-09T04:07:34.631386+0000","last_active":"2023-10-09T04:07:43.614137+0000","last_peered":"2023-10-09T04:07:43.614137+0000","last_clean":"2023-10-09T04:07:43.614137+0000","last_became_active":"2023-10-09T04:07:34.631090+0000","last_became_peered":"2023-10-09T04:07:34.631090+0000","last_unstale":"2023-10-09T04:07:43.614137+0000","last_undegraded":"2023-10-09T04:07:43.614137+0000","last_fullsized":"2023-10-09T04:07:43.614137+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:05.633909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.648111+0000","last_change":"2023-10-09T04:07:34.659338+0000","last_active":"2023-10-09T04:07:35.648111+0000","last_peered":"2023-10-09T04:07:35.648111+0000","last_clean":"2023-10-09T04:07:35.648111+0000","last_became_active":"2023-10-09T04:07:34.658997+0000","last_became_peered":"2023-10-09T04:07:34.658997+0000","last_unstale":"2023-10-09T04:07:35.648111+0000","last_undegraded":"2023-10-09T04:07:35.648111+0000","last_fullsized":"2023-10-09T04:07:35.648111+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:38.217792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646451+0000","last_change":"2023-10-09T04:07:34.642042+0000","last_active":"2023-10-09T04:07:36.646451+0000","last_peered":"2023-10-09T04:07:36.646451+0000","last_clean":"2023-10-09T04:07:36.646451+0000","last_became_active":"2023-10-09T04:07:34.641579+0000","last_became_peered":"2023-10-09T04:07:34.641579+0000","last_unstale":"2023-10-09T04:07:36.646451+0000","last_undegraded":"2023-10-09T04:07:36.646451+0000","last_fullsized":"2023-10-09T04:07:36.646451+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:44.869504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283572+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:07:36.283572+0000","last_peered":"2023-10-09T04:07:36.283572+0000","last_clean":"2023-10-09T04:07:36.283572+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:07:36.283572+0000","last_undegraded":"2023-10-09T04:07:36.283572+0000","last_fullsized":"2023-10-09T04:07:36.283572+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646736+0000","last_change":"2023-10-09T04:07:34.645613+0000","last_active":"2023-10-09T04:07:36.646736+0000","last_peered":"2023-10-09T04:07:36.646736+0000","last_clean":"2023-10-09T04:07:36.646736+0000","last_became_active":"2023-10-09T04:07:34.645309+0000","last_became_peered":"2023-10-09T04:07:34.645309+0000","last_unstale":"2023-10-09T04:07:36.646736+0000","last_undegraded":"2023-10-09T04:07:36.646736+0000","last_fullsized":"2023-10-09T04:07:36.646736+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:07.674226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.724317+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:07:35.724317+0000","last_peered":"2023-10-09T04:07:35.724317+0000","last_clean":"2023-10-09T04:07:35.724317+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:07:35.724317+0000","last_undegraded":"2023-10-09T04:07:35.724317+0000","last_fullsized":"2023-10-09T04:07:35.724317+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030979+0000","last_change":"2023-10-09T04:07:36.030076+0000","last_active":"2023-10-09T04:07:36.030979+0000","last_peered":"2023-10-09T04:07:36.030979+0000","last_clean":"2023-10-09T04:07:36.030979+0000","last_became_active":"2023-10-09T04:07:36.029271+0000","last_became_peered":"2023-10-09T04:07:36.029271+0000","last_unstale":"2023-10-09T04:07:36.030979+0000","last_undegraded":"2023-10-09T04:07:36.030979+0000","last_fullsized":"2023-10-09T04:07:36.030979+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:12.758916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799325+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:07:35.799325+0000","last_peered":"2023-10-09T04:07:35.799325+0000","last_clean":"2023-10-09T04:07:35.799325+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:07:35.799325+0000","last_undegraded":"2023-10-09T04:07:35.799325+0000","last_fullsized":"2023-10-09T04:07:35.799325+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.615068+0000","last_change":"2023-10-09T04:07:34.643380+0000","last_active":"2023-10-09T04:07:43.615068+0000","last_peered":"2023-10-09T04:07:43.615068+0000","last_clean":"2023-10-09T04:07:43.615068+0000","last_became_active":"2023-10-09T04:07:34.643086+0000","last_became_peered":"2023-10-09T04:07:34.643086+0000","last_unstale":"2023-10-09T04:07:43.615068+0000","last_undegraded":"2023-10-09T04:07:43.615068+0000","last_fullsized":"2023-10-09T04:07:43.615068+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:13.152038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030708+0000","last_change":"2023-10-09T04:07:36.030238+0000","last_active":"2023-10-09T04:07:36.030708+0000","last_peered":"2023-10-09T04:07:36.030708+0000","last_clean":"2023-10-09T04:07:36.030708+0000","last_became_active":"2023-10-09T04:07:36.029932+0000","last_became_peered":"2023-10-09T04:07:36.029932+0000","last_unstale":"2023-10-09T04:07:36.030708+0000","last_undegraded":"2023-10-09T04:07:36.030708+0000","last_fullsized":"2023-10-09T04:07:36.030708+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:25.269223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799494+0000","last_change":"2023-10-09T04:07:34.633885+0000","last_active":"2023-10-09T04:07:35.799494+0000","last_peered":"2023-10-09T04:07:35.799494+0000","last_clean":"2023-10-09T04:07:35.799494+0000","last_became_active":"2023-10-09T04:07:34.633345+0000","last_became_peered":"2023-10-09T04:07:34.633345+0000","last_unstale":"2023-10-09T04:07:35.799494+0000","last_undegraded":"2023-10-09T04:07:35.799494+0000","last_fullsized":"2023-10-09T04:07:35.799494+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:42.728405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.029518+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:07:36.029518+0000","last_peered":"2023-10-09T04:07:36.029518+0000","last_clean":"2023-10-09T04:07:36.029518+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:07:36.029518+0000","last_undegraded":"2023-10-09T04:07:36.029518+0000","last_fullsized":"2023-10-09T04:07:36.029518+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.802076+0000","last_change":"2023-10-09T04:07:35.800119+0000","last_active":"2023-10-09T04:07:35.802076+0000","last_peered":"2023-10-09T04:07:35.802076+0000","last_clean":"2023-10-09T04:07:35.802076+0000","last_became_active":"2023-10-09T04:07:35.799858+0000","last_became_peered":"2023-10-09T04:07:35.799858+0000","last_unstale":"2023-10-09T04:07:35.802076+0000","last_undegraded":"2023-10-09T04:07:35.802076+0000","last_fullsized":"2023-10-09T04:07:35.802076+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:04.465377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.651006+0000","last_change":"2023-10-09T04:07:34.645437+0000","last_active":"2023-10-09T04:07:35.651006+0000","last_peered":"2023-10-09T04:07:35.651006+0000","last_clean":"2023-10-09T04:07:35.651006+0000","last_became_active":"2023-10-09T04:07:34.645140+0000","last_became_peered":"2023-10-09T04:07:34.645140+0000","last_unstale":"2023-10-09T04:07:35.651006+0000","last_undegraded":"2023-10-09T04:07:35.651006+0000","last_fullsized":"2023-10-09T04:07:35.651006+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:45.140472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.028456+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:07:36.028456+0000","last_peered":"2023-10-09T04:07:36.028456+0000","last_clean":"2023-10-09T04:07:36.028456+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:07:36.028456+0000","last_undegraded":"2023-10-09T04:07:36.028456+0000","last_fullsized":"2023-10-09T04:07:36.028456+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.724200+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:07:35.724200+0000","last_peered":"2023-10-09T04:07:35.724200+0000","last_clean":"2023-10-09T04:07:35.724200+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:07:35.724200+0000","last_undegraded":"2023-10-09T04:07:35.724200+0000","last_fullsized":"2023-10-09T04:07:35.724200+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652736+0000","last_change":"2023-10-09T04:07:34.639655+0000","last_active":"2023-10-09T04:07:35.652736+0000","last_peered":"2023-10-09T04:07:35.652736+0000","last_clean":"2023-10-09T04:07:35.652736+0000","last_became_active":"2023-10-09T04:07:34.639321+0000","last_became_peered":"2023-10-09T04:07:34.639321+0000","last_unstale":"2023-10-09T04:07:35.652736+0000","last_undegraded":"2023-10-09T04:07:35.652736+0000","last_fullsized":"2023-10-09T04:07:35.652736+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:25.904866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":276,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.028444+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:07:36.028444+0000","last_peered":"2023-10-09T04:07:36.028444+0000","last_clean":"2023-10-09T04:07:36.028444+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:07:36.028444+0000","last_undegraded":"2023-10-09T04:07:36.028444+0000","last_fullsized":"2023-10-09T04:07:36.028444+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.645176+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:07:35.645176+0000","last_peered":"2023-10-09T04:07:35.645176+0000","last_clean":"2023-10-09T04:07:35.645176+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:07:35.645176+0000","last_undegraded":"2023-10-09T04:07:35.645176+0000","last_fullsized":"2023-10-09T04:07:35.645176+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799334+0000","last_change":"2023-10-09T04:07:34.676079+0000","last_active":"2023-10-09T04:07:35.799334+0000","last_peered":"2023-10-09T04:07:35.799334+0000","last_clean":"2023-10-09T04:07:35.799334+0000","last_became_active":"2023-10-09T04:07:34.675776+0000","last_became_peered":"2023-10-09T04:07:34.675776+0000","last_unstale":"2023-10-09T04:07:35.799334+0000","last_undegraded":"2023-10-09T04:07:35.799334+0000","last_fullsized":"2023-10-09T04:07:35.799334+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:26.867406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799080+0000","last_change":"2023-10-09T04:07:34.632817+0000","last_active":"2023-10-09T04:07:35.799080+0000","last_peered":"2023-10-09T04:07:35.799080+0000","last_clean":"2023-10-09T04:07:35.799080+0000","last_became_active":"2023-10-09T04:07:34.632434+0000","last_became_peered":"2023-10-09T04:07:34.632434+0000","last_unstale":"2023-10-09T04:07:35.799080+0000","last_undegraded":"2023-10-09T04:07:35.799080+0000","last_fullsized":"2023-10-09T04:07:35.799080+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:37.018559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.647880+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:07:35.647880+0000","last_peered":"2023-10-09T04:07:35.647880+0000","last_clean":"2023-10-09T04:07:35.647880+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:07:35.647880+0000","last_undegraded":"2023-10-09T04:07:35.647880+0000","last_fullsized":"2023-10-09T04:07:35.647880+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799388+0000","last_change":"2023-10-09T04:07:34.659865+0000","last_active":"2023-10-09T04:07:35.799388+0000","last_peered":"2023-10-09T04:07:35.799388+0000","last_clean":"2023-10-09T04:07:35.799388+0000","last_became_active":"2023-10-09T04:07:34.659515+0000","last_became_peered":"2023-10-09T04:07:34.659515+0000","last_unstale":"2023-10-09T04:07:35.799388+0000","last_undegraded":"2023-10-09T04:07:35.799388+0000","last_fullsized":"2023-10-09T04:07:35.799388+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:07.703247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283814+0000","last_change":"2023-10-09T04:07:34.641445+0000","last_active":"2023-10-09T04:07:36.283814+0000","last_peered":"2023-10-09T04:07:36.283814+0000","last_clean":"2023-10-09T04:07:36.283814+0000","last_became_active":"2023-10-09T04:07:34.641181+0000","last_became_peered":"2023-10-09T04:07:34.641181+0000","last_unstale":"2023-10-09T04:07:36.283814+0000","last_undegraded":"2023-10-09T04:07:36.283814+0000","last_fullsized":"2023-10-09T04:07:36.283814+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.897577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.645492+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:07:35.645492+0000","last_peered":"2023-10-09T04:07:35.645492+0000","last_clean":"2023-10-09T04:07:35.645492+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:07:35.645492+0000","last_undegraded":"2023-10-09T04:07:35.645492+0000","last_fullsized":"2023-10-09T04:07:35.645492+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799348+0000","last_change":"2023-10-09T04:07:34.675845+0000","last_active":"2023-10-09T04:07:35.799348+0000","last_peered":"2023-10-09T04:07:35.799348+0000","last_clean":"2023-10-09T04:07:35.799348+0000","last_became_active":"2023-10-09T04:07:34.675371+0000","last_became_peered":"2023-10-09T04:07:34.675371+0000","last_unstale":"2023-10-09T04:07:35.799348+0000","last_undegraded":"2023-10-09T04:07:35.799348+0000","last_fullsized":"2023-10-09T04:07:35.799348+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:45.429941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.645547+0000","last_change":"2023-10-09T04:07:34.640999+0000","last_active":"2023-10-09T04:07:35.645547+0000","last_peered":"2023-10-09T04:07:35.645547+0000","last_clean":"2023-10-09T04:07:35.645547+0000","last_became_active":"2023-10-09T04:07:34.640725+0000","last_became_peered":"2023-10-09T04:07:34.640725+0000","last_unstale":"2023-10-09T04:07:35.645547+0000","last_undegraded":"2023-10-09T04:07:35.645547+0000","last_fullsized":"2023-10-09T04:07:35.645547+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:45.119257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799306+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:07:35.799306+0000","last_peered":"2023-10-09T04:07:35.799306+0000","last_clean":"2023-10-09T04:07:35.799306+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:07:35.799306+0000","last_undegraded":"2023-10-09T04:07:35.799306+0000","last_fullsized":"2023-10-09T04:07:35.799306+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799413+0000","last_change":"2023-10-09T04:07:34.662253+0000","last_active":"2023-10-09T04:07:35.799413+0000","last_peered":"2023-10-09T04:07:35.799413+0000","last_clean":"2023-10-09T04:07:35.799413+0000","last_became_active":"2023-10-09T04:07:34.661919+0000","last_became_peered":"2023-10-09T04:07:34.661919+0000","last_unstale":"2023-10-09T04:07:35.799413+0000","last_undegraded":"2023-10-09T04:07:35.799413+0000","last_fullsized":"2023-10-09T04:07:35.799413+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:35.167033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652246+0000","last_change":"2023-10-09T04:07:34.651433+0000","last_active":"2023-10-09T04:07:35.652246+0000","last_peered":"2023-10-09T04:07:35.652246+0000","last_clean":"2023-10-09T04:07:35.652246+0000","last_became_active":"2023-10-09T04:07:34.650984+0000","last_became_peered":"2023-10-09T04:07:34.650984+0000","last_unstale":"2023-10-09T04:07:35.652246+0000","last_undegraded":"2023-10-09T04:07:35.652246+0000","last_fullsized":"2023-10-09T04:07:35.652246+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:57.513271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283719+0000","last_change":"2023-10-09T04:07:34.659514+0000","last_active":"2023-10-09T04:07:36.283719+0000","last_peered":"2023-10-09T04:07:36.283719+0000","last_clean":"2023-10-09T04:07:36.283719+0000","last_became_active":"2023-10-09T04:07:34.659212+0000","last_became_peered":"2023-10-09T04:07:34.659212+0000","last_unstale":"2023-10-09T04:07:36.283719+0000","last_undegraded":"2023-10-09T04:07:36.283719+0000","last_fullsized":"2023-10-09T04:07:36.283719+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:30.189193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283658+0000","last_change":"2023-10-09T04:07:34.630774+0000","last_active":"2023-10-09T04:07:36.283658+0000","last_peered":"2023-10-09T04:07:36.283658+0000","last_clean":"2023-10-09T04:07:36.283658+0000","last_became_active":"2023-10-09T04:07:34.630525+0000","last_became_peered":"2023-10-09T04:07:34.630525+0000","last_unstale":"2023-10-09T04:07:36.283658+0000","last_undegraded":"2023-10-09T04:07:36.283658+0000","last_fullsized":"2023-10-09T04:07:36.283658+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:05.578748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283766+0000","last_change":"2023-10-09T04:07:34.659260+0000","last_active":"2023-10-09T04:07:36.283766+0000","last_peered":"2023-10-09T04:07:36.283766+0000","last_clean":"2023-10-09T04:07:36.283766+0000","last_became_active":"2023-10-09T04:07:34.658976+0000","last_became_peered":"2023-10-09T04:07:34.658976+0000","last_unstale":"2023-10-09T04:07:36.283766+0000","last_undegraded":"2023-10-09T04:07:36.283766+0000","last_fullsized":"2023-10-09T04:07:36.283766+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:35.014675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.724284+0000","last_change":"2023-10-09T04:07:34.634417+0000","last_active":"2023-10-09T04:07:35.724284+0000","last_peered":"2023-10-09T04:07:35.724284+0000","last_clean":"2023-10-09T04:07:35.724284+0000","last_became_active":"2023-10-09T04:07:34.634183+0000","last_became_peered":"2023-10-09T04:07:34.634183+0000","last_unstale":"2023-10-09T04:07:35.724284+0000","last_undegraded":"2023-10-09T04:07:35.724284+0000","last_fullsized":"2023-10-09T04:07:35.724284+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:13.162009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283491+0000","last_change":"2023-10-09T04:07:34.655573+0000","last_active":"2023-10-09T04:07:36.283491+0000","last_peered":"2023-10-09T04:07:36.283491+0000","last_clean":"2023-10-09T04:07:36.283491+0000","last_became_active":"2023-10-09T04:07:34.655326+0000","last_became_peered":"2023-10-09T04:07:34.655326+0000","last_unstale":"2023-10-09T04:07:36.283491+0000","last_undegraded":"2023-10-09T04:07:36.283491+0000","last_fullsized":"2023-10-09T04:07:36.283491+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:07.510114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614867+0000","last_change":"2023-10-09T04:07:34.643373+0000","last_active":"2023-10-09T04:07:43.614867+0000","last_peered":"2023-10-09T04:07:43.614867+0000","last_clean":"2023-10-09T04:07:43.614867+0000","last_became_active":"2023-10-09T04:07:34.643033+0000","last_became_peered":"2023-10-09T04:07:34.643033+0000","last_unstale":"2023-10-09T04:07:43.614867+0000","last_undegraded":"2023-10-09T04:07:43.614867+0000","last_fullsized":"2023-10-09T04:07:43.614867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:57.618154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.285588+0000","last_change":"2023-10-09T04:07:36.284733+0000","last_active":"2023-10-09T04:07:36.285588+0000","last_peered":"2023-10-09T04:07:36.285588+0000","last_clean":"2023-10-09T04:07:36.285588+0000","last_became_active":"2023-10-09T04:07:36.284375+0000","last_became_peered":"2023-10-09T04:07:36.284375+0000","last_unstale":"2023-10-09T04:07:36.285588+0000","last_undegraded":"2023-10-09T04:07:36.285588+0000","last_fullsized":"2023-10-09T04:07:36.285588+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:10.421745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.645638+0000","last_change":"2023-10-09T04:07:34.640164+0000","last_active":"2023-10-09T04:07:35.645638+0000","last_peered":"2023-10-09T04:07:35.645638+0000","last_clean":"2023-10-09T04:07:35.645638+0000","last_became_active":"2023-10-09T04:07:34.639871+0000","last_became_peered":"2023-10-09T04:07:34.639871+0000","last_unstale":"2023-10-09T04:07:35.645638+0000","last_undegraded":"2023-10-09T04:07:35.645638+0000","last_fullsized":"2023-10-09T04:07:35.645638+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:59.659674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.724038+0000","last_change":"2023-10-09T04:07:34.677291+0000","last_active":"2023-10-09T04:07:35.724038+0000","last_peered":"2023-10-09T04:07:35.724038+0000","last_clean":"2023-10-09T04:07:35.724038+0000","last_became_active":"2023-10-09T04:07:34.677024+0000","last_became_peered":"2023-10-09T04:07:34.677024+0000","last_unstale":"2023-10-09T04:07:35.724038+0000","last_undegraded":"2023-10-09T04:07:35.724038+0000","last_fullsized":"2023-10-09T04:07:35.724038+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:11.684880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.060105+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:07:37.060105+0000","last_peered":"2023-10-09T04:07:37.060105+0000","last_clean":"2023-10-09T04:07:37.060105+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:07:37.060105+0000","last_undegraded":"2023-10-09T04:07:37.060105+0000","last_fullsized":"2023-10-09T04:07:37.060105+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283381+0000","last_change":"2023-10-09T04:07:34.659608+0000","last_active":"2023-10-09T04:07:36.283381+0000","last_peered":"2023-10-09T04:07:36.283381+0000","last_clean":"2023-10-09T04:07:36.283381+0000","last_became_active":"2023-10-09T04:07:34.658918+0000","last_became_peered":"2023-10-09T04:07:34.658918+0000","last_unstale":"2023-10-09T04:07:36.283381+0000","last_undegraded":"2023-10-09T04:07:36.283381+0000","last_fullsized":"2023-10-09T04:07:36.283381+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:42.508338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616107+0000","last_change":"2023-10-09T04:07:34.631609+0000","last_active":"2023-10-09T04:07:43.616107+0000","last_peered":"2023-10-09T04:07:43.616107+0000","last_clean":"2023-10-09T04:07:43.616107+0000","last_became_active":"2023-10-09T04:07:34.631327+0000","last_became_peered":"2023-10-09T04:07:34.631327+0000","last_unstale":"2023-10-09T04:07:43.616107+0000","last_undegraded":"2023-10-09T04:07:43.616107+0000","last_fullsized":"2023-10-09T04:07:43.616107+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:51.678421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.029830+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:07:36.029830+0000","last_peered":"2023-10-09T04:07:36.029830+0000","last_clean":"2023-10-09T04:07:36.029830+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:07:36.029830+0000","last_undegraded":"2023-10-09T04:07:36.029830+0000","last_fullsized":"2023-10-09T04:07:36.029830+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.060105+0000","last_change":"2023-10-09T04:07:34.642317+0000","last_active":"2023-10-09T04:07:37.060105+0000","last_peered":"2023-10-09T04:07:37.060105+0000","last_clean":"2023-10-09T04:07:37.060105+0000","last_became_active":"2023-10-09T04:07:34.642028+0000","last_became_peered":"2023-10-09T04:07:34.642028+0000","last_unstale":"2023-10-09T04:07:37.060105+0000","last_undegraded":"2023-10-09T04:07:37.060105+0000","last_fullsized":"2023-10-09T04:07:37.060105+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:12.984276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030869+0000","last_change":"2023-10-09T04:07:36.029846+0000","last_active":"2023-10-09T04:07:36.030869+0000","last_peered":"2023-10-09T04:07:36.030869+0000","last_clean":"2023-10-09T04:07:36.030869+0000","last_became_active":"2023-10-09T04:07:36.029223+0000","last_became_peered":"2023-10-09T04:07:36.029223+0000","last_unstale":"2023-10-09T04:07:36.030869+0000","last_undegraded":"2023-10-09T04:07:36.030869+0000","last_fullsized":"2023-10-09T04:07:36.030869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:12.111167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616068+0000","last_change":"2023-10-09T04:07:34.640376+0000","last_active":"2023-10-09T04:07:43.616068+0000","last_peered":"2023-10-09T04:07:43.616068+0000","last_clean":"2023-10-09T04:07:43.616068+0000","last_became_active":"2023-10-09T04:07:34.640073+0000","last_became_peered":"2023-10-09T04:07:34.640073+0000","last_unstale":"2023-10-09T04:07:43.616068+0000","last_undegraded":"2023-10-09T04:07:43.616068+0000","last_fullsized":"2023-10-09T04:07:43.616068+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:16.684588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030722+0000","last_change":"2023-10-09T04:07:36.029535+0000","last_active":"2023-10-09T04:07:36.030722+0000","last_peered":"2023-10-09T04:07:36.030722+0000","last_clean":"2023-10-09T04:07:36.030722+0000","last_became_active":"2023-10-09T04:07:36.029175+0000","last_became_peered":"2023-10-09T04:07:36.029175+0000","last_unstale":"2023-10-09T04:07:36.030722+0000","last_undegraded":"2023-10-09T04:07:36.030722+0000","last_fullsized":"2023-10-09T04:07:36.030722+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:34.595934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.029171+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:07:36.029171+0000","last_peered":"2023-10-09T04:07:36.029171+0000","last_clean":"2023-10-09T04:07:36.029171+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:07:36.029171+0000","last_undegraded":"2023-10-09T04:07:36.029171+0000","last_fullsized":"2023-10-09T04:07:36.029171+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.029772+0000","last_change":"2023-10-09T04:07:36.029411+0000","last_active":"2023-10-09T04:07:36.029772+0000","last_peered":"2023-10-09T04:07:36.029772+0000","last_clean":"2023-10-09T04:07:36.029772+0000","last_became_active":"2023-10-09T04:07:36.029109+0000","last_became_peered":"2023-10-09T04:07:36.029109+0000","last_unstale":"2023-10-09T04:07:36.029772+0000","last_undegraded":"2023-10-09T04:07:36.029772+0000","last_fullsized":"2023-10-09T04:07:36.029772+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:05.311520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.648366+0000","last_change":"2023-10-09T04:07:34.646539+0000","last_active":"2023-10-09T04:07:35.648366+0000","last_peered":"2023-10-09T04:07:35.648366+0000","last_clean":"2023-10-09T04:07:35.648366+0000","last_became_active":"2023-10-09T04:07:34.646243+0000","last_became_peered":"2023-10-09T04:07:34.646243+0000","last_unstale":"2023-10-09T04:07:35.648366+0000","last_undegraded":"2023-10-09T04:07:35.648366+0000","last_fullsized":"2023-10-09T04:07:35.648366+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:01.969093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030827+0000","last_change":"2023-10-09T04:07:36.029585+0000","last_active":"2023-10-09T04:07:36.030827+0000","last_peered":"2023-10-09T04:07:36.030827+0000","last_clean":"2023-10-09T04:07:36.030827+0000","last_became_active":"2023-10-09T04:07:36.029157+0000","last_became_peered":"2023-10-09T04:07:36.029157+0000","last_unstale":"2023-10-09T04:07:36.030827+0000","last_undegraded":"2023-10-09T04:07:36.030827+0000","last_fullsized":"2023-10-09T04:07:36.030827+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:19.819664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616905+0000","last_change":"2023-10-09T04:07:34.632203+0000","last_active":"2023-10-09T04:07:43.616905+0000","last_peered":"2023-10-09T04:07:43.616905+0000","last_clean":"2023-10-09T04:07:43.616905+0000","last_became_active":"2023-10-09T04:07:34.631969+0000","last_became_peered":"2023-10-09T04:07:34.631969+0000","last_unstale":"2023-10-09T04:07:43.616905+0000","last_undegraded":"2023-10-09T04:07:43.616905+0000","last_fullsized":"2023-10-09T04:07:43.616905+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:01.619892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.283537+0000","last_change":"2023-10-09T04:07:34.659790+0000","last_active":"2023-10-09T04:07:36.283537+0000","last_peered":"2023-10-09T04:07:36.283537+0000","last_clean":"2023-10-09T04:07:36.283537+0000","last_became_active":"2023-10-09T04:07:34.659565+0000","last_became_peered":"2023-10-09T04:07:34.659565+0000","last_unstale":"2023-10-09T04:07:36.283537+0000","last_undegraded":"2023-10-09T04:07:36.283537+0000","last_fullsized":"2023-10-09T04:07:36.283537+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:32.044413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.615997+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:07:43.615997+0000","last_peered":"2023-10-09T04:07:43.615997+0000","last_clean":"2023-10-09T04:07:43.615997+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:07:43.615997+0000","last_undegraded":"2023-10-09T04:07:43.615997+0000","last_fullsized":"2023-10-09T04:07:43.615997+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.799225+0000","last_change":"2023-10-09T04:07:34.661425+0000","last_active":"2023-10-09T04:07:35.799225+0000","last_peered":"2023-10-09T04:07:35.799225+0000","last_clean":"2023-10-09T04:07:35.799225+0000","last_became_active":"2023-10-09T04:07:34.661099+0000","last_became_peered":"2023-10-09T04:07:34.661099+0000","last_unstale":"2023-10-09T04:07:35.799225+0000","last_undegraded":"2023-10-09T04:07:35.799225+0000","last_fullsized":"2023-10-09T04:07:35.799225+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:53.588610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616156+0000","last_change":"2023-10-09T04:07:34.641392+0000","last_active":"2023-10-09T04:07:43.616156+0000","last_peered":"2023-10-09T04:07:43.616156+0000","last_clean":"2023-10-09T04:07:43.616156+0000","last_became_active":"2023-10-09T04:07:34.641037+0000","last_became_peered":"2023-10-09T04:07:34.641037+0000","last_unstale":"2023-10-09T04:07:43.616156+0000","last_undegraded":"2023-10-09T04:07:43.616156+0000","last_fullsized":"2023-10-09T04:07:43.616156+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:43.467265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.031551+0000","last_change":"2023-10-09T04:07:36.031224+0000","last_active":"2023-10-09T04:07:36.031551+0000","last_peered":"2023-10-09T04:07:36.031551+0000","last_clean":"2023-10-09T04:07:36.031551+0000","last_became_active":"2023-10-09T04:07:36.030834+0000","last_became_peered":"2023-10-09T04:07:36.030834+0000","last_unstale":"2023-10-09T04:07:36.031551+0000","last_undegraded":"2023-10-09T04:07:36.031551+0000","last_fullsized":"2023-10-09T04:07:36.031551+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:21.909762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.645270+0000","last_change":"2023-10-09T04:07:34.631933+0000","last_active":"2023-10-09T04:07:35.645270+0000","last_peered":"2023-10-09T04:07:35.645270+0000","last_clean":"2023-10-09T04:07:35.645270+0000","last_became_active":"2023-10-09T04:07:34.631667+0000","last_became_peered":"2023-10-09T04:07:34.631667+0000","last_unstale":"2023-10-09T04:07:35.645270+0000","last_undegraded":"2023-10-09T04:07:35.645270+0000","last_fullsized":"2023-10-09T04:07:35.645270+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:58.644048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.801084+0000","last_change":"2023-10-09T04:07:35.800509+0000","last_active":"2023-10-09T04:07:35.801084+0000","last_peered":"2023-10-09T04:07:35.801084+0000","last_clean":"2023-10-09T04:07:35.801084+0000","last_became_active":"2023-10-09T04:07:35.800155+0000","last_became_peered":"2023-10-09T04:07:35.800155+0000","last_unstale":"2023-10-09T04:07:35.801084+0000","last_undegraded":"2023-10-09T04:07:35.801084+0000","last_fullsized":"2023-10-09T04:07:35.801084+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:50.929412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.648560+0000","last_change":"2023-10-09T04:07:34.646997+0000","last_active":"2023-10-09T04:07:35.648560+0000","last_peered":"2023-10-09T04:07:35.648560+0000","last_clean":"2023-10-09T04:07:35.648560+0000","last_became_active":"2023-10-09T04:07:34.645625+0000","last_became_peered":"2023-10-09T04:07:34.645625+0000","last_unstale":"2023-10-09T04:07:35.648560+0000","last_undegraded":"2023-10-09T04:07:35.648560+0000","last_fullsized":"2023-10-09T04:07:35.648560+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:40.584254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.724087+0000","last_change":"2023-10-09T04:07:34.676137+0000","last_active":"2023-10-09T04:07:35.724087+0000","last_peered":"2023-10-09T04:07:35.724087+0000","last_clean":"2023-10-09T04:07:35.724087+0000","last_became_active":"2023-10-09T04:07:34.675822+0000","last_became_peered":"2023-10-09T04:07:34.675822+0000","last_unstale":"2023-10-09T04:07:35.724087+0000","last_undegraded":"2023-10-09T04:07:35.724087+0000","last_fullsized":"2023-10-09T04:07:35.724087+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:06.758028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652383+0000","last_change":"2023-10-09T04:07:34.646983+0000","last_active":"2023-10-09T04:07:35.652383+0000","last_peered":"2023-10-09T04:07:35.652383+0000","last_clean":"2023-10-09T04:07:35.652383+0000","last_became_active":"2023-10-09T04:07:34.646623+0000","last_became_peered":"2023-10-09T04:07:34.646623+0000","last_unstale":"2023-10-09T04:07:35.652383+0000","last_undegraded":"2023-10-09T04:07:35.652383+0000","last_fullsized":"2023-10-09T04:07:35.652383+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.119527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030287+0000","last_change":"2023-10-09T04:07:36.029481+0000","last_active":"2023-10-09T04:07:36.030287+0000","last_peered":"2023-10-09T04:07:36.030287+0000","last_clean":"2023-10-09T04:07:36.030287+0000","last_became_active":"2023-10-09T04:07:36.029167+0000","last_became_peered":"2023-10-09T04:07:36.029167+0000","last_unstale":"2023-10-09T04:07:36.030287+0000","last_undegraded":"2023-10-09T04:07:36.030287+0000","last_fullsized":"2023-10-09T04:07:36.030287+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:42.409682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.798991+0000","last_change":"2023-10-09T04:07:34.645964+0000","last_active":"2023-10-09T04:07:35.798991+0000","last_peered":"2023-10-09T04:07:35.798991+0000","last_clean":"2023-10-09T04:07:35.798991+0000","last_became_active":"2023-10-09T04:07:34.645619+0000","last_became_peered":"2023-10-09T04:07:34.645619+0000","last_unstale":"2023-10-09T04:07:35.798991+0000","last_undegraded":"2023-10-09T04:07:35.798991+0000","last_fullsized":"2023-10-09T04:07:35.798991+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:26.090109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.285502+0000","last_change":"2023-10-09T04:07:36.284775+0000","last_active":"2023-10-09T04:07:36.285502+0000","last_peered":"2023-10-09T04:07:36.285502+0000","last_clean":"2023-10-09T04:07:36.285502+0000","last_became_active":"2023-10-09T04:07:36.284501+0000","last_became_peered":"2023-10-09T04:07:36.284501+0000","last_unstale":"2023-10-09T04:07:36.285502+0000","last_undegraded":"2023-10-09T04:07:36.285502+0000","last_fullsized":"2023-10-09T04:07:36.285502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:17.182374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785699+0000","last_change":"2023-10-09T04:07:34.630544+0000","last_active":"2023-10-09T04:07:36.785699+0000","last_peered":"2023-10-09T04:07:36.785699+0000","last_clean":"2023-10-09T04:07:36.785699+0000","last_became_active":"2023-10-09T04:07:34.630230+0000","last_became_peered":"2023-10-09T04:07:34.630230+0000","last_unstale":"2023-10-09T04:07:36.785699+0000","last_undegraded":"2023-10-09T04:07:36.785699+0000","last_fullsized":"2023-10-09T04:07:36.785699+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:32.002588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234018,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0329999999999999,"15min":1.123},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76500000000000001,"5min":10.557,"15min":10.557},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.63900000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.83999999999999997,"15min":0.88800000000000001},"min":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71099999999999997,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.498},{"interface":"front","average":{"1min":0.503,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.81599999999999995,"15min":0.86699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69799999999999995,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.68300000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72999999999999998,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67100000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.73999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.64300000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.52000000000000002,"15min":0.53400000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.58399999999999996,"5min":1.391,"15min":1.391},"last":0.81999999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.71599999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.63500000000000001,"15min":0.65400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69099999999999995,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.76600000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495652,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1268,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1298432,"data_stored":891208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54900000000000004,"15min":0.55400000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.751,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.505},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69399999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.57999999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56999999999999995,"15min":0.57799999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73299999999999998,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.70199999999999996,"5min":0.748,"15min":0.748},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73199999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.54000000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.42999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.376},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48599999999999999,"15min":0.48199999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.39700000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48799999999999999,"15min":0.48699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.89200000000000002,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.77300000000000002,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.51500000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.39800000000000002,"15min":0.372},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.435,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.41399999999999998,"15min":0.42299999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.51500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.36499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:07:28 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.499,"15min":0.50700000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61499999999999999,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.44,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.75600000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.70099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58699999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.60899999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70899999999999996,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69899999999999995,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.751}]},{"osd":6,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.62,"15min":0.628},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.85999999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66400000000000003,"15min":0.67600000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.78600000000000003,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.629}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.65100000000000002,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.48099999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.68700000000000006}]}]},{"osd":0,"up_from":8,"seq":34359738434,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:07:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.32800000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.308},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.38400000000000001,"15min":0.34599999999999997},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53600000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.35399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.58599999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.46800000000000003,"15min":0.47599999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.53200000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.372},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.50900000000000001,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.373,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77800000000000002,"5min":0.878,"15min":0.878},"last":0.752},{"interface":"front","average":{"1min":0.626,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84299999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.75800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61099999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.622,"15min":0.63800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77600000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80600000000000005,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79600000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.624}]},{"osd":7,"last update":"Mon Oct 9 04:07:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65100000000000002,"15min":0.65900000000000003},"min":{"1min":0.46600000000000003,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80900000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.69099999999999995,"15min":0.68700000000000006},"min":{"1min":0.505,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.70799999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411384,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70699999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.64900000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.378},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.436}]},{"osd":3,"last update":"Mon Oct 9 04:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.47599999999999998,"15min":0.47699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.48599999999999999,"15min":0.48399999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.46600000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.45400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.46999999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.499}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.61599999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79400000000000004,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.63400000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.62,"15min":0.61799999999999999},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.60599999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67400000000000004},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98099999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.629,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.67200000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182452,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:26 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.40400000000000003,"15min":0.42099999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.53400000000000003,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.43099999999999999,"15min":0.45400000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.57799999999999996,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.52700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:07:26 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.45100000000000001,"15min":0.46999999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55300000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.35899999999999999,"5min":0.45500000000000002,"15min":0.48299999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.50600000000000001,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.50700000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:07:26 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.48399999999999999,"15min":0.503},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.74399999999999999,"5min":0.745,"15min":0.745},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.48999999999999999,"15min":0.51200000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.749,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54700000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.73399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.626,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.76300000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.84799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.624,"15min":0.63500000000000001},"min":{"1min":0.433,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77100000000000002,"5min":0.874,"15min":0.874},"last":0.754}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.63,"15min":0.63900000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.748,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.61399999999999999,"15min":0.622},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76400000000000001,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.86799999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73599999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.878},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.46500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81200000000000006,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.72199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018926,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.61599999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.503,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.54800000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.52600000000000002,"15min":0.51700000000000002},"min":{"1min":0.40200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.53500000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.432,"5min":0.37,"15min":0.37},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.5},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54100000000000004,"15min":0.52300000000000002},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.073,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.51600000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.439,"15min":0.42299999999999999},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.44400000000000001,"15min":0.42399999999999999},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.47799999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.47899999999999998,"15min":0.47299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68899999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69099999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.439},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.57199999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757290,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53000000000000003,"15min":0.52400000000000002},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.51200000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65000000000000002,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.36899999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71799999999999997,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.442}]},{"osd":3,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55800000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.443,"15min":0.41899999999999998},"min":{"1min":0.35499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.45400000000000001,"15min":0.42799999999999999},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.60899999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.45500000000000002,"15min":0.442},"min":{"1min":0.38700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.45900000000000002,"15min":0.443},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.65400000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.629}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:08:27.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:08:27.704 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[101043]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[119131]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-10-09T04:08:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-10-09T04:08:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:27 smithi177 ceph-mon[106070]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-10-09T04:08:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2023-10-09T04:08:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-10-09T04:08:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:28.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[101043]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[101043]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[101043]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[119131]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-10-09T04:08:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-10-09T04:08:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[119131]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:08:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:28 smithi029 ceph-mon[119131]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:08:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:28 smithi177 ceph-mon[106070]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:28 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-10-09T04:08:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:28 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2023-10-09T04:08:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:28 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-10-09T04:08:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:28 smithi177 ceph-mon[106070]: osdmap e69: 8 total, 8 up, 8 in 2023-10-09T04:08:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:28 smithi177 ceph-mon[106070]: osdmap e70: 8 total, 8 up, 8 in 2023-10-09T04:08:29.596 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:29.598 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:08:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:29 smithi029 ceph-mon[101043]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:29 smithi029 ceph-mon[101043]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:29 smithi029 ceph-mon[119131]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:29 smithi029 ceph-mon[119131]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:29 smithi177 ceph-mon[106070]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:29 smithi177 ceph-mon[106070]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:30.390 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":75,"stamp":"2023-10-09T04:08:28.775620+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385544,"kb_used_data":7240,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575672,"statfs":{"total":767960285184,"available":765517488128,"internally_reserved":0,"allocated":7413760,"data_stored":4174155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001650"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2023-10-09T04:08:27.631298+0000","last_change":"2023-10-09T04:08:27.631472+0000","last_active":"2023-10-09T04:08:27.631278+0000","last_peered":"2023-10-09T04:07:36.785613+0000","last_clean":"2023-10-09T04:07:36.785613+0000","last_became_active":"2023-10-09T04:07:34.658916+0000","last_became_peered":"2023-10-09T04:07:34.658916+0000","last_unstale":"2023-10-09T04:08:27.631298+0000","last_undegraded":"2023-10-09T04:08:27.631298+0000","last_fullsized":"2023-10-09T04:08:27.631298+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:50.983204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614490+0000","last_change":"2023-10-09T04:07:34.646776+0000","last_active":"2023-10-09T04:07:43.614490+0000","last_peered":"2023-10-09T04:07:43.614490+0000","last_clean":"2023-10-09T04:07:43.614490+0000","last_became_active":"2023-10-09T04:07:34.644325+0000","last_became_peered":"2023-10-09T04:07:34.644325+0000","last_unstale":"2023-10-09T04:07:43.614490+0000","last_undegraded":"2023-10-09T04:07:43.614490+0000","last_fullsized":"2023-10-09T04:07:43.614490+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:23.379485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.653353+0000","last_change":"2023-10-09T04:07:35.653186+0000","last_active":"2023-10-09T04:07:35.653353+0000","last_peered":"2023-10-09T04:07:35.653353+0000","last_clean":"2023-10-09T04:07:35.653353+0000","last_became_active":"2023-10-09T04:07:35.652908+0000","last_became_peered":"2023-10-09T04:07:35.652908+0000","last_unstale":"2023-10-09T04:07:35.653353+0000","last_undegraded":"2023-10-09T04:07:35.653353+0000","last_fullsized":"2023-10-09T04:07:35.653353+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:42.975764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286552+0000","last_change":"2023-10-09T04:07:34.638165+0000","last_active":"2023-10-09T04:07:37.286552+0000","last_peered":"2023-10-09T04:07:37.286552+0000","last_clean":"2023-10-09T04:07:37.286552+0000","last_became_active":"2023-10-09T04:07:34.637823+0000","last_became_peered":"2023-10-09T04:07:34.637823+0000","last_unstale":"2023-10-09T04:07:37.286552+0000","last_undegraded":"2023-10-09T04:07:37.286552+0000","last_fullsized":"2023-10-09T04:07:37.286552+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:13.118351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785812+0000","last_change":"2023-10-09T04:07:34.676759+0000","last_active":"2023-10-09T04:07:36.785812+0000","last_peered":"2023-10-09T04:07:36.785812+0000","last_clean":"2023-10-09T04:07:36.785812+0000","last_became_active":"2023-10-09T04:07:34.676294+0000","last_became_peered":"2023-10-09T04:07:34.676294+0000","last_unstale":"2023-10-09T04:07:36.785812+0000","last_undegraded":"2023-10-09T04:07:36.785812+0000","last_fullsized":"2023-10-09T04:07:36.785812+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:44.387110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614227+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:07:43.614227+0000","last_peered":"2023-10-09T04:07:43.614227+0000","last_clean":"2023-10-09T04:07:43.614227+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:07:43.614227+0000","last_undegraded":"2023-10-09T04:07:43.614227+0000","last_fullsized":"2023-10-09T04:07:43.614227+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.786034+0000","last_change":"2023-10-09T04:07:34.659699+0000","last_active":"2023-10-09T04:07:36.786034+0000","last_peered":"2023-10-09T04:07:36.786034+0000","last_clean":"2023-10-09T04:07:36.786034+0000","last_became_active":"2023-10-09T04:07:34.659189+0000","last_became_peered":"2023-10-09T04:07:34.659189+0000","last_unstale":"2023-10-09T04:07:36.786034+0000","last_undegraded":"2023-10-09T04:07:36.786034+0000","last_fullsized":"2023-10-09T04:07:36.786034+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:44.119093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614262+0000","last_change":"2023-10-09T04:07:34.631386+0000","last_active":"2023-10-09T04:07:43.614262+0000","last_peered":"2023-10-09T04:07:43.614262+0000","last_clean":"2023-10-09T04:07:43.614262+0000","last_became_active":"2023-10-09T04:07:34.631090+0000","last_became_peered":"2023-10-09T04:07:34.631090+0000","last_unstale":"2023-10-09T04:07:43.614262+0000","last_undegraded":"2023-10-09T04:07:43.614262+0000","last_fullsized":"2023-10-09T04:07:43.614262+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:05.633909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.974493+0000","last_change":"2023-10-09T04:07:34.659338+0000","last_active":"2023-10-09T04:07:36.974493+0000","last_peered":"2023-10-09T04:07:36.974493+0000","last_clean":"2023-10-09T04:07:36.974493+0000","last_became_active":"2023-10-09T04:07:34.658997+0000","last_became_peered":"2023-10-09T04:07:34.658997+0000","last_unstale":"2023-10-09T04:07:36.974493+0000","last_undegraded":"2023-10-09T04:07:36.974493+0000","last_fullsized":"2023-10-09T04:07:36.974493+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:38.217792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614676+0000","last_change":"2023-10-09T04:07:34.642042+0000","last_active":"2023-10-09T04:07:43.614676+0000","last_peered":"2023-10-09T04:07:43.614676+0000","last_clean":"2023-10-09T04:07:43.614676+0000","last_became_active":"2023-10-09T04:07:34.641579+0000","last_became_peered":"2023-10-09T04:07:34.641579+0000","last_unstale":"2023-10-09T04:07:43.614676+0000","last_undegraded":"2023-10-09T04:07:43.614676+0000","last_fullsized":"2023-10-09T04:07:43.614676+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:44.869504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785755+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:07:36.785755+0000","last_peered":"2023-10-09T04:07:36.785755+0000","last_clean":"2023-10-09T04:07:36.785755+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:07:36.785755+0000","last_undegraded":"2023-10-09T04:07:36.785755+0000","last_fullsized":"2023-10-09T04:07:36.785755+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:48.392870+0000","last_change":"2023-10-09T04:07:34.645613+0000","last_active":"2023-10-09T04:07:48.392870+0000","last_peered":"2023-10-09T04:07:48.392870+0000","last_clean":"2023-10-09T04:07:48.392870+0000","last_became_active":"2023-10-09T04:07:34.645309+0000","last_became_peered":"2023-10-09T04:07:34.645309+0000","last_unstale":"2023-10-09T04:07:48.392870+0000","last_undegraded":"2023-10-09T04:07:48.392870+0000","last_fullsized":"2023-10-09T04:07:48.392870+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:07.674226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286308+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:07:37.286308+0000","last_peered":"2023-10-09T04:07:37.286308+0000","last_clean":"2023-10-09T04:07:37.286308+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:07:37.286308+0000","last_undegraded":"2023-10-09T04:07:37.286308+0000","last_fullsized":"2023-10-09T04:07:37.286308+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030979+0000","last_change":"2023-10-09T04:07:36.030076+0000","last_active":"2023-10-09T04:07:36.030979+0000","last_peered":"2023-10-09T04:07:36.030979+0000","last_clean":"2023-10-09T04:07:36.030979+0000","last_became_active":"2023-10-09T04:07:36.029271+0000","last_became_peered":"2023-10-09T04:07:36.029271+0000","last_unstale":"2023-10-09T04:07:36.030979+0000","last_undegraded":"2023-10-09T04:07:36.030979+0000","last_fullsized":"2023-10-09T04:07:36.030979+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:12.758916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647183+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:07:36.647183+0000","last_peered":"2023-10-09T04:07:36.647183+0000","last_clean":"2023-10-09T04:07:36.647183+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:07:36.647183+0000","last_undegraded":"2023-10-09T04:07:36.647183+0000","last_fullsized":"2023-10-09T04:07:36.647183+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.615068+0000","last_change":"2023-10-09T04:07:34.643380+0000","last_active":"2023-10-09T04:07:43.615068+0000","last_peered":"2023-10-09T04:07:43.615068+0000","last_clean":"2023-10-09T04:07:43.615068+0000","last_became_active":"2023-10-09T04:07:34.643086+0000","last_became_peered":"2023-10-09T04:07:34.643086+0000","last_unstale":"2023-10-09T04:07:43.615068+0000","last_undegraded":"2023-10-09T04:07:43.615068+0000","last_fullsized":"2023-10-09T04:07:43.615068+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:13.152038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646307+0000","last_change":"2023-10-09T04:07:36.030238+0000","last_active":"2023-10-09T04:07:36.646307+0000","last_peered":"2023-10-09T04:07:36.646307+0000","last_clean":"2023-10-09T04:07:36.646307+0000","last_became_active":"2023-10-09T04:07:36.029932+0000","last_became_peered":"2023-10-09T04:07:36.029932+0000","last_unstale":"2023-10-09T04:07:36.646307+0000","last_undegraded":"2023-10-09T04:07:36.646307+0000","last_fullsized":"2023-10-09T04:07:36.646307+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:25.269223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647031+0000","last_change":"2023-10-09T04:07:34.633885+0000","last_active":"2023-10-09T04:07:36.647031+0000","last_peered":"2023-10-09T04:07:36.647031+0000","last_clean":"2023-10-09T04:07:36.647031+0000","last_became_active":"2023-10-09T04:07:34.633345+0000","last_became_peered":"2023-10-09T04:07:34.633345+0000","last_unstale":"2023-10-09T04:07:36.647031+0000","last_undegraded":"2023-10-09T04:07:36.647031+0000","last_fullsized":"2023-10-09T04:07:36.647031+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:42.728405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.060025+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:07:37.060025+0000","last_peered":"2023-10-09T04:07:37.060025+0000","last_clean":"2023-10-09T04:07:37.060025+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:07:37.060025+0000","last_undegraded":"2023-10-09T04:07:37.060025+0000","last_fullsized":"2023-10-09T04:07:37.060025+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646816+0000","last_change":"2023-10-09T04:07:35.800119+0000","last_active":"2023-10-09T04:07:36.646816+0000","last_peered":"2023-10-09T04:07:36.646816+0000","last_clean":"2023-10-09T04:07:36.646816+0000","last_became_active":"2023-10-09T04:07:35.799858+0000","last_became_peered":"2023-10-09T04:07:35.799858+0000","last_unstale":"2023-10-09T04:07:36.646816+0000","last_undegraded":"2023-10-09T04:07:36.646816+0000","last_fullsized":"2023-10-09T04:07:36.646816+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:04.465377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646162+0000","last_change":"2023-10-09T04:07:34.645437+0000","last_active":"2023-10-09T04:07:36.646162+0000","last_peered":"2023-10-09T04:07:36.646162+0000","last_clean":"2023-10-09T04:07:36.646162+0000","last_became_active":"2023-10-09T04:07:34.645140+0000","last_became_peered":"2023-10-09T04:07:34.645140+0000","last_unstale":"2023-10-09T04:07:36.646162+0000","last_undegraded":"2023-10-09T04:07:36.646162+0000","last_fullsized":"2023-10-09T04:07:36.646162+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:45.140472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.059769+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:07:37.059769+0000","last_peered":"2023-10-09T04:07:37.059769+0000","last_clean":"2023-10-09T04:07:37.059769+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:07:37.059769+0000","last_undegraded":"2023-10-09T04:07:37.059769+0000","last_fullsized":"2023-10-09T04:07:37.059769+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286341+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:07:37.286341+0000","last_peered":"2023-10-09T04:07:37.286341+0000","last_clean":"2023-10-09T04:07:37.286341+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:07:37.286341+0000","last_undegraded":"2023-10-09T04:07:37.286341+0000","last_fullsized":"2023-10-09T04:07:37.286341+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652736+0000","last_change":"2023-10-09T04:07:34.639655+0000","last_active":"2023-10-09T04:07:35.652736+0000","last_peered":"2023-10-09T04:07:35.652736+0000","last_clean":"2023-10-09T04:07:35.652736+0000","last_became_active":"2023-10-09T04:07:34.639321+0000","last_became_peered":"2023-10-09T04:07:34.639321+0000","last_unstale":"2023-10-09T04:07:35.652736+0000","last_undegraded":"2023-10-09T04:07:35.652736+0000","last_fullsized":"2023-10-09T04:07:35.652736+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:25.904866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":277,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.059724+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:07:37.059724+0000","last_peered":"2023-10-09T04:07:37.059724+0000","last_clean":"2023-10-09T04:07:37.059724+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:07:37.059724+0000","last_undegraded":"2023-10-09T04:07:37.059724+0000","last_fullsized":"2023-10-09T04:07:37.059724+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647114+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:07:36.647114+0000","last_peered":"2023-10-09T04:07:36.647114+0000","last_clean":"2023-10-09T04:07:36.647114+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:07:36.647114+0000","last_undegraded":"2023-10-09T04:07:36.647114+0000","last_fullsized":"2023-10-09T04:07:36.647114+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647234+0000","last_change":"2023-10-09T04:07:34.676079+0000","last_active":"2023-10-09T04:07:36.647234+0000","last_peered":"2023-10-09T04:07:36.647234+0000","last_clean":"2023-10-09T04:07:36.647234+0000","last_became_active":"2023-10-09T04:07:34.675776+0000","last_became_peered":"2023-10-09T04:07:34.675776+0000","last_unstale":"2023-10-09T04:07:36.647234+0000","last_undegraded":"2023-10-09T04:07:36.647234+0000","last_fullsized":"2023-10-09T04:07:36.647234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:26.867406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646483+0000","last_change":"2023-10-09T04:07:34.632817+0000","last_active":"2023-10-09T04:07:36.646483+0000","last_peered":"2023-10-09T04:07:36.646483+0000","last_clean":"2023-10-09T04:07:36.646483+0000","last_became_active":"2023-10-09T04:07:34.632434+0000","last_became_peered":"2023-10-09T04:07:34.632434+0000","last_unstale":"2023-10-09T04:07:36.646483+0000","last_undegraded":"2023-10-09T04:07:36.646483+0000","last_fullsized":"2023-10-09T04:07:36.646483+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:37.018559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.974369+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:07:36.974369+0000","last_peered":"2023-10-09T04:07:36.974369+0000","last_clean":"2023-10-09T04:07:36.974369+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:07:36.974369+0000","last_undegraded":"2023-10-09T04:07:36.974369+0000","last_fullsized":"2023-10-09T04:07:36.974369+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646941+0000","last_change":"2023-10-09T04:07:34.659865+0000","last_active":"2023-10-09T04:07:36.646941+0000","last_peered":"2023-10-09T04:07:36.646941+0000","last_clean":"2023-10-09T04:07:36.646941+0000","last_became_active":"2023-10-09T04:07:34.659515+0000","last_became_peered":"2023-10-09T04:07:34.659515+0000","last_unstale":"2023-10-09T04:07:36.646941+0000","last_undegraded":"2023-10-09T04:07:36.646941+0000","last_fullsized":"2023-10-09T04:07:36.646941+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:07.703247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785841+0000","last_change":"2023-10-09T04:07:34.641445+0000","last_active":"2023-10-09T04:07:36.785841+0000","last_peered":"2023-10-09T04:07:36.785841+0000","last_clean":"2023-10-09T04:07:36.785841+0000","last_became_active":"2023-10-09T04:07:34.641181+0000","last_became_peered":"2023-10-09T04:07:34.641181+0000","last_unstale":"2023-10-09T04:07:36.785841+0000","last_undegraded":"2023-10-09T04:07:36.785841+0000","last_fullsized":"2023-10-09T04:07:36.785841+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.897577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647089+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:07:36.647089+0000","last_peered":"2023-10-09T04:07:36.647089+0000","last_clean":"2023-10-09T04:07:36.647089+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:07:36.647089+0000","last_undegraded":"2023-10-09T04:07:36.647089+0000","last_fullsized":"2023-10-09T04:07:36.647089+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646911+0000","last_change":"2023-10-09T04:07:34.675845+0000","last_active":"2023-10-09T04:07:36.646911+0000","last_peered":"2023-10-09T04:07:36.646911+0000","last_clean":"2023-10-09T04:07:36.646911+0000","last_became_active":"2023-10-09T04:07:34.675371+0000","last_became_peered":"2023-10-09T04:07:34.675371+0000","last_unstale":"2023-10-09T04:07:36.646911+0000","last_undegraded":"2023-10-09T04:07:36.646911+0000","last_fullsized":"2023-10-09T04:07:36.646911+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:45.429941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647238+0000","last_change":"2023-10-09T04:07:34.640999+0000","last_active":"2023-10-09T04:07:36.647238+0000","last_peered":"2023-10-09T04:07:36.647238+0000","last_clean":"2023-10-09T04:07:36.647238+0000","last_became_active":"2023-10-09T04:07:34.640725+0000","last_became_peered":"2023-10-09T04:07:34.640725+0000","last_unstale":"2023-10-09T04:07:36.647238+0000","last_undegraded":"2023-10-09T04:07:36.647238+0000","last_fullsized":"2023-10-09T04:07:36.647238+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:45.119257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646853+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:07:36.646853+0000","last_peered":"2023-10-09T04:07:36.646853+0000","last_clean":"2023-10-09T04:07:36.646853+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:07:36.646853+0000","last_undegraded":"2023-10-09T04:07:36.646853+0000","last_fullsized":"2023-10-09T04:07:36.646853+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647296+0000","last_change":"2023-10-09T04:07:34.662253+0000","last_active":"2023-10-09T04:07:36.647296+0000","last_peered":"2023-10-09T04:07:36.647296+0000","last_clean":"2023-10-09T04:07:36.647296+0000","last_became_active":"2023-10-09T04:07:34.661919+0000","last_became_peered":"2023-10-09T04:07:34.661919+0000","last_unstale":"2023-10-09T04:07:36.647296+0000","last_undegraded":"2023-10-09T04:07:36.647296+0000","last_fullsized":"2023-10-09T04:07:36.647296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:35.167033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652246+0000","last_change":"2023-10-09T04:07:34.651433+0000","last_active":"2023-10-09T04:07:35.652246+0000","last_peered":"2023-10-09T04:07:35.652246+0000","last_clean":"2023-10-09T04:07:35.652246+0000","last_became_active":"2023-10-09T04:07:34.650984+0000","last_became_peered":"2023-10-09T04:07:34.650984+0000","last_unstale":"2023-10-09T04:07:35.652246+0000","last_undegraded":"2023-10-09T04:07:35.652246+0000","last_fullsized":"2023-10-09T04:07:35.652246+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:57.513271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.786015+0000","last_change":"2023-10-09T04:07:34.659514+0000","last_active":"2023-10-09T04:07:36.786015+0000","last_peered":"2023-10-09T04:07:36.786015+0000","last_clean":"2023-10-09T04:07:36.786015+0000","last_became_active":"2023-10-09T04:07:34.659212+0000","last_became_peered":"2023-10-09T04:07:34.659212+0000","last_unstale":"2023-10-09T04:07:36.786015+0000","last_undegraded":"2023-10-09T04:07:36.786015+0000","last_fullsized":"2023-10-09T04:07:36.786015+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:30.189193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785776+0000","last_change":"2023-10-09T04:07:34.630774+0000","last_active":"2023-10-09T04:07:36.785776+0000","last_peered":"2023-10-09T04:07:36.785776+0000","last_clean":"2023-10-09T04:07:36.785776+0000","last_became_active":"2023-10-09T04:07:34.630525+0000","last_became_peered":"2023-10-09T04:07:34.630525+0000","last_unstale":"2023-10-09T04:07:36.785776+0000","last_undegraded":"2023-10-09T04:07:36.785776+0000","last_fullsized":"2023-10-09T04:07:36.785776+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:05.578748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.786134+0000","last_change":"2023-10-09T04:07:34.659260+0000","last_active":"2023-10-09T04:07:36.786134+0000","last_peered":"2023-10-09T04:07:36.786134+0000","last_clean":"2023-10-09T04:07:36.786134+0000","last_became_active":"2023-10-09T04:07:34.658976+0000","last_became_peered":"2023-10-09T04:07:34.658976+0000","last_unstale":"2023-10-09T04:07:36.786134+0000","last_undegraded":"2023-10-09T04:07:36.786134+0000","last_fullsized":"2023-10-09T04:07:36.786134+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:35.014675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286183+0000","last_change":"2023-10-09T04:07:34.634417+0000","last_active":"2023-10-09T04:07:37.286183+0000","last_peered":"2023-10-09T04:07:37.286183+0000","last_clean":"2023-10-09T04:07:37.286183+0000","last_became_active":"2023-10-09T04:07:34.634183+0000","last_became_peered":"2023-10-09T04:07:34.634183+0000","last_unstale":"2023-10-09T04:07:37.286183+0000","last_undegraded":"2023-10-09T04:07:37.286183+0000","last_fullsized":"2023-10-09T04:07:37.286183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:13.162009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785888+0000","last_change":"2023-10-09T04:07:34.655573+0000","last_active":"2023-10-09T04:07:36.785888+0000","last_peered":"2023-10-09T04:07:36.785888+0000","last_clean":"2023-10-09T04:07:36.785888+0000","last_became_active":"2023-10-09T04:07:34.655326+0000","last_became_peered":"2023-10-09T04:07:34.655326+0000","last_unstale":"2023-10-09T04:07:36.785888+0000","last_undegraded":"2023-10-09T04:07:36.785888+0000","last_fullsized":"2023-10-09T04:07:36.785888+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:07.510114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614867+0000","last_change":"2023-10-09T04:07:34.643373+0000","last_active":"2023-10-09T04:07:43.614867+0000","last_peered":"2023-10-09T04:07:43.614867+0000","last_clean":"2023-10-09T04:07:43.614867+0000","last_became_active":"2023-10-09T04:07:34.643033+0000","last_became_peered":"2023-10-09T04:07:34.643033+0000","last_unstale":"2023-10-09T04:07:43.614867+0000","last_undegraded":"2023-10-09T04:07:43.614867+0000","last_fullsized":"2023-10-09T04:07:43.614867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:57.618154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2023-10-09T04:08:27.629117+0000","last_change":"2023-10-09T04:08:27.629346+0000","last_active":"2023-10-09T04:08:27.629086+0000","last_peered":"2023-10-09T04:07:36.646602+0000","last_clean":"2023-10-09T04:07:36.646602+0000","last_became_active":"2023-10-09T04:07:36.284375+0000","last_became_peered":"2023-10-09T04:07:36.284375+0000","last_unstale":"2023-10-09T04:08:27.629117+0000","last_undegraded":"2023-10-09T04:08:27.629117+0000","last_fullsized":"2023-10-09T04:08:27.629117+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:40.978215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4,5],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647202+0000","last_change":"2023-10-09T04:07:34.640164+0000","last_active":"2023-10-09T04:07:36.647202+0000","last_peered":"2023-10-09T04:07:36.647202+0000","last_clean":"2023-10-09T04:07:36.647202+0000","last_became_active":"2023-10-09T04:07:34.639871+0000","last_became_peered":"2023-10-09T04:07:34.639871+0000","last_unstale":"2023-10-09T04:07:36.647202+0000","last_undegraded":"2023-10-09T04:07:36.647202+0000","last_fullsized":"2023-10-09T04:07:36.647202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:59.659674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2023-10-09T04:08:27.634388+0000","last_change":"2023-10-09T04:08:27.634650+0000","last_active":"2023-10-09T04:08:27.634356+0000","last_peered":"2023-10-09T04:07:37.287391+0000","last_clean":"2023-10-09T04:07:37.287391+0000","last_became_active":"2023-10-09T04:07:34.677024+0000","last_became_peered":"2023-10-09T04:07:34.677024+0000","last_unstale":"2023-10-09T04:08:27.634388+0000","last_undegraded":"2023-10-09T04:08:27.634388+0000","last_fullsized":"2023-10-09T04:08:27.634388+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:45.749116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.617262+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:07:43.617262+0000","last_peered":"2023-10-09T04:07:43.617262+0000","last_clean":"2023-10-09T04:07:43.617262+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:07:43.617262+0000","last_undegraded":"2023-10-09T04:07:43.617262+0000","last_fullsized":"2023-10-09T04:07:43.617262+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785688+0000","last_change":"2023-10-09T04:07:34.659608+0000","last_active":"2023-10-09T04:07:36.785688+0000","last_peered":"2023-10-09T04:07:36.785688+0000","last_clean":"2023-10-09T04:07:36.785688+0000","last_became_active":"2023-10-09T04:07:34.658918+0000","last_became_peered":"2023-10-09T04:07:34.658918+0000","last_unstale":"2023-10-09T04:07:36.785688+0000","last_undegraded":"2023-10-09T04:07:36.785688+0000","last_fullsized":"2023-10-09T04:07:36.785688+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:42.508338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616507+0000","last_change":"2023-10-09T04:07:34.631609+0000","last_active":"2023-10-09T04:07:43.616507+0000","last_peered":"2023-10-09T04:07:43.616507+0000","last_clean":"2023-10-09T04:07:43.616507+0000","last_became_active":"2023-10-09T04:07:34.631327+0000","last_became_peered":"2023-10-09T04:07:34.631327+0000","last_unstale":"2023-10-09T04:07:43.616507+0000","last_undegraded":"2023-10-09T04:07:43.616507+0000","last_fullsized":"2023-10-09T04:07:43.616507+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:51.678421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647005+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:07:36.647005+0000","last_peered":"2023-10-09T04:07:36.647005+0000","last_clean":"2023-10-09T04:07:36.647005+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:07:36.647005+0000","last_undegraded":"2023-10-09T04:07:36.647005+0000","last_fullsized":"2023-10-09T04:07:36.647005+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616462+0000","last_change":"2023-10-09T04:07:34.642317+0000","last_active":"2023-10-09T04:07:43.616462+0000","last_peered":"2023-10-09T04:07:43.616462+0000","last_clean":"2023-10-09T04:07:43.616462+0000","last_became_active":"2023-10-09T04:07:34.642028+0000","last_became_peered":"2023-10-09T04:07:34.642028+0000","last_unstale":"2023-10-09T04:07:43.616462+0000","last_undegraded":"2023-10-09T04:07:43.616462+0000","last_fullsized":"2023-10-09T04:07:43.616462+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:12.984276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030869+0000","last_change":"2023-10-09T04:07:36.029846+0000","last_active":"2023-10-09T04:07:36.030869+0000","last_peered":"2023-10-09T04:07:36.030869+0000","last_clean":"2023-10-09T04:07:36.030869+0000","last_became_active":"2023-10-09T04:07:36.029223+0000","last_became_peered":"2023-10-09T04:07:36.029223+0000","last_unstale":"2023-10-09T04:07:36.030869+0000","last_undegraded":"2023-10-09T04:07:36.030869+0000","last_fullsized":"2023-10-09T04:07:36.030869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:12.111167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616194+0000","last_change":"2023-10-09T04:07:34.640376+0000","last_active":"2023-10-09T04:07:43.616194+0000","last_peered":"2023-10-09T04:07:43.616194+0000","last_clean":"2023-10-09T04:07:43.616194+0000","last_became_active":"2023-10-09T04:07:34.640073+0000","last_became_peered":"2023-10-09T04:07:34.640073+0000","last_unstale":"2023-10-09T04:07:43.616194+0000","last_undegraded":"2023-10-09T04:07:43.616194+0000","last_fullsized":"2023-10-09T04:07:43.616194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:16.684588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646576+0000","last_change":"2023-10-09T04:07:36.029535+0000","last_active":"2023-10-09T04:07:36.646576+0000","last_peered":"2023-10-09T04:07:36.646576+0000","last_clean":"2023-10-09T04:07:36.646576+0000","last_became_active":"2023-10-09T04:07:36.029175+0000","last_became_peered":"2023-10-09T04:07:36.029175+0000","last_unstale":"2023-10-09T04:07:36.646576+0000","last_undegraded":"2023-10-09T04:07:36.646576+0000","last_fullsized":"2023-10-09T04:07:36.646576+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:34.595934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.059723+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:07:37.059723+0000","last_peered":"2023-10-09T04:07:37.059723+0000","last_clean":"2023-10-09T04:07:37.059723+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:07:37.059723+0000","last_undegraded":"2023-10-09T04:07:37.059723+0000","last_fullsized":"2023-10-09T04:07:37.059723+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647210+0000","last_change":"2023-10-09T04:07:36.029411+0000","last_active":"2023-10-09T04:07:36.647210+0000","last_peered":"2023-10-09T04:07:36.647210+0000","last_clean":"2023-10-09T04:07:36.647210+0000","last_became_active":"2023-10-09T04:07:36.029109+0000","last_became_peered":"2023-10-09T04:07:36.029109+0000","last_unstale":"2023-10-09T04:07:36.647210+0000","last_undegraded":"2023-10-09T04:07:36.647210+0000","last_fullsized":"2023-10-09T04:07:36.647210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:05.311520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646301+0000","last_change":"2023-10-09T04:07:34.646539+0000","last_active":"2023-10-09T04:07:36.646301+0000","last_peered":"2023-10-09T04:07:36.646301+0000","last_clean":"2023-10-09T04:07:36.646301+0000","last_became_active":"2023-10-09T04:07:34.646243+0000","last_became_peered":"2023-10-09T04:07:34.646243+0000","last_unstale":"2023-10-09T04:07:36.646301+0000","last_undegraded":"2023-10-09T04:07:36.646301+0000","last_fullsized":"2023-10-09T04:07:36.646301+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:01.969093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030827+0000","last_change":"2023-10-09T04:07:36.029585+0000","last_active":"2023-10-09T04:07:36.030827+0000","last_peered":"2023-10-09T04:07:36.030827+0000","last_clean":"2023-10-09T04:07:36.030827+0000","last_became_active":"2023-10-09T04:07:36.029157+0000","last_became_peered":"2023-10-09T04:07:36.029157+0000","last_unstale":"2023-10-09T04:07:36.030827+0000","last_undegraded":"2023-10-09T04:07:36.030827+0000","last_fullsized":"2023-10-09T04:07:36.030827+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:19.819664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.617011+0000","last_change":"2023-10-09T04:07:34.632203+0000","last_active":"2023-10-09T04:07:43.617011+0000","last_peered":"2023-10-09T04:07:43.617011+0000","last_clean":"2023-10-09T04:07:43.617011+0000","last_became_active":"2023-10-09T04:07:34.631969+0000","last_became_peered":"2023-10-09T04:07:34.631969+0000","last_unstale":"2023-10-09T04:07:43.617011+0000","last_undegraded":"2023-10-09T04:07:43.617011+0000","last_fullsized":"2023-10-09T04:07:43.617011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:01.619892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785865+0000","last_change":"2023-10-09T04:07:34.659790+0000","last_active":"2023-10-09T04:07:36.785865+0000","last_peered":"2023-10-09T04:07:36.785865+0000","last_clean":"2023-10-09T04:07:36.785865+0000","last_became_active":"2023-10-09T04:07:34.659565+0000","last_became_peered":"2023-10-09T04:07:34.659565+0000","last_unstale":"2023-10-09T04:07:36.785865+0000","last_undegraded":"2023-10-09T04:07:36.785865+0000","last_fullsized":"2023-10-09T04:07:36.785865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:32.044413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.617433+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:07:43.617433+0000","last_peered":"2023-10-09T04:07:43.617433+0000","last_clean":"2023-10-09T04:07:43.617433+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:07:43.617433+0000","last_undegraded":"2023-10-09T04:07:43.617433+0000","last_fullsized":"2023-10-09T04:07:43.617433+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646837+0000","last_change":"2023-10-09T04:07:34.661425+0000","last_active":"2023-10-09T04:07:36.646837+0000","last_peered":"2023-10-09T04:07:36.646837+0000","last_clean":"2023-10-09T04:07:36.646837+0000","last_became_active":"2023-10-09T04:07:34.661099+0000","last_became_peered":"2023-10-09T04:07:34.661099+0000","last_unstale":"2023-10-09T04:07:36.646837+0000","last_undegraded":"2023-10-09T04:07:36.646837+0000","last_fullsized":"2023-10-09T04:07:36.646837+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:53.588610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616956+0000","last_change":"2023-10-09T04:07:34.641392+0000","last_active":"2023-10-09T04:07:43.616956+0000","last_peered":"2023-10-09T04:07:43.616956+0000","last_clean":"2023-10-09T04:07:43.616956+0000","last_became_active":"2023-10-09T04:07:34.641037+0000","last_became_peered":"2023-10-09T04:07:34.641037+0000","last_unstale":"2023-10-09T04:07:43.616956+0000","last_undegraded":"2023-10-09T04:07:43.616956+0000","last_fullsized":"2023-10-09T04:07:43.616956+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:43.467265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.031551+0000","last_change":"2023-10-09T04:07:36.031224+0000","last_active":"2023-10-09T04:07:36.031551+0000","last_peered":"2023-10-09T04:07:36.031551+0000","last_clean":"2023-10-09T04:07:36.031551+0000","last_became_active":"2023-10-09T04:07:36.030834+0000","last_became_peered":"2023-10-09T04:07:36.030834+0000","last_unstale":"2023-10-09T04:07:36.031551+0000","last_undegraded":"2023-10-09T04:07:36.031551+0000","last_fullsized":"2023-10-09T04:07:36.031551+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:21.909762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647243+0000","last_change":"2023-10-09T04:07:34.631933+0000","last_active":"2023-10-09T04:07:36.647243+0000","last_peered":"2023-10-09T04:07:36.647243+0000","last_clean":"2023-10-09T04:07:36.647243+0000","last_became_active":"2023-10-09T04:07:34.631667+0000","last_became_peered":"2023-10-09T04:07:34.631667+0000","last_unstale":"2023-10-09T04:07:36.647243+0000","last_undegraded":"2023-10-09T04:07:36.647243+0000","last_fullsized":"2023-10-09T04:07:36.647243+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:58.644048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.801084+0000","last_change":"2023-10-09T04:07:35.800509+0000","last_active":"2023-10-09T04:07:35.801084+0000","last_peered":"2023-10-09T04:07:35.801084+0000","last_clean":"2023-10-09T04:07:35.801084+0000","last_became_active":"2023-10-09T04:07:35.800155+0000","last_became_peered":"2023-10-09T04:07:35.800155+0000","last_unstale":"2023-10-09T04:07:35.801084+0000","last_undegraded":"2023-10-09T04:07:35.801084+0000","last_fullsized":"2023-10-09T04:07:35.801084+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:50.929412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647129+0000","last_change":"2023-10-09T04:07:34.646997+0000","last_active":"2023-10-09T04:07:36.647129+0000","last_peered":"2023-10-09T04:07:36.647129+0000","last_clean":"2023-10-09T04:07:36.647129+0000","last_became_active":"2023-10-09T04:07:34.645625+0000","last_became_peered":"2023-10-09T04:07:34.645625+0000","last_unstale":"2023-10-09T04:07:36.647129+0000","last_undegraded":"2023-10-09T04:07:36.647129+0000","last_fullsized":"2023-10-09T04:07:36.647129+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:40.584254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286258+0000","last_change":"2023-10-09T04:07:34.676137+0000","last_active":"2023-10-09T04:07:37.286258+0000","last_peered":"2023-10-09T04:07:37.286258+0000","last_clean":"2023-10-09T04:07:37.286258+0000","last_became_active":"2023-10-09T04:07:34.675822+0000","last_became_peered":"2023-10-09T04:07:34.675822+0000","last_unstale":"2023-10-09T04:07:37.286258+0000","last_undegraded":"2023-10-09T04:07:37.286258+0000","last_fullsized":"2023-10-09T04:07:37.286258+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:06.758028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652383+0000","last_change":"2023-10-09T04:07:34.646983+0000","last_active":"2023-10-09T04:07:35.652383+0000","last_peered":"2023-10-09T04:07:35.652383+0000","last_clean":"2023-10-09T04:07:35.652383+0000","last_became_active":"2023-10-09T04:07:34.646623+0000","last_became_peered":"2023-10-09T04:07:34.646623+0000","last_unstale":"2023-10-09T04:07:35.652383+0000","last_undegraded":"2023-10-09T04:07:35.652383+0000","last_fullsized":"2023-10-09T04:07:35.652383+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.119527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646658+0000","last_change":"2023-10-09T04:07:36.029481+0000","last_active":"2023-10-09T04:07:36.646658+0000","last_peered":"2023-10-09T04:07:36.646658+0000","last_clean":"2023-10-09T04:07:36.646658+0000","last_became_active":"2023-10-09T04:07:36.029167+0000","last_became_peered":"2023-10-09T04:07:36.029167+0000","last_unstale":"2023-10-09T04:07:36.646658+0000","last_undegraded":"2023-10-09T04:07:36.646658+0000","last_fullsized":"2023-10-09T04:07:36.646658+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:42.409682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647521+0000","last_change":"2023-10-09T04:07:34.645964+0000","last_active":"2023-10-09T04:07:36.647521+0000","last_peered":"2023-10-09T04:07:36.647521+0000","last_clean":"2023-10-09T04:07:36.647521+0000","last_became_active":"2023-10-09T04:07:34.645619+0000","last_became_peered":"2023-10-09T04:07:34.645619+0000","last_unstale":"2023-10-09T04:07:36.647521+0000","last_undegraded":"2023-10-09T04:07:36.647521+0000","last_fullsized":"2023-10-09T04:07:36.647521+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:26.090109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.285502+0000","last_change":"2023-10-09T04:07:36.284775+0000","last_active":"2023-10-09T04:07:36.285502+0000","last_peered":"2023-10-09T04:07:36.285502+0000","last_clean":"2023-10-09T04:07:36.285502+0000","last_became_active":"2023-10-09T04:07:36.284501+0000","last_became_peered":"2023-10-09T04:07:36.284501+0000","last_unstale":"2023-10-09T04:07:36.285502+0000","last_undegraded":"2023-10-09T04:07:36.285502+0000","last_fullsized":"2023-10-09T04:07:36.285502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:17.182374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614371+0000","last_change":"2023-10-09T04:07:34.630544+0000","last_active":"2023-10-09T04:07:43.614371+0000","last_peered":"2023-10-09T04:07:43.614371+0000","last_clean":"2023-10-09T04:07:43.614371+0000","last_became_active":"2023-10-09T04:07:34.630230+0000","last_became_peered":"2023-10-09T04:07:34.630230+0000","last_unstale":"2023-10-09T04:07:43.614371+0000","last_undegraded":"2023-10-09T04:07:43.614371+0000","last_fullsized":"2023-10-09T04:07:43.614371+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:32.002588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0329999999999999,"15min":1.123},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76500000000000001,"5min":10.557,"15min":10.557},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.62}]},{"osd":1,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.83999999999999997,"15min":0.88800000000000001},"min":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71099999999999997,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.81599999999999995,"15min":0.86699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69799999999999995,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72999999999999998,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67100000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.64300000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.52000000000000002,"15min":0.53400000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.58399999999999996,"5min":1.391,"15min":1.391},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.63500000000000001,"15min":0.65400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69099999999999995,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.56799999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495653,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1268,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1298432,"data_stored":891208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54900000000000004,"15min":0.55400000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.751,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.505},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69399999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.63400000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56999999999999995,"15min":0.57799999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73299999999999998,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.70199999999999996,"5min":0.748,"15min":0.748},"last":0.622}]},{"osd":2,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73199999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.60299999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.65400000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48599999999999999,"15min":0.48199999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.70699999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48799999999999999,"15min":0.48699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.89200000000000002,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.75900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.77300000000000002,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.58399999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574910,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.39800000000000002,"15min":0.372},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.435,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.41399999999999998,"15min":0.42299999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.51500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.30499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:07:28 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.499,"15min":0.50700000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61499999999999999,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.44,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.75600000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.70099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.60899999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70899999999999996,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.504},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69899999999999995,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.62,"15min":0.628},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.85999999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66400000000000003,"15min":0.67600000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.78600000000000003,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.55700000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.65100000000000002,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.48099999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.59799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738435,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:07:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.32800000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.38400000000000001,"15min":0.34599999999999997},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53600000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.25600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.58599999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.308}]},{"osd":3,"last update":"Mon Oct 9 04:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.46800000000000003,"15min":0.47599999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.53200000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.378},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.50900000000000001,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.373,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77800000000000002,"5min":0.878,"15min":0.878},"last":0.498},{"interface":"front","average":{"1min":0.626,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84299999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.52200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61099999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.622,"15min":0.63800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77600000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80600000000000005,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79600000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:07:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65100000000000002,"15min":0.65900000000000003},"min":{"1min":0.46600000000000003,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80900000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.69099999999999995,"15min":0.68700000000000006},"min":{"1min":0.505,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.42499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70699999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.37},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.43099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.47599999999999998,"15min":0.47699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.48599999999999999,"15min":0.48399999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.35299999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.45400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.46999999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.70099999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.61599999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79400000000000004,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.62,"15min":0.61799999999999999},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.53900000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67400000000000004},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98099999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.629,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.60199999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182453,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.41899999999999998},"min":{"1min":0.20399999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.308},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41699999999999998,"15min":0.44900000000000001},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.27300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.43099999999999999,"15min":0.46300000000000002},"min":{"1min":0.255,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.54800000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.43099999999999999,"15min":0.47499999999999998},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56999999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.46400000000000002,"15min":0.496},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54000000000000004,"5min":0.745,"15min":0.745},"last":0.375},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.46400000000000002,"15min":0.503},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.63300000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.73399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.626,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.76300000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.624,"15min":0.63500000000000001},"min":{"1min":0.433,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77100000000000002,"5min":0.874,"15min":0.874},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.63,"15min":0.63900000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.748,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.61399999999999999,"15min":0.622},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76400000000000001,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.622}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73599999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.46500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81200000000000006,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018927,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.503,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.622},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.61099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.52600000000000002,"15min":0.51700000000000002},"min":{"1min":0.40200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.70799999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.432,"5min":0.37,"15min":0.37},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54100000000000004,"15min":0.52300000000000002},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.073,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.69799999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.439,"15min":0.42299999999999999},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.44400000000000001,"15min":0.42399999999999999},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.47899999999999998,"15min":0.47299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68899999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.66700000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69099999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.68400000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757291,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53000000000000003,"15min":0.52400000000000002},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65000000000000002,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.308}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71799999999999997,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.57699999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.443,"15min":0.41899999999999998},"min":{"1min":0.35499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.45400000000000001,"15min":0.42799999999999999},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.59199999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.45500000000000002,"15min":0.442},"min":{"1min":0.38700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.45900000000000002,"15min":0.443},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:08:30.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:08:30.766 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:32.511 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:32.512 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:32 smithi029 ceph-mon[101043]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:32.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:32 smithi029 ceph-mon[119131]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:32 smithi177 ceph-mon[106070]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:33.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:08:32 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:08:33.247 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":76,"stamp":"2023-10-09T04:08:30.776063+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385544,"kb_used_data":7240,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575672,"statfs":{"total":767960285184,"available":765517488128,"internally_reserved":0,"allocated":7413760,"data_stored":4174155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001756"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2023-10-09T04:08:27.631298+0000","last_change":"2023-10-09T04:08:27.631472+0000","last_active":"2023-10-09T04:08:27.631278+0000","last_peered":"2023-10-09T04:07:36.785613+0000","last_clean":"2023-10-09T04:07:36.785613+0000","last_became_active":"2023-10-09T04:07:34.658916+0000","last_became_peered":"2023-10-09T04:07:34.658916+0000","last_unstale":"2023-10-09T04:08:27.631298+0000","last_undegraded":"2023-10-09T04:08:27.631298+0000","last_fullsized":"2023-10-09T04:08:27.631298+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:50.983204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614490+0000","last_change":"2023-10-09T04:07:34.646776+0000","last_active":"2023-10-09T04:07:43.614490+0000","last_peered":"2023-10-09T04:07:43.614490+0000","last_clean":"2023-10-09T04:07:43.614490+0000","last_became_active":"2023-10-09T04:07:34.644325+0000","last_became_peered":"2023-10-09T04:07:34.644325+0000","last_unstale":"2023-10-09T04:07:43.614490+0000","last_undegraded":"2023-10-09T04:07:43.614490+0000","last_fullsized":"2023-10-09T04:07:43.614490+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:23.379485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.653353+0000","last_change":"2023-10-09T04:07:35.653186+0000","last_active":"2023-10-09T04:07:35.653353+0000","last_peered":"2023-10-09T04:07:35.653353+0000","last_clean":"2023-10-09T04:07:35.653353+0000","last_became_active":"2023-10-09T04:07:35.652908+0000","last_became_peered":"2023-10-09T04:07:35.652908+0000","last_unstale":"2023-10-09T04:07:35.653353+0000","last_undegraded":"2023-10-09T04:07:35.653353+0000","last_fullsized":"2023-10-09T04:07:35.653353+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:42.975764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286552+0000","last_change":"2023-10-09T04:07:34.638165+0000","last_active":"2023-10-09T04:07:37.286552+0000","last_peered":"2023-10-09T04:07:37.286552+0000","last_clean":"2023-10-09T04:07:37.286552+0000","last_became_active":"2023-10-09T04:07:34.637823+0000","last_became_peered":"2023-10-09T04:07:34.637823+0000","last_unstale":"2023-10-09T04:07:37.286552+0000","last_undegraded":"2023-10-09T04:07:37.286552+0000","last_fullsized":"2023-10-09T04:07:37.286552+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:13.118351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785812+0000","last_change":"2023-10-09T04:07:34.676759+0000","last_active":"2023-10-09T04:07:36.785812+0000","last_peered":"2023-10-09T04:07:36.785812+0000","last_clean":"2023-10-09T04:07:36.785812+0000","last_became_active":"2023-10-09T04:07:34.676294+0000","last_became_peered":"2023-10-09T04:07:34.676294+0000","last_unstale":"2023-10-09T04:07:36.785812+0000","last_undegraded":"2023-10-09T04:07:36.785812+0000","last_fullsized":"2023-10-09T04:07:36.785812+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:44.387110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614227+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:07:43.614227+0000","last_peered":"2023-10-09T04:07:43.614227+0000","last_clean":"2023-10-09T04:07:43.614227+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:07:43.614227+0000","last_undegraded":"2023-10-09T04:07:43.614227+0000","last_fullsized":"2023-10-09T04:07:43.614227+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.786034+0000","last_change":"2023-10-09T04:07:34.659699+0000","last_active":"2023-10-09T04:07:36.786034+0000","last_peered":"2023-10-09T04:07:36.786034+0000","last_clean":"2023-10-09T04:07:36.786034+0000","last_became_active":"2023-10-09T04:07:34.659189+0000","last_became_peered":"2023-10-09T04:07:34.659189+0000","last_unstale":"2023-10-09T04:07:36.786034+0000","last_undegraded":"2023-10-09T04:07:36.786034+0000","last_fullsized":"2023-10-09T04:07:36.786034+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:44.119093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614262+0000","last_change":"2023-10-09T04:07:34.631386+0000","last_active":"2023-10-09T04:07:43.614262+0000","last_peered":"2023-10-09T04:07:43.614262+0000","last_clean":"2023-10-09T04:07:43.614262+0000","last_became_active":"2023-10-09T04:07:34.631090+0000","last_became_peered":"2023-10-09T04:07:34.631090+0000","last_unstale":"2023-10-09T04:07:43.614262+0000","last_undegraded":"2023-10-09T04:07:43.614262+0000","last_fullsized":"2023-10-09T04:07:43.614262+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:05.633909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.974493+0000","last_change":"2023-10-09T04:07:34.659338+0000","last_active":"2023-10-09T04:07:36.974493+0000","last_peered":"2023-10-09T04:07:36.974493+0000","last_clean":"2023-10-09T04:07:36.974493+0000","last_became_active":"2023-10-09T04:07:34.658997+0000","last_became_peered":"2023-10-09T04:07:34.658997+0000","last_unstale":"2023-10-09T04:07:36.974493+0000","last_undegraded":"2023-10-09T04:07:36.974493+0000","last_fullsized":"2023-10-09T04:07:36.974493+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:38.217792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614676+0000","last_change":"2023-10-09T04:07:34.642042+0000","last_active":"2023-10-09T04:07:43.614676+0000","last_peered":"2023-10-09T04:07:43.614676+0000","last_clean":"2023-10-09T04:07:43.614676+0000","last_became_active":"2023-10-09T04:07:34.641579+0000","last_became_peered":"2023-10-09T04:07:34.641579+0000","last_unstale":"2023-10-09T04:07:43.614676+0000","last_undegraded":"2023-10-09T04:07:43.614676+0000","last_fullsized":"2023-10-09T04:07:43.614676+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:44.869504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785755+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:07:36.785755+0000","last_peered":"2023-10-09T04:07:36.785755+0000","last_clean":"2023-10-09T04:07:36.785755+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:07:36.785755+0000","last_undegraded":"2023-10-09T04:07:36.785755+0000","last_fullsized":"2023-10-09T04:07:36.785755+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:48.392870+0000","last_change":"2023-10-09T04:07:34.645613+0000","last_active":"2023-10-09T04:07:48.392870+0000","last_peered":"2023-10-09T04:07:48.392870+0000","last_clean":"2023-10-09T04:07:48.392870+0000","last_became_active":"2023-10-09T04:07:34.645309+0000","last_became_peered":"2023-10-09T04:07:34.645309+0000","last_unstale":"2023-10-09T04:07:48.392870+0000","last_undegraded":"2023-10-09T04:07:48.392870+0000","last_fullsized":"2023-10-09T04:07:48.392870+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:07.674226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286308+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:07:37.286308+0000","last_peered":"2023-10-09T04:07:37.286308+0000","last_clean":"2023-10-09T04:07:37.286308+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:07:37.286308+0000","last_undegraded":"2023-10-09T04:07:37.286308+0000","last_fullsized":"2023-10-09T04:07:37.286308+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030979+0000","last_change":"2023-10-09T04:07:36.030076+0000","last_active":"2023-10-09T04:07:36.030979+0000","last_peered":"2023-10-09T04:07:36.030979+0000","last_clean":"2023-10-09T04:07:36.030979+0000","last_became_active":"2023-10-09T04:07:36.029271+0000","last_became_peered":"2023-10-09T04:07:36.029271+0000","last_unstale":"2023-10-09T04:07:36.030979+0000","last_undegraded":"2023-10-09T04:07:36.030979+0000","last_fullsized":"2023-10-09T04:07:36.030979+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:12.758916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647183+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:07:36.647183+0000","last_peered":"2023-10-09T04:07:36.647183+0000","last_clean":"2023-10-09T04:07:36.647183+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:07:36.647183+0000","last_undegraded":"2023-10-09T04:07:36.647183+0000","last_fullsized":"2023-10-09T04:07:36.647183+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.615068+0000","last_change":"2023-10-09T04:07:34.643380+0000","last_active":"2023-10-09T04:07:43.615068+0000","last_peered":"2023-10-09T04:07:43.615068+0000","last_clean":"2023-10-09T04:07:43.615068+0000","last_became_active":"2023-10-09T04:07:34.643086+0000","last_became_peered":"2023-10-09T04:07:34.643086+0000","last_unstale":"2023-10-09T04:07:43.615068+0000","last_undegraded":"2023-10-09T04:07:43.615068+0000","last_fullsized":"2023-10-09T04:07:43.615068+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:13.152038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646307+0000","last_change":"2023-10-09T04:07:36.030238+0000","last_active":"2023-10-09T04:07:36.646307+0000","last_peered":"2023-10-09T04:07:36.646307+0000","last_clean":"2023-10-09T04:07:36.646307+0000","last_became_active":"2023-10-09T04:07:36.029932+0000","last_became_peered":"2023-10-09T04:07:36.029932+0000","last_unstale":"2023-10-09T04:07:36.646307+0000","last_undegraded":"2023-10-09T04:07:36.646307+0000","last_fullsized":"2023-10-09T04:07:36.646307+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:25.269223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647031+0000","last_change":"2023-10-09T04:07:34.633885+0000","last_active":"2023-10-09T04:07:36.647031+0000","last_peered":"2023-10-09T04:07:36.647031+0000","last_clean":"2023-10-09T04:07:36.647031+0000","last_became_active":"2023-10-09T04:07:34.633345+0000","last_became_peered":"2023-10-09T04:07:34.633345+0000","last_unstale":"2023-10-09T04:07:36.647031+0000","last_undegraded":"2023-10-09T04:07:36.647031+0000","last_fullsized":"2023-10-09T04:07:36.647031+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:42.728405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.060025+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:07:37.060025+0000","last_peered":"2023-10-09T04:07:37.060025+0000","last_clean":"2023-10-09T04:07:37.060025+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:07:37.060025+0000","last_undegraded":"2023-10-09T04:07:37.060025+0000","last_fullsized":"2023-10-09T04:07:37.060025+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646816+0000","last_change":"2023-10-09T04:07:35.800119+0000","last_active":"2023-10-09T04:07:36.646816+0000","last_peered":"2023-10-09T04:07:36.646816+0000","last_clean":"2023-10-09T04:07:36.646816+0000","last_became_active":"2023-10-09T04:07:35.799858+0000","last_became_peered":"2023-10-09T04:07:35.799858+0000","last_unstale":"2023-10-09T04:07:36.646816+0000","last_undegraded":"2023-10-09T04:07:36.646816+0000","last_fullsized":"2023-10-09T04:07:36.646816+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:04.465377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646162+0000","last_change":"2023-10-09T04:07:34.645437+0000","last_active":"2023-10-09T04:07:36.646162+0000","last_peered":"2023-10-09T04:07:36.646162+0000","last_clean":"2023-10-09T04:07:36.646162+0000","last_became_active":"2023-10-09T04:07:34.645140+0000","last_became_peered":"2023-10-09T04:07:34.645140+0000","last_unstale":"2023-10-09T04:07:36.646162+0000","last_undegraded":"2023-10-09T04:07:36.646162+0000","last_fullsized":"2023-10-09T04:07:36.646162+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:45.140472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.059769+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:07:37.059769+0000","last_peered":"2023-10-09T04:07:37.059769+0000","last_clean":"2023-10-09T04:07:37.059769+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:07:37.059769+0000","last_undegraded":"2023-10-09T04:07:37.059769+0000","last_fullsized":"2023-10-09T04:07:37.059769+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286341+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:07:37.286341+0000","last_peered":"2023-10-09T04:07:37.286341+0000","last_clean":"2023-10-09T04:07:37.286341+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:07:37.286341+0000","last_undegraded":"2023-10-09T04:07:37.286341+0000","last_fullsized":"2023-10-09T04:07:37.286341+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652736+0000","last_change":"2023-10-09T04:07:34.639655+0000","last_active":"2023-10-09T04:07:35.652736+0000","last_peered":"2023-10-09T04:07:35.652736+0000","last_clean":"2023-10-09T04:07:35.652736+0000","last_became_active":"2023-10-09T04:07:34.639321+0000","last_became_peered":"2023-10-09T04:07:34.639321+0000","last_unstale":"2023-10-09T04:07:35.652736+0000","last_undegraded":"2023-10-09T04:07:35.652736+0000","last_fullsized":"2023-10-09T04:07:35.652736+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:25.904866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":277,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.059724+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:07:37.059724+0000","last_peered":"2023-10-09T04:07:37.059724+0000","last_clean":"2023-10-09T04:07:37.059724+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:07:37.059724+0000","last_undegraded":"2023-10-09T04:07:37.059724+0000","last_fullsized":"2023-10-09T04:07:37.059724+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647114+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:07:36.647114+0000","last_peered":"2023-10-09T04:07:36.647114+0000","last_clean":"2023-10-09T04:07:36.647114+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:07:36.647114+0000","last_undegraded":"2023-10-09T04:07:36.647114+0000","last_fullsized":"2023-10-09T04:07:36.647114+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647234+0000","last_change":"2023-10-09T04:07:34.676079+0000","last_active":"2023-10-09T04:07:36.647234+0000","last_peered":"2023-10-09T04:07:36.647234+0000","last_clean":"2023-10-09T04:07:36.647234+0000","last_became_active":"2023-10-09T04:07:34.675776+0000","last_became_peered":"2023-10-09T04:07:34.675776+0000","last_unstale":"2023-10-09T04:07:36.647234+0000","last_undegraded":"2023-10-09T04:07:36.647234+0000","last_fullsized":"2023-10-09T04:07:36.647234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:26.867406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646483+0000","last_change":"2023-10-09T04:07:34.632817+0000","last_active":"2023-10-09T04:07:36.646483+0000","last_peered":"2023-10-09T04:07:36.646483+0000","last_clean":"2023-10-09T04:07:36.646483+0000","last_became_active":"2023-10-09T04:07:34.632434+0000","last_became_peered":"2023-10-09T04:07:34.632434+0000","last_unstale":"2023-10-09T04:07:36.646483+0000","last_undegraded":"2023-10-09T04:07:36.646483+0000","last_fullsized":"2023-10-09T04:07:36.646483+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:37.018559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.974369+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:07:36.974369+0000","last_peered":"2023-10-09T04:07:36.974369+0000","last_clean":"2023-10-09T04:07:36.974369+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:07:36.974369+0000","last_undegraded":"2023-10-09T04:07:36.974369+0000","last_fullsized":"2023-10-09T04:07:36.974369+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646941+0000","last_change":"2023-10-09T04:07:34.659865+0000","last_active":"2023-10-09T04:07:36.646941+0000","last_peered":"2023-10-09T04:07:36.646941+0000","last_clean":"2023-10-09T04:07:36.646941+0000","last_became_active":"2023-10-09T04:07:34.659515+0000","last_became_peered":"2023-10-09T04:07:34.659515+0000","last_unstale":"2023-10-09T04:07:36.646941+0000","last_undegraded":"2023-10-09T04:07:36.646941+0000","last_fullsized":"2023-10-09T04:07:36.646941+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:07.703247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785841+0000","last_change":"2023-10-09T04:07:34.641445+0000","last_active":"2023-10-09T04:07:36.785841+0000","last_peered":"2023-10-09T04:07:36.785841+0000","last_clean":"2023-10-09T04:07:36.785841+0000","last_became_active":"2023-10-09T04:07:34.641181+0000","last_became_peered":"2023-10-09T04:07:34.641181+0000","last_unstale":"2023-10-09T04:07:36.785841+0000","last_undegraded":"2023-10-09T04:07:36.785841+0000","last_fullsized":"2023-10-09T04:07:36.785841+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.897577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647089+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:07:36.647089+0000","last_peered":"2023-10-09T04:07:36.647089+0000","last_clean":"2023-10-09T04:07:36.647089+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:07:36.647089+0000","last_undegraded":"2023-10-09T04:07:36.647089+0000","last_fullsized":"2023-10-09T04:07:36.647089+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646911+0000","last_change":"2023-10-09T04:07:34.675845+0000","last_active":"2023-10-09T04:07:36.646911+0000","last_peered":"2023-10-09T04:07:36.646911+0000","last_clean":"2023-10-09T04:07:36.646911+0000","last_became_active":"2023-10-09T04:07:34.675371+0000","last_became_peered":"2023-10-09T04:07:34.675371+0000","last_unstale":"2023-10-09T04:07:36.646911+0000","last_undegraded":"2023-10-09T04:07:36.646911+0000","last_fullsized":"2023-10-09T04:07:36.646911+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:45.429941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647238+0000","last_change":"2023-10-09T04:07:34.640999+0000","last_active":"2023-10-09T04:07:36.647238+0000","last_peered":"2023-10-09T04:07:36.647238+0000","last_clean":"2023-10-09T04:07:36.647238+0000","last_became_active":"2023-10-09T04:07:34.640725+0000","last_became_peered":"2023-10-09T04:07:34.640725+0000","last_unstale":"2023-10-09T04:07:36.647238+0000","last_undegraded":"2023-10-09T04:07:36.647238+0000","last_fullsized":"2023-10-09T04:07:36.647238+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:45.119257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646853+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:07:36.646853+0000","last_peered":"2023-10-09T04:07:36.646853+0000","last_clean":"2023-10-09T04:07:36.646853+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:07:36.646853+0000","last_undegraded":"2023-10-09T04:07:36.646853+0000","last_fullsized":"2023-10-09T04:07:36.646853+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647296+0000","last_change":"2023-10-09T04:07:34.662253+0000","last_active":"2023-10-09T04:07:36.647296+0000","last_peered":"2023-10-09T04:07:36.647296+0000","last_clean":"2023-10-09T04:07:36.647296+0000","last_became_active":"2023-10-09T04:07:34.661919+0000","last_became_peered":"2023-10-09T04:07:34.661919+0000","last_unstale":"2023-10-09T04:07:36.647296+0000","last_undegraded":"2023-10-09T04:07:36.647296+0000","last_fullsized":"2023-10-09T04:07:36.647296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:35.167033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652246+0000","last_change":"2023-10-09T04:07:34.651433+0000","last_active":"2023-10-09T04:07:35.652246+0000","last_peered":"2023-10-09T04:07:35.652246+0000","last_clean":"2023-10-09T04:07:35.652246+0000","last_became_active":"2023-10-09T04:07:34.650984+0000","last_became_peered":"2023-10-09T04:07:34.650984+0000","last_unstale":"2023-10-09T04:07:35.652246+0000","last_undegraded":"2023-10-09T04:07:35.652246+0000","last_fullsized":"2023-10-09T04:07:35.652246+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:57.513271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.786015+0000","last_change":"2023-10-09T04:07:34.659514+0000","last_active":"2023-10-09T04:07:36.786015+0000","last_peered":"2023-10-09T04:07:36.786015+0000","last_clean":"2023-10-09T04:07:36.786015+0000","last_became_active":"2023-10-09T04:07:34.659212+0000","last_became_peered":"2023-10-09T04:07:34.659212+0000","last_unstale":"2023-10-09T04:07:36.786015+0000","last_undegraded":"2023-10-09T04:07:36.786015+0000","last_fullsized":"2023-10-09T04:07:36.786015+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:30.189193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785776+0000","last_change":"2023-10-09T04:07:34.630774+0000","last_active":"2023-10-09T04:07:36.785776+0000","last_peered":"2023-10-09T04:07:36.785776+0000","last_clean":"2023-10-09T04:07:36.785776+0000","last_became_active":"2023-10-09T04:07:34.630525+0000","last_became_peered":"2023-10-09T04:07:34.630525+0000","last_unstale":"2023-10-09T04:07:36.785776+0000","last_undegraded":"2023-10-09T04:07:36.785776+0000","last_fullsized":"2023-10-09T04:07:36.785776+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:05.578748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.786134+0000","last_change":"2023-10-09T04:07:34.659260+0000","last_active":"2023-10-09T04:07:36.786134+0000","last_peered":"2023-10-09T04:07:36.786134+0000","last_clean":"2023-10-09T04:07:36.786134+0000","last_became_active":"2023-10-09T04:07:34.658976+0000","last_became_peered":"2023-10-09T04:07:34.658976+0000","last_unstale":"2023-10-09T04:07:36.786134+0000","last_undegraded":"2023-10-09T04:07:36.786134+0000","last_fullsized":"2023-10-09T04:07:36.786134+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:35.014675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286183+0000","last_change":"2023-10-09T04:07:34.634417+0000","last_active":"2023-10-09T04:07:37.286183+0000","last_peered":"2023-10-09T04:07:37.286183+0000","last_clean":"2023-10-09T04:07:37.286183+0000","last_became_active":"2023-10-09T04:07:34.634183+0000","last_became_peered":"2023-10-09T04:07:34.634183+0000","last_unstale":"2023-10-09T04:07:37.286183+0000","last_undegraded":"2023-10-09T04:07:37.286183+0000","last_fullsized":"2023-10-09T04:07:37.286183+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:13.162009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785888+0000","last_change":"2023-10-09T04:07:34.655573+0000","last_active":"2023-10-09T04:07:36.785888+0000","last_peered":"2023-10-09T04:07:36.785888+0000","last_clean":"2023-10-09T04:07:36.785888+0000","last_became_active":"2023-10-09T04:07:34.655326+0000","last_became_peered":"2023-10-09T04:07:34.655326+0000","last_unstale":"2023-10-09T04:07:36.785888+0000","last_undegraded":"2023-10-09T04:07:36.785888+0000","last_fullsized":"2023-10-09T04:07:36.785888+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:07.510114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614867+0000","last_change":"2023-10-09T04:07:34.643373+0000","last_active":"2023-10-09T04:07:43.614867+0000","last_peered":"2023-10-09T04:07:43.614867+0000","last_clean":"2023-10-09T04:07:43.614867+0000","last_became_active":"2023-10-09T04:07:34.643033+0000","last_became_peered":"2023-10-09T04:07:34.643033+0000","last_unstale":"2023-10-09T04:07:43.614867+0000","last_undegraded":"2023-10-09T04:07:43.614867+0000","last_fullsized":"2023-10-09T04:07:43.614867+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:57.618154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2023-10-09T04:08:27.629117+0000","last_change":"2023-10-09T04:08:27.629346+0000","last_active":"2023-10-09T04:08:27.629086+0000","last_peered":"2023-10-09T04:07:36.646602+0000","last_clean":"2023-10-09T04:07:36.646602+0000","last_became_active":"2023-10-09T04:07:36.284375+0000","last_became_peered":"2023-10-09T04:07:36.284375+0000","last_unstale":"2023-10-09T04:08:27.629117+0000","last_undegraded":"2023-10-09T04:08:27.629117+0000","last_fullsized":"2023-10-09T04:08:27.629117+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:16:40.978215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4,5],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647202+0000","last_change":"2023-10-09T04:07:34.640164+0000","last_active":"2023-10-09T04:07:36.647202+0000","last_peered":"2023-10-09T04:07:36.647202+0000","last_clean":"2023-10-09T04:07:36.647202+0000","last_became_active":"2023-10-09T04:07:34.639871+0000","last_became_peered":"2023-10-09T04:07:34.639871+0000","last_unstale":"2023-10-09T04:07:36.647202+0000","last_undegraded":"2023-10-09T04:07:36.647202+0000","last_fullsized":"2023-10-09T04:07:36.647202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:59.659674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"peering","last_fresh":"2023-10-09T04:08:27.634388+0000","last_change":"2023-10-09T04:08:27.634650+0000","last_active":"2023-10-09T04:08:27.634356+0000","last_peered":"2023-10-09T04:07:37.287391+0000","last_clean":"2023-10-09T04:07:37.287391+0000","last_became_active":"2023-10-09T04:07:34.677024+0000","last_became_peered":"2023-10-09T04:07:34.677024+0000","last_unstale":"2023-10-09T04:08:27.634388+0000","last_undegraded":"2023-10-09T04:08:27.634388+0000","last_fullsized":"2023-10-09T04:08:27.634388+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:45.749116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.617262+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:07:43.617262+0000","last_peered":"2023-10-09T04:07:43.617262+0000","last_clean":"2023-10-09T04:07:43.617262+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:07:43.617262+0000","last_undegraded":"2023-10-09T04:07:43.617262+0000","last_fullsized":"2023-10-09T04:07:43.617262+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785688+0000","last_change":"2023-10-09T04:07:34.659608+0000","last_active":"2023-10-09T04:07:36.785688+0000","last_peered":"2023-10-09T04:07:36.785688+0000","last_clean":"2023-10-09T04:07:36.785688+0000","last_became_active":"2023-10-09T04:07:34.658918+0000","last_became_peered":"2023-10-09T04:07:34.658918+0000","last_unstale":"2023-10-09T04:07:36.785688+0000","last_undegraded":"2023-10-09T04:07:36.785688+0000","last_fullsized":"2023-10-09T04:07:36.785688+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:42.508338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616507+0000","last_change":"2023-10-09T04:07:34.631609+0000","last_active":"2023-10-09T04:07:43.616507+0000","last_peered":"2023-10-09T04:07:43.616507+0000","last_clean":"2023-10-09T04:07:43.616507+0000","last_became_active":"2023-10-09T04:07:34.631327+0000","last_became_peered":"2023-10-09T04:07:34.631327+0000","last_unstale":"2023-10-09T04:07:43.616507+0000","last_undegraded":"2023-10-09T04:07:43.616507+0000","last_fullsized":"2023-10-09T04:07:43.616507+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:51.678421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647005+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:07:36.647005+0000","last_peered":"2023-10-09T04:07:36.647005+0000","last_clean":"2023-10-09T04:07:36.647005+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:07:36.647005+0000","last_undegraded":"2023-10-09T04:07:36.647005+0000","last_fullsized":"2023-10-09T04:07:36.647005+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616462+0000","last_change":"2023-10-09T04:07:34.642317+0000","last_active":"2023-10-09T04:07:43.616462+0000","last_peered":"2023-10-09T04:07:43.616462+0000","last_clean":"2023-10-09T04:07:43.616462+0000","last_became_active":"2023-10-09T04:07:34.642028+0000","last_became_peered":"2023-10-09T04:07:34.642028+0000","last_unstale":"2023-10-09T04:07:43.616462+0000","last_undegraded":"2023-10-09T04:07:43.616462+0000","last_fullsized":"2023-10-09T04:07:43.616462+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:12.984276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030869+0000","last_change":"2023-10-09T04:07:36.029846+0000","last_active":"2023-10-09T04:07:36.030869+0000","last_peered":"2023-10-09T04:07:36.030869+0000","last_clean":"2023-10-09T04:07:36.030869+0000","last_became_active":"2023-10-09T04:07:36.029223+0000","last_became_peered":"2023-10-09T04:07:36.029223+0000","last_unstale":"2023-10-09T04:07:36.030869+0000","last_undegraded":"2023-10-09T04:07:36.030869+0000","last_fullsized":"2023-10-09T04:07:36.030869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:12.111167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616194+0000","last_change":"2023-10-09T04:07:34.640376+0000","last_active":"2023-10-09T04:07:43.616194+0000","last_peered":"2023-10-09T04:07:43.616194+0000","last_clean":"2023-10-09T04:07:43.616194+0000","last_became_active":"2023-10-09T04:07:34.640073+0000","last_became_peered":"2023-10-09T04:07:34.640073+0000","last_unstale":"2023-10-09T04:07:43.616194+0000","last_undegraded":"2023-10-09T04:07:43.616194+0000","last_fullsized":"2023-10-09T04:07:43.616194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:16.684588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646576+0000","last_change":"2023-10-09T04:07:36.029535+0000","last_active":"2023-10-09T04:07:36.646576+0000","last_peered":"2023-10-09T04:07:36.646576+0000","last_clean":"2023-10-09T04:07:36.646576+0000","last_became_active":"2023-10-09T04:07:36.029175+0000","last_became_peered":"2023-10-09T04:07:36.029175+0000","last_unstale":"2023-10-09T04:07:36.646576+0000","last_undegraded":"2023-10-09T04:07:36.646576+0000","last_fullsized":"2023-10-09T04:07:36.646576+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:34.595934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.059723+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:07:37.059723+0000","last_peered":"2023-10-09T04:07:37.059723+0000","last_clean":"2023-10-09T04:07:37.059723+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:07:37.059723+0000","last_undegraded":"2023-10-09T04:07:37.059723+0000","last_fullsized":"2023-10-09T04:07:37.059723+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647210+0000","last_change":"2023-10-09T04:07:36.029411+0000","last_active":"2023-10-09T04:07:36.647210+0000","last_peered":"2023-10-09T04:07:36.647210+0000","last_clean":"2023-10-09T04:07:36.647210+0000","last_became_active":"2023-10-09T04:07:36.029109+0000","last_became_peered":"2023-10-09T04:07:36.029109+0000","last_unstale":"2023-10-09T04:07:36.647210+0000","last_undegraded":"2023-10-09T04:07:36.647210+0000","last_fullsized":"2023-10-09T04:07:36.647210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:05.311520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646301+0000","last_change":"2023-10-09T04:07:34.646539+0000","last_active":"2023-10-09T04:07:36.646301+0000","last_peered":"2023-10-09T04:07:36.646301+0000","last_clean":"2023-10-09T04:07:36.646301+0000","last_became_active":"2023-10-09T04:07:34.646243+0000","last_became_peered":"2023-10-09T04:07:34.646243+0000","last_unstale":"2023-10-09T04:07:36.646301+0000","last_undegraded":"2023-10-09T04:07:36.646301+0000","last_fullsized":"2023-10-09T04:07:36.646301+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:01.969093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.030827+0000","last_change":"2023-10-09T04:07:36.029585+0000","last_active":"2023-10-09T04:07:36.030827+0000","last_peered":"2023-10-09T04:07:36.030827+0000","last_clean":"2023-10-09T04:07:36.030827+0000","last_became_active":"2023-10-09T04:07:36.029157+0000","last_became_peered":"2023-10-09T04:07:36.029157+0000","last_unstale":"2023-10-09T04:07:36.030827+0000","last_undegraded":"2023-10-09T04:07:36.030827+0000","last_fullsized":"2023-10-09T04:07:36.030827+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:19.819664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.617011+0000","last_change":"2023-10-09T04:07:34.632203+0000","last_active":"2023-10-09T04:07:43.617011+0000","last_peered":"2023-10-09T04:07:43.617011+0000","last_clean":"2023-10-09T04:07:43.617011+0000","last_became_active":"2023-10-09T04:07:34.631969+0000","last_became_peered":"2023-10-09T04:07:34.631969+0000","last_unstale":"2023-10-09T04:07:43.617011+0000","last_undegraded":"2023-10-09T04:07:43.617011+0000","last_fullsized":"2023-10-09T04:07:43.617011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:01.619892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.785865+0000","last_change":"2023-10-09T04:07:34.659790+0000","last_active":"2023-10-09T04:07:36.785865+0000","last_peered":"2023-10-09T04:07:36.785865+0000","last_clean":"2023-10-09T04:07:36.785865+0000","last_became_active":"2023-10-09T04:07:34.659565+0000","last_became_peered":"2023-10-09T04:07:34.659565+0000","last_unstale":"2023-10-09T04:07:36.785865+0000","last_undegraded":"2023-10-09T04:07:36.785865+0000","last_fullsized":"2023-10-09T04:07:36.785865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:32.044413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.617433+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:07:43.617433+0000","last_peered":"2023-10-09T04:07:43.617433+0000","last_clean":"2023-10-09T04:07:43.617433+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:07:43.617433+0000","last_undegraded":"2023-10-09T04:07:43.617433+0000","last_fullsized":"2023-10-09T04:07:43.617433+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646837+0000","last_change":"2023-10-09T04:07:34.661425+0000","last_active":"2023-10-09T04:07:36.646837+0000","last_peered":"2023-10-09T04:07:36.646837+0000","last_clean":"2023-10-09T04:07:36.646837+0000","last_became_active":"2023-10-09T04:07:34.661099+0000","last_became_peered":"2023-10-09T04:07:34.661099+0000","last_unstale":"2023-10-09T04:07:36.646837+0000","last_undegraded":"2023-10-09T04:07:36.646837+0000","last_fullsized":"2023-10-09T04:07:36.646837+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:53.588610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.616956+0000","last_change":"2023-10-09T04:07:34.641392+0000","last_active":"2023-10-09T04:07:43.616956+0000","last_peered":"2023-10-09T04:07:43.616956+0000","last_clean":"2023-10-09T04:07:43.616956+0000","last_became_active":"2023-10-09T04:07:34.641037+0000","last_became_peered":"2023-10-09T04:07:34.641037+0000","last_unstale":"2023-10-09T04:07:43.616956+0000","last_undegraded":"2023-10-09T04:07:43.616956+0000","last_fullsized":"2023-10-09T04:07:43.616956+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:43.467265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.031551+0000","last_change":"2023-10-09T04:07:36.031224+0000","last_active":"2023-10-09T04:07:36.031551+0000","last_peered":"2023-10-09T04:07:36.031551+0000","last_clean":"2023-10-09T04:07:36.031551+0000","last_became_active":"2023-10-09T04:07:36.030834+0000","last_became_peered":"2023-10-09T04:07:36.030834+0000","last_unstale":"2023-10-09T04:07:36.031551+0000","last_undegraded":"2023-10-09T04:07:36.031551+0000","last_fullsized":"2023-10-09T04:07:36.031551+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:21.909762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647243+0000","last_change":"2023-10-09T04:07:34.631933+0000","last_active":"2023-10-09T04:07:36.647243+0000","last_peered":"2023-10-09T04:07:36.647243+0000","last_clean":"2023-10-09T04:07:36.647243+0000","last_became_active":"2023-10-09T04:07:34.631667+0000","last_became_peered":"2023-10-09T04:07:34.631667+0000","last_unstale":"2023-10-09T04:07:36.647243+0000","last_undegraded":"2023-10-09T04:07:36.647243+0000","last_fullsized":"2023-10-09T04:07:36.647243+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:58.644048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.801084+0000","last_change":"2023-10-09T04:07:35.800509+0000","last_active":"2023-10-09T04:07:35.801084+0000","last_peered":"2023-10-09T04:07:35.801084+0000","last_clean":"2023-10-09T04:07:35.801084+0000","last_became_active":"2023-10-09T04:07:35.800155+0000","last_became_peered":"2023-10-09T04:07:35.800155+0000","last_unstale":"2023-10-09T04:07:35.801084+0000","last_undegraded":"2023-10-09T04:07:35.801084+0000","last_fullsized":"2023-10-09T04:07:35.801084+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:50.929412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647129+0000","last_change":"2023-10-09T04:07:34.646997+0000","last_active":"2023-10-09T04:07:36.647129+0000","last_peered":"2023-10-09T04:07:36.647129+0000","last_clean":"2023-10-09T04:07:36.647129+0000","last_became_active":"2023-10-09T04:07:34.645625+0000","last_became_peered":"2023-10-09T04:07:34.645625+0000","last_unstale":"2023-10-09T04:07:36.647129+0000","last_undegraded":"2023-10-09T04:07:36.647129+0000","last_fullsized":"2023-10-09T04:07:36.647129+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:40.584254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:37.286258+0000","last_change":"2023-10-09T04:07:34.676137+0000","last_active":"2023-10-09T04:07:37.286258+0000","last_peered":"2023-10-09T04:07:37.286258+0000","last_clean":"2023-10-09T04:07:37.286258+0000","last_became_active":"2023-10-09T04:07:34.675822+0000","last_became_peered":"2023-10-09T04:07:34.675822+0000","last_unstale":"2023-10-09T04:07:37.286258+0000","last_undegraded":"2023-10-09T04:07:37.286258+0000","last_fullsized":"2023-10-09T04:07:37.286258+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:06.758028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:35.652383+0000","last_change":"2023-10-09T04:07:34.646983+0000","last_active":"2023-10-09T04:07:35.652383+0000","last_peered":"2023-10-09T04:07:35.652383+0000","last_clean":"2023-10-09T04:07:35.652383+0000","last_became_active":"2023-10-09T04:07:34.646623+0000","last_became_peered":"2023-10-09T04:07:34.646623+0000","last_unstale":"2023-10-09T04:07:35.652383+0000","last_undegraded":"2023-10-09T04:07:35.652383+0000","last_fullsized":"2023-10-09T04:07:35.652383+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.119527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.646658+0000","last_change":"2023-10-09T04:07:36.029481+0000","last_active":"2023-10-09T04:07:36.646658+0000","last_peered":"2023-10-09T04:07:36.646658+0000","last_clean":"2023-10-09T04:07:36.646658+0000","last_became_active":"2023-10-09T04:07:36.029167+0000","last_became_peered":"2023-10-09T04:07:36.029167+0000","last_unstale":"2023-10-09T04:07:36.646658+0000","last_undegraded":"2023-10-09T04:07:36.646658+0000","last_fullsized":"2023-10-09T04:07:36.646658+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:42.409682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.647521+0000","last_change":"2023-10-09T04:07:34.645964+0000","last_active":"2023-10-09T04:07:36.647521+0000","last_peered":"2023-10-09T04:07:36.647521+0000","last_clean":"2023-10-09T04:07:36.647521+0000","last_became_active":"2023-10-09T04:07:34.645619+0000","last_became_peered":"2023-10-09T04:07:34.645619+0000","last_unstale":"2023-10-09T04:07:36.647521+0000","last_undegraded":"2023-10-09T04:07:36.647521+0000","last_fullsized":"2023-10-09T04:07:36.647521+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:26.090109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-10-09T04:07:36.285502+0000","last_change":"2023-10-09T04:07:36.284775+0000","last_active":"2023-10-09T04:07:36.285502+0000","last_peered":"2023-10-09T04:07:36.285502+0000","last_clean":"2023-10-09T04:07:36.285502+0000","last_became_active":"2023-10-09T04:07:36.284501+0000","last_became_peered":"2023-10-09T04:07:36.284501+0000","last_unstale":"2023-10-09T04:07:36.285502+0000","last_undegraded":"2023-10-09T04:07:36.285502+0000","last_fullsized":"2023-10-09T04:07:36.285502+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:17.182374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-10-09T04:07:43.614371+0000","last_change":"2023-10-09T04:07:34.630544+0000","last_active":"2023-10-09T04:07:43.614371+0000","last_peered":"2023-10-09T04:07:43.614371+0000","last_clean":"2023-10-09T04:07:43.614371+0000","last_became_active":"2023-10-09T04:07:34.630230+0000","last_became_peered":"2023-10-09T04:07:34.630230+0000","last_unstale":"2023-10-09T04:07:43.614371+0000","last_undegraded":"2023-10-09T04:07:43.614371+0000","last_fullsized":"2023-10-09T04:07:43.614371+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:32.002588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298572,"kb_used_data":1284,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446580,"statfs":{"total":95995035648,"available":95689297920,"internally_reserved":0,"allocated":1314816,"data_stored":891677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0329999999999999,"15min":1.123},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76500000000000001,"5min":10.557,"15min":10.557},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.62}]},{"osd":1,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.83999999999999997,"15min":0.88800000000000001},"min":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71099999999999997,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.65100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.81599999999999995,"15min":0.86699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69799999999999995,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.63200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72999999999999998,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67100000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.55900000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.64300000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.52000000000000002,"15min":0.53400000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.58399999999999996,"5min":1.391,"15min":1.391},"last":0.38300000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.63500000000000001,"15min":0.65400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69099999999999995,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.56799999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495653,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1268,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1298432,"data_stored":891208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54900000000000004,"15min":0.55400000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.751,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.505},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69399999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.63400000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56999999999999995,"15min":0.57799999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73299999999999998,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.70199999999999996,"5min":0.748,"15min":0.748},"last":0.622}]},{"osd":2,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.73199999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.60299999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.65400000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.48599999999999999,"15min":0.48199999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.70699999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48799999999999999,"15min":0.48699999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.60899999999999999,"15min":0.621},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.89200000000000002,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.75900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.77300000000000002,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.58399999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574910,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297964,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447188,"statfs":{"total":95995035648,"available":95689920512,"internally_reserved":0,"allocated":692224,"data_stored":299557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.39800000000000002,"15min":0.372},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.435,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.41399999999999998,"15min":0.42299999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.51500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.30499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:07:28 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.499,"15min":0.50700000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61499999999999999,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.44,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.75600000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.70099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.60899999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70899999999999996,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.504},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69899999999999995,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.63}]},{"osd":6,"last update":"Mon Oct 9 04:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.62,"15min":0.628},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.85999999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66400000000000003,"15min":0.67600000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.78600000000000003,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.55700000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.65100000000000002,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.48099999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.59799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738435,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1294336,"data_stored":890501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:07:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.32800000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.38400000000000001,"15min":0.34599999999999997},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53600000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.25600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.58599999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.308}]},{"osd":3,"last update":"Mon Oct 9 04:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.46800000000000003,"15min":0.47599999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.53200000000000003,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.378},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.50900000000000001,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60999999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.40100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.373,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77800000000000002,"5min":0.878,"15min":0.878},"last":0.498},{"interface":"front","average":{"1min":0.626,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84299999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.52200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61099999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.622,"15min":0.63800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77600000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80600000000000005,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79600000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:07:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65100000000000002,"15min":0.65900000000000003},"min":{"1min":0.46600000000000003,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80900000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.69099999999999995,"15min":0.68700000000000006},"min":{"1min":0.505,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.42499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70699999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.37},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.43099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:30 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.47599999999999998,"15min":0.47699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.48599999999999999,"15min":0.48399999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.35299999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.45400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.46999999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.70099999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.61599999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79400000000000004,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.62,"15min":0.61799999999999999},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.53900000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67400000000000004},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98099999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.629,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.60199999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182453,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297968,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447184,"statfs":{"total":95995035648,"available":95689916416,"internally_reserved":0,"allocated":696320,"data_stored":299647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.41899999999999998},"min":{"1min":0.20399999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.308},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41699999999999998,"15min":0.44900000000000001},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.27300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.43099999999999999,"15min":0.46300000000000002},"min":{"1min":0.255,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.54800000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.43099999999999999,"15min":0.47499999999999998},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56999999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.32500000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.46400000000000002,"15min":0.496},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54000000000000004,"5min":0.745,"15min":0.745},"last":0.375},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.46400000000000002,"15min":0.503},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.63300000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.73399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.626,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.76300000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.624,"15min":0.63500000000000001},"min":{"1min":0.433,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77100000000000002,"5min":0.874,"15min":0.874},"last":0.46200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.63,"15min":0.63900000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.748,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.61399999999999999,"15min":0.622},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76400000000000001,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.622}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73599999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.46500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81200000000000006,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.59099999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018927,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":301239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.64700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.503,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.622},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.61099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.52600000000000002,"15min":0.51700000000000002},"min":{"1min":0.40200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.70799999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.432,"5min":0.37,"15min":0.37},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54100000000000004,"15min":0.52300000000000002},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.073,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.69799999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.439,"15min":0.42299999999999999},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.44400000000000001,"15min":0.42399999999999999},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.47899999999999998,"15min":0.47299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68899999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.66700000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69099999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.68400000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757291,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":684,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":700416,"data_stored":300181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53000000000000003,"15min":0.52400000000000002},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65000000000000002,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.308}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71799999999999997,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.57699999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.443,"15min":0.41899999999999998},"min":{"1min":0.35499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.45400000000000001,"15min":0.42799999999999999},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.59199999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.45500000000000002,"15min":0.442},"min":{"1min":0.38700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.45900000000000002,"15min":0.443},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60499999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:08:33.253 INFO:tasks.cephadm.ceph_manager.ceph:PG 4.18 is not active+clean 2023-10-09T04:08:33.253 INFO:tasks.cephadm.ceph_manager.ceph:{'pgid': '4.18', 'version': "0'0", 'reported_seq': 14, 'reported_epoch': 69, 'state': 'peering', 'last_fresh': '2023-10-09T04:08:27.631298+0000', 'last_change': '2023-10-09T04:08:27.631472+0000', 'last_active': '2023-10-09T04:08:27.631278+0000', 'last_peered': '2023-10-09T04:07:36.785613+0000', 'last_clean': '2023-10-09T04:07:36.785613+0000', 'last_became_active': '2023-10-09T04:07:34.658916+0000', 'last_became_peered': '2023-10-09T04:07:34.658916+0000', 'last_unstale': '2023-10-09T04:08:27.631298+0000', 'last_undegraded': '2023-10-09T04:08:27.631298+0000', 'last_fullsized': '2023-10-09T04:08:27.631298+0000', 'mapping_epoch': 69, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 65, 'last_epoch_clean': 66, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'last_clean_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-10-10T14:15:50.983204+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [4, 6, 2], 'acting': [4, 6, 2], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 2, 6], 'up_primary': 4, 'acting_primary': 4, 'purged_snaps': []} 2023-10-09T04:08:33.253 INFO:tasks.cephadm.ceph_manager.ceph:PG 4.b is not active+clean 2023-10-09T04:08:33.253 INFO:tasks.cephadm.ceph_manager.ceph:{'pgid': '4.b', 'version': "0'0", 'reported_seq': 14, 'reported_epoch': 69, 'state': 'peering', 'last_fresh': '2023-10-09T04:08:27.629117+0000', 'last_change': '2023-10-09T04:08:27.629346+0000', 'last_active': '2023-10-09T04:08:27.629086+0000', 'last_peered': '2023-10-09T04:07:36.646602+0000', 'last_clean': '2023-10-09T04:07:36.646602+0000', 'last_became_active': '2023-10-09T04:07:36.284375+0000', 'last_became_peered': '2023-10-09T04:07:36.284375+0000', 'last_unstale': '2023-10-09T04:08:27.629117+0000', 'last_undegraded': '2023-10-09T04:08:27.629117+0000', 'last_fullsized': '2023-10-09T04:08:27.629117+0000', 'mapping_epoch': 69, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 65, 'last_epoch_clean': 67, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'last_clean_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-10-10T08:16:40.978215+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [0, 5, 4], 'acting': [0, 5, 4], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 4, 5], 'up_primary': 0, 'acting_primary': 0, 'purged_snaps': []} 2023-10-09T04:08:33.253 INFO:tasks.cephadm.ceph_manager.ceph:PG 4.a is not active+clean 2023-10-09T04:08:33.253 INFO:tasks.cephadm.ceph_manager.ceph:{'pgid': '4.a', 'version': "0'0", 'reported_seq': 14, 'reported_epoch': 69, 'state': 'peering', 'last_fresh': '2023-10-09T04:08:27.634388+0000', 'last_change': '2023-10-09T04:08:27.634650+0000', 'last_active': '2023-10-09T04:08:27.634356+0000', 'last_peered': '2023-10-09T04:07:37.287391+0000', 'last_clean': '2023-10-09T04:07:37.287391+0000', 'last_became_active': '2023-10-09T04:07:34.677024+0000', 'last_became_peered': '2023-10-09T04:07:34.677024+0000', 'last_unstale': '2023-10-09T04:08:27.634388+0000', 'last_undegraded': '2023-10-09T04:08:27.634388+0000', 'last_fullsized': '2023-10-09T04:08:27.634388+0000', 'mapping_epoch': 69, 'log_start': "0'0", 'ondisk_log_start': "0'0", 'created': 65, 'last_epoch_clean': 66, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'last_clean_scrub_stamp': '2023-10-09T04:07:34.617315+0000', 'objects_scrubbed': 0, 'log_size': 0, 'log_dups_size': 0, 'ondisk_log_size': 0, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-10-10T05:16:45.749116+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 0, 'num_objects': 0, 'num_object_clones': 0, 'num_object_copies': 0, 'num_objects_missing_on_primary': 0, 'num_objects_missing': 0, 'num_objects_degraded': 0, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 0, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 0, 'num_write_kb': 0, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [6, 2, 7], 'acting': [6, 2, 7], 'avail_no_missing': [], 'object_location_counts': [], 'blocked_by': [1, 2, 7], 'up_primary': 6, 'acting_primary': 6, 'purged_snaps': []} 2023-10-09T04:08:33.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:08:33.613 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:33.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:33 smithi029 ceph-mon[101043]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:33.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:33 smithi029 ceph-mon[119131]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:33 smithi177 ceph-mon[106070]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:34.612 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:34 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:34] "GET /metrics HTTP/1.1" 200 238772 "" "Prometheus/2.43.0" 2023-10-09T04:08:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:34 smithi029 ceph-mon[101043]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:34.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:34 smithi029 ceph-mon[119131]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:34 smithi177 ceph-mon[106070]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:35.495 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:35.495 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":14,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":364,"monmap":{"epoch":3,"min_mon_release_name":"quincy","num_mons":3},"osdmap":{"epoch":70,"num_osds":8,"num_up_osds":8,"osd_up_since":1696824350,"num_in_osds":8,"osd_in_since":1696824334,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":592674,"bytes_used":2442993664,"bytes_avail":765517291520,"bytes_total":767960285184},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":12,"modified":"2023-10-09T04:07:28.762087+0000","services":{}},"progress_events":{}} 2023-10-09T04:08:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:35 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1708550637' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:08:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:35 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1708550637' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:08:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:35 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1708550637' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:08:36.288 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T04:08:36.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:08:36.649 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:36.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:36 smithi029 ceph-mon[101043]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:36.887 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:36 smithi029 ceph-mon[119131]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:36 smithi177 ceph-mon[106070]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:38.469 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:38.471 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:08:38.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:38 smithi029 ceph-mon[101043]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:38 smithi029 ceph-mon[119131]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:38 smithi177 ceph-mon[106070]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:39.239 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":79,"stamp":"2023-10-09T04:08:36.777160+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385736,"kb_used_data":7432,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575480,"statfs":{"total":767960285184,"available":765517291520,"internally_reserved":0,"allocated":7610368,"data_stored":4286899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.150307"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.636477+0000","last_change":"2023-10-09T04:08:28.636477+0000","last_active":"2023-10-09T04:08:28.636477+0000","last_peered":"2023-10-09T04:08:28.636477+0000","last_clean":"2023-10-09T04:08:28.636477+0000","last_became_active":"2023-10-09T04:08:28.635995+0000","last_became_peered":"2023-10-09T04:08:28.635995+0000","last_unstale":"2023-10-09T04:08:28.636477+0000","last_undegraded":"2023-10-09T04:08:28.636477+0000","last_fullsized":"2023-10-09T04:08:28.636477+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:15.972599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628792+0000","last_change":"2023-10-09T04:07:34.646776+0000","last_active":"2023-10-09T04:08:27.628792+0000","last_peered":"2023-10-09T04:08:27.628792+0000","last_clean":"2023-10-09T04:08:27.628792+0000","last_became_active":"2023-10-09T04:07:34.644325+0000","last_became_peered":"2023-10-09T04:07:34.644325+0000","last_unstale":"2023-10-09T04:08:27.628792+0000","last_undegraded":"2023-10-09T04:08:27.628792+0000","last_fullsized":"2023-10-09T04:08:27.628792+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:23.379485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022721+0000","last_change":"2023-10-09T04:07:35.653186+0000","last_active":"2023-10-09T04:08:28.022721+0000","last_peered":"2023-10-09T04:08:28.022721+0000","last_clean":"2023-10-09T04:08:28.022721+0000","last_became_active":"2023-10-09T04:07:35.652908+0000","last_became_peered":"2023-10-09T04:07:35.652908+0000","last_unstale":"2023-10-09T04:08:28.022721+0000","last_undegraded":"2023-10-09T04:08:28.022721+0000","last_fullsized":"2023-10-09T04:08:28.022721+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:42.975764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.023033+0000","last_change":"2023-10-09T04:07:34.638165+0000","last_active":"2023-10-09T04:08:28.023033+0000","last_peered":"2023-10-09T04:08:28.023033+0000","last_clean":"2023-10-09T04:08:28.023033+0000","last_became_active":"2023-10-09T04:07:34.637823+0000","last_became_peered":"2023-10-09T04:07:34.637823+0000","last_unstale":"2023-10-09T04:08:28.023033+0000","last_undegraded":"2023-10-09T04:08:28.023033+0000","last_fullsized":"2023-10-09T04:08:28.023033+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:13.118351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631244+0000","last_change":"2023-10-09T04:07:34.676759+0000","last_active":"2023-10-09T04:08:27.631244+0000","last_peered":"2023-10-09T04:08:27.631244+0000","last_clean":"2023-10-09T04:08:27.631244+0000","last_became_active":"2023-10-09T04:07:34.676294+0000","last_became_peered":"2023-10-09T04:07:34.676294+0000","last_unstale":"2023-10-09T04:08:27.631244+0000","last_undegraded":"2023-10-09T04:08:27.631244+0000","last_fullsized":"2023-10-09T04:08:27.631244+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:44.387110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629025+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:08:27.629025+0000","last_peered":"2023-10-09T04:08:27.629025+0000","last_clean":"2023-10-09T04:08:27.629025+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:08:27.629025+0000","last_undegraded":"2023-10-09T04:08:27.629025+0000","last_fullsized":"2023-10-09T04:08:27.629025+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631988+0000","last_change":"2023-10-09T04:07:34.659699+0000","last_active":"2023-10-09T04:08:27.631988+0000","last_peered":"2023-10-09T04:08:27.631988+0000","last_clean":"2023-10-09T04:08:27.631988+0000","last_became_active":"2023-10-09T04:07:34.659189+0000","last_became_peered":"2023-10-09T04:07:34.659189+0000","last_unstale":"2023-10-09T04:08:27.631988+0000","last_undegraded":"2023-10-09T04:08:27.631988+0000","last_fullsized":"2023-10-09T04:08:27.631988+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:44.119093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628741+0000","last_change":"2023-10-09T04:07:34.631386+0000","last_active":"2023-10-09T04:08:27.628741+0000","last_peered":"2023-10-09T04:08:27.628741+0000","last_clean":"2023-10-09T04:08:27.628741+0000","last_became_active":"2023-10-09T04:07:34.631090+0000","last_became_peered":"2023-10-09T04:07:34.631090+0000","last_unstale":"2023-10-09T04:08:27.628741+0000","last_undegraded":"2023-10-09T04:08:27.628741+0000","last_fullsized":"2023-10-09T04:08:27.628741+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:05.633909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636714+0000","last_change":"2023-10-09T04:07:34.659338+0000","last_active":"2023-10-09T04:08:27.636714+0000","last_peered":"2023-10-09T04:08:27.636714+0000","last_clean":"2023-10-09T04:08:27.636714+0000","last_became_active":"2023-10-09T04:07:34.658997+0000","last_became_peered":"2023-10-09T04:07:34.658997+0000","last_unstale":"2023-10-09T04:08:27.636714+0000","last_undegraded":"2023-10-09T04:08:27.636714+0000","last_fullsized":"2023-10-09T04:08:27.636714+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:38.217792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628581+0000","last_change":"2023-10-09T04:07:34.642042+0000","last_active":"2023-10-09T04:08:27.628581+0000","last_peered":"2023-10-09T04:08:27.628581+0000","last_clean":"2023-10-09T04:08:27.628581+0000","last_became_active":"2023-10-09T04:07:34.641579+0000","last_became_peered":"2023-10-09T04:07:34.641579+0000","last_unstale":"2023-10-09T04:08:27.628581+0000","last_undegraded":"2023-10-09T04:08:27.628581+0000","last_fullsized":"2023-10-09T04:08:27.628581+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:44.869504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631210+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:08:27.631210+0000","last_peered":"2023-10-09T04:08:27.631210+0000","last_clean":"2023-10-09T04:08:27.631210+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:08:27.631210+0000","last_undegraded":"2023-10-09T04:08:27.631210+0000","last_fullsized":"2023-10-09T04:08:27.631210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628551+0000","last_change":"2023-10-09T04:07:34.645613+0000","last_active":"2023-10-09T04:08:27.628551+0000","last_peered":"2023-10-09T04:08:27.628551+0000","last_clean":"2023-10-09T04:08:27.628551+0000","last_became_active":"2023-10-09T04:07:34.645309+0000","last_became_peered":"2023-10-09T04:07:34.645309+0000","last_unstale":"2023-10-09T04:08:27.628551+0000","last_undegraded":"2023-10-09T04:08:27.628551+0000","last_fullsized":"2023-10-09T04:08:27.628551+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:07.674226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634715+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:08:27.634715+0000","last_peered":"2023-10-09T04:08:27.634715+0000","last_clean":"2023-10-09T04:08:27.634715+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:08:27.634715+0000","last_undegraded":"2023-10-09T04:08:27.634715+0000","last_fullsized":"2023-10-09T04:08:27.634715+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022540+0000","last_change":"2023-10-09T04:07:36.030076+0000","last_active":"2023-10-09T04:08:28.022540+0000","last_peered":"2023-10-09T04:08:28.022540+0000","last_clean":"2023-10-09T04:08:28.022540+0000","last_became_active":"2023-10-09T04:07:36.029271+0000","last_became_peered":"2023-10-09T04:07:36.029271+0000","last_unstale":"2023-10-09T04:08:28.022540+0000","last_undegraded":"2023-10-09T04:08:28.022540+0000","last_fullsized":"2023-10-09T04:08:28.022540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:12.758916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629037+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:08:27.629037+0000","last_peered":"2023-10-09T04:08:27.629037+0000","last_clean":"2023-10-09T04:08:27.629037+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:08:27.629037+0000","last_undegraded":"2023-10-09T04:08:27.629037+0000","last_fullsized":"2023-10-09T04:08:27.629037+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.023114+0000","last_change":"2023-10-09T04:07:34.643380+0000","last_active":"2023-10-09T04:08:28.023114+0000","last_peered":"2023-10-09T04:08:28.023114+0000","last_clean":"2023-10-09T04:08:28.023114+0000","last_became_active":"2023-10-09T04:07:34.643086+0000","last_became_peered":"2023-10-09T04:07:34.643086+0000","last_unstale":"2023-10-09T04:08:28.023114+0000","last_undegraded":"2023-10-09T04:08:28.023114+0000","last_fullsized":"2023-10-09T04:08:28.023114+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:13.152038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628441+0000","last_change":"2023-10-09T04:07:36.030238+0000","last_active":"2023-10-09T04:08:27.628441+0000","last_peered":"2023-10-09T04:08:27.628441+0000","last_clean":"2023-10-09T04:08:27.628441+0000","last_became_active":"2023-10-09T04:07:36.029932+0000","last_became_peered":"2023-10-09T04:07:36.029932+0000","last_unstale":"2023-10-09T04:08:27.628441+0000","last_undegraded":"2023-10-09T04:08:27.628441+0000","last_fullsized":"2023-10-09T04:08:27.628441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:25.269223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630408+0000","last_change":"2023-10-09T04:07:34.633885+0000","last_active":"2023-10-09T04:08:27.630408+0000","last_peered":"2023-10-09T04:08:27.630408+0000","last_clean":"2023-10-09T04:08:27.630408+0000","last_became_active":"2023-10-09T04:07:34.633345+0000","last_became_peered":"2023-10-09T04:07:34.633345+0000","last_unstale":"2023-10-09T04:08:27.630408+0000","last_undegraded":"2023-10-09T04:08:27.630408+0000","last_fullsized":"2023-10-09T04:08:27.630408+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:42.728405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637020+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:08:27.637020+0000","last_peered":"2023-10-09T04:08:27.637020+0000","last_clean":"2023-10-09T04:08:27.637020+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:08:27.637020+0000","last_undegraded":"2023-10-09T04:08:27.637020+0000","last_fullsized":"2023-10-09T04:08:27.637020+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628630+0000","last_change":"2023-10-09T04:07:35.800119+0000","last_active":"2023-10-09T04:08:27.628630+0000","last_peered":"2023-10-09T04:08:27.628630+0000","last_clean":"2023-10-09T04:08:27.628630+0000","last_became_active":"2023-10-09T04:07:35.799858+0000","last_became_peered":"2023-10-09T04:07:35.799858+0000","last_unstale":"2023-10-09T04:08:27.628630+0000","last_undegraded":"2023-10-09T04:08:27.628630+0000","last_fullsized":"2023-10-09T04:08:27.628630+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:04.465377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628499+0000","last_change":"2023-10-09T04:07:34.645437+0000","last_active":"2023-10-09T04:08:27.628499+0000","last_peered":"2023-10-09T04:08:27.628499+0000","last_clean":"2023-10-09T04:08:27.628499+0000","last_became_active":"2023-10-09T04:07:34.645140+0000","last_became_peered":"2023-10-09T04:07:34.645140+0000","last_unstale":"2023-10-09T04:08:27.628499+0000","last_undegraded":"2023-10-09T04:08:27.628499+0000","last_fullsized":"2023-10-09T04:08:27.628499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:45.140472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636396+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:08:27.636396+0000","last_peered":"2023-10-09T04:08:27.636396+0000","last_clean":"2023-10-09T04:08:27.636396+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:08:27.636396+0000","last_undegraded":"2023-10-09T04:08:27.636396+0000","last_fullsized":"2023-10-09T04:08:27.636396+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634402+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:08:27.634402+0000","last_peered":"2023-10-09T04:08:27.634402+0000","last_clean":"2023-10-09T04:08:27.634402+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:08:27.634402+0000","last_undegraded":"2023-10-09T04:08:27.634402+0000","last_fullsized":"2023-10-09T04:08:27.634402+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022762+0000","last_change":"2023-10-09T04:07:34.639655+0000","last_active":"2023-10-09T04:08:28.022762+0000","last_peered":"2023-10-09T04:08:28.022762+0000","last_clean":"2023-10-09T04:08:28.022762+0000","last_became_active":"2023-10-09T04:07:34.639321+0000","last_became_peered":"2023-10-09T04:07:34.639321+0000","last_unstale":"2023-10-09T04:08:28.022762+0000","last_undegraded":"2023-10-09T04:08:28.022762+0000","last_fullsized":"2023-10-09T04:08:28.022762+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:25.904866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":278,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636308+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:08:27.636308+0000","last_peered":"2023-10-09T04:08:27.636308+0000","last_clean":"2023-10-09T04:08:27.636308+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:08:27.636308+0000","last_undegraded":"2023-10-09T04:08:27.636308+0000","last_fullsized":"2023-10-09T04:08:27.636308+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628728+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:08:27.628728+0000","last_peered":"2023-10-09T04:08:27.628728+0000","last_clean":"2023-10-09T04:08:27.628728+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:08:27.628728+0000","last_undegraded":"2023-10-09T04:08:27.628728+0000","last_fullsized":"2023-10-09T04:08:27.628728+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630252+0000","last_change":"2023-10-09T04:07:34.676079+0000","last_active":"2023-10-09T04:08:27.630252+0000","last_peered":"2023-10-09T04:08:27.630252+0000","last_clean":"2023-10-09T04:08:27.630252+0000","last_became_active":"2023-10-09T04:07:34.675776+0000","last_became_peered":"2023-10-09T04:07:34.675776+0000","last_unstale":"2023-10-09T04:08:27.630252+0000","last_undegraded":"2023-10-09T04:08:27.630252+0000","last_fullsized":"2023-10-09T04:08:27.630252+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:26.867406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629037+0000","last_change":"2023-10-09T04:07:34.632817+0000","last_active":"2023-10-09T04:08:27.629037+0000","last_peered":"2023-10-09T04:08:27.629037+0000","last_clean":"2023-10-09T04:08:27.629037+0000","last_became_active":"2023-10-09T04:07:34.632434+0000","last_became_peered":"2023-10-09T04:07:34.632434+0000","last_unstale":"2023-10-09T04:08:27.629037+0000","last_undegraded":"2023-10-09T04:08:27.629037+0000","last_fullsized":"2023-10-09T04:08:27.629037+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:37.018559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636392+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:08:27.636392+0000","last_peered":"2023-10-09T04:08:27.636392+0000","last_clean":"2023-10-09T04:08:27.636392+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:08:27.636392+0000","last_undegraded":"2023-10-09T04:08:27.636392+0000","last_fullsized":"2023-10-09T04:08:27.636392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630324+0000","last_change":"2023-10-09T04:07:34.659865+0000","last_active":"2023-10-09T04:08:27.630324+0000","last_peered":"2023-10-09T04:08:27.630324+0000","last_clean":"2023-10-09T04:08:27.630324+0000","last_became_active":"2023-10-09T04:07:34.659515+0000","last_became_peered":"2023-10-09T04:07:34.659515+0000","last_unstale":"2023-10-09T04:08:27.630324+0000","last_undegraded":"2023-10-09T04:08:27.630324+0000","last_fullsized":"2023-10-09T04:08:27.630324+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:07.703247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631810+0000","last_change":"2023-10-09T04:07:34.641445+0000","last_active":"2023-10-09T04:08:27.631810+0000","last_peered":"2023-10-09T04:08:27.631810+0000","last_clean":"2023-10-09T04:08:27.631810+0000","last_became_active":"2023-10-09T04:07:34.641181+0000","last_became_peered":"2023-10-09T04:07:34.641181+0000","last_unstale":"2023-10-09T04:08:27.631810+0000","last_undegraded":"2023-10-09T04:08:27.631810+0000","last_fullsized":"2023-10-09T04:08:27.631810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.897577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628723+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:08:27.628723+0000","last_peered":"2023-10-09T04:08:27.628723+0000","last_clean":"2023-10-09T04:08:27.628723+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:08:27.628723+0000","last_undegraded":"2023-10-09T04:08:27.628723+0000","last_fullsized":"2023-10-09T04:08:27.628723+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629441+0000","last_change":"2023-10-09T04:07:34.675845+0000","last_active":"2023-10-09T04:08:27.629441+0000","last_peered":"2023-10-09T04:08:27.629441+0000","last_clean":"2023-10-09T04:08:27.629441+0000","last_became_active":"2023-10-09T04:07:34.675371+0000","last_became_peered":"2023-10-09T04:07:34.675371+0000","last_unstale":"2023-10-09T04:08:27.629441+0000","last_undegraded":"2023-10-09T04:08:27.629441+0000","last_fullsized":"2023-10-09T04:08:27.629441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:45.429941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628955+0000","last_change":"2023-10-09T04:07:34.640999+0000","last_active":"2023-10-09T04:08:27.628955+0000","last_peered":"2023-10-09T04:08:27.628955+0000","last_clean":"2023-10-09T04:08:27.628955+0000","last_became_active":"2023-10-09T04:07:34.640725+0000","last_became_peered":"2023-10-09T04:07:34.640725+0000","last_unstale":"2023-10-09T04:08:27.628955+0000","last_undegraded":"2023-10-09T04:08:27.628955+0000","last_fullsized":"2023-10-09T04:08:27.628955+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:45.119257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630240+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:08:27.630240+0000","last_peered":"2023-10-09T04:08:27.630240+0000","last_clean":"2023-10-09T04:08:27.630240+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:08:27.630240+0000","last_undegraded":"2023-10-09T04:08:27.630240+0000","last_fullsized":"2023-10-09T04:08:27.630240+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630330+0000","last_change":"2023-10-09T04:07:34.662253+0000","last_active":"2023-10-09T04:08:27.630330+0000","last_peered":"2023-10-09T04:08:27.630330+0000","last_clean":"2023-10-09T04:08:27.630330+0000","last_became_active":"2023-10-09T04:07:34.661919+0000","last_became_peered":"2023-10-09T04:07:34.661919+0000","last_unstale":"2023-10-09T04:08:27.630330+0000","last_undegraded":"2023-10-09T04:08:27.630330+0000","last_fullsized":"2023-10-09T04:08:27.630330+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:35.167033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022567+0000","last_change":"2023-10-09T04:07:34.651433+0000","last_active":"2023-10-09T04:08:28.022567+0000","last_peered":"2023-10-09T04:08:28.022567+0000","last_clean":"2023-10-09T04:08:28.022567+0000","last_became_active":"2023-10-09T04:07:34.650984+0000","last_became_peered":"2023-10-09T04:07:34.650984+0000","last_unstale":"2023-10-09T04:08:28.022567+0000","last_undegraded":"2023-10-09T04:08:28.022567+0000","last_fullsized":"2023-10-09T04:08:28.022567+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:57.513271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.632096+0000","last_change":"2023-10-09T04:07:34.659514+0000","last_active":"2023-10-09T04:08:27.632096+0000","last_peered":"2023-10-09T04:08:27.632096+0000","last_clean":"2023-10-09T04:08:27.632096+0000","last_became_active":"2023-10-09T04:07:34.659212+0000","last_became_peered":"2023-10-09T04:07:34.659212+0000","last_unstale":"2023-10-09T04:08:27.632096+0000","last_undegraded":"2023-10-09T04:08:27.632096+0000","last_fullsized":"2023-10-09T04:08:27.632096+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:30.189193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631243+0000","last_change":"2023-10-09T04:07:34.630774+0000","last_active":"2023-10-09T04:08:27.631243+0000","last_peered":"2023-10-09T04:08:27.631243+0000","last_clean":"2023-10-09T04:08:27.631243+0000","last_became_active":"2023-10-09T04:07:34.630525+0000","last_became_peered":"2023-10-09T04:07:34.630525+0000","last_unstale":"2023-10-09T04:08:27.631243+0000","last_undegraded":"2023-10-09T04:08:27.631243+0000","last_fullsized":"2023-10-09T04:08:27.631243+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:05.578748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.632243+0000","last_change":"2023-10-09T04:07:34.659260+0000","last_active":"2023-10-09T04:08:27.632243+0000","last_peered":"2023-10-09T04:08:27.632243+0000","last_clean":"2023-10-09T04:08:27.632243+0000","last_became_active":"2023-10-09T04:07:34.658976+0000","last_became_peered":"2023-10-09T04:07:34.658976+0000","last_unstale":"2023-10-09T04:08:27.632243+0000","last_undegraded":"2023-10-09T04:08:27.632243+0000","last_fullsized":"2023-10-09T04:08:27.632243+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:35.014675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634308+0000","last_change":"2023-10-09T04:07:34.634417+0000","last_active":"2023-10-09T04:08:27.634308+0000","last_peered":"2023-10-09T04:08:27.634308+0000","last_clean":"2023-10-09T04:08:27.634308+0000","last_became_active":"2023-10-09T04:07:34.634183+0000","last_became_peered":"2023-10-09T04:07:34.634183+0000","last_unstale":"2023-10-09T04:08:27.634308+0000","last_undegraded":"2023-10-09T04:08:27.634308+0000","last_fullsized":"2023-10-09T04:08:27.634308+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:13.162009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631571+0000","last_change":"2023-10-09T04:07:34.655573+0000","last_active":"2023-10-09T04:08:27.631571+0000","last_peered":"2023-10-09T04:08:27.631571+0000","last_clean":"2023-10-09T04:08:27.631571+0000","last_became_active":"2023-10-09T04:07:34.655326+0000","last_became_peered":"2023-10-09T04:07:34.655326+0000","last_unstale":"2023-10-09T04:08:27.631571+0000","last_undegraded":"2023-10-09T04:08:27.631571+0000","last_fullsized":"2023-10-09T04:08:27.631571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:07.510114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022787+0000","last_change":"2023-10-09T04:07:34.643373+0000","last_active":"2023-10-09T04:08:28.022787+0000","last_peered":"2023-10-09T04:08:28.022787+0000","last_clean":"2023-10-09T04:08:28.022787+0000","last_became_active":"2023-10-09T04:07:34.643033+0000","last_became_peered":"2023-10-09T04:07:34.643033+0000","last_unstale":"2023-10-09T04:08:28.022787+0000","last_undegraded":"2023-10-09T04:08:28.022787+0000","last_fullsized":"2023-10-09T04:08:28.022787+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:57.618154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.896496+0000","last_change":"2023-10-09T04:08:28.896496+0000","last_active":"2023-10-09T04:08:28.896496+0000","last_peered":"2023-10-09T04:08:28.896496+0000","last_clean":"2023-10-09T04:08:28.896496+0000","last_became_active":"2023-10-09T04:08:28.896152+0000","last_became_peered":"2023-10-09T04:08:28.896152+0000","last_unstale":"2023-10-09T04:08:28.896496+0000","last_undegraded":"2023-10-09T04:08:28.896496+0000","last_fullsized":"2023-10-09T04:08:28.896496+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:57.253137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628810+0000","last_change":"2023-10-09T04:07:34.640164+0000","last_active":"2023-10-09T04:08:27.628810+0000","last_peered":"2023-10-09T04:08:27.628810+0000","last_clean":"2023-10-09T04:08:27.628810+0000","last_became_active":"2023-10-09T04:07:34.639871+0000","last_became_peered":"2023-10-09T04:07:34.639871+0000","last_unstale":"2023-10-09T04:08:27.628810+0000","last_undegraded":"2023-10-09T04:08:27.628810+0000","last_fullsized":"2023-10-09T04:08:27.628810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:59.659674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.633592+0000","last_change":"2023-10-09T04:08:28.633592+0000","last_active":"2023-10-09T04:08:28.633592+0000","last_peered":"2023-10-09T04:08:28.633592+0000","last_clean":"2023-10-09T04:08:28.633592+0000","last_became_active":"2023-10-09T04:08:28.633218+0000","last_became_peered":"2023-10-09T04:08:28.633218+0000","last_unstale":"2023-10-09T04:08:28.633592+0000","last_undegraded":"2023-10-09T04:08:28.633592+0000","last_fullsized":"2023-10-09T04:08:28.633592+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:07.571024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637077+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:08:27.637077+0000","last_peered":"2023-10-09T04:08:27.637077+0000","last_clean":"2023-10-09T04:08:27.637077+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:08:27.637077+0000","last_undegraded":"2023-10-09T04:08:27.637077+0000","last_fullsized":"2023-10-09T04:08:27.637077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631119+0000","last_change":"2023-10-09T04:07:34.659608+0000","last_active":"2023-10-09T04:08:27.631119+0000","last_peered":"2023-10-09T04:08:27.631119+0000","last_clean":"2023-10-09T04:08:27.631119+0000","last_became_active":"2023-10-09T04:07:34.658918+0000","last_became_peered":"2023-10-09T04:07:34.658918+0000","last_unstale":"2023-10-09T04:08:27.631119+0000","last_undegraded":"2023-10-09T04:08:27.631119+0000","last_fullsized":"2023-10-09T04:08:27.631119+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:42.508338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637114+0000","last_change":"2023-10-09T04:07:34.631609+0000","last_active":"2023-10-09T04:08:27.637114+0000","last_peered":"2023-10-09T04:08:27.637114+0000","last_clean":"2023-10-09T04:08:27.637114+0000","last_became_active":"2023-10-09T04:07:34.631327+0000","last_became_peered":"2023-10-09T04:07:34.631327+0000","last_unstale":"2023-10-09T04:08:27.637114+0000","last_undegraded":"2023-10-09T04:08:27.637114+0000","last_fullsized":"2023-10-09T04:08:27.637114+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:51.678421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628586+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:08:27.628586+0000","last_peered":"2023-10-09T04:08:27.628586+0000","last_clean":"2023-10-09T04:08:27.628586+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:08:27.628586+0000","last_undegraded":"2023-10-09T04:08:27.628586+0000","last_fullsized":"2023-10-09T04:08:27.628586+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637222+0000","last_change":"2023-10-09T04:07:34.642317+0000","last_active":"2023-10-09T04:08:27.637222+0000","last_peered":"2023-10-09T04:08:27.637222+0000","last_clean":"2023-10-09T04:08:27.637222+0000","last_became_active":"2023-10-09T04:07:34.642028+0000","last_became_peered":"2023-10-09T04:07:34.642028+0000","last_unstale":"2023-10-09T04:08:27.637222+0000","last_undegraded":"2023-10-09T04:08:27.637222+0000","last_fullsized":"2023-10-09T04:08:27.637222+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:12.984276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.023045+0000","last_change":"2023-10-09T04:07:36.029846+0000","last_active":"2023-10-09T04:08:28.023045+0000","last_peered":"2023-10-09T04:08:28.023045+0000","last_clean":"2023-10-09T04:08:28.023045+0000","last_became_active":"2023-10-09T04:07:36.029223+0000","last_became_peered":"2023-10-09T04:07:36.029223+0000","last_unstale":"2023-10-09T04:08:28.023045+0000","last_undegraded":"2023-10-09T04:08:28.023045+0000","last_fullsized":"2023-10-09T04:08:28.023045+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:12.111167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634708+0000","last_change":"2023-10-09T04:07:34.640376+0000","last_active":"2023-10-09T04:08:27.634708+0000","last_peered":"2023-10-09T04:08:27.634708+0000","last_clean":"2023-10-09T04:08:27.634708+0000","last_became_active":"2023-10-09T04:07:34.640073+0000","last_became_peered":"2023-10-09T04:07:34.640073+0000","last_unstale":"2023-10-09T04:08:27.634708+0000","last_undegraded":"2023-10-09T04:08:27.634708+0000","last_fullsized":"2023-10-09T04:08:27.634708+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:16.684588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628374+0000","last_change":"2023-10-09T04:07:36.029535+0000","last_active":"2023-10-09T04:08:27.628374+0000","last_peered":"2023-10-09T04:08:27.628374+0000","last_clean":"2023-10-09T04:08:27.628374+0000","last_became_active":"2023-10-09T04:07:36.029175+0000","last_became_peered":"2023-10-09T04:07:36.029175+0000","last_unstale":"2023-10-09T04:08:27.628374+0000","last_undegraded":"2023-10-09T04:08:27.628374+0000","last_fullsized":"2023-10-09T04:08:27.628374+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:34.595934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636321+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:08:27.636321+0000","last_peered":"2023-10-09T04:08:27.636321+0000","last_clean":"2023-10-09T04:08:27.636321+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:08:27.636321+0000","last_undegraded":"2023-10-09T04:08:27.636321+0000","last_fullsized":"2023-10-09T04:08:27.636321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628931+0000","last_change":"2023-10-09T04:07:36.029411+0000","last_active":"2023-10-09T04:08:27.628931+0000","last_peered":"2023-10-09T04:08:27.628931+0000","last_clean":"2023-10-09T04:08:27.628931+0000","last_became_active":"2023-10-09T04:07:36.029109+0000","last_became_peered":"2023-10-09T04:07:36.029109+0000","last_unstale":"2023-10-09T04:08:27.628931+0000","last_undegraded":"2023-10-09T04:08:27.628931+0000","last_fullsized":"2023-10-09T04:08:27.628931+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:05.311520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628217+0000","last_change":"2023-10-09T04:07:34.646539+0000","last_active":"2023-10-09T04:08:27.628217+0000","last_peered":"2023-10-09T04:08:27.628217+0000","last_clean":"2023-10-09T04:08:27.628217+0000","last_became_active":"2023-10-09T04:07:34.646243+0000","last_became_peered":"2023-10-09T04:07:34.646243+0000","last_unstale":"2023-10-09T04:08:27.628217+0000","last_undegraded":"2023-10-09T04:08:27.628217+0000","last_fullsized":"2023-10-09T04:08:27.628217+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:01.969093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022936+0000","last_change":"2023-10-09T04:07:36.029585+0000","last_active":"2023-10-09T04:08:28.022936+0000","last_peered":"2023-10-09T04:08:28.022936+0000","last_clean":"2023-10-09T04:08:28.022936+0000","last_became_active":"2023-10-09T04:07:36.029157+0000","last_became_peered":"2023-10-09T04:07:36.029157+0000","last_unstale":"2023-10-09T04:08:28.022936+0000","last_undegraded":"2023-10-09T04:08:28.022936+0000","last_fullsized":"2023-10-09T04:08:28.022936+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:19.819664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636654+0000","last_change":"2023-10-09T04:07:34.632203+0000","last_active":"2023-10-09T04:08:27.636654+0000","last_peered":"2023-10-09T04:08:27.636654+0000","last_clean":"2023-10-09T04:08:27.636654+0000","last_became_active":"2023-10-09T04:07:34.631969+0000","last_became_peered":"2023-10-09T04:07:34.631969+0000","last_unstale":"2023-10-09T04:08:27.636654+0000","last_undegraded":"2023-10-09T04:08:27.636654+0000","last_fullsized":"2023-10-09T04:08:27.636654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:01.619892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631738+0000","last_change":"2023-10-09T04:07:34.659790+0000","last_active":"2023-10-09T04:08:27.631738+0000","last_peered":"2023-10-09T04:08:27.631738+0000","last_clean":"2023-10-09T04:08:27.631738+0000","last_became_active":"2023-10-09T04:07:34.659565+0000","last_became_peered":"2023-10-09T04:07:34.659565+0000","last_unstale":"2023-10-09T04:08:27.631738+0000","last_undegraded":"2023-10-09T04:08:27.631738+0000","last_fullsized":"2023-10-09T04:08:27.631738+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:32.044413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631785+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:08:27.631785+0000","last_peered":"2023-10-09T04:08:27.631785+0000","last_clean":"2023-10-09T04:08:27.631785+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:08:27.631785+0000","last_undegraded":"2023-10-09T04:08:27.631785+0000","last_fullsized":"2023-10-09T04:08:27.631785+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629366+0000","last_change":"2023-10-09T04:07:34.661425+0000","last_active":"2023-10-09T04:08:27.629366+0000","last_peered":"2023-10-09T04:08:27.629366+0000","last_clean":"2023-10-09T04:08:27.629366+0000","last_became_active":"2023-10-09T04:07:34.661099+0000","last_became_peered":"2023-10-09T04:07:34.661099+0000","last_unstale":"2023-10-09T04:08:27.629366+0000","last_undegraded":"2023-10-09T04:08:27.629366+0000","last_fullsized":"2023-10-09T04:08:27.629366+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:53.588610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636962+0000","last_change":"2023-10-09T04:07:34.641392+0000","last_active":"2023-10-09T04:08:27.636962+0000","last_peered":"2023-10-09T04:08:27.636962+0000","last_clean":"2023-10-09T04:08:27.636962+0000","last_became_active":"2023-10-09T04:07:34.641037+0000","last_became_peered":"2023-10-09T04:07:34.641037+0000","last_unstale":"2023-10-09T04:08:27.636962+0000","last_undegraded":"2023-10-09T04:08:27.636962+0000","last_fullsized":"2023-10-09T04:08:27.636962+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:43.467265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022724+0000","last_change":"2023-10-09T04:07:36.031224+0000","last_active":"2023-10-09T04:08:28.022724+0000","last_peered":"2023-10-09T04:08:28.022724+0000","last_clean":"2023-10-09T04:08:28.022724+0000","last_became_active":"2023-10-09T04:07:36.030834+0000","last_became_peered":"2023-10-09T04:07:36.030834+0000","last_unstale":"2023-10-09T04:08:28.022724+0000","last_undegraded":"2023-10-09T04:08:28.022724+0000","last_fullsized":"2023-10-09T04:08:28.022724+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:21.909762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628958+0000","last_change":"2023-10-09T04:07:34.631933+0000","last_active":"2023-10-09T04:08:27.628958+0000","last_peered":"2023-10-09T04:08:27.628958+0000","last_clean":"2023-10-09T04:08:27.628958+0000","last_became_active":"2023-10-09T04:07:34.631667+0000","last_became_peered":"2023-10-09T04:07:34.631667+0000","last_unstale":"2023-10-09T04:08:27.628958+0000","last_undegraded":"2023-10-09T04:08:27.628958+0000","last_fullsized":"2023-10-09T04:08:27.628958+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:58.644048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022596+0000","last_change":"2023-10-09T04:07:35.800509+0000","last_active":"2023-10-09T04:08:28.022596+0000","last_peered":"2023-10-09T04:08:28.022596+0000","last_clean":"2023-10-09T04:08:28.022596+0000","last_became_active":"2023-10-09T04:07:35.800155+0000","last_became_peered":"2023-10-09T04:07:35.800155+0000","last_unstale":"2023-10-09T04:08:28.022596+0000","last_undegraded":"2023-10-09T04:08:28.022596+0000","last_fullsized":"2023-10-09T04:08:28.022596+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:50.929412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628872+0000","last_change":"2023-10-09T04:07:34.646997+0000","last_active":"2023-10-09T04:08:27.628872+0000","last_peered":"2023-10-09T04:08:27.628872+0000","last_clean":"2023-10-09T04:08:27.628872+0000","last_became_active":"2023-10-09T04:07:34.645625+0000","last_became_peered":"2023-10-09T04:07:34.645625+0000","last_unstale":"2023-10-09T04:08:27.628872+0000","last_undegraded":"2023-10-09T04:08:27.628872+0000","last_fullsized":"2023-10-09T04:08:27.628872+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:40.584254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634306+0000","last_change":"2023-10-09T04:07:34.676137+0000","last_active":"2023-10-09T04:08:27.634306+0000","last_peered":"2023-10-09T04:08:27.634306+0000","last_clean":"2023-10-09T04:08:27.634306+0000","last_became_active":"2023-10-09T04:07:34.675822+0000","last_became_peered":"2023-10-09T04:07:34.675822+0000","last_unstale":"2023-10-09T04:08:27.634306+0000","last_undegraded":"2023-10-09T04:08:27.634306+0000","last_fullsized":"2023-10-09T04:08:27.634306+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:06.758028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022654+0000","last_change":"2023-10-09T04:07:34.646983+0000","last_active":"2023-10-09T04:08:28.022654+0000","last_peered":"2023-10-09T04:08:28.022654+0000","last_clean":"2023-10-09T04:08:28.022654+0000","last_became_active":"2023-10-09T04:07:34.646623+0000","last_became_peered":"2023-10-09T04:07:34.646623+0000","last_unstale":"2023-10-09T04:08:28.022654+0000","last_undegraded":"2023-10-09T04:08:28.022654+0000","last_fullsized":"2023-10-09T04:08:28.022654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.119527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628454+0000","last_change":"2023-10-09T04:07:36.029481+0000","last_active":"2023-10-09T04:08:27.628454+0000","last_peered":"2023-10-09T04:08:27.628454+0000","last_clean":"2023-10-09T04:08:27.628454+0000","last_became_active":"2023-10-09T04:07:36.029167+0000","last_became_peered":"2023-10-09T04:07:36.029167+0000","last_unstale":"2023-10-09T04:08:27.628454+0000","last_undegraded":"2023-10-09T04:08:27.628454+0000","last_fullsized":"2023-10-09T04:08:27.628454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:42.409682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629155+0000","last_change":"2023-10-09T04:07:34.645964+0000","last_active":"2023-10-09T04:08:27.629155+0000","last_peered":"2023-10-09T04:08:27.629155+0000","last_clean":"2023-10-09T04:08:27.629155+0000","last_became_active":"2023-10-09T04:07:34.645619+0000","last_became_peered":"2023-10-09T04:07:34.645619+0000","last_unstale":"2023-10-09T04:08:27.629155+0000","last_undegraded":"2023-10-09T04:08:27.629155+0000","last_fullsized":"2023-10-09T04:08:27.629155+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:26.090109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022969+0000","last_change":"2023-10-09T04:07:36.284775+0000","last_active":"2023-10-09T04:08:28.022969+0000","last_peered":"2023-10-09T04:08:28.022969+0000","last_clean":"2023-10-09T04:08:28.022969+0000","last_became_active":"2023-10-09T04:07:36.284501+0000","last_became_peered":"2023-10-09T04:07:36.284501+0000","last_unstale":"2023-10-09T04:08:28.022969+0000","last_undegraded":"2023-10-09T04:08:28.022969+0000","last_fullsized":"2023-10-09T04:08:28.022969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:17.182374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631184+0000","last_change":"2023-10-09T04:07:34.630544+0000","last_active":"2023-10-09T04:08:27.631184+0000","last_peered":"2023-10-09T04:08:27.631184+0000","last_clean":"2023-10-09T04:08:27.631184+0000","last_became_active":"2023-10-09T04:07:34.630230+0000","last_became_peered":"2023-10-09T04:07:34.630230+0000","last_unstale":"2023-10-09T04:08:27.631184+0000","last_undegraded":"2023-10-09T04:08:27.631184+0000","last_fullsized":"2023-10-09T04:08:27.631184+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:32.002588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1308,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1339392,"data_stored":905770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0329999999999999,"15min":1.123},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76500000000000001,"5min":10.557,"15min":10.557},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.74399999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.83999999999999997,"15min":0.88800000000000001},"min":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71099999999999997,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.82499999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.81599999999999995,"15min":0.86699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69799999999999995,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.76200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72999999999999998,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67100000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.79600000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.64300000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.44,"5min":0.52000000000000002,"15min":0.53400000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.58399999999999996,"5min":1.391,"15min":1.391},"last":0.73299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.77400000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.63500000000000001,"15min":0.65400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69099999999999995,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.85999999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495654,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298580,"kb_used_data":1292,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446572,"statfs":{"total":95995035648,"available":95689289728,"internally_reserved":0,"allocated":1323008,"data_stored":905301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.53700000000000003,"15min":0.55000000000000004},"min":{"1min":0.372,"5min":0.312,"15min":0.312},"max":{"1min":0.70499999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54000000000000004,"15min":0.54500000000000004},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.64800000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.57399999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56100000000000005,"15min":0.57499999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.56000000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69899999999999995,"5min":0.748,"15min":0.748},"last":0.68999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55700000000000005,"15min":0.56100000000000005},"min":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78300000000000003,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57399999999999995,"15min":0.57699999999999996},"min":{"1min":0.41299999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.628}]},{"osd":3,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.58399999999999996,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.5839999999999996,"5min":4.5839999999999996,"15min":4.5839999999999996},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.64300000000000002,"15min":0.61599999999999999},"min":{"1min":0.42999999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.524,"5min":4.524,"15min":4.524},"last":0.61099999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45300000000000001},"min":{"1min":0.313,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.502,"15min":0.48799999999999999},"min":{"1min":0.37,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":1.0229999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.35899999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.75700000000000001,"5min":0.877,"15min":0.877},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.59699999999999998,"15min":0.61699999999999999},"min":{"1min":0.34999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.88700000000000001,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.88700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.77300000000000002,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.96299999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":313650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.249},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.39800000000000002,"15min":0.372},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.37}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.435,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.41399999999999998,"15min":0.42299999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.51500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.33700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.47599999999999998,"15min":0.5},"min":{"1min":0.31900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.47499999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.439,"15min":0.45300000000000001},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.59699999999999998,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.32500000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.44,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.75600000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.70099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58199999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.60899999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70899999999999996,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69899999999999995,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.55000000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.61099999999999999,"15min":0.625},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81399999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.629},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65600000000000003,"15min":0.67400000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.79700000000000004,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.67300000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.65100000000000002,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.48099999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.60899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298576,"kb_used_data":1288,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446576,"statfs":{"total":95995035648,"available":95689293824,"internally_reserved":0,"allocated":1318912,"data_stored":904594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:07:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.32800000000000001},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53700000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.38400000000000001,"15min":0.34599999999999997},"min":{"1min":0.30599999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.53600000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.44}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.58599999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:08:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.44500000000000001,"15min":0.46800000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.51000000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.376},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.48799999999999999,"15min":0.51400000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.626,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.373,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77800000000000002,"5min":0.878,"15min":0.878},"last":0.75},{"interface":"front","average":{"1min":0.626,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84299999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.71299999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61099999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.622,"15min":0.63800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77600000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.76800000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80600000000000005,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.623},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79600000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.86099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:07:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65100000000000002,"15min":0.65900000000000003},"min":{"1min":0.46600000000000003,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80900000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.69099999999999995,"15min":0.68700000000000006},"min":{"1min":0.505,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.81200000000000006}]}]},{"osd":2,"up_from":18,"seq":77309411386,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297996,"kb_used_data":708,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447156,"statfs":{"total":95995035648,"available":95689887744,"internally_reserved":0,"allocated":724992,"data_stored":314238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70699999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.371}]},{"osd":1,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.503,"15min":0.48599999999999999},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.6659999999999999,"5min":3.6659999999999999,"15min":3.6659999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.50800000000000001,"15min":0.49099999999999999},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.5779999999999998,"5min":3.5779999999999998,"15min":3.5779999999999998},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.45400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.46999999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.82999999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.61599999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79400000000000004,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.88}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.62,"15min":0.61799999999999999},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.85999999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67400000000000004},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98099999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.629,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.625}]}]},{"osd":3,"up_from":25,"seq":107374182454,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297992,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447160,"statfs":{"total":95995035648,"available":95689891840,"internally_reserved":0,"allocated":720896,"data_stored":313740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.41899999999999998},"min":{"1min":0.20399999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41699999999999998,"15min":0.44900000000000001},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.43099999999999999,"15min":0.46300000000000002},"min":{"1min":0.255,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.54800000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.43099999999999999,"15min":0.47499999999999998},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56999999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.376}]},{"osd":2,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.46400000000000002,"15min":0.496},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54000000000000004,"5min":0.745,"15min":0.745},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.46400000000000002,"15min":0.503},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.63300000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.39800000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.73399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.626,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.76300000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.504}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.624,"15min":0.63500000000000001},"min":{"1min":0.433,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77100000000000002,"5min":0.874,"15min":0.874},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.63,"15min":0.63900000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.748,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.61399999999999999,"15min":0.622},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.76400000000000001,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.54500000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73599999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.46500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81200000000000006,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.45900000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018928,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":741376,"data_stored":315332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.48699999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.503,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.60199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.52600000000000002,"15min":0.51700000000000002},"min":{"1min":0.40200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.58299999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.432,"5min":0.37,"15min":0.37},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54100000000000004,"15min":0.52300000000000002},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.073,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.439,"15min":0.42299999999999999},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.503,"5min":0.44400000000000001,"15min":0.42399999999999999},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.68000000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.47899999999999998,"15min":0.47299999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.68899999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.73099999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69099999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.79700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757292,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297996,"kb_used_data":708,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447156,"statfs":{"total":95995035648,"available":95689887744,"internally_reserved":0,"allocated":724992,"data_stored":314274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53000000000000003,"15min":0.52400000000000002},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65000000000000002,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.66300000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71799999999999997,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.47599999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.53000000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.443,"15min":0.41899999999999998},"min":{"1min":0.35499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.45400000000000001,"15min":0.42799999999999999},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.45500000000000002,"15min":0.442},"min":{"1min":0.38700000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.45900000000000002,"15min":0.443},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.63700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.73199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:08:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:39 smithi029 ceph-mon[101043]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:39 smithi029 ceph-mon[119131]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:39 smithi177 ceph-mon[106070]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:40 smithi029 ceph-mon[101043]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:40 smithi029 ceph-mon[119131]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:40 smithi177 ceph-mon[106070]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:42.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:08:42.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:42.883 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:42 smithi029 ceph-mon[101043]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:42.883 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:42.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:42 smithi029 ceph-mon[119131]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:42.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:42 smithi177 ceph-mon[106070]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:43.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:08:42 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:08:44.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:44 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:44] "GET /metrics HTTP/1.1" 200 238848 "" "Prometheus/2.43.0" 2023-10-09T04:08:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:44.587 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:08:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:44 smithi029 ceph-mon[101043]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:44 smithi029 ceph-mon[119131]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:44 smithi177 ceph-mon[106070]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:45.398 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":82,"stamp":"2023-10-09T04:08:42.778839+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385736,"kb_used_data":7432,"kb_used_omap":0,"kb_used_meta":2378240,"kb_avail":747575480,"statfs":{"total":767960285184,"available":765517291520,"internally_reserved":0,"allocated":7610368,"data_stored":4286899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2435317760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002776"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.636477+0000","last_change":"2023-10-09T04:08:28.636477+0000","last_active":"2023-10-09T04:08:28.636477+0000","last_peered":"2023-10-09T04:08:28.636477+0000","last_clean":"2023-10-09T04:08:28.636477+0000","last_became_active":"2023-10-09T04:08:28.635995+0000","last_became_peered":"2023-10-09T04:08:28.635995+0000","last_unstale":"2023-10-09T04:08:28.636477+0000","last_undegraded":"2023-10-09T04:08:28.636477+0000","last_fullsized":"2023-10-09T04:08:28.636477+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:15.972599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628792+0000","last_change":"2023-10-09T04:07:34.646776+0000","last_active":"2023-10-09T04:08:27.628792+0000","last_peered":"2023-10-09T04:08:27.628792+0000","last_clean":"2023-10-09T04:08:27.628792+0000","last_became_active":"2023-10-09T04:07:34.644325+0000","last_became_peered":"2023-10-09T04:07:34.644325+0000","last_unstale":"2023-10-09T04:08:27.628792+0000","last_undegraded":"2023-10-09T04:08:27.628792+0000","last_fullsized":"2023-10-09T04:08:27.628792+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:23.379485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022721+0000","last_change":"2023-10-09T04:07:35.653186+0000","last_active":"2023-10-09T04:08:28.022721+0000","last_peered":"2023-10-09T04:08:28.022721+0000","last_clean":"2023-10-09T04:08:28.022721+0000","last_became_active":"2023-10-09T04:07:35.652908+0000","last_became_peered":"2023-10-09T04:07:35.652908+0000","last_unstale":"2023-10-09T04:08:28.022721+0000","last_undegraded":"2023-10-09T04:08:28.022721+0000","last_fullsized":"2023-10-09T04:08:28.022721+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:42.975764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.023033+0000","last_change":"2023-10-09T04:07:34.638165+0000","last_active":"2023-10-09T04:08:28.023033+0000","last_peered":"2023-10-09T04:08:28.023033+0000","last_clean":"2023-10-09T04:08:28.023033+0000","last_became_active":"2023-10-09T04:07:34.637823+0000","last_became_peered":"2023-10-09T04:07:34.637823+0000","last_unstale":"2023-10-09T04:08:28.023033+0000","last_undegraded":"2023-10-09T04:08:28.023033+0000","last_fullsized":"2023-10-09T04:08:28.023033+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:13.118351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631244+0000","last_change":"2023-10-09T04:07:34.676759+0000","last_active":"2023-10-09T04:08:27.631244+0000","last_peered":"2023-10-09T04:08:27.631244+0000","last_clean":"2023-10-09T04:08:27.631244+0000","last_became_active":"2023-10-09T04:07:34.676294+0000","last_became_peered":"2023-10-09T04:07:34.676294+0000","last_unstale":"2023-10-09T04:08:27.631244+0000","last_undegraded":"2023-10-09T04:08:27.631244+0000","last_fullsized":"2023-10-09T04:08:27.631244+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:44.387110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629025+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:08:27.629025+0000","last_peered":"2023-10-09T04:08:27.629025+0000","last_clean":"2023-10-09T04:08:27.629025+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:08:27.629025+0000","last_undegraded":"2023-10-09T04:08:27.629025+0000","last_fullsized":"2023-10-09T04:08:27.629025+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631988+0000","last_change":"2023-10-09T04:07:34.659699+0000","last_active":"2023-10-09T04:08:27.631988+0000","last_peered":"2023-10-09T04:08:27.631988+0000","last_clean":"2023-10-09T04:08:27.631988+0000","last_became_active":"2023-10-09T04:07:34.659189+0000","last_became_peered":"2023-10-09T04:07:34.659189+0000","last_unstale":"2023-10-09T04:08:27.631988+0000","last_undegraded":"2023-10-09T04:08:27.631988+0000","last_fullsized":"2023-10-09T04:08:27.631988+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:44.119093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628741+0000","last_change":"2023-10-09T04:07:34.631386+0000","last_active":"2023-10-09T04:08:27.628741+0000","last_peered":"2023-10-09T04:08:27.628741+0000","last_clean":"2023-10-09T04:08:27.628741+0000","last_became_active":"2023-10-09T04:07:34.631090+0000","last_became_peered":"2023-10-09T04:07:34.631090+0000","last_unstale":"2023-10-09T04:08:27.628741+0000","last_undegraded":"2023-10-09T04:08:27.628741+0000","last_fullsized":"2023-10-09T04:08:27.628741+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:05.633909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636714+0000","last_change":"2023-10-09T04:07:34.659338+0000","last_active":"2023-10-09T04:08:27.636714+0000","last_peered":"2023-10-09T04:08:27.636714+0000","last_clean":"2023-10-09T04:08:27.636714+0000","last_became_active":"2023-10-09T04:07:34.658997+0000","last_became_peered":"2023-10-09T04:07:34.658997+0000","last_unstale":"2023-10-09T04:08:27.636714+0000","last_undegraded":"2023-10-09T04:08:27.636714+0000","last_fullsized":"2023-10-09T04:08:27.636714+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:38.217792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628581+0000","last_change":"2023-10-09T04:07:34.642042+0000","last_active":"2023-10-09T04:08:27.628581+0000","last_peered":"2023-10-09T04:08:27.628581+0000","last_clean":"2023-10-09T04:08:27.628581+0000","last_became_active":"2023-10-09T04:07:34.641579+0000","last_became_peered":"2023-10-09T04:07:34.641579+0000","last_unstale":"2023-10-09T04:08:27.628581+0000","last_undegraded":"2023-10-09T04:08:27.628581+0000","last_fullsized":"2023-10-09T04:08:27.628581+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:44.869504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631210+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:08:27.631210+0000","last_peered":"2023-10-09T04:08:27.631210+0000","last_clean":"2023-10-09T04:08:27.631210+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:08:27.631210+0000","last_undegraded":"2023-10-09T04:08:27.631210+0000","last_fullsized":"2023-10-09T04:08:27.631210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628551+0000","last_change":"2023-10-09T04:07:34.645613+0000","last_active":"2023-10-09T04:08:27.628551+0000","last_peered":"2023-10-09T04:08:27.628551+0000","last_clean":"2023-10-09T04:08:27.628551+0000","last_became_active":"2023-10-09T04:07:34.645309+0000","last_became_peered":"2023-10-09T04:07:34.645309+0000","last_unstale":"2023-10-09T04:08:27.628551+0000","last_undegraded":"2023-10-09T04:08:27.628551+0000","last_fullsized":"2023-10-09T04:08:27.628551+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:58:07.674226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634715+0000","last_change":"2023-10-09T04:06:28.908019+0000","last_active":"2023-10-09T04:08:27.634715+0000","last_peered":"2023-10-09T04:08:27.634715+0000","last_clean":"2023-10-09T04:08:27.634715+0000","last_became_active":"2023-10-09T04:06:28.907736+0000","last_became_peered":"2023-10-09T04:06:28.907736+0000","last_unstale":"2023-10-09T04:08:27.634715+0000","last_undegraded":"2023-10-09T04:08:27.634715+0000","last_fullsized":"2023-10-09T04:08:27.634715+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:13:40.663976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022540+0000","last_change":"2023-10-09T04:07:36.030076+0000","last_active":"2023-10-09T04:08:28.022540+0000","last_peered":"2023-10-09T04:08:28.022540+0000","last_clean":"2023-10-09T04:08:28.022540+0000","last_became_active":"2023-10-09T04:07:36.029271+0000","last_became_peered":"2023-10-09T04:07:36.029271+0000","last_unstale":"2023-10-09T04:08:28.022540+0000","last_undegraded":"2023-10-09T04:08:28.022540+0000","last_fullsized":"2023-10-09T04:08:28.022540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:06:12.758916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629037+0000","last_change":"2023-10-09T04:06:28.986921+0000","last_active":"2023-10-09T04:08:27.629037+0000","last_peered":"2023-10-09T04:08:27.629037+0000","last_clean":"2023-10-09T04:08:27.629037+0000","last_became_active":"2023-10-09T04:06:28.986662+0000","last_became_peered":"2023-10-09T04:06:28.986662+0000","last_unstale":"2023-10-09T04:08:27.629037+0000","last_undegraded":"2023-10-09T04:08:27.629037+0000","last_fullsized":"2023-10-09T04:08:27.629037+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:58:18.613517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.023114+0000","last_change":"2023-10-09T04:07:34.643380+0000","last_active":"2023-10-09T04:08:28.023114+0000","last_peered":"2023-10-09T04:08:28.023114+0000","last_clean":"2023-10-09T04:08:28.023114+0000","last_became_active":"2023-10-09T04:07:34.643086+0000","last_became_peered":"2023-10-09T04:07:34.643086+0000","last_unstale":"2023-10-09T04:08:28.023114+0000","last_undegraded":"2023-10-09T04:08:28.023114+0000","last_fullsized":"2023-10-09T04:08:28.023114+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:36:13.152038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628441+0000","last_change":"2023-10-09T04:07:36.030238+0000","last_active":"2023-10-09T04:08:27.628441+0000","last_peered":"2023-10-09T04:08:27.628441+0000","last_clean":"2023-10-09T04:08:27.628441+0000","last_became_active":"2023-10-09T04:07:36.029932+0000","last_became_peered":"2023-10-09T04:07:36.029932+0000","last_unstale":"2023-10-09T04:08:27.628441+0000","last_undegraded":"2023-10-09T04:08:27.628441+0000","last_fullsized":"2023-10-09T04:08:27.628441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:34:25.269223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630408+0000","last_change":"2023-10-09T04:07:34.633885+0000","last_active":"2023-10-09T04:08:27.630408+0000","last_peered":"2023-10-09T04:08:27.630408+0000","last_clean":"2023-10-09T04:08:27.630408+0000","last_became_active":"2023-10-09T04:07:34.633345+0000","last_became_peered":"2023-10-09T04:07:34.633345+0000","last_unstale":"2023-10-09T04:08:27.630408+0000","last_undegraded":"2023-10-09T04:08:27.630408+0000","last_fullsized":"2023-10-09T04:08:27.630408+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:29:42.728405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637020+0000","last_change":"2023-10-09T04:06:29.273673+0000","last_active":"2023-10-09T04:08:27.637020+0000","last_peered":"2023-10-09T04:08:27.637020+0000","last_clean":"2023-10-09T04:08:27.637020+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:08:27.637020+0000","last_undegraded":"2023-10-09T04:08:27.637020+0000","last_fullsized":"2023-10-09T04:08:27.637020+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:19:47.034548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628630+0000","last_change":"2023-10-09T04:07:35.800119+0000","last_active":"2023-10-09T04:08:27.628630+0000","last_peered":"2023-10-09T04:08:27.628630+0000","last_clean":"2023-10-09T04:08:27.628630+0000","last_became_active":"2023-10-09T04:07:35.799858+0000","last_became_peered":"2023-10-09T04:07:35.799858+0000","last_unstale":"2023-10-09T04:08:27.628630+0000","last_undegraded":"2023-10-09T04:08:27.628630+0000","last_fullsized":"2023-10-09T04:08:27.628630+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:04.465377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628499+0000","last_change":"2023-10-09T04:07:34.645437+0000","last_active":"2023-10-09T04:08:27.628499+0000","last_peered":"2023-10-09T04:08:27.628499+0000","last_clean":"2023-10-09T04:08:27.628499+0000","last_became_active":"2023-10-09T04:07:34.645140+0000","last_became_peered":"2023-10-09T04:07:34.645140+0000","last_unstale":"2023-10-09T04:08:27.628499+0000","last_undegraded":"2023-10-09T04:08:27.628499+0000","last_fullsized":"2023-10-09T04:08:27.628499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:45.140472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636396+0000","last_change":"2023-10-09T04:06:29.273687+0000","last_active":"2023-10-09T04:08:27.636396+0000","last_peered":"2023-10-09T04:08:27.636396+0000","last_clean":"2023-10-09T04:08:27.636396+0000","last_became_active":"2023-10-09T04:06:29.273349+0000","last_became_peered":"2023-10-09T04:06:29.273349+0000","last_unstale":"2023-10-09T04:08:27.636396+0000","last_undegraded":"2023-10-09T04:08:27.636396+0000","last_fullsized":"2023-10-09T04:08:27.636396+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:52:17.939519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634402+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:08:27.634402+0000","last_peered":"2023-10-09T04:08:27.634402+0000","last_clean":"2023-10-09T04:08:27.634402+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:08:27.634402+0000","last_undegraded":"2023-10-09T04:08:27.634402+0000","last_fullsized":"2023-10-09T04:08:27.634402+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022762+0000","last_change":"2023-10-09T04:07:34.639655+0000","last_active":"2023-10-09T04:08:28.022762+0000","last_peered":"2023-10-09T04:08:28.022762+0000","last_clean":"2023-10-09T04:08:28.022762+0000","last_became_active":"2023-10-09T04:07:34.639321+0000","last_became_peered":"2023-10-09T04:07:34.639321+0000","last_unstale":"2023-10-09T04:08:28.022762+0000","last_undegraded":"2023-10-09T04:08:28.022762+0000","last_fullsized":"2023-10-09T04:08:28.022762+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:13:25.904866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":278,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636308+0000","last_change":"2023-10-09T04:05:54.579980+0000","last_active":"2023-10-09T04:08:27.636308+0000","last_peered":"2023-10-09T04:08:27.636308+0000","last_clean":"2023-10-09T04:08:27.636308+0000","last_became_active":"2023-10-09T04:05:54.579581+0000","last_became_peered":"2023-10-09T04:05:54.579581+0000","last_unstale":"2023-10-09T04:08:27.636308+0000","last_undegraded":"2023-10-09T04:08:27.636308+0000","last_fullsized":"2023-10-09T04:08:27.636308+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:43.156581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628728+0000","last_change":"2023-10-09T04:06:28.906978+0000","last_active":"2023-10-09T04:08:27.628728+0000","last_peered":"2023-10-09T04:08:27.628728+0000","last_clean":"2023-10-09T04:08:27.628728+0000","last_became_active":"2023-10-09T04:06:28.906540+0000","last_became_peered":"2023-10-09T04:06:28.906540+0000","last_unstale":"2023-10-09T04:08:27.628728+0000","last_undegraded":"2023-10-09T04:08:27.628728+0000","last_fullsized":"2023-10-09T04:08:27.628728+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:05.118528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630252+0000","last_change":"2023-10-09T04:07:34.676079+0000","last_active":"2023-10-09T04:08:27.630252+0000","last_peered":"2023-10-09T04:08:27.630252+0000","last_clean":"2023-10-09T04:08:27.630252+0000","last_became_active":"2023-10-09T04:07:34.675776+0000","last_became_peered":"2023-10-09T04:07:34.675776+0000","last_unstale":"2023-10-09T04:08:27.630252+0000","last_undegraded":"2023-10-09T04:08:27.630252+0000","last_fullsized":"2023-10-09T04:08:27.630252+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:26.867406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629037+0000","last_change":"2023-10-09T04:07:34.632817+0000","last_active":"2023-10-09T04:08:27.629037+0000","last_peered":"2023-10-09T04:08:27.629037+0000","last_clean":"2023-10-09T04:08:27.629037+0000","last_became_active":"2023-10-09T04:07:34.632434+0000","last_became_peered":"2023-10-09T04:07:34.632434+0000","last_unstale":"2023-10-09T04:08:27.629037+0000","last_undegraded":"2023-10-09T04:08:27.629037+0000","last_fullsized":"2023-10-09T04:08:27.629037+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:37.018559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636392+0000","last_change":"2023-10-09T04:06:29.273597+0000","last_active":"2023-10-09T04:08:27.636392+0000","last_peered":"2023-10-09T04:08:27.636392+0000","last_clean":"2023-10-09T04:08:27.636392+0000","last_became_active":"2023-10-09T04:06:29.273224+0000","last_became_peered":"2023-10-09T04:06:29.273224+0000","last_unstale":"2023-10-09T04:08:27.636392+0000","last_undegraded":"2023-10-09T04:08:27.636392+0000","last_fullsized":"2023-10-09T04:08:27.636392+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:52.719321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630324+0000","last_change":"2023-10-09T04:07:34.659865+0000","last_active":"2023-10-09T04:08:27.630324+0000","last_peered":"2023-10-09T04:08:27.630324+0000","last_clean":"2023-10-09T04:08:27.630324+0000","last_became_active":"2023-10-09T04:07:34.659515+0000","last_became_peered":"2023-10-09T04:07:34.659515+0000","last_unstale":"2023-10-09T04:08:27.630324+0000","last_undegraded":"2023-10-09T04:08:27.630324+0000","last_fullsized":"2023-10-09T04:08:27.630324+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:07.703247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631810+0000","last_change":"2023-10-09T04:07:34.641445+0000","last_active":"2023-10-09T04:08:27.631810+0000","last_peered":"2023-10-09T04:08:27.631810+0000","last_clean":"2023-10-09T04:08:27.631810+0000","last_became_active":"2023-10-09T04:07:34.641181+0000","last_became_peered":"2023-10-09T04:07:34.641181+0000","last_unstale":"2023-10-09T04:08:27.631810+0000","last_undegraded":"2023-10-09T04:08:27.631810+0000","last_fullsized":"2023-10-09T04:08:27.631810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:08.897577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628723+0000","last_change":"2023-10-09T04:06:28.907512+0000","last_active":"2023-10-09T04:08:27.628723+0000","last_peered":"2023-10-09T04:08:27.628723+0000","last_clean":"2023-10-09T04:08:27.628723+0000","last_became_active":"2023-10-09T04:06:28.906953+0000","last_became_peered":"2023-10-09T04:06:28.906953+0000","last_unstale":"2023-10-09T04:08:27.628723+0000","last_undegraded":"2023-10-09T04:08:27.628723+0000","last_fullsized":"2023-10-09T04:08:27.628723+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:16.817208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629441+0000","last_change":"2023-10-09T04:07:34.675845+0000","last_active":"2023-10-09T04:08:27.629441+0000","last_peered":"2023-10-09T04:08:27.629441+0000","last_clean":"2023-10-09T04:08:27.629441+0000","last_became_active":"2023-10-09T04:07:34.675371+0000","last_became_peered":"2023-10-09T04:07:34.675371+0000","last_unstale":"2023-10-09T04:08:27.629441+0000","last_undegraded":"2023-10-09T04:08:27.629441+0000","last_fullsized":"2023-10-09T04:08:27.629441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:45.429941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628955+0000","last_change":"2023-10-09T04:07:34.640999+0000","last_active":"2023-10-09T04:08:27.628955+0000","last_peered":"2023-10-09T04:08:27.628955+0000","last_clean":"2023-10-09T04:08:27.628955+0000","last_became_active":"2023-10-09T04:07:34.640725+0000","last_became_peered":"2023-10-09T04:07:34.640725+0000","last_unstale":"2023-10-09T04:08:27.628955+0000","last_undegraded":"2023-10-09T04:08:27.628955+0000","last_fullsized":"2023-10-09T04:08:27.628955+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:50:45.119257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630240+0000","last_change":"2023-10-09T04:06:29.273537+0000","last_active":"2023-10-09T04:08:27.630240+0000","last_peered":"2023-10-09T04:08:27.630240+0000","last_clean":"2023-10-09T04:08:27.630240+0000","last_became_active":"2023-10-09T04:06:29.273215+0000","last_became_peered":"2023-10-09T04:06:29.273215+0000","last_unstale":"2023-10-09T04:08:27.630240+0000","last_undegraded":"2023-10-09T04:08:27.630240+0000","last_fullsized":"2023-10-09T04:08:27.630240+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:25.010041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.630330+0000","last_change":"2023-10-09T04:07:34.662253+0000","last_active":"2023-10-09T04:08:27.630330+0000","last_peered":"2023-10-09T04:08:27.630330+0000","last_clean":"2023-10-09T04:08:27.630330+0000","last_became_active":"2023-10-09T04:07:34.661919+0000","last_became_peered":"2023-10-09T04:07:34.661919+0000","last_unstale":"2023-10-09T04:08:27.630330+0000","last_undegraded":"2023-10-09T04:08:27.630330+0000","last_fullsized":"2023-10-09T04:08:27.630330+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:35.167033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022567+0000","last_change":"2023-10-09T04:07:34.651433+0000","last_active":"2023-10-09T04:08:28.022567+0000","last_peered":"2023-10-09T04:08:28.022567+0000","last_clean":"2023-10-09T04:08:28.022567+0000","last_became_active":"2023-10-09T04:07:34.650984+0000","last_became_peered":"2023-10-09T04:07:34.650984+0000","last_unstale":"2023-10-09T04:08:28.022567+0000","last_undegraded":"2023-10-09T04:08:28.022567+0000","last_fullsized":"2023-10-09T04:08:28.022567+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:57.513271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.632096+0000","last_change":"2023-10-09T04:07:34.659514+0000","last_active":"2023-10-09T04:08:27.632096+0000","last_peered":"2023-10-09T04:08:27.632096+0000","last_clean":"2023-10-09T04:08:27.632096+0000","last_became_active":"2023-10-09T04:07:34.659212+0000","last_became_peered":"2023-10-09T04:07:34.659212+0000","last_unstale":"2023-10-09T04:08:27.632096+0000","last_undegraded":"2023-10-09T04:08:27.632096+0000","last_fullsized":"2023-10-09T04:08:27.632096+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:30.189193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631243+0000","last_change":"2023-10-09T04:07:34.630774+0000","last_active":"2023-10-09T04:08:27.631243+0000","last_peered":"2023-10-09T04:08:27.631243+0000","last_clean":"2023-10-09T04:08:27.631243+0000","last_became_active":"2023-10-09T04:07:34.630525+0000","last_became_peered":"2023-10-09T04:07:34.630525+0000","last_unstale":"2023-10-09T04:08:27.631243+0000","last_undegraded":"2023-10-09T04:08:27.631243+0000","last_fullsized":"2023-10-09T04:08:27.631243+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:05.578748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.632243+0000","last_change":"2023-10-09T04:07:34.659260+0000","last_active":"2023-10-09T04:08:27.632243+0000","last_peered":"2023-10-09T04:08:27.632243+0000","last_clean":"2023-10-09T04:08:27.632243+0000","last_became_active":"2023-10-09T04:07:34.658976+0000","last_became_peered":"2023-10-09T04:07:34.658976+0000","last_unstale":"2023-10-09T04:08:27.632243+0000","last_undegraded":"2023-10-09T04:08:27.632243+0000","last_fullsized":"2023-10-09T04:08:27.632243+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:30:35.014675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634308+0000","last_change":"2023-10-09T04:07:34.634417+0000","last_active":"2023-10-09T04:08:27.634308+0000","last_peered":"2023-10-09T04:08:27.634308+0000","last_clean":"2023-10-09T04:08:27.634308+0000","last_became_active":"2023-10-09T04:07:34.634183+0000","last_became_peered":"2023-10-09T04:07:34.634183+0000","last_unstale":"2023-10-09T04:08:27.634308+0000","last_undegraded":"2023-10-09T04:08:27.634308+0000","last_fullsized":"2023-10-09T04:08:27.634308+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:04:13.162009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631571+0000","last_change":"2023-10-09T04:07:34.655573+0000","last_active":"2023-10-09T04:08:27.631571+0000","last_peered":"2023-10-09T04:08:27.631571+0000","last_clean":"2023-10-09T04:08:27.631571+0000","last_became_active":"2023-10-09T04:07:34.655326+0000","last_became_peered":"2023-10-09T04:07:34.655326+0000","last_unstale":"2023-10-09T04:08:27.631571+0000","last_undegraded":"2023-10-09T04:08:27.631571+0000","last_fullsized":"2023-10-09T04:08:27.631571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:07.510114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022787+0000","last_change":"2023-10-09T04:07:34.643373+0000","last_active":"2023-10-09T04:08:28.022787+0000","last_peered":"2023-10-09T04:08:28.022787+0000","last_clean":"2023-10-09T04:08:28.022787+0000","last_became_active":"2023-10-09T04:07:34.643033+0000","last_became_peered":"2023-10-09T04:07:34.643033+0000","last_unstale":"2023-10-09T04:08:28.022787+0000","last_undegraded":"2023-10-09T04:08:28.022787+0000","last_fullsized":"2023-10-09T04:08:28.022787+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:37:57.618154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.896496+0000","last_change":"2023-10-09T04:08:28.896496+0000","last_active":"2023-10-09T04:08:28.896496+0000","last_peered":"2023-10-09T04:08:28.896496+0000","last_clean":"2023-10-09T04:08:28.896496+0000","last_became_active":"2023-10-09T04:08:28.896152+0000","last_became_peered":"2023-10-09T04:08:28.896152+0000","last_unstale":"2023-10-09T04:08:28.896496+0000","last_undegraded":"2023-10-09T04:08:28.896496+0000","last_fullsized":"2023-10-09T04:08:28.896496+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:57.253137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628810+0000","last_change":"2023-10-09T04:07:34.640164+0000","last_active":"2023-10-09T04:08:27.628810+0000","last_peered":"2023-10-09T04:08:27.628810+0000","last_clean":"2023-10-09T04:08:27.628810+0000","last_became_active":"2023-10-09T04:07:34.639871+0000","last_became_peered":"2023-10-09T04:07:34.639871+0000","last_unstale":"2023-10-09T04:08:27.628810+0000","last_undegraded":"2023-10-09T04:08:27.628810+0000","last_fullsized":"2023-10-09T04:08:27.628810+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:59:59.659674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.633592+0000","last_change":"2023-10-09T04:08:28.633592+0000","last_active":"2023-10-09T04:08:28.633592+0000","last_peered":"2023-10-09T04:08:28.633592+0000","last_clean":"2023-10-09T04:08:28.633592+0000","last_became_active":"2023-10-09T04:08:28.633218+0000","last_became_peered":"2023-10-09T04:08:28.633218+0000","last_unstale":"2023-10-09T04:08:28.633592+0000","last_undegraded":"2023-10-09T04:08:28.633592+0000","last_fullsized":"2023-10-09T04:08:28.633592+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:07.571024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637077+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:08:27.637077+0000","last_peered":"2023-10-09T04:08:27.637077+0000","last_clean":"2023-10-09T04:08:27.637077+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:08:27.637077+0000","last_undegraded":"2023-10-09T04:08:27.637077+0000","last_fullsized":"2023-10-09T04:08:27.637077+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631119+0000","last_change":"2023-10-09T04:07:34.659608+0000","last_active":"2023-10-09T04:08:27.631119+0000","last_peered":"2023-10-09T04:08:27.631119+0000","last_clean":"2023-10-09T04:08:27.631119+0000","last_became_active":"2023-10-09T04:07:34.658918+0000","last_became_peered":"2023-10-09T04:07:34.658918+0000","last_unstale":"2023-10-09T04:08:27.631119+0000","last_undegraded":"2023-10-09T04:08:27.631119+0000","last_fullsized":"2023-10-09T04:08:27.631119+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:42.508338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637114+0000","last_change":"2023-10-09T04:07:34.631609+0000","last_active":"2023-10-09T04:08:27.637114+0000","last_peered":"2023-10-09T04:08:27.637114+0000","last_clean":"2023-10-09T04:08:27.637114+0000","last_became_active":"2023-10-09T04:07:34.631327+0000","last_became_peered":"2023-10-09T04:07:34.631327+0000","last_unstale":"2023-10-09T04:08:27.637114+0000","last_undegraded":"2023-10-09T04:08:27.637114+0000","last_fullsized":"2023-10-09T04:08:27.637114+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:51.678421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628586+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:08:27.628586+0000","last_peered":"2023-10-09T04:08:27.628586+0000","last_clean":"2023-10-09T04:08:27.628586+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:08:27.628586+0000","last_undegraded":"2023-10-09T04:08:27.628586+0000","last_fullsized":"2023-10-09T04:08:27.628586+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.637222+0000","last_change":"2023-10-09T04:07:34.642317+0000","last_active":"2023-10-09T04:08:27.637222+0000","last_peered":"2023-10-09T04:08:27.637222+0000","last_clean":"2023-10-09T04:08:27.637222+0000","last_became_active":"2023-10-09T04:07:34.642028+0000","last_became_peered":"2023-10-09T04:07:34.642028+0000","last_unstale":"2023-10-09T04:08:27.637222+0000","last_undegraded":"2023-10-09T04:08:27.637222+0000","last_fullsized":"2023-10-09T04:08:27.637222+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:12.984276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.023045+0000","last_change":"2023-10-09T04:07:36.029846+0000","last_active":"2023-10-09T04:08:28.023045+0000","last_peered":"2023-10-09T04:08:28.023045+0000","last_clean":"2023-10-09T04:08:28.023045+0000","last_became_active":"2023-10-09T04:07:36.029223+0000","last_became_peered":"2023-10-09T04:07:36.029223+0000","last_unstale":"2023-10-09T04:08:28.023045+0000","last_undegraded":"2023-10-09T04:08:28.023045+0000","last_fullsized":"2023-10-09T04:08:28.023045+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:12.111167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":22,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634708+0000","last_change":"2023-10-09T04:07:34.640376+0000","last_active":"2023-10-09T04:08:27.634708+0000","last_peered":"2023-10-09T04:08:27.634708+0000","last_clean":"2023-10-09T04:08:27.634708+0000","last_became_active":"2023-10-09T04:07:34.640073+0000","last_became_peered":"2023-10-09T04:07:34.640073+0000","last_unstale":"2023-10-09T04:08:27.634708+0000","last_undegraded":"2023-10-09T04:08:27.634708+0000","last_fullsized":"2023-10-09T04:08:27.634708+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:16.684588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628374+0000","last_change":"2023-10-09T04:07:36.029535+0000","last_active":"2023-10-09T04:08:27.628374+0000","last_peered":"2023-10-09T04:08:27.628374+0000","last_clean":"2023-10-09T04:08:27.628374+0000","last_became_active":"2023-10-09T04:07:36.029175+0000","last_became_peered":"2023-10-09T04:07:36.029175+0000","last_unstale":"2023-10-09T04:08:27.628374+0000","last_undegraded":"2023-10-09T04:08:27.628374+0000","last_fullsized":"2023-10-09T04:08:27.628374+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:02:34.595934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636321+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:08:27.636321+0000","last_peered":"2023-10-09T04:08:27.636321+0000","last_clean":"2023-10-09T04:08:27.636321+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:08:27.636321+0000","last_undegraded":"2023-10-09T04:08:27.636321+0000","last_fullsized":"2023-10-09T04:08:27.636321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628931+0000","last_change":"2023-10-09T04:07:36.029411+0000","last_active":"2023-10-09T04:08:27.628931+0000","last_peered":"2023-10-09T04:08:27.628931+0000","last_clean":"2023-10-09T04:08:27.628931+0000","last_became_active":"2023-10-09T04:07:36.029109+0000","last_became_peered":"2023-10-09T04:07:36.029109+0000","last_unstale":"2023-10-09T04:08:27.628931+0000","last_undegraded":"2023-10-09T04:08:27.628931+0000","last_fullsized":"2023-10-09T04:08:27.628931+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:05.311520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628217+0000","last_change":"2023-10-09T04:07:34.646539+0000","last_active":"2023-10-09T04:08:27.628217+0000","last_peered":"2023-10-09T04:08:27.628217+0000","last_clean":"2023-10-09T04:08:27.628217+0000","last_became_active":"2023-10-09T04:07:34.646243+0000","last_became_peered":"2023-10-09T04:07:34.646243+0000","last_unstale":"2023-10-09T04:08:27.628217+0000","last_undegraded":"2023-10-09T04:08:27.628217+0000","last_fullsized":"2023-10-09T04:08:27.628217+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:01.969093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022936+0000","last_change":"2023-10-09T04:07:36.029585+0000","last_active":"2023-10-09T04:08:28.022936+0000","last_peered":"2023-10-09T04:08:28.022936+0000","last_clean":"2023-10-09T04:08:28.022936+0000","last_became_active":"2023-10-09T04:07:36.029157+0000","last_became_peered":"2023-10-09T04:07:36.029157+0000","last_unstale":"2023-10-09T04:08:28.022936+0000","last_undegraded":"2023-10-09T04:08:28.022936+0000","last_fullsized":"2023-10-09T04:08:28.022936+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:19.819664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636654+0000","last_change":"2023-10-09T04:07:34.632203+0000","last_active":"2023-10-09T04:08:27.636654+0000","last_peered":"2023-10-09T04:08:27.636654+0000","last_clean":"2023-10-09T04:08:27.636654+0000","last_became_active":"2023-10-09T04:07:34.631969+0000","last_became_peered":"2023-10-09T04:07:34.631969+0000","last_unstale":"2023-10-09T04:08:27.636654+0000","last_undegraded":"2023-10-09T04:08:27.636654+0000","last_fullsized":"2023-10-09T04:08:27.636654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:46:01.619892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631738+0000","last_change":"2023-10-09T04:07:34.659790+0000","last_active":"2023-10-09T04:08:27.631738+0000","last_peered":"2023-10-09T04:08:27.631738+0000","last_clean":"2023-10-09T04:08:27.631738+0000","last_became_active":"2023-10-09T04:07:34.659565+0000","last_became_peered":"2023-10-09T04:07:34.659565+0000","last_unstale":"2023-10-09T04:08:27.631738+0000","last_undegraded":"2023-10-09T04:08:27.631738+0000","last_fullsized":"2023-10-09T04:08:27.631738+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:32.044413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":20,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631785+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:08:27.631785+0000","last_peered":"2023-10-09T04:08:27.631785+0000","last_clean":"2023-10-09T04:08:27.631785+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:08:27.631785+0000","last_undegraded":"2023-10-09T04:08:27.631785+0000","last_fullsized":"2023-10-09T04:08:27.631785+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629366+0000","last_change":"2023-10-09T04:07:34.661425+0000","last_active":"2023-10-09T04:08:27.629366+0000","last_peered":"2023-10-09T04:08:27.629366+0000","last_clean":"2023-10-09T04:08:27.629366+0000","last_became_active":"2023-10-09T04:07:34.661099+0000","last_became_peered":"2023-10-09T04:07:34.661099+0000","last_unstale":"2023-10-09T04:08:27.629366+0000","last_undegraded":"2023-10-09T04:08:27.629366+0000","last_fullsized":"2023-10-09T04:08:27.629366+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:53.588610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.636962+0000","last_change":"2023-10-09T04:07:34.641392+0000","last_active":"2023-10-09T04:08:27.636962+0000","last_peered":"2023-10-09T04:08:27.636962+0000","last_clean":"2023-10-09T04:08:27.636962+0000","last_became_active":"2023-10-09T04:07:34.641037+0000","last_became_peered":"2023-10-09T04:07:34.641037+0000","last_unstale":"2023-10-09T04:08:27.636962+0000","last_undegraded":"2023-10-09T04:08:27.636962+0000","last_fullsized":"2023-10-09T04:08:27.636962+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:43.467265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022724+0000","last_change":"2023-10-09T04:07:36.031224+0000","last_active":"2023-10-09T04:08:28.022724+0000","last_peered":"2023-10-09T04:08:28.022724+0000","last_clean":"2023-10-09T04:08:28.022724+0000","last_became_active":"2023-10-09T04:07:36.030834+0000","last_became_peered":"2023-10-09T04:07:36.030834+0000","last_unstale":"2023-10-09T04:08:28.022724+0000","last_undegraded":"2023-10-09T04:08:28.022724+0000","last_fullsized":"2023-10-09T04:08:28.022724+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:21.909762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628958+0000","last_change":"2023-10-09T04:07:34.631933+0000","last_active":"2023-10-09T04:08:27.628958+0000","last_peered":"2023-10-09T04:08:27.628958+0000","last_clean":"2023-10-09T04:08:27.628958+0000","last_became_active":"2023-10-09T04:07:34.631667+0000","last_became_peered":"2023-10-09T04:07:34.631667+0000","last_unstale":"2023-10-09T04:08:27.628958+0000","last_undegraded":"2023-10-09T04:08:27.628958+0000","last_fullsized":"2023-10-09T04:08:27.628958+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:58.644048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022596+0000","last_change":"2023-10-09T04:07:35.800509+0000","last_active":"2023-10-09T04:08:28.022596+0000","last_peered":"2023-10-09T04:08:28.022596+0000","last_clean":"2023-10-09T04:08:28.022596+0000","last_became_active":"2023-10-09T04:07:35.800155+0000","last_became_peered":"2023-10-09T04:07:35.800155+0000","last_unstale":"2023-10-09T04:08:28.022596+0000","last_undegraded":"2023-10-09T04:08:28.022596+0000","last_fullsized":"2023-10-09T04:08:28.022596+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:50.929412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628872+0000","last_change":"2023-10-09T04:07:34.646997+0000","last_active":"2023-10-09T04:08:27.628872+0000","last_peered":"2023-10-09T04:08:27.628872+0000","last_clean":"2023-10-09T04:08:27.628872+0000","last_became_active":"2023-10-09T04:07:34.645625+0000","last_became_peered":"2023-10-09T04:07:34.645625+0000","last_unstale":"2023-10-09T04:08:27.628872+0000","last_undegraded":"2023-10-09T04:08:27.628872+0000","last_fullsized":"2023-10-09T04:08:27.628872+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:10:40.584254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.634306+0000","last_change":"2023-10-09T04:07:34.676137+0000","last_active":"2023-10-09T04:08:27.634306+0000","last_peered":"2023-10-09T04:08:27.634306+0000","last_clean":"2023-10-09T04:08:27.634306+0000","last_became_active":"2023-10-09T04:07:34.675822+0000","last_became_peered":"2023-10-09T04:07:34.675822+0000","last_unstale":"2023-10-09T04:08:27.634306+0000","last_undegraded":"2023-10-09T04:08:27.634306+0000","last_fullsized":"2023-10-09T04:08:27.634306+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:06.758028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022654+0000","last_change":"2023-10-09T04:07:34.646983+0000","last_active":"2023-10-09T04:08:28.022654+0000","last_peered":"2023-10-09T04:08:28.022654+0000","last_clean":"2023-10-09T04:08:28.022654+0000","last_became_active":"2023-10-09T04:07:34.646623+0000","last_became_peered":"2023-10-09T04:07:34.646623+0000","last_unstale":"2023-10-09T04:08:28.022654+0000","last_undegraded":"2023-10-09T04:08:28.022654+0000","last_fullsized":"2023-10-09T04:08:28.022654+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.119527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.628454+0000","last_change":"2023-10-09T04:07:36.029481+0000","last_active":"2023-10-09T04:08:27.628454+0000","last_peered":"2023-10-09T04:08:27.628454+0000","last_clean":"2023-10-09T04:08:27.628454+0000","last_became_active":"2023-10-09T04:07:36.029167+0000","last_became_peered":"2023-10-09T04:07:36.029167+0000","last_unstale":"2023-10-09T04:08:27.628454+0000","last_undegraded":"2023-10-09T04:08:27.628454+0000","last_fullsized":"2023-10-09T04:08:27.628454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:42.409682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.629155+0000","last_change":"2023-10-09T04:07:34.645964+0000","last_active":"2023-10-09T04:08:27.629155+0000","last_peered":"2023-10-09T04:08:27.629155+0000","last_clean":"2023-10-09T04:08:27.629155+0000","last_became_active":"2023-10-09T04:07:34.645619+0000","last_became_peered":"2023-10-09T04:07:34.645619+0000","last_unstale":"2023-10-09T04:08:27.629155+0000","last_undegraded":"2023-10-09T04:08:27.629155+0000","last_fullsized":"2023-10-09T04:08:27.629155+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:57:26.090109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:28.022969+0000","last_change":"2023-10-09T04:07:36.284775+0000","last_active":"2023-10-09T04:08:28.022969+0000","last_peered":"2023-10-09T04:08:28.022969+0000","last_clean":"2023-10-09T04:08:28.022969+0000","last_became_active":"2023-10-09T04:07:36.284501+0000","last_became_peered":"2023-10-09T04:07:36.284501+0000","last_unstale":"2023-10-09T04:08:28.022969+0000","last_undegraded":"2023-10-09T04:08:28.022969+0000","last_fullsized":"2023-10-09T04:08:28.022969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:51:17.182374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-10-09T04:08:27.631184+0000","last_change":"2023-10-09T04:07:34.630544+0000","last_active":"2023-10-09T04:08:27.631184+0000","last_peered":"2023-10-09T04:08:27.631184+0000","last_clean":"2023-10-09T04:08:27.631184+0000","last_became_active":"2023-10-09T04:07:34.630230+0000","last_became_peered":"2023-10-09T04:07:34.630230+0000","last_unstale":"2023-10-09T04:08:27.631184+0000","last_undegraded":"2023-10-09T04:08:27.631184+0000","last_fullsized":"2023-10-09T04:08:27.631184+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:55:32.002588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234022,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1308,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1339392,"data_stored":905770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.0329999999999999,"15min":1.123},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76500000000000001,"5min":10.557,"15min":10.557},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.64400000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.83999999999999997,"15min":0.88800000000000001},"min":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71099999999999997,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.503,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.81599999999999995,"15min":0.86699999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69799999999999995,"5min":5.6970000000000001,"15min":5.6970000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.57599999999999996,"15min":0.58599999999999997},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69799999999999995,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72999999999999998,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.67100000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.54600000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.64300000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.52000000000000002,"15min":0.53400000000000003},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.58399999999999996,"5min":1.391,"15min":1.391},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.53800000000000003,"15min":0.54700000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.62}]},{"osd":6,"last update":"Mon Oct 9 04:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.5},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.63500000000000001,"15min":0.65400000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69099999999999995,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.51900000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495656,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298580,"kb_used_data":1292,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446572,"statfs":{"total":95995035648,"available":95689289728,"internally_reserved":0,"allocated":1323008,"data_stored":905301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.53700000000000003,"15min":0.55000000000000004},"min":{"1min":0.372,"5min":0.312,"15min":0.312},"max":{"1min":0.70499999999999996,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54000000000000004,"15min":0.54500000000000004},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.64800000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.67800000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56100000000000005,"15min":0.57499999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.56000000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69899999999999995,"5min":0.748,"15min":0.748},"last":0.46500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55700000000000005,"15min":0.56100000000000005},"min":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78300000000000003,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57399999999999995,"15min":0.57699999999999996},"min":{"1min":0.41299999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.50800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.58399999999999996,"15min":0.55600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.5839999999999996,"5min":4.5839999999999996,"15min":4.5839999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.64300000000000002,"15min":0.61599999999999999},"min":{"1min":0.42999999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.524,"5min":4.524,"15min":4.524},"last":0.64500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45300000000000001},"min":{"1min":0.313,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.502,"15min":0.48799999999999999},"min":{"1min":0.37,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.47899999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.35899999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.75700000000000001,"5min":0.877,"15min":0.877},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.59699999999999998,"15min":0.61699999999999999},"min":{"1min":0.34999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.88700000000000001,"5min":2.1779999999999999,"15min":2.1779999999999999},"last":0.66400000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.54400000000000004,"15min":0.54500000000000004},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.77300000000000002,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81100000000000005,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.63300000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297988,"kb_used_data":700,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447164,"statfs":{"total":95995035648,"available":95689895936,"internally_reserved":0,"allocated":716800,"data_stored":313650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:38 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.54600000000000004,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.20000000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.39800000000000002,"15min":0.372},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.214}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.45800000000000002,"15min":0.45200000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.435,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.41399999999999998,"15min":0.42299999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":0.51500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.318}]},{"osd":3,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.47599999999999998,"15min":0.5},"min":{"1min":0.31900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.47499999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.439,"15min":0.45300000000000001},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.59699999999999998,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.30199999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59999999999999998,"15min":0.60699999999999998},"min":{"1min":0.44,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.75600000000000001,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.59599999999999997,"15min":0.60499999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.70099999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.47699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.60899999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70899999999999996,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69899999999999995,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:08:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.61099999999999999,"15min":0.625},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81399999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65600000000000003,"15min":0.67400000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.79700000000000004,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.501}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.65100000000000002,"15min":0.66000000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.72699999999999998,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64700000000000002,"15min":0.65500000000000003},"min":{"1min":0.48099999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.77500000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.57999999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298576,"kb_used_data":1288,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93446576,"statfs":{"total":95995035648,"available":95689293824,"internally_reserved":0,"allocated":1318912,"data_stored":904594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:08:40 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.372,"15min":0.33000000000000002},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.55000000000000004,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.248},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.38600000000000001,"15min":0.34599999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58099999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.28599999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:08:02 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41599999999999998,"15min":0.42799999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.432,"15min":0.42999999999999999},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.58599999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.32800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:08:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.44500000000000001,"15min":0.46800000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.51000000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.48799999999999999,"15min":0.51400000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.626,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.628,"15min":0.63200000000000001},"min":{"1min":0.373,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77800000000000002,"5min":0.878,"15min":0.878},"last":0.624},{"interface":"front","average":{"1min":0.626,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84299999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.61299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.61099999999999999,"15min":0.622},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.622,"15min":0.63800000000000001},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77600000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.59699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:08:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.65100000000000002,"15min":0.66200000000000003},"min":{"1min":0.44600000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.651,"5min":1.651,"15min":1.651},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.63700000000000001,"15min":0.64600000000000002},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.86099999999999999,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.56100000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:07:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.65100000000000002,"15min":0.65900000000000003},"min":{"1min":0.46600000000000003,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80900000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.69099999999999995,"15min":0.68700000000000006},"min":{"1min":0.505,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.53800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297996,"kb_used_data":708,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447156,"statfs":{"total":95995035648,"available":95689887744,"internally_reserved":0,"allocated":724992,"data_stored":314238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.70699999999999996,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.377},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.28799999999999998,"5min":0.217,"15min":0.217},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.40100000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:05 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.503,"15min":0.48599999999999999},"min":{"1min":0.33700000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.6659999999999999,"5min":3.6659999999999999,"15min":3.6659999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.50800000000000001,"15min":0.49099999999999999},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.5779999999999998,"5min":3.5779999999999998,"15min":3.5779999999999998},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 04:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.45400000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.46999999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.69999999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.61599999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79400000000000004,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.753}]},{"osd":6,"last update":"Mon Oct 9 04:08:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.624,"15min":0.627},"min":{"1min":0.46700000000000003,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79300000000000004,"5min":0.998,"15min":0.998},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.64100000000000001,"15min":0.625},"min":{"1min":0.47399999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":2.1890000000000001,"5min":2.1890000000000001,"15min":2.1890000000000001},"last":0.79400000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:07:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67400000000000004},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98099999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.629,"15min":0.627},"min":{"1min":0.49399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.77600000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297992,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447160,"statfs":{"total":95995035648,"available":95689891840,"internally_reserved":0,"allocated":720896,"data_stored":313740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.41899999999999998},"min":{"1min":0.20399999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41699999999999998,"15min":0.44900000000000001},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.43099999999999999,"15min":0.46300000000000002},"min":{"1min":0.255,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.54800000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.317},{"interface":"front","average":{"1min":0.378,"5min":0.43099999999999999,"15min":0.47499999999999998},"min":{"1min":0.26600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56999999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.36799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:08:27 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.46400000000000002,"15min":0.496},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.54000000000000004,"5min":0.745,"15min":0.745},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.46400000000000002,"15min":0.503},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.63300000000000001,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.34799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.73399999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.626,"15min":0.64500000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.76300000000000001,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.71299999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:08:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82399999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.624,"15min":0.63500000000000001},"min":{"1min":0.433,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77100000000000002,"5min":0.874,"15min":0.874},"last":0.75700000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:08:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.621,"15min":0.63600000000000001},"min":{"1min":0.43099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.61499999999999999,"15min":0.622},"min":{"1min":0.48799999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.65100000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63300000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.73599999999999999,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.64100000000000001,"15min":0.64500000000000002},"min":{"1min":0.46500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81200000000000006,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.72999999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":724,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":741376,"data_stored":315332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.68300000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.503,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.76000000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.52600000000000002,"15min":0.51700000000000002},"min":{"1min":0.40200000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.71399999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.432,"5min":0.37,"15min":0.37},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54100000000000004,"15min":0.52300000000000002},"min":{"1min":0.40400000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.073,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:08:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.439,"15min":0.42299999999999999},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.503},{"interface":"front","average":{"1min":0.503,"5min":0.44400000000000001,"15min":0.42399999999999999},"min":{"1min":0.36299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.48799999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:08:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.50600000000000001,"15min":0.49299999999999999},"min":{"1min":0.34200000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.77200000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.49399999999999999,"15min":0.47799999999999998},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.75,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.54000000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:07:52 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32300000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69099999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.57799999999999996,"15min":0.58199999999999996},"min":{"1min":0.38300000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.73799999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757294,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297996,"kb_used_data":708,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93447156,"statfs":{"total":95995035648,"available":95689887744,"internally_reserved":0,"allocated":724992,"data_stored":314274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.39300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71099999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53000000000000003,"15min":0.52400000000000002},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59899999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65000000000000002,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.314,"15min":0.314},"max":{"1min":0.68200000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55400000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.56100000000000005},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71799999999999997,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54400000000000004,"15min":0.54700000000000004},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68500000000000005,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42499999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77800000000000002,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.443,"15min":0.41899999999999998},"min":{"1min":0.35499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.378},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.45400000000000001,"15min":0.42799999999999999},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:08:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.48399999999999999,"15min":0.45200000000000001},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.439},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.49399999999999999,"15min":0.45500000000000002},"min":{"1min":0.438,"5min":0.312,"15min":0.312},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.622}]},{"osd":7,"last update":"Mon Oct 9 04:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.51600000000000001,"15min":0.503},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54900000000000004,"15min":0.53900000000000003},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.497}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:08:45.404 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:08:45.404 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:08:45.404 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:08:45.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:08:45.768 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:45 smithi029 ceph-mon[101043]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:45 smithi029 ceph-mon[119131]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:45 smithi177 ceph-mon[106070]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:08:46.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:46 smithi029 ceph-mon[101043]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:46 smithi029 ceph-mon[119131]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:46 smithi177 ceph-mon[106070]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:47 smithi177 ceph-mon[106070]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:48.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:47 smithi029 ceph-mon[101043]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:48.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:47 smithi029 ceph-mon[119131]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:49 smithi029 ceph-mon[101043]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:49 smithi029 ceph-mon[119131]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:49 smithi177 ceph-mon[106070]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:51.928 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:08:51.928 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:08:52.745 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:08:52.745 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:08:52.755 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-09T04:08:52.756 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T04:08:52.769 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T04:08:52.769 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:08:52.770 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T04:08:52.770 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:08:52.772 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:08:52.772 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-09T04:08:52.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-09T04:08:52.773 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:08:52.773 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-09T04:08:52.773 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi029.front.sepia.ceph.com 2023-10-09T04:08:52.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-09T04:08:52.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:52 smithi029 ceph-mon[101043]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:52.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:52 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2642082944' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:08:52.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:52 smithi029 ceph-mon[119131]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:52.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:52 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2642082944' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:08:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:52 smithi177 ceph-mon[106070]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:52 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2642082944' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:08:53.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:08:52 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:08:53.149 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:53.202 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:54.613 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:54 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:08:54] "GET /metrics HTTP/1.1" 200 238848 "" "Prometheus/2.43.0" 2023-10-09T04:08:54.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:54 smithi029 ceph-mon[101043]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:54.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:54 smithi029 ceph-mon[119131]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:54 smithi177 ceph-mon[106070]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:56 smithi029 conmon[101466]: [09/Oct/2023:04:08:56] ENGINE Bus STOPPING 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:56 smithi029 conmon[101466]: [09/Oct/2023:04:08:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:56 smithi029 conmon[101466]: [09/Oct/2023:04:08:56] ENGINE Bus STOPPED 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:56 smithi029 conmon[101466]: [09/Oct/2023:04:08:56] ENGINE Bus STARTING 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:56 smithi029 conmon[101466]: [09/Oct/2023:04:08:56] ENGINE Serving on http://:::9283 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:08:56 smithi029 conmon[101466]: [09/Oct/2023:04:08:56] ENGINE Bus STARTED 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[101043]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:56.623 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/470906565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:08:56.624 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' 2023-10-09T04:08:56.624 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:56.624 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_0' created 2023-10-09T04:08:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[119131]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/470906565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:08:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' 2023-10-09T04:08:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:56 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:56 smithi177 ceph-mon[106070]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:56 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/470906565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-09T04:08:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:56 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' 2023-10-09T04:08:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:56 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:08:57.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-09T04:08:57.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/470906565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:08:57.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[101043]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:08:57.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:57.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[101043]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:08:57.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/470906565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:08:57.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[119131]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:08:57.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:57.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:57 smithi029 ceph-mon[119131]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:08:57.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-09T04:08:57.863 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:57 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/470906565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-09T04:08:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:57 smithi177 ceph-mon[106070]: osdmap e71: 8 total, 8 up, 8 in 2023-10-09T04:08:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:57 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:08:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:57 smithi177 ceph-mon[106070]: osdmap e72: 8 total, 8 up, 8 in 2023-10-09T04:08:58.531 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:08:58.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:58 smithi029 ceph-mon[101043]: pgmap v90: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:58 smithi029 ceph-mon[101043]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:08:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:58 smithi029 ceph-mon[101043]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:08:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:58 smithi029 ceph-mon[119131]: pgmap v90: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:58 smithi029 ceph-mon[119131]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:08:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:58 smithi029 ceph-mon[119131]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:08:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:58 smithi177 ceph-mon[106070]: pgmap v90: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:08:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:58 smithi177 ceph-mon[106070]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2023-10-09T04:08:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:58 smithi177 ceph-mon[106070]: osdmap e73: 8 total, 8 up, 8 in 2023-10-09T04:09:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[101043]: pgmap v93: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[119131]: pgmap v93: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:08:59 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:59 smithi177 ceph-mon[106070]: pgmap v93: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:59 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:59 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:59 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:59 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:08:59 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:01.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-09T04:09:01.989 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:02.748 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-09T04:09:02.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:02 smithi029 ceph-mon[101043]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:02 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2082133442' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:02 smithi029 ceph-mon[119131]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:02 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2082133442' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:03.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:09:02 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:09:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:02 smithi177 ceph-mon[106070]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:02 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2082133442' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:09:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:03 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2082133442' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:03 smithi029 ceph-mon[101043]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:09:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:03 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2082133442' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:03 smithi029 ceph-mon[119131]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:09:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:03 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2082133442' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:09:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:03 smithi177 ceph-mon[106070]: osdmap e74: 8 total, 8 up, 8 in 2023-10-09T04:09:04.382 INFO:teuthology.orchestra.run.smithi029.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:04.592 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:09:04 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:04] "GET /metrics HTTP/1.1" 200 241126 "" "Prometheus/2.43.0" 2023-10-09T04:09:04.788 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-09T04:09:04.791 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:09:04.791 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:09:04.841 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:09:04.841 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:09:04.842 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_194350 2023-10-09T04:09:04.842 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:09:04.842 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: Cluster is now healthy 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: Cluster is now healthy 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:05.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: pgmap v96: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:05.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2023-10-09T04:09:05.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: Cluster is now healthy 2023-10-09T04:09:05.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:05.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:09:05.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:09:05.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:09:05.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:09:05.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:09:05.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:05 smithi029 ceph-mon[101043]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:05.632 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:05 smithi029 ceph-mon[101043]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:05.632 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:05 smithi029 ceph-mon[101043]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:05.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:05 smithi029 ceph-mon[119131]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:05.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:05 smithi029 ceph-mon[119131]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:05.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:05 smithi029 ceph-mon[119131]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:05.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3633 3617 225.471 226.062 0.00278317 0.00442154 2023-10-09T04:09:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:05 smithi177 ceph-mon[106070]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:05 smithi177 ceph-mon[106070]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:05 smithi177 ceph-mon[106070]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 2023-10-09T04:09:06.062 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:06 smithi029 ceph-mon[101043]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:06 smithi029 ceph-mon[119131]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:06.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6598 6582 205.409 185.312 0.00346856 0.00486009 2023-10-09T04:09:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:06 smithi177 ceph-mon[106070]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:07.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 9825 9809 204.146 201.688 0.00775668 0.00488928 2023-10-09T04:09:08.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 12772 12756 199.156 184.188 0.00367634 0.00501536 2023-10-09T04:09:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:08 smithi029 ceph-mon[101043]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:08 smithi029 ceph-mon[119131]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:08 smithi177 ceph-mon[106070]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:09:09.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 15539 15523 193.912 172.938 0.00481925 0.00514873 2023-10-09T04:09:09.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:09 smithi029 ceph-mon[101043]: pgmap v99: 89 pgs: 89 active+clean; 600 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-09T04:09:09.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:09 smithi029 ceph-mon[119131]: pgmap v99: 89 pgs: 89 active+clean; 600 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-09T04:09:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:09 smithi177 ceph-mon[106070]: pgmap v99: 89 pgs: 89 active+clean; 600 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-09T04:09:10.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 17954 17938 186.751 150.938 0.00600424 0.00534734 2023-10-09T04:09:11.845 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 20429 20413 182.162 154.688 0.0116308 0.0054842 2023-10-09T04:09:12.845 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 22886 22870 178.587 153.562 0.00374775 0.00559325 2023-10-09T04:09:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:12 smithi029 ceph-mon[101043]: pgmap v100: 89 pgs: 89 active+clean; 600 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-09T04:09:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:12 smithi029 ceph-mon[119131]: pgmap v100: 89 pgs: 89 active+clean; 600 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-09T04:09:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:13.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:09:12 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:09:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:12 smithi177 ceph-mon[106070]: pgmap v100: 89 pgs: 89 active+clean; 600 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-09T04:09:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:13.846 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 25257 25241 175.183 148.188 0.00467737 0.00570203 2023-10-09T04:09:14.625 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:09:14 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:14] "GET /metrics HTTP/1.1" 200 241814 "" "Prometheus/2.43.0" 2023-10-09T04:09:14.846 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 27517 27501 171.79 141.25 0.00399121 0.00581683 2023-10-09T04:09:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:14 smithi029 ceph-mon[101043]: pgmap v101: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-10-09T04:09:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:14 smithi029 ceph-mon[119131]: pgmap v101: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-10-09T04:09:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:14 smithi177 ceph-mon[106070]: pgmap v101: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-10-09T04:09:15.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 29647 29631 168.11 133.125 0.00497847 0.00594382 2023-10-09T04:09:16.793 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:09:16.793 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (107s) 71s ago 2m 16.5M - 0.25.0 c8568f914cd2 6a4c4bc4e5b3 2023-10-09T04:09:16.793 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi177 *:3000 running (2m) 17s ago 2m 93.7M - 9.4.7 2c41d148cca3 3d5a415c7f4f 2023-10-09T04:09:16.793 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.mozfhx smithi029 running (94s) 71s ago 93s 16.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 e346a0e9dcdd 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi177.pbovgs smithi177 running (91s) 17s ago 90s 13.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 a20b9123e852 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi177 *:8443 running (6m) 17s ago 6m 440M - 17.2.6-1303-gb2903cb0 5517396c50d4 caddb7c690a3 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:9283 running (8m) 71s ago 8m 496M - 17.2.6-1303-gb2903cb0 5517396c50d4 308b3b05d6d1 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (8m) 71s ago 8m 81.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 5d294109024f 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi177 running (6m) 17s ago 6m 59.8M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 f19800b51616 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (6m) 71s ago 6m 56.7M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c8851494e8fd 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (2m) 71s ago 2m 15.8M - 1.5.0 0da6a335fe13 15156916bd10 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi177 *:9100 running (2m) 17s ago 119s 9881k - 1.5.0 0da6a335fe13 b7a4d2d246cc 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (6m) 71s ago 6m 56.2M 4037M 17.2.6-1303-gb2903cb0 5517396c50d4 2c8a6318fb47 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (5m) 71s ago 5m 59.6M 4037M 17.2.6-1303-gb2903cb0 5517396c50d4 951ea50b247d 2023-10-09T04:09:16.794 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (5m) 71s ago 5m 52.0M 4037M 17.2.6-1303-gb2903cb0 5517396c50d4 7d2e4b8000f3 2023-10-09T04:09:16.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (5m) 71s ago 5m 50.3M 4037M 17.2.6-1303-gb2903cb0 5517396c50d4 e4ab91bdb38b 2023-10-09T04:09:16.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi177 running (4m) 17s ago 4m 54.5M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 d4b592bd1322 2023-10-09T04:09:16.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi177 running (4m) 17s ago 4m 52.2M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 28d4ab6e8bfe 2023-10-09T04:09:16.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi177 running (3m) 17s ago 3m 55.6M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 03ede6ceccc0 2023-10-09T04:09:16.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi177 running (3m) 17s ago 3m 53.5M 4036M 17.2.6-1303-gb2903cb0 5517396c50d4 0f3217cc8fa1 2023-10-09T04:09:16.795 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi177 *:9095 running (102s) 17s ago 2m 48.2M - 2.43.0 a07b618ecd1d 51d99d79686f 2023-10-09T04:09:16.857 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 31629 31613 164.427 123.875 0.00449083 0.00607641 2023-10-09T04:09:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:16 smithi029 ceph-mon[119131]: pgmap v102: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:09:16.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:16 smithi029 ceph-mon[101043]: pgmap v102: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:09:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:16 smithi177 ceph-mon[106070]: pgmap v102: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:09:17.857 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 33688 33672 161.681 128.688 0.00712811 0.00618028 2023-10-09T04:09:18.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 35740 35724 159.295 128.25 0.00855823 0.00627284 2023-10-09T04:09:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:18 smithi029 ceph-mon[101043]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:18 smithi029 ceph-mon[101043]: pgmap v103: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:09:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:18 smithi029 ceph-mon[119131]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:18 smithi029 ceph-mon[119131]: pgmap v103: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:09:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:18 smithi177 ceph-mon[106070]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:09:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:18 smithi177 ceph-mon[106070]: pgmap v103: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:09:19.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 37665 37649 156.698 120.312 0.00800842 0.00637598 2023-10-09T04:09:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:19 smithi177 ceph-mon[106070]: pgmap v104: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2023-10-09T04:09:20.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:19 smithi029 ceph-mon[101043]: pgmap v104: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2023-10-09T04:09:20.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:19 smithi029 ceph-mon[119131]: pgmap v104: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 174 MiB/s wr, 2.79k op/s 2023-10-09T04:09:20.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 39543 39527 154.242 117.375 0.0179205 0.0064783 2023-10-09T04:09:21.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 41361 41345 151.854 113.625 0.00202507 0.00657859 2023-10-09T04:09:22.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 43079 43063 149.379 107.375 0.00867541 0.00668899 2023-10-09T04:09:22.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:22 smithi029 ceph-mon[101043]: pgmap v105: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T04:09:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:22 smithi029 ceph-mon[119131]: pgmap v105: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T04:09:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:22 smithi177 ceph-mon[106070]: pgmap v105: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-09T04:09:23.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:09:22 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:09:23.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 44805 44789 147.196 107.875 0.0074491 0.00678855 2023-10-09T04:09:24.636 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:09:24 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:24] "GET /metrics HTTP/1.1" 200 241814 "" "Prometheus/2.43.0" 2023-10-09T04:09:24.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:09:24.859670+0000 min lat: 0.00115846 max lat: 0.0901032 avg lat: 0.00688705 2023-10-09T04:09:24.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:09:24.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 46485 46469 145.086 105 0.00799513 0.00688705 2023-10-09T04:09:24.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:24 smithi029 ceph-mon[101043]: pgmap v106: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2023-10-09T04:09:24.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:24 smithi029 ceph-mon[119131]: pgmap v106: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2023-10-09T04:09:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:24 smithi177 ceph-mon[106070]: pgmap v106: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 173 MiB/s wr, 2.77k op/s 2023-10-09T04:09:25.859 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 48172 48156 143.199 105.438 0.012076 0.00698 2023-10-09T04:09:26.860 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 49788 49772 141.279 101 0.00621627 0.00707286 2023-10-09T04:09:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:26 smithi029 ceph-mon[101043]: pgmap v107: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:09:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:26 smithi029 ceph-mon[119131]: pgmap v107: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:09:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:26 smithi177 ceph-mon[106070]: pgmap v107: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:09:27.860 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 51308 51292 139.268 95 0.0106526 0.00717573 2023-10-09T04:09:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:27 smithi029 ceph-mon[101043]: pgmap v108: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:09:27.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:27 smithi029 ceph-mon[119131]: pgmap v108: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:09:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:27 smithi177 ceph-mon[106070]: pgmap v108: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-09T04:09:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:28.863 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 52793 52777 137.319 92.8125 0.0117748 0.00727721 2023-10-09T04:09:29.862 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 54252 54236 135.474 91.1875 0.015419 0.00737562 2023-10-09T04:09:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:29 smithi029 ceph-mon[101043]: pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-10-09T04:09:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:29 smithi029 ceph-mon[119131]: pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-10-09T04:09:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:29 smithi177 ceph-mon[106070]: pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-10-09T04:09:30.865 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 55668 55652 133.659 88.5 0.0106499 0.00747743 2023-10-09T04:09:31.865 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 56938 56922 131.65 79.375 0.008453 0.00759111 2023-10-09T04:09:32.868 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 58276 58260 129.921 83.625 0.00926127 0.00769191 2023-10-09T04:09:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:32 smithi029 ceph-mon[101043]: pgmap v110: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:09:32.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:32 smithi029 ceph-mon[119131]: pgmap v110: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:09:33.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:09:32 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:09:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:32 smithi177 ceph-mon[106070]: pgmap v110: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:09:33.868 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 15 59650 59635 128.405 85.9375 0.0096961 0.00778321 2023-10-09T04:09:34.635 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:09:34 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:34] "GET /metrics HTTP/1.1" 200 242213 "" "Prometheus/2.43.0" 2023-10-09T04:09:34.636 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:34 smithi029 ceph-mon[101043]: pgmap v111: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T04:09:34.868 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 60974 60958 126.882 82.6875 0.00971025 0.00787726 2023-10-09T04:09:34.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:34 smithi029 ceph-mon[119131]: pgmap v111: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T04:09:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:34 smithi177 ceph-mon[106070]: pgmap v111: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-09T04:09:35.868 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 62243 62227 125.349 79.3125 0.014015 0.00797292 2023-10-09T04:09:36.871 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 63478 63462 123.832 77.1875 0.0107004 0.00807066 2023-10-09T04:09:36.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:36 smithi029 ceph-mon[101043]: pgmap v112: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:09:36.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:36 smithi029 ceph-mon[119131]: pgmap v112: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:09:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:36 smithi177 ceph-mon[106070]: pgmap v112: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:09:37.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 64728 64712 122.447 78.125 0.0127869 0.00816178 2023-10-09T04:09:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:37 smithi177 ceph-mon[106070]: pgmap v113: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:09:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:37 smithi029 ceph-mon[101043]: pgmap v113: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:09:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:37 smithi029 ceph-mon[119131]: pgmap v113: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:09:38.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 65987 65971 121.161 78.6875 0.00616716 0.0082488 2023-10-09T04:09:39.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 67094 67078 119.677 69.1875 0.00550009 0.0083504 2023-10-09T04:09:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:39 smithi029 ceph-mon[119131]: pgmap v114: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:09:40.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:39 smithi029 ceph-mon[101043]: pgmap v114: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:09:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:39 smithi177 ceph-mon[106070]: pgmap v114: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:09:40.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-09T04:09:40.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 15 68379 68364 118.586 80.375 0.0110925 0.00842804 2023-10-09T04:09:41.080 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:09:41.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 69500 69484 117.274 70 0.0102969 0.00852322 2023-10-09T04:09:42.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 70856 70840 116.418 84.75 0.0120381 0.00858561 2023-10-09T04:09:42.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:42 smithi029 ceph-mon[101043]: pgmap v115: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:09:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:42 smithi029 ceph-mon[119131]: pgmap v115: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:09:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:43.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:09:42 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:09:43.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:42 smithi177 ceph-mon[106070]: pgmap v115: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-09T04:09:43.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:43.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 72232 72216 115.639 86 0.0141401 0.00864318 2023-10-09T04:09:44.642 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:09:44 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:44] "GET /metrics HTTP/1.1" 200 242301 "" "Prometheus/2.43.0" 2023-10-09T04:09:44.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:44 smithi029 ceph-mon[119131]: pgmap v116: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:09:44.643 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:44 smithi029 ceph-mon[101043]: pgmap v116: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:09:44.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:09:44.872895+0000 min lat: 0.00115846 max lat: 0.179741 avg lat: 0.00870145 2023-10-09T04:09:44.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:09:44.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 73592 73576 114.873 85 0.0091457 0.00870145 2023-10-09T04:09:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:44 smithi177 ceph-mon[106070]: pgmap v116: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:09:45.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 74932 74916 114.114 83.75 0.00650427 0.00875873 2023-10-09T04:09:46.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 76143 76127 113.2 75.6875 0.00327372 0.00880902 2023-10-09T04:09:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:46 smithi029 ceph-mon[101043]: pgmap v117: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:09:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:46 smithi029 ceph-mon[119131]: pgmap v117: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:09:47.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:46 smithi177 ceph-mon[106070]: pgmap v117: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:09:47.873 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 77406 77390 112.403 78.9375 0.0159266 0.00889235 2023-10-09T04:09:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:47 smithi177 ceph-mon[106070]: pgmap v118: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:09:48.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:47 smithi029 ceph-mon[101043]: pgmap v118: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:09:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:47 smithi029 ceph-mon[119131]: pgmap v118: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:09:48.873 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 78703 78687 111.691 81.0625 0.00988671 0.008949 2023-10-09T04:09:49.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 79983 79967 110.977 80 0.0209988 0.00900634 2023-10-09T04:09:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:49 smithi029 ceph-mon[101043]: pgmap v119: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:09:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:49 smithi029 ceph-mon[119131]: pgmap v119: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:09:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:49 smithi177 ceph-mon[106070]: pgmap v119: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:09:50.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 81255 81239 110.293 79.5 0.0124221 0.00906255 2023-10-09T04:09:51.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 82498 82482 109.6 77.6875 0.0104403 0.00911934 2023-10-09T04:09:52.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 83727 83711 108.917 76.8125 0.0139294 0.00917665 2023-10-09T04:09:52.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:52 smithi029 ceph-mon[119131]: pgmap v120: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:09:52.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:52 smithi029 ceph-mon[101043]: pgmap v120: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:09:53.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:09:52 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:09:53.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:52 smithi177 ceph-mon[106070]: pgmap v120: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-09T04:09:53.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 84952 84936 108.257 76.5625 0.00854702 0.009233 2023-10-09T04:09:54.645 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:09:54 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:09:54] "GET /metrics HTTP/1.1" 200 242301 "" "Prometheus/2.43.0" 2023-10-09T04:09:54.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 86151 86135 107.591 74.9375 0.0119507 0.00928987 2023-10-09T04:09:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:54 smithi029 ceph-mon[101043]: pgmap v121: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:09:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:54 smithi029 ceph-mon[119131]: pgmap v121: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:09:55.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:54 smithi177 ceph-mon[106070]: pgmap v121: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:09:55.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 87359 87343 106.962 75.5 0.0074614 0.00934545 2023-10-09T04:09:56.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 88496 88480 106.272 71.0625 0.00509483 0.0094052 2023-10-09T04:09:56.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:56 smithi029 ceph-mon[119131]: pgmap v122: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:09:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:56 smithi029 ceph-mon[101043]: pgmap v122: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:09:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:56 smithi177 ceph-mon[106070]: pgmap v122: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:09:57.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 89670 89654 105.649 73.375 0.00996234 0.00946054 2023-10-09T04:09:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:57 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:57 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:57 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:09:58.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 90827 90811 105.032 72.3125 0.0121308 0.00951609 2023-10-09T04:09:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:58 smithi177 ceph-mon[106070]: pgmap v123: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:09:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:58 smithi029 ceph-mon[101043]: pgmap v123: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:09:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:58 smithi029 ceph-mon[119131]: pgmap v123: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:09:59.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 91996 91980 104.451 73.0625 0.0110826 0.00956961 2023-10-09T04:10:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:09:59 smithi177 ceph-mon[106070]: pgmap v124: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:10:00.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:09:59 smithi029 ceph-mon[101043]: pgmap v124: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:10:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:09:59 smithi029 ceph-mon[119131]: pgmap v124: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:10:00.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 93117 93101 103.837 70.0625 0.0170692 0.00962571 2023-10-09T04:10:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:00 smithi177 ceph-mon[106070]: overall HEALTH_OK 2023-10-09T04:10:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:00 smithi029 ceph-mon[119131]: overall HEALTH_OK 2023-10-09T04:10:01.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:00 smithi029 ceph-mon[101043]: overall HEALTH_OK 2023-10-09T04:10:01.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 94234 94218 103.24 69.8125 0.0133997 0.00968107 2023-10-09T04:10:02.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 95350 95334 102.663 69.75 0.00690881 0.0097354 2023-10-09T04:10:02.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:02 smithi029 ceph-mon[101043]: pgmap v125: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:10:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:02 smithi029 ceph-mon[119131]: pgmap v125: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:10:03.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:10:02 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:10:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:02 smithi177 ceph-mon[106070]: pgmap v125: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-09T04:10:03.879 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 96473 96457 102.113 70.1875 0.0110587 0.0097883 2023-10-09T04:10:04.659 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:10:04 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:04] "GET /metrics HTTP/1.1" 200 242356 "" "Prometheus/2.43.0" 2023-10-09T04:10:04.660 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:04 smithi029 ceph-mon[119131]: pgmap v126: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:10:04.883 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:10:04.882033+0000 min lat: 0.00115846 max lat: 0.179741 avg lat: 0.00984187 2023-10-09T04:10:04.884 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:10:04.884 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 97574 97558 101.555 68.8125 0.00992777 0.00984187 2023-10-09T04:10:04.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:04 smithi029 ceph-mon[101043]: pgmap v126: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:10:05.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:04 smithi177 ceph-mon[106070]: pgmap v126: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:10:05.882 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 98662 98646 101.003 68 0.00816663 0.00989581 2023-10-09T04:10:06.882 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 99737 99721 100.458 67.1875 0.0123193 0.00994979 2023-10-09T04:10:07.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:06 smithi177 ceph-mon[106070]: pgmap v127: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T04:10:07.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:06 smithi029 ceph-mon[101043]: pgmap v127: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T04:10:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:06 smithi029 ceph-mon[119131]: pgmap v127: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T04:10:07.882 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 100809 100793 99.9274 67 0.0161073 0.0100026 2023-10-09T04:10:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:07 smithi177 ceph-mon[106070]: pgmap v128: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T04:10:08.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:07 smithi029 ceph-mon[101043]: pgmap v128: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T04:10:08.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:07 smithi029 ceph-mon[119131]: pgmap v128: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-09T04:10:08.882 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 15 101856 101841 99.3897 65.5 0.00821322 0.0100564 2023-10-09T04:10:09.883 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 102863 102847 98.8281 62.875 0.0157326 0.0101146 2023-10-09T04:10:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:09 smithi029 ceph-mon[119131]: pgmap v129: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:10:10.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:09 smithi029 ceph-mon[101043]: pgmap v129: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:10:10.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:09 smithi177 ceph-mon[106070]: pgmap v129: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-09T04:10:10.885 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 103888 103872 98.2988 64.0625 0.0127532 0.0101691 2023-10-09T04:10:11.885 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 104913 104897 97.788 64.0625 0.0113786 0.010221 2023-10-09T04:10:12.885 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 105924 105908 97.2793 63.1875 0.0121313 0.0102747 2023-10-09T04:10:12.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:12 smithi029 ceph-mon[101043]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2023-10-09T04:10:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:12 smithi029 ceph-mon[119131]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2023-10-09T04:10:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:13.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:10:12 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:10:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:12 smithi177 ceph-mon[106070]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2023-10-09T04:10:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:13.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 106945 106929 96.7918 63.8125 0.0161063 0.0103269 2023-10-09T04:10:14.655 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:10:14 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:14] "GET /metrics HTTP/1.1" 200 242382 "" "Prometheus/2.43.0" 2023-10-09T04:10:14.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 107942 107926 96.2994 62.3125 0.00796694 0.0103797 2023-10-09T04:10:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:14 smithi029 ceph-mon[119131]: pgmap v131: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:10:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:14 smithi029 ceph-mon[101043]: pgmap v131: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:10:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:14 smithi177 ceph-mon[106070]: pgmap v131: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-09T04:10:15.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 108950 108934 95.8306 63 0.0287827 0.0104305 2023-10-09T04:10:16.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 109945 109929 95.3635 62.1875 0.0199482 0.0104817 2023-10-09T04:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:16 smithi029 ceph-mon[101043]: pgmap v132: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T04:10:16.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:16 smithi029 ceph-mon[119131]: pgmap v132: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T04:10:17.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:16 smithi177 ceph-mon[106070]: pgmap v132: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T04:10:17.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 110903 110887 94.8775 59.875 0.00370103 0.0105301 2023-10-09T04:10:18.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 111419 111403 94.0316 32.25 0.011315 0.0106294 2023-10-09T04:10:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:18 smithi177 ceph-mon[106070]: pgmap v133: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T04:10:19.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:18 smithi029 ceph-mon[101043]: pgmap v133: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T04:10:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:18 smithi029 ceph-mon[119131]: pgmap v133: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-09T04:10:19.409 INFO:teuthology.orchestra.run.smithi029.stdout:wait for minority of mons to upgrade 2023-10-09T04:10:19.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 112240 112224 93.4623 51.3125 0.00795738 0.0106928 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: pgmap v134: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:10:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[101043]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi177 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: pgmap v134: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:10:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:19 smithi029 ceph-mon[119131]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi177 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: pgmap v134: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: Upgrade: Target is version 18.2.0-598-gb4fa663f (unknown) 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c'] 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-09T04:10:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:19 smithi177 ceph-mon[106070]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c on smithi177 2023-10-09T04:10:20.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-09T04:10:20.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 112895 112879 92.7715 40.9375 0.0205366 0.010774 2023-10-09T04:10:21.345 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:10:21.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 113731 113715 92.2454 52.25 0.015606 0.010836 2023-10-09T04:10:22.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 114396 114380 91.5959 41.5625 0.0309271 0.0109125 2023-10-09T04:10:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:22 smithi029 ceph-mon[101043]: pgmap v135: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-10-09T04:10:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:22 smithi029 ceph-mon[119131]: pgmap v135: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-10-09T04:10:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:22 smithi177 ceph-mon[106070]: pgmap v135: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-10-09T04:10:23.030 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:10:22 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:10:23.889 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 115252 115236 91.1139 53.5 0.0132904 0.0109706 2023-10-09T04:10:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:23 smithi029 ceph-mon[101043]: pgmap v136: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:10:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:23 smithi029 ceph-mon[119131]: pgmap v136: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:10:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:23 smithi177 ceph-mon[106070]: pgmap v136: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-09T04:10:24.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:10:24 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:24] "GET /metrics HTTP/1.1" 200 242382 "" "Prometheus/2.43.0" 2023-10-09T04:10:24.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:10:24.888567+0000 min lat: 0.00115846 max lat: 0.33895 avg lat: 0.0110274 2023-10-09T04:10:24.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:10:24.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 116104 116088 90.6407 53.25 0.00997438 0.0110274 2023-10-09T04:10:25.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:24 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1350265468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:25.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:24 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1350265468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:24 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1350265468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:25.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 116931 116915 90.16 51.6875 0.0106229 0.0110866 2023-10-09T04:10:26.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:25 smithi177 ceph-mon[106070]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T04:10:26.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:25 smithi029 ceph-mon[101043]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T04:10:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:25 smithi029 ceph-mon[119131]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T04:10:26.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 117733 117717 89.6719 50.125 0.0251822 0.0111473 2023-10-09T04:10:27.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 118544 118528 89.2024 50.6875 0.0159167 0.0112054 2023-10-09T04:10:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:27 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:27 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/996620110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:27 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:27 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/996620110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:27 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:27 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/996620110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:28.888 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 119375 119359 88.7589 51.9375 0.0210327 0.0112619 2023-10-09T04:10:28.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:28 smithi029 ceph-mon[119131]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T04:10:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:28 smithi029 ceph-mon[101043]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T04:10:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:28 smithi177 ceph-mon[106070]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 706 op/s 2023-10-09T04:10:29.894 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 120162 120146 88.2875 49.1875 0.0197673 0.0113214 2023-10-09T04:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:29 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1313185280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:29 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1313185280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:29 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1313185280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:30.894 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 120985 120969 87.8591 51.4375 0.0282188 0.0113767 2023-10-09T04:10:31.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:30 smithi177 ceph-mon[106070]: pgmap v139: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:10:31.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:30 smithi029 ceph-mon[101043]: pgmap v139: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:10:31.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:30 smithi029 ceph-mon[119131]: pgmap v139: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:10:31.894 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 121794 121778 87.4306 50.5625 0.0180512 0.0114328 2023-10-09T04:10:32.894 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 122572 122556 86.9898 48.625 0.00968699 0.0114912 2023-10-09T04:10:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:32 smithi029 ceph-mon[101043]: pgmap v140: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s 2023-10-09T04:10:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:32 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2877055428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:32 smithi029 ceph-mon[119131]: pgmap v140: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s 2023-10-09T04:10:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:32 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2877055428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:33.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:10:32 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:10:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:32 smithi177 ceph-mon[106070]: pgmap v140: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 40 MiB/s wr, 634 op/s 2023-10-09T04:10:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:32 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2877055428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:33.894 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 123383 123367 86.5821 50.6875 0.00906655 0.0115446 2023-10-09T04:10:34.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:10:34 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:34] "GET /metrics HTTP/1.1" 200 242409 "" "Prometheus/2.43.0" 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0132 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 124101 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 86.1686 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 37.7435 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 226.062 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 32.25 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1378 2023-10-09T04:10:34.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 603.896 2023-10-09T04:10:34.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3617 2023-10-09T04:10:34.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 516 2023-10-09T04:10:34.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0116013 2023-10-09T04:10:34.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.00877964 2023-10-09T04:10:34.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 0.33895 2023-10-09T04:10:34.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00115846 2023-10-09T04:10:34.900 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:10:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:34 smithi177 ceph-mon[106070]: pgmap v141: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T04:10:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:34 smithi029 ceph-mon[101043]: pgmap v141: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T04:10:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:34 smithi029 ceph-mon[119131]: pgmap v141: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2023-10-09T04:10:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:35 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2236480252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:36.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:35 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2236480252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:36.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:35 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2236480252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:37.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:36 smithi177 ceph-mon[106070]: pgmap v142: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2023-10-09T04:10:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:36 smithi029 ceph-mon[101043]: pgmap v142: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2023-10-09T04:10:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:36 smithi029 ceph-mon[119131]: pgmap v142: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s 2023-10-09T04:10:38.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:37 smithi177 ceph-mon[106070]: pgmap v143: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-10-09T04:10:38.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:37 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/77387990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:38.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:37 smithi029 ceph-mon[101043]: pgmap v143: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-10-09T04:10:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:37 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/77387990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:37 smithi029 ceph-mon[119131]: pgmap v143: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-10-09T04:10:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:37 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/77387990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:40.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:39 smithi029 ceph-mon[101043]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 1.01k op/s 2023-10-09T04:10:40.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:39 smithi029 ceph-mon[119131]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 1.01k op/s 2023-10-09T04:10:40.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:39 smithi177 ceph-mon[106070]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 1.01k op/s 2023-10-09T04:10:41.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:40 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3153142072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:40 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3153142072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:41.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:40 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3153142072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:42.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:42 smithi029 ceph-mon[101043]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 665 op/s 2023-10-09T04:10:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:42 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:42 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3550588202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:42 smithi029 ceph-mon[119131]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 665 op/s 2023-10-09T04:10:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:42 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:42 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3550588202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:42 smithi177 ceph-mon[106070]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 665 op/s 2023-10-09T04:10:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:42 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:42 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3550588202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:43.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:10:42 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:10:44.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:10:44 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:44] "GET /metrics HTTP/1.1" 200 242474 "" "Prometheus/2.43.0" 2023-10-09T04:10:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:44 smithi029 ceph-mon[101043]: pgmap v146: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 761 op/s 2023-10-09T04:10:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:44 smithi029 ceph-mon[119131]: pgmap v146: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 761 op/s 2023-10-09T04:10:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:44 smithi177 ceph-mon[106070]: pgmap v146: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 761 op/s 2023-10-09T04:10:46.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:45 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4261034459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:46.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:45 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4261034459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:46.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:45 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4261034459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:46 smithi177 ceph-mon[106070]: pgmap v147: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 649 op/s 2023-10-09T04:10:47.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:46 smithi029 ceph-mon[119131]: pgmap v147: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 649 op/s 2023-10-09T04:10:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:46 smithi029 ceph-mon[101043]: pgmap v147: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 649 op/s 2023-10-09T04:10:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:47 smithi177 ceph-mon[106070]: pgmap v148: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 649 op/s 2023-10-09T04:10:48.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:47 smithi029 ceph-mon[119131]: pgmap v148: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 649 op/s 2023-10-09T04:10:48.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:47 smithi029 ceph-mon[101043]: pgmap v148: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 649 op/s 2023-10-09T04:10:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:48 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3098067789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:49.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:48 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3098067789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:48 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3098067789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:49 smithi177 ceph-mon[106070]: pgmap v149: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 962 op/s 2023-10-09T04:10:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:49 smithi029 ceph-mon[101043]: pgmap v149: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 962 op/s 2023-10-09T04:10:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:49 smithi029 ceph-mon[119131]: pgmap v149: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 962 op/s 2023-10-09T04:10:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:50 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1634356686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:51.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:50 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1634356686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:51.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:50 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1634356686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:52.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:52 smithi029 ceph-mon[101043]: pgmap v150: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T04:10:52.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:52 smithi029 ceph-mon[119131]: pgmap v150: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T04:10:53.031 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:10:52 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:10:53.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:52 smithi177 ceph-mon[106070]: pgmap v150: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-09T04:10:53.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:53 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1589931097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:53.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:53 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1589931097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:53 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1589931097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:54.662 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:10:54 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:10:54] "GET /metrics HTTP/1.1" 200 242474 "" "Prometheus/2.43.0" 2023-10-09T04:10:54.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:54 smithi029 ceph-mon[119131]: pgmap v151: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T04:10:54.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:54 smithi029 ceph-mon[101043]: pgmap v151: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T04:10:55.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:54 smithi177 ceph-mon[106070]: pgmap v151: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-09T04:10:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:56 smithi029 ceph-mon[101043]: pgmap v152: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-09T04:10:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:56 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1108770205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:56 smithi029 ceph-mon[119131]: pgmap v152: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-09T04:10:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:56 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1108770205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:56 smithi177 ceph-mon[106070]: pgmap v152: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-09T04:10:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:56 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1108770205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:57 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:57 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:57 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:10:59.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:58 smithi177 ceph-mon[106070]: pgmap v153: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T04:10:59.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:58 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3280059425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:58 smithi029 ceph-mon[101043]: pgmap v153: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T04:10:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:58 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3280059425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:10:59.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:58 smithi029 ceph-mon[119131]: pgmap v153: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-09T04:10:59.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:58 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3280059425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:00.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:10:59 smithi177 ceph-mon[106070]: pgmap v154: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2023-10-09T04:11:00.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:10:59 smithi029 ceph-mon[101043]: pgmap v154: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2023-10-09T04:11:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:10:59 smithi029 ceph-mon[119131]: pgmap v154: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2023-10-09T04:11:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:01 smithi029 ceph-mon[101043]: pgmap v155: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-09T04:11:02.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:01 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2962793044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:02.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:01 smithi029 ceph-mon[119131]: pgmap v155: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-09T04:11:02.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:01 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2962793044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:02.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:01 smithi177 ceph-mon[106070]: pgmap v155: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-09T04:11:02.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:01 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2962793044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:03.279 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:02 smithi177 conmon[109257]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:11:04.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: pgmap v156: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T04:11:04.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: Upgrade: Updating mgr.x 2023-10-09T04:11:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:11:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:11:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: Deploying daemon mgr.x on smithi177 2023-10-09T04:11:04.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:04 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1234667403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:04.328 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: pgmap v156: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: Upgrade: Updating mgr.x 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: Deploying daemon mgr.x on smithi177 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1234667403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:04.329 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: pgmap v156: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: Upgrade: Updating mgr.x 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: Deploying daemon mgr.x on smithi177 2023-10-09T04:11:04.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:04 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1234667403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:04.331 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:04 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:04] "GET /metrics HTTP/1.1" 200 242500 "" "Prometheus/2.43.0" 2023-10-09T04:11:05.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:05 smithi177 systemd[1]: Stopping Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:11:06.282 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:06 smithi177 bash[160568]: caddb7c690a30bb509b9ab07c89b2724b490eb04561a6d53eb7b286abf6d9914 2023-10-09T04:11:06.285 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:06 smithi177 ceph-mon[106070]: pgmap v157: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T04:11:06.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:06 smithi029 ceph-mon[101043]: pgmap v157: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T04:11:06.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:06 smithi029 ceph-mon[119131]: pgmap v157: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T04:11:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:07 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3102775866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:07.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:07 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:11:07.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:07 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T04:11:07.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:07 smithi177 systemd[1]: Stopped Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:11:07.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:07 smithi177 systemd[1]: Starting Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:11:07.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:07 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3102775866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:07 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3102775866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:08.533 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 bash[160911]: ffa2fda2ab93a2500a750459d9999143fb156bfd62cb7734ff4e3fc3a47eb8f2 2023-10-09T04:11:08.533 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 ceph-mgr[160975]: -- 172.21.15.177:0/3777669637 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fd183e51e0 con 0x55fd183bcc00 2023-10-09T04:11:08.533 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 systemd[1]: Started Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:11:08.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:08 smithi177 ceph-mon[106070]: pgmap v158: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T04:11:08.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:08 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:08.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:08 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:08.829 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:08 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:08.830 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 conmon[160950]: 2023-10-09T04:11:08.557+0000 7efe16b9e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:11:08.830 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 conmon[160950]: 2023-10-09T04:11:08.682+0000 7efe16b9e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:11:08.830 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 conmon[160950]: 2023-10-09T04:11:08.827+0000 7efe16b9e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:11:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[101043]: pgmap v158: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T04:11:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:09.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[119131]: pgmap v158: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-09T04:11:09.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:09.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:09.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:08 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:09.262 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:08 smithi177 conmon[160950]: 2023-10-09T04:11:08.980+0000 7efe16b9e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:11:09.262 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:09 smithi177 conmon[160950]: 2023-10-09T04:11:09.261+0000 7efe16b9e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:11:10.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:09 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1618009319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:10.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:09 smithi177 conmon[160950]: 2023-10-09T04:11:09.866+0000 7efe16b9e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:11:10.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:09 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1618009319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:09 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1618009319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:10.379 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:10 smithi177 conmon[160950]: 2023-10-09T04:11:10.070+0000 7efe16b9e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:11:10.379 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:10 smithi177 conmon[160950]: 2023-10-09T04:11:10.202+0000 7efe16b9e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:11:10.380 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:10 smithi177 conmon[160950]: 2023-10-09T04:11:10.357+0000 7efe16b9e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:11:10.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:10 smithi177 ceph-mon[106070]: pgmap v159: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2023-10-09T04:11:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:10 smithi029 ceph-mon[101043]: pgmap v159: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2023-10-09T04:11:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:10 smithi029 ceph-mon[119131]: pgmap v159: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2023-10-09T04:11:11.534 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:11 smithi177 conmon[160950]: 2023-10-09T04:11:11.105+0000 7efe16b9e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:11:12.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:11 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1888075591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:12.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:11 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1888075591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:12.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:11 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1888075591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:12.530 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:12 smithi177 conmon[160950]: 2023-10-09T04:11:12.162+0000 7efe16b9e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:11:12.530 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:12 smithi177 conmon[160950]: 2023-10-09T04:11:12.298+0000 7efe16b9e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:11:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:12 smithi177 ceph-mon[106070]: pgmap v160: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:11:13.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:13.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:13.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:12 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:13.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[101043]: pgmap v160: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[119131]: pgmap v160: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:12 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:13.994 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:13 smithi177 ceph-mon[106070]: pgmap v161: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T04:11:13.995 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:13 smithi177 conmon[160950]: 2023-10-09T04:11:13.551+0000 7efe16b9e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:11:13.995 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:13 smithi177 conmon[160950]: 2023-10-09T04:11:13.716+0000 7efe16b9e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:11:13.995 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:13 smithi177 conmon[160950]: 2023-10-09T04:11:13.993+0000 7efe16b9e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:11:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:13 smithi029 ceph-mon[101043]: pgmap v161: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T04:11:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:13 smithi029 ceph-mon[119131]: pgmap v161: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-09T04:11:14.672 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:14 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:14] "GET /metrics HTTP/1.1" 200 242511 "" "Prometheus/2.43.0" 2023-10-09T04:11:14.762 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:14 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3643744821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:14.763 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:14 smithi177 conmon[160950]: 2023-10-09T04:11:14.494+0000 7efe16b9e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:11:15.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:14 smithi177 conmon[160950]: 2023-10-09T04:11:14.849+0000 7efe16b9e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:11:15.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:14 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3643744821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:15.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:14 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3643744821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:15.533 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:15 smithi177 conmon[160950]: 2023-10-09T04:11:15.094+0000 7efe16b9e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:11:16.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:16 smithi177 ceph-mon[106070]: pgmap v162: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T04:11:16.022 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:16 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:16.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:16 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:16.023 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:15 smithi177 conmon[160950]: 2023-10-09T04:11:15.571+0000 7efe16b9e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:11:16.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:16 smithi029 ceph-mon[101043]: pgmap v162: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T04:11:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:16 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:16 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:16 smithi029 ceph-mon[119131]: pgmap v162: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T04:11:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:16 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:16 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:17.031 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:16 smithi177 conmon[160950]: 2023-10-09T04:11:16.701+0000 7efe16b9e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:11:17.031 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:16 smithi177 conmon[160950]: 2023-10-09T04:11:16.917+0000 7efe16b9e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:11:17.530 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:17 smithi177 conmon[160950]: 2023-10-09T04:11:17.258+0000 7efe16b9e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:11:17.530 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:17 smithi177 conmon[160950]: 2023-10-09T04:11:17.454+0000 7efe16b9e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:11:18.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:17 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:18.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:17 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:18.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:17 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2672581644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:18.004 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:17 smithi177 conmon[160950]: 2023-10-09T04:11:17.817+0000 7efe16b9e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:11:18.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:17 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:18.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:17 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:18.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:17 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2672581644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:18.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:17 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:18.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:17 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:18.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:17 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2672581644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:18.697 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:18 smithi177 ceph-mon[106070]: pgmap v163: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:11:19.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:18 smithi029 ceph-mon[101043]: pgmap v163: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:11:19.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:18 smithi029 ceph-mon[119131]: pgmap v163: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:11:19.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: 2023-10-09T04:11:19.183+0000 7efe16b9e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:11:19.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: [09/Oct/2023:04:11:19] ENGINE Bus STARTING 2023-10-09T04:11:19.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: CherryPy Checker: 2023-10-09T04:11:19.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: The Application mounted at '' has an empty config. 2023-10-09T04:11:19.529 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: 2023-10-09T04:11:19.529 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: [09/Oct/2023:04:11:19] ENGINE Serving on http://:::9283 2023-10-09T04:11:19.529 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:19 smithi177 conmon[160950]: [09/Oct/2023:04:11:19] ENGINE Bus STARTED 2023-10-09T04:11:20.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:11:20.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: Standby manager daemon x restarted 2023-10-09T04:11:20.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: Standby manager daemon x started 2023-10-09T04:11:20.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:11:20.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:11:20.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:11:20.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:19 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/692864357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: Standby manager daemon x restarted 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: Standby manager daemon x started 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:11:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/692864357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:11:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: Standby manager daemon x restarted 2023-10-09T04:11:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: Standby manager daemon x started 2023-10-09T04:11:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:11:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:11:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:11:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:19 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/692864357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:20 smithi177 ceph-mon[106070]: pgmap v164: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-10-09T04:11:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:20 smithi177 ceph-mon[106070]: mgrmap e21: y(active, since 4m), standbys: x 2023-10-09T04:11:21.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:20 smithi029 ceph-mon[119131]: pgmap v164: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-10-09T04:11:21.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:20 smithi029 ceph-mon[119131]: mgrmap e21: y(active, since 4m), standbys: x 2023-10-09T04:11:21.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:20 smithi029 ceph-mon[101043]: pgmap v164: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-10-09T04:11:21.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:20 smithi029 ceph-mon[101043]: mgrmap e21: y(active, since 4m), standbys: x 2023-10-09T04:11:22.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:22 smithi029 ceph-mon[119131]: pgmap v165: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:11:22.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:22 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1307956667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:22.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:22 smithi029 ceph-mon[101043]: pgmap v165: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:11:22.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:22 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1307956667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:23.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:22 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:11:23.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:22 smithi177 ceph-mon[106070]: pgmap v165: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:11:23.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:22 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1307956667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:24.673 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:24 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:24] "GET /metrics HTTP/1.1" 200 242511 "" "Prometheus/2.43.0" 2023-10-09T04:11:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:24 smithi177 ceph-mon[106070]: pgmap v166: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:11:25.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:24 smithi029 ceph-mon[101043]: pgmap v166: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:11:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:24 smithi029 ceph-mon[119131]: pgmap v166: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:11:25.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:25 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1948095422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:25.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:25 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:25 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:25 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:25.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:25 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:25.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:25 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1948095422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1948095422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:25.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:25.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:25.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:25.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' 2023-10-09T04:11:26.207 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:25 smithi177 conmon[160950]: [09/Oct/2023:04:11:25] ENGINE Bus STOPPING 2023-10-09T04:11:26.207 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:25 smithi177 conmon[160950]: [09/Oct/2023:04:11:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:11:26.207 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:25 smithi177 conmon[160950]: [09/Oct/2023:04:11:25] ENGINE Bus STOPPED 2023-10-09T04:11:26.208 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: [09/Oct/2023:04:11:26] ENGINE Bus STARTING 2023-10-09T04:11:26.208 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: [09/Oct/2023:04:11:26] ENGINE Serving on http://:::9283 2023-10-09T04:11:26.208 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: [09/Oct/2023:04:11:26] ENGINE Bus STARTED 2023-10-09T04:11:26.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:25 smithi029 conmon[101466]: ignoring --setuser ceph since I am not root 2023-10-09T04:11:26.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:25 smithi029 conmon[101466]: ignoring --setgroup ceph since I am not root 2023-10-09T04:11:26.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:25 smithi029 ceph-mgr[101490]: -- 172.21.15.29:0/3438505619 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d131f749c0 con 0x55d131f4d000 2023-10-09T04:11:26.231 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:26 smithi029 conmon[101466]: 2023-10-09T04:11:26.151+0000 7f14f6b9b1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:11:26.606 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:26 smithi029 conmon[101466]: 2023-10-09T04:11:26.290+0000 7f14f6b9b1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:11:26.606 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:26 smithi029 conmon[101466]: 2023-10-09T04:11:26.434+0000 7f14f6b9b1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:11:26.607 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:26 smithi029 conmon[101466]: 2023-10-09T04:11:26.597+0000 7f14f6b9b1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:11:26.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: pgmap v167: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-10-09T04:11:26.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:26.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:11:26.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:11:26.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: mgrmap e22: x(active, starting, since 0.0306983s) 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:11:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: Manager daemon x is now available 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: pgmap v167: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:11:26.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: mgrmap e22: x(active, starting, since 0.0306983s) 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:11:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: Manager daemon x is now available 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:11:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:11:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:11:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:26 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:11:26.925 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:26 smithi029 conmon[101466]: 2023-10-09T04:11:26.907+0000 7f14f6b9b1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: pgmap v167: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: osdmap e75: 8 total, 8 up, 8 in 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.14448 172.21.15.29:0/271093914' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: mgrmap e22: x(active, starting, since 0.0306983s) 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:11:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:11:27.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: Manager daemon x is now available 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:11:27.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:26 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.589+0000 7efd76f9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.589+0000 7efd76f9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.589+0000 7efd76f9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.589+0000 7efd76f9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.589+0000 7efd76f9c700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.655+0000 7efd6f78d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.655+0000 7efd6f78d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.032 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.655+0000 7efd6f78d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.033 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.655+0000 7efd6f78d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.033 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:26 smithi177 conmon[160950]: 2023-10-09T04:11:26.655+0000 7efd6f78d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:11:27.833 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:27 smithi029 conmon[101466]: 2023-10-09T04:11:27.482+0000 7f14f6b9b1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:11:27.833 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:27 smithi029 conmon[101466]: 2023-10-09T04:11:27.635+0000 7f14f6b9b1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:11:27.833 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:27 smithi029 conmon[101466]: 2023-10-09T04:11:27.832+0000 7f14f6b9b1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:11:28.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: Deploying cephadm binary to smithi029 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: Deploying cephadm binary to smithi177 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: mgrmap e23: x(active, since 1.07115s) 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: pgmap v3: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2761498425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: Deploying cephadm binary to smithi029 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: Deploying cephadm binary to smithi177 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: mgrmap e23: x(active, since 1.07115s) 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: pgmap v3: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:27 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2761498425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:28.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:27 smithi029 conmon[101466]: 2023-10-09T04:11:27.987+0000 7f14f6b9b1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: Deploying cephadm binary to smithi029 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: Deploying cephadm binary to smithi177 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: mgrmap e23: x(active, since 1.07115s) 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: pgmap v3: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:27 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2761498425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:29.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:29 smithi029 ceph-mon[101043]: pgmap v4: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:29.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:29 smithi029 ceph-mon[101043]: mgrmap e24: x(active, since 2s) 2023-10-09T04:11:29.154 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:28 smithi029 conmon[101466]: 2023-10-09T04:11:28.678+0000 7f14f6b9b1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:11:29.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:29 smithi029 ceph-mon[119131]: pgmap v4: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:29 smithi029 ceph-mon[119131]: mgrmap e24: x(active, since 2s) 2023-10-09T04:11:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:29 smithi177 ceph-mon[106070]: pgmap v4: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:29 smithi177 ceph-mon[106070]: mgrmap e24: x(active, since 2s) 2023-10-09T04:11:29.889 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:29 smithi029 conmon[101466]: 2023-10-09T04:11:29.616+0000 7f14f6b9b1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:11:29.890 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:29 smithi029 conmon[101466]: 2023-10-09T04:11:29.887+0000 7f14f6b9b1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:11:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:30 smithi177 ceph-mon[106070]: [09/Oct/2023:04:11:28] ENGINE Bus STARTING 2023-10-09T04:11:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:30 smithi177 ceph-mon[106070]: [09/Oct/2023:04:11:28] ENGINE Serving on https://172.21.15.177:7150 2023-10-09T04:11:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:30 smithi177 ceph-mon[106070]: [09/Oct/2023:04:11:28] ENGINE Serving on http://172.21.15.177:8765 2023-10-09T04:11:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:30 smithi177 ceph-mon[106070]: [09/Oct/2023:04:11:28] ENGINE Bus STARTED 2023-10-09T04:11:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[101043]: [09/Oct/2023:04:11:28] ENGINE Bus STARTING 2023-10-09T04:11:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[101043]: [09/Oct/2023:04:11:28] ENGINE Serving on https://172.21.15.177:7150 2023-10-09T04:11:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[101043]: [09/Oct/2023:04:11:28] ENGINE Serving on http://172.21.15.177:8765 2023-10-09T04:11:30.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[101043]: [09/Oct/2023:04:11:28] ENGINE Bus STARTED 2023-10-09T04:11:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[119131]: [09/Oct/2023:04:11:28] ENGINE Bus STARTING 2023-10-09T04:11:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[119131]: [09/Oct/2023:04:11:28] ENGINE Serving on https://172.21.15.177:7150 2023-10-09T04:11:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[119131]: [09/Oct/2023:04:11:28] ENGINE Serving on http://172.21.15.177:8765 2023-10-09T04:11:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:30 smithi029 ceph-mon[119131]: [09/Oct/2023:04:11:28] ENGINE Bus STARTED 2023-10-09T04:11:31.497 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:31 smithi029 ceph-mon[101043]: pgmap v5: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:31.497 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:31 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/5127129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:31.498 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:31 smithi029 ceph-mon[119131]: pgmap v5: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:31.498 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:31 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/5127129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:31.498 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:31 smithi029 conmon[101466]: 2023-10-09T04:11:31.337+0000 7f14f6b9b1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:11:31.498 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:31 smithi029 conmon[101466]: 2023-10-09T04:11:31.496+0000 7f14f6b9b1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:11:31.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:31 smithi177 ceph-mon[106070]: pgmap v5: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:31.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:31 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/5127129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:32.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:31 smithi029 conmon[101466]: 2023-10-09T04:11:31.803+0000 7f14f6b9b1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:11:32.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:32 smithi029 conmon[101466]: 2023-10-09T04:11:32.325+0000 7f14f6b9b1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:11:32.675 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:32 smithi029 conmon[101466]: 2023-10-09T04:11:32.503+0000 7f14f6b9b1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:11:33.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:32 smithi029 ceph-mon[101043]: pgmap v6: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:33.015 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:33 smithi029 conmon[101466]: 2023-10-09T04:11:33.009+0000 7f14f6b9b1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:11:33.016 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:32 smithi029 ceph-mon[119131]: pgmap v6: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:33.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:32 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:32] "GET /metrics HTTP/1.1" 200 33286 "" "Prometheus/2.43.0" 2023-10-09T04:11:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:32 smithi177 ceph-mon[106070]: pgmap v6: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-10-09T04:11:33.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:33 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3018533608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:33.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:33 smithi029 conmon[101466]: 2023-10-09T04:11:33.523+0000 7f14f6b9b1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:11:33.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:33 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3018533608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:33 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3018533608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:34 smithi177 ceph-mon[106070]: pgmap v7: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s 2023-10-09T04:11:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:34 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:34 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:34 smithi029 ceph-mon[101043]: pgmap v7: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s 2023-10-09T04:11:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:34 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:34 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:34 smithi029 ceph-mon[119131]: pgmap v7: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 479 op/s 2023-10-09T04:11:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:34 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:34 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:35.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:34 smithi029 conmon[101466]: 2023-10-09T04:11:34.734+0000 7f14f6b9b1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:11:35.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:34 smithi029 conmon[101466]: 2023-10-09T04:11:34.974+0000 7f14f6b9b1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:11:35.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:35 smithi029 conmon[101466]: 2023-10-09T04:11:35.335+0000 7f14f6b9b1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:11:35.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:35 smithi029 conmon[101466]: 2023-10-09T04:11:35.549+0000 7f14f6b9b1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:11:36.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:35 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1136855937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:36.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:35 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1136855937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:35 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1136855937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:36.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:35 smithi029 conmon[101466]: 2023-10-09T04:11:35.942+0000 7f14f6b9b1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:11:37.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[119131]: pgmap v8: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 371 op/s 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:36 smithi029 ceph-mon[101043]: pgmap v8: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 371 op/s 2023-10-09T04:11:37.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:36 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:36 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:36 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:36 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:37.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:36 smithi177 ceph-mon[106070]: pgmap v8: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 371 op/s 2023-10-09T04:11:37.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: 2023-10-09T04:11:37.466+0000 7f14f6b9b1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:11:37.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: [09/Oct/2023:04:11:37] ENGINE Bus STARTING 2023-10-09T04:11:37.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: CherryPy Checker: 2023-10-09T04:11:37.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: The Application mounted at '' has an empty config. 2023-10-09T04:11:37.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: 2023-10-09T04:11:37.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: [09/Oct/2023:04:11:37] ENGINE Serving on http://:::9283 2023-10-09T04:11:37.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:37 smithi029 conmon[101466]: [09/Oct/2023:04:11:37] ENGINE Bus STARTED 2023-10-09T04:11:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi029 to 2501M 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: Standby manager daemon y started 2023-10-09T04:11:38.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:11:38.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:11:38.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:11:38.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi029 to 2501M 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: Standby manager daemon y started 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:11:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi029 to 2501M 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: Standby manager daemon y started 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:11:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/3297359481' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:11:39.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:39 smithi177 ceph-mon[106070]: pgmap v9: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:11:39.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:39 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3910790358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:39.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:39 smithi177 ceph-mon[106070]: mgrmap e25: x(active, since 12s), standbys: y 2023-10-09T04:11:39.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:39 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:11:39.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[101043]: pgmap v9: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3910790358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[101043]: mgrmap e25: x(active, since 12s), standbys: y 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[119131]: pgmap v9: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3910790358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[119131]: mgrmap e25: x(active, since 12s), standbys: y 2023-10-09T04:11:39.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:39 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:11:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:41 smithi177 ceph-mon[106070]: pgmap v10: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:11:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:41 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/673652618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:41 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:41.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:41 smithi029 ceph-mon[101043]: pgmap v10: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:11:41.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:41 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/673652618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:41.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:41 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:41.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:41 smithi029 ceph-mon[119131]: pgmap v10: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:11:41.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:41 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/673652618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:41.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:41 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:42 smithi177 ceph-mon[106070]: pgmap v11: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:11:43.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:42 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:42] "GET /metrics HTTP/1.1" 200 36221 "" "Prometheus/2.43.0" 2023-10-09T04:11:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:42 smithi029 ceph-mon[101043]: pgmap v11: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:11:43.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:42 smithi029 ceph-mon[119131]: pgmap v11: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:11:44.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:44 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:11:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2542839334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: Adjusting osd_memory_target on smithi177 to 2500M 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: pgmap v12: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:11:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2542839334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: Adjusting osd_memory_target on smithi177 to 2500M 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: pgmap v12: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:11:44.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:44 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2542839334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: Adjusting osd_memory_target on smithi177 to 2500M 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: pgmap v12: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:11:45.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:44 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:11:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:11:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:45 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:45 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:11:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:45 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:11:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:45 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:11:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:45 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:45 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:11:47.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: pgmap v13: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-10-09T04:11:47.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2024126204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[119131]: Reconfiguring daemon alertmanager.a on smithi029 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:11:47.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: pgmap v13: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2024126204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:11:47.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:47 smithi029 ceph-mon[101043]: Reconfiguring daemon alertmanager.a on smithi029 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: pgmap v13: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2024126204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:11:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: Reconfiguring alertmanager.a (dependencies changed)... 2023-10-09T04:11:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:47 smithi177 ceph-mon[106070]: Reconfiguring daemon alertmanager.a on smithi029 2023-10-09T04:11:49.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:49 smithi029 ceph-mon[101043]: pgmap v14: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:11:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:49 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3900538616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:49 smithi029 ceph-mon[119131]: pgmap v14: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:11:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:49 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3900538616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:49.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:49 smithi177 ceph-mon[106070]: pgmap v14: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2023-10-09T04:11:49.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:49 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3900538616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:51 smithi029 ceph-mon[119131]: pgmap v15: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:11:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:51 smithi029 ceph-mon[101043]: pgmap v15: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:11:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:51 smithi177 ceph-mon[106070]: pgmap v15: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:11:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:52 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3336344221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:52 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3336344221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:52.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:52 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3336344221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:53.094 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:53 smithi177 ceph-mon[106070]: pgmap v16: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:11:53.095 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:11:52 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:52] "GET /metrics HTTP/1.1" 200 36221 "" "Prometheus/2.43.0" 2023-10-09T04:11:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:53 smithi029 ceph-mon[101043]: pgmap v16: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:11:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:53 smithi029 ceph-mon[119131]: pgmap v16: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:11:54.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:11:54 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:11:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:11:55.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:55 smithi029 ceph-mon[101043]: pgmap v17: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-10-09T04:11:55.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:55 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3844918366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:55.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:55 smithi029 ceph-mon[119131]: pgmap v17: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-10-09T04:11:55.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:55 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3844918366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:55 smithi177 ceph-mon[106070]: pgmap v17: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-10-09T04:11:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:55 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3844918366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:56 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:56.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:56 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:56 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:11:57.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:57 smithi029 ceph-mon[101043]: pgmap v18: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:11:57.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:57 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1045872654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:57.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:57 smithi029 ceph-mon[119131]: pgmap v18: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:11:57.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:57 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1045872654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:57 smithi177 ceph-mon[106070]: pgmap v18: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:11:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:57 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1045872654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:11:58.425 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:11:58 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:11:58.887 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:11:58 smithi029 conmon[176538]: ts=2023-10-09T04:11:58.560Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:11:58.887 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:11:58 smithi029 bash[200315]: 6a4c4bc4e5b31060c5c22b8f00913e104f69932c91365fc552c5b1ebc2463d5b 2023-10-09T04:11:59.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:11:59 smithi029 ceph-mon[101043]: pgmap v19: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-10-09T04:11:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:11:59 smithi029 ceph-mon[119131]: pgmap v19: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-10-09T04:11:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:11:59 smithi177 ceph-mon[106070]: pgmap v19: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-10-09T04:12:00.173 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:11:59 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T04:12:00.173 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:11:59 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:00.173 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:11:59 smithi029 systemd[1]: Starting Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:00.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:00 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2032287163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:00.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:00 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2032287163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:00.656 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:00 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2032287163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:01.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 bash[200647]: 7ab788eff7bee282e96ae2441cd3b9015c7100c966a0a4742e28efdc23d9cd5d 2023-10-09T04:12:01.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.790Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-10-09T04:12:01.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.790Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-10-09T04:12:01.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.792Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2023-10-09T04:12:01.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.793Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-09T04:12:01.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 systemd[1]: Started Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:01.172 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.839Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:12:01.172 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.840Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-09T04:12:01.172 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.844Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-10-09T04:12:01.172 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:00 smithi029 conmon[200683]: ts=2023-10-09T04:12:00.844Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-10-09T04:12:01.458 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:01 smithi029 ceph-mon[101043]: pgmap v20: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:12:01.458 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:01 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:01.458 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:01 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:01.459 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:01 smithi029 ceph-mon[119131]: pgmap v20: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:12:01.459 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:01 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:01.459 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:01 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:01.460 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:01 smithi029 systemd[1]: Stopping Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:01 smithi177 ceph-mon[106070]: pgmap v20: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:12:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:01 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:01.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:01 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:02 smithi177 ceph-mon[106070]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:12:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:02 smithi177 ceph-mon[106070]: Reconfiguring daemon node-exporter.a on smithi029 2023-10-09T04:12:02.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:02 smithi029 ceph-mon[101043]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:12:02.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:02 smithi029 ceph-mon[101043]: Reconfiguring daemon node-exporter.a on smithi029 2023-10-09T04:12:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:02 smithi029 ceph-mon[119131]: Reconfiguring node-exporter.a (dependencies changed)... 2023-10-09T04:12:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:02 smithi029 ceph-mon[119131]: Reconfiguring daemon node-exporter.a on smithi029 2023-10-09T04:12:02.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:02 smithi029 bash[200927]: 15156916bd109a8c5bc71eb23637984df313af7a5770caf814e289a9b0457088 2023-10-09T04:12:03.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:02 smithi029 conmon[200683]: ts=2023-10-09T04:12:02.794Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000913088s 2023-10-09T04:12:03.211 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:02 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:12:02] "GET /metrics HTTP/1.1" 200 36223 "" "Prometheus/2.43.0" 2023-10-09T04:12:03.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:03 smithi177 ceph-mon[106070]: pgmap v21: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:12:03.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:03 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2820395966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:03.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:03 smithi029 ceph-mon[101043]: pgmap v21: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:12:03.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:03 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2820395966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:03.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:03 smithi029 ceph-mon[119131]: pgmap v21: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:12:03.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:03 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2820395966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:04.327 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[101466]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:12:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:12:04.327 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:03 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:12:04.327 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:03 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T04:12:04.328 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:03 smithi029 systemd[1]: Stopped Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:04.328 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:03 smithi029 systemd[1]: Starting Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.923Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.923Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.924Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.924Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.924Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:12:05.216 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:12:05.217 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:12:05.218 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:12:05.218 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:12:05.219 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:12:05.220 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.925Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.926Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 conmon[201298]: ts=2023-10-09T04:12:04.926Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 bash[201246]: 2be0e618a0f3fdc1dbacb71fc95d571446c64f53d14083b20c4ca1954510e13a 2023-10-09T04:12:05.221 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 04:12:04 smithi029 systemd[1]: Started Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:05.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[101043]: pgmap v22: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T04:12:05.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2282166389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:05.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:05.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:05.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:05 smithi177 ceph-mon[106070]: pgmap v22: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T04:12:05.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:05 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2282166389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:05.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:05 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:05.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:05 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:05.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[119131]: pgmap v22: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-09T04:12:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2282166389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:05 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:06.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:06 smithi177 ceph-mon[106070]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:12:06.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:06 smithi177 ceph-mon[106070]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:12:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:06 smithi029 ceph-mon[101043]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:12:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:06 smithi029 ceph-mon[101043]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:12:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:06 smithi029 ceph-mon[119131]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:12:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:06 smithi029 ceph-mon[119131]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:12:07.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:07 smithi177 ceph-mon[106070]: pgmap v23: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T04:12:07.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:07 smithi029 ceph-mon[101043]: pgmap v23: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T04:12:07.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:07 smithi029 ceph-mon[119131]: pgmap v23: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-09T04:12:08.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:08 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2334080309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:08.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:08 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2334080309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:08.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:08 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2334080309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:09.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:09 smithi177 ceph-mon[106070]: pgmap v24: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2023-10-09T04:12:09.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:09 smithi029 ceph-mon[101043]: pgmap v24: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2023-10-09T04:12:09.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:09 smithi029 ceph-mon[119131]: pgmap v24: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2023-10-09T04:12:10.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:10 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1743075961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:10.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:10 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1743075961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:10 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1743075961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:11.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 04:12:10 smithi029 conmon[200683]: ts=2023-10-09T04:12:10.798Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004374836s 2023-10-09T04:12:11.531 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 systemd[1]: Stopping Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:11.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:11 smithi177 ceph-mon[106070]: pgmap v25: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:12:11.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:11 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:11.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:11 smithi029 ceph-mon[101043]: pgmap v25: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:12:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:11 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:11 smithi029 ceph-mon[119131]: pgmap v25: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:12:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:11 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:12.265 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.917Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:12:12.265 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:12:12.265 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:12:12.265 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.918Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.919Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.919Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.919Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:11 smithi177 conmon[154629]: ts=2023-10-09T04:12:11.919Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:12:12.266 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:12 smithi177 bash[173224]: 51d99d79686facde26f1219241f16a432b447b605378f7988cc5fe051213c58c 2023-10-09T04:12:12.960 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:12 smithi177 ceph-mon[106070]: pgmap v26: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:12:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:12 smithi029 ceph-mon[119131]: pgmap v26: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:12:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:12 smithi029 ceph-mon[101043]: pgmap v26: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:12:13.280 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:12 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:12:13.280 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:12 smithi177 systemd[1]: Stopped Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:13.280 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:12 smithi177 systemd[1]: Starting Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:14.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:13 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/776862649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:13 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/776862649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:13 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/776862649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:14.528 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 bash[173541]: bb8553476ffeb1498124fb6d8f506e330816564ef5518e65fbeeda7bbe94d55d 2023-10-09T04:12:14.528 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 systemd[1]: Started Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:14.528 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.076Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.076Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.076Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi177 (none))" 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.076Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.076Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.078Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.079Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.101Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.101Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.103Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.103Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.269µs 2023-10-09T04:12:14.529 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.103Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.104Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.133Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.133Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.133Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.887µs wal_replay_duration=29.997742ms wbl_replay_duration=89ns total_replay_duration=30.063786ms 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.136Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.136Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.136Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.166Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.311526ms db_storage=1.067µs remote_storage=1.856µs web_handler=447ns query_engine=925ns scrape=258.52µs scrape_sd=109.837µs notify=16.79µs notify_sd=14.198µs rules=28.592574ms tracing=16.161µs 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.166Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:12:14.530 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:14 smithi177 conmon[173579]: ts=2023-10-09T04:12:14.166Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:12:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[101043]: pgmap v27: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[101043]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[101043]: Reconfiguring daemon grafana.a on smithi177 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[119131]: pgmap v27: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[119131]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:12:15.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:15 smithi029 ceph-mon[119131]: Reconfiguring daemon grafana.a on smithi177 2023-10-09T04:12:15.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:15 smithi177 ceph-mon[106070]: pgmap v27: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-10-09T04:12:15.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:15 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:15.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:15 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:15.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:15 smithi177 ceph-mon[106070]: Reconfiguring grafana.a (dependencies changed)... 2023-10-09T04:12:15.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:15 smithi177 ceph-mon[106070]: Reconfiguring daemon grafana.a on smithi177 2023-10-09T04:12:16.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:16 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/870288733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:16 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/870288733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:16.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:16 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/870288733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:17.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:17 smithi029 ceph-mon[101043]: pgmap v28: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:12:17.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:17 smithi029 ceph-mon[119131]: pgmap v28: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:12:17.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:17 smithi177 ceph-mon[106070]: pgmap v28: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:12:18.278 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:17 smithi177 systemd[1]: Stopping Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:19.036 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:18 smithi177 bash[174037]: 3d5a415c7f4fa886850c4452ddfb01020cf1db82aa986091d58d0c65a97be922 2023-10-09T04:12:19.413 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:19 smithi177 ceph-mon[106070]: pgmap v29: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T04:12:19.414 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:19 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3695002723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:19.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:19 smithi029 ceph-mon[119131]: pgmap v29: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T04:12:19.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:19 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3695002723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:19 smithi029 ceph-mon[101043]: pgmap v29: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T04:12:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:19 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3695002723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:19.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:19 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:12:19.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:19 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T04:12:19.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:19 smithi177 systemd[1]: Stopped Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:19.778 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:19 smithi177 systemd[1]: Starting Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:20.782 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 bash[174350]: dbe79334ca87f7b53849e7539d7a21ed2a796e24d5636db57bb29bade77a1c9f 2023-10-09T04:12:20.782 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 systemd[1]: Started Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:21.164 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:21 smithi177 ceph-mon[106070]: pgmap v30: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:12:21.164 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:21 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:21.164 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:21 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:21.164 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:21 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/4044551704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775019253Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775280016Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775304059Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775314724Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775324135Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775333383Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775342453Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.77535142Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-09T04:12:21.165 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775360411Z level=info msg="Path Home" path=/usr/share/grafana 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775370413Z level=info msg="Path Data" path=/var/lib/grafana 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775378952Z level=info msg="Path Logs" path=/var/log/grafana 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775387641Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.775396532Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=settings t=2023-10-09T04:12:20.77540561Z level=info msg="App mode production" 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=sqlstore t=2023-10-09T04:12:20.7754797Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=sqlstore t=2023-10-09T04:12:20.775502801Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=migrator t=2023-10-09T04:12:20.798160554Z level=info msg="Starting DB migrations" 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=migrator t=2023-10-09T04:12:20.805483535Z level=info msg="migrations completed" performed=0 skipped=481 duration=536.751µs 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=sqlstore t=2023-10-09T04:12:20.806113589Z level=info msg="Created default organization" 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=plugin.loader t=2023-10-09T04:12:20.890636716Z level=info msg="Plugin registered" pluginID=input 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=plugin.loader t=2023-10-09T04:12:20.934196948Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-10-09T04:12:21.166 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=plugin.loader t=2023-10-09T04:12:20.934237459Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-10-09T04:12:21.167 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=secrets t=2023-10-09T04:12:20.934326936Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-10-09T04:12:21.167 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=query_data t=2023-10-09T04:12:20.936860688Z level=info msg="Query Service initialization" 2023-10-09T04:12:21.167 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:20 smithi177 conmon[174387]: logger=live.push_http t=2023-10-09T04:12:20.945804151Z level=info msg="Live Push Gateway initialization" 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[101043]: pgmap v30: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/4044551704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[119131]: pgmap v30: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:21 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/4044551704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:21.528 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:21 smithi177 systemd[1]: Stopping Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:22.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:22 smithi177 bash[174612]: b7a4d2d246ccd755dd0a1651ba8eb4e826e8783e0f8d627e463ca6b8fe69f877 2023-10-09T04:12:22.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:22 smithi177 ceph-mon[106070]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:12:22.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:22 smithi177 ceph-mon[106070]: Reconfiguring daemon node-exporter.b on smithi177 2023-10-09T04:12:22.535 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:22 smithi177 conmon[174387]: logger=infra.usagestats.collector t=2023-10-09T04:12:22.246868138Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-10-09T04:12:22.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:22 smithi029 ceph-mon[101043]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:12:22.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:22 smithi029 ceph-mon[101043]: Reconfiguring daemon node-exporter.b on smithi177 2023-10-09T04:12:22.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:22 smithi029 ceph-mon[119131]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-09T04:12:22.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:22 smithi029 ceph-mon[119131]: Reconfiguring daemon node-exporter.b on smithi177 2023-10-09T04:12:23.178 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:22 smithi177 conmon[174387]: logger=provisioning.datasources t=2023-10-09T04:12:22.795047169Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-10-09T04:12:23.178 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:22 smithi177 conmon[174387]: logger=provisioning.datasources t=2023-10-09T04:12:22.795098504Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-10-09T04:12:23.178 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:22 smithi177 conmon[174387]: logger=provisioning.datasources t=2023-10-09T04:12:22.809145909Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-09T04:12:23.530 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=provisioning.alerting t=2023-10-09T04:12:23.18125951Z level=info msg="starting to provision alerting" 2023-10-09T04:12:23.530 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=provisioning.alerting t=2023-10-09T04:12:23.181291095Z level=info msg="finished to provision alerting" 2023-10-09T04:12:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=grafanaStorageLogger t=2023-10-09T04:12:23.181500213Z level=info msg="storage starting" 2023-10-09T04:12:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=ngalert.state.manager t=2023-10-09T04:12:23.183296252Z level=info msg="Warming state cache for startup" 2023-10-09T04:12:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=http.server t=2023-10-09T04:12:23.183789381Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-10-09T04:12:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=ngalert.state.manager t=2023-10-09T04:12:23.250166734Z level=info msg="State cache has been initialized" states=0 duration=66.866667ms 2023-10-09T04:12:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=ticker t=2023-10-09T04:12:23.250273962Z level=info msg=starting first_tick=2023-10-09T04:12:30Z 2023-10-09T04:12:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:12:23 smithi177 conmon[174387]: logger=ngalert.multiorg.alertmanager t=2023-10-09T04:12:23.250299546Z level=info msg="starting MultiOrg Alertmanager" 2023-10-09T04:12:23.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:23 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:12:23.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:23 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T04:12:23.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:23 smithi177 systemd[1]: Stopped Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:23.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:23 smithi177 systemd[1]: Starting Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:23 smithi177 ceph-mon[106070]: pgmap v31: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:12:23.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:23 smithi029 ceph-mon[101043]: pgmap v31: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:12:23.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:23 smithi029 ceph-mon[119131]: pgmap v31: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-09T04:12:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:24 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3997511805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:24.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[160950]: [09/Oct/2023:04:12:24] ENGINE Bus STOPPING 2023-10-09T04:12:24.528 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.305Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-10-09T04:12:24.528 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.305Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.306Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.306Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.306Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.306Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=arp 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=bcache 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=bonding 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=btrfs 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=conntrack 2023-10-09T04:12:24.529 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=cpu 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=diskstats 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=dmi 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=edac 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=entropy 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=filefd 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=filesystem 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=hwmon 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=infiniband 2023-10-09T04:12:24.530 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=ipvs 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=loadavg 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=mdadm 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=meminfo 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=netclass 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=netdev 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=netstat 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=nfs 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=nfsd 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=nvme 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=os 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-10-09T04:12:24.531 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=pressure 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=rapl 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=schedstat 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=selinux 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=sockstat 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=softnet 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=stat 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=tapestats 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=textfile 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=time 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-10-09T04:12:24.532 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=uname 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=vmstat 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=xfs 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.307Z caller=node_exporter.go:117 level=info collector=zfs 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.308Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[174959]: ts=2023-10-09T04:12:24.308Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 bash[174919]: 2adf0c28b4540e065c2489e1d32ae43e882c4903b94c6c5bcc91f0c9478932ce 2023-10-09T04:12:24.533 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 04:12:24 smithi177 systemd[1]: Started Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:24 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3997511805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:24.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:24 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3997511805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:24.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[160950]: [09/Oct/2023:04:12:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:12:24.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[160950]: [09/Oct/2023:04:12:24] ENGINE Bus STOPPED 2023-10-09T04:12:24.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[160950]: [09/Oct/2023:04:12:24] ENGINE Bus STARTING 2023-10-09T04:12:24.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[160950]: [09/Oct/2023:04:12:24] ENGINE Serving on http://:::9283 2023-10-09T04:12:24.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:24 smithi177 conmon[160950]: [09/Oct/2023:04:12:24] ENGINE Bus STARTED 2023-10-09T04:12:25.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: pgmap v32: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:12:25.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: pgmap v32: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:12:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:12:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:12:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:12:25.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:25 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:25.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: pgmap v32: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:12:25.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:25 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:26.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:12:26.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:12:26.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:12:26.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:12:26.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:12:26.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:12:26.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:26.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:26 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3884916722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3884916722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:26 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3884916722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:27 smithi177 ceph-mon[106070]: pgmap v33: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:12:27.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:27 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:12:27] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2023-10-09T04:12:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:27 smithi029 ceph-mon[101043]: pgmap v33: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:12:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:27 smithi029 ceph-mon[119131]: pgmap v33: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-10-09T04:12:28.781 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:28 smithi177 conmon[173579]: ts=2023-10-09T04:12:28.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:12:28.784 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:28 smithi177 ceph-mon[106070]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-10-09T04:12:28.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:28 smithi029 ceph-mon[101043]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-10-09T04:12:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:28 smithi029 ceph-mon[119131]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-10-09T04:12:29.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:29 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1513037731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:29 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1513037731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:29.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:29 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1513037731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:30 smithi177 ceph-mon[106070]: pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:12:30.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:30 smithi029 ceph-mon[101043]: pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:12:30.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:30 smithi029 ceph-mon[119131]: pgmap v35: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:12:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:31 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/895891110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:31 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/895891110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:32.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:31 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/895891110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:32 smithi177 ceph-mon[106070]: pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:12:33.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:32 smithi029 ceph-mon[101043]: pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:12:33.144 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:32 smithi029 ceph-mon[119131]: pgmap v36: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: Upgrade: Updating mgr.y 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[101043]: Deploying daemon mgr.y on smithi029 2023-10-09T04:12:34.388 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: Upgrade: Updating mgr.y 2023-10-09T04:12:34.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:34.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:12:34.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:12:34.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:12:34.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:34.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:34 smithi029 ceph-mon[119131]: Deploying daemon mgr.y on smithi029 2023-10-09T04:12:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: Upgrade: Updating mgr.y 2023-10-09T04:12:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:12:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:12:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:12:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:34.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:34 smithi177 ceph-mon[106070]: Deploying daemon mgr.y on smithi029 2023-10-09T04:12:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:35 smithi177 ceph-mon[106070]: pgmap v37: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2023-10-09T04:12:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:35 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1299372084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:35 smithi029 ceph-mon[101043]: pgmap v37: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2023-10-09T04:12:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:35 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1299372084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:35.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:35 smithi029 ceph-mon[119131]: pgmap v37: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2023-10-09T04:12:35.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:35 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1299372084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:36.278 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:36 smithi029 systemd[1]: Stopping Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:37.248 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:36 smithi029 bash[202199]: 308b3b05d6d1a4abf0bb3b0b704d13b5b7f2b20d2fcefeef9adb96ba731184fe 2023-10-09T04:12:37.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:37 smithi029 ceph-mon[101043]: pgmap v38: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T04:12:37.248 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:37 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3090327575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:37 smithi177 ceph-mon[106070]: pgmap v38: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T04:12:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:37 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3090327575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:37.531 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:37 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:12:37] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2023-10-09T04:12:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:37 smithi029 ceph-mon[119131]: pgmap v38: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-09T04:12:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:37 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3090327575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:38.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:38 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:12:38.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:38 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-09T04:12:38.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:38 smithi029 systemd[1]: Stopped Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:38.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:38 smithi029 systemd[1]: Starting Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:12:39.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:38 smithi177 conmon[173579]: ts=2023-10-09T04:12:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:12:39.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:39 smithi177 ceph-mon[106070]: pgmap v39: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T04:12:39.587 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:39 smithi029 ceph-mon[101043]: pgmap v39: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T04:12:39.587 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:39 smithi029 bash[202541]: 239eea6ccc4c263316dcd1f74f4ace30cd9882972f93301957506df0a616abc7 2023-10-09T04:12:39.587 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:39 smithi029 systemd[1]: Started Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:12:39.587 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:39 smithi029 conmon[202582]: 2023-10-09T04:12:39.583+0000 7fd501d9d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:12:39.590 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:39 smithi029 ceph-mon[119131]: pgmap v39: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 945 op/s 2023-10-09T04:12:39.892 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:39 smithi029 conmon[202582]: 2023-10-09T04:12:39.730+0000 7fd501d9d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:12:39.893 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:39 smithi029 conmon[202582]: 2023-10-09T04:12:39.885+0000 7fd501d9d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:12:40.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:40 smithi029 conmon[202582]: 2023-10-09T04:12:40.062+0000 7fd501d9d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:12:40.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1399781785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: pgmap v40: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:12:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:40 smithi029 conmon[202582]: 2023-10-09T04:12:40.390+0000 7fd501d9d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1399781785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: pgmap v40: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:12:40.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:40 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1399781785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: pgmap v40: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:40 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:41.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:41 smithi029 conmon[202582]: 2023-10-09T04:12:41.092+0000 7fd501d9d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:12:41.423 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:41 smithi029 conmon[202582]: 2023-10-09T04:12:41.314+0000 7fd501d9d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:12:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:41 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:41.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:41 smithi029 conmon[202582]: 2023-10-09T04:12:41.474+0000 7fd501d9d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:12:41.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:41 smithi029 conmon[202582]: 2023-10-09T04:12:41.657+0000 7fd501d9d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:12:41.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:41 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:41 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:42.755 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:42 smithi029 conmon[202582]: 2023-10-09T04:12:42.449+0000 7fd501d9d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:12:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:42 smithi177 ceph-mon[106070]: pgmap v41: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:12:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:42 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/917694368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:42 smithi029 ceph-mon[119131]: pgmap v41: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:12:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:42 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/917694368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:43.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:42 smithi029 ceph-mon[101043]: pgmap v41: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:12:43.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:42 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/917694368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:43.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:43 smithi029 conmon[202582]: 2023-10-09T04:12:43.579+0000 7fd501d9d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:12:43.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:43 smithi029 conmon[202582]: 2023-10-09T04:12:43.745+0000 7fd501d9d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:12:45.370 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:45 smithi029 ceph-mon[101043]: pgmap v42: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T04:12:45.370 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:45 smithi029 conmon[202582]: 2023-10-09T04:12:45.196+0000 7fd501d9d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:12:45.370 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:45 smithi029 conmon[202582]: 2023-10-09T04:12:45.361+0000 7fd501d9d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:12:45.371 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:45 smithi029 ceph-mon[119131]: pgmap v42: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T04:12:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:45 smithi177 ceph-mon[106070]: pgmap v42: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-10-09T04:12:46.054 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:45 smithi029 conmon[202582]: 2023-10-09T04:12:45.688+0000 7fd501d9d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:12:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3657056841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:46 smithi029 conmon[202582]: 2023-10-09T04:12:46.232+0000 7fd501d9d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:12:46.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3657056841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:46 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:46 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:46 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:46 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3657056841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:46 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:46 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:46.875 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:46 smithi029 conmon[202582]: 2023-10-09T04:12:46.586+0000 7fd501d9d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:12:46.908 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:46 smithi029 conmon[202582]: 2023-10-09T04:12:46.872+0000 7fd501d9d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:12:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:47 smithi029 ceph-mon[119131]: pgmap v43: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:12:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:47 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:47.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:47 smithi029 ceph-mon[101043]: pgmap v43: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:12:47.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:47.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:47 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:47.338 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:47 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:12:47] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:12:47.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:47 smithi177 ceph-mon[106070]: pgmap v43: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-09T04:12:47.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:47.338 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:47 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:47.675 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:47 smithi029 conmon[202582]: 2023-10-09T04:12:47.379+0000 7fd501d9d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:12:48.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:48 smithi029 conmon[202582]: 2023-10-09T04:12:48.683+0000 7fd501d9d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:12:48.927 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:48 smithi029 conmon[202582]: 2023-10-09T04:12:48.920+0000 7fd501d9d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:12:49.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:48 smithi177 conmon[173579]: ts=2023-10-09T04:12:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:12:49.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:49 smithi029 ceph-mon[101043]: pgmap v44: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T04:12:49.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:49 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2031089318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:49.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:49 smithi029 ceph-mon[119131]: pgmap v44: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T04:12:49.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:49 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2031089318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:49.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:49 smithi177 ceph-mon[106070]: pgmap v44: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-10-09T04:12:49.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:49 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2031089318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:49.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:49 smithi029 conmon[202582]: 2023-10-09T04:12:49.276+0000 7fd501d9d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:12:49.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:49 smithi029 conmon[202582]: 2023-10-09T04:12:49.487+0000 7fd501d9d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:12:50.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:49 smithi029 conmon[202582]: 2023-10-09T04:12:49.896+0000 7fd501d9d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:12:51.391 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:51 smithi029 ceph-mon[101043]: pgmap v45: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-09T04:12:51.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:51 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3309360411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:51.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:51 smithi029 ceph-mon[119131]: pgmap v45: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-09T04:12:51.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:51 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3309360411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:51.393 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: 2023-10-09T04:12:51.390+0000 7fd501d9d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:12:51.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:51 smithi177 ceph-mon[106070]: pgmap v45: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-09T04:12:51.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:51 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3309360411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:51.673 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: [09/Oct/2023:04:12:51] ENGINE Bus STARTING 2023-10-09T04:12:51.673 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: CherryPy Checker: 2023-10-09T04:12:51.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: The Application mounted at '' has an empty config. 2023-10-09T04:12:51.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: 2023-10-09T04:12:51.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: [09/Oct/2023:04:12:51] ENGINE Serving on http://:::9283 2023-10-09T04:12:51.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:51 smithi029 conmon[202582]: [09/Oct/2023:04:12:51] ENGINE Bus STARTED 2023-10-09T04:12:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[119131]: Standby manager daemon y restarted 2023-10-09T04:12:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[119131]: Standby manager daemon y started 2023-10-09T04:12:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:12:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:12:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:12:52.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:12:52.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[101043]: Standby manager daemon y restarted 2023-10-09T04:12:52.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[101043]: Standby manager daemon y started 2023-10-09T04:12:52.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:12:52.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:12:52.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:12:52.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:52 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:12:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:52 smithi177 ceph-mon[106070]: Standby manager daemon y restarted 2023-10-09T04:12:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:52 smithi177 ceph-mon[106070]: Standby manager daemon y started 2023-10-09T04:12:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:52 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-09T04:12:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:52 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:12:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:52 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-09T04:12:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:52 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:12:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:53 smithi029 ceph-mon[101043]: pgmap v46: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-09T04:12:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:53 smithi029 ceph-mon[101043]: mgrmap e26: x(active, since 86s), standbys: y 2023-10-09T04:12:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:53 smithi029 ceph-mon[119131]: pgmap v46: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-09T04:12:53.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:53 smithi029 ceph-mon[119131]: mgrmap e26: x(active, since 86s), standbys: y 2023-10-09T04:12:53.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:53 smithi177 ceph-mon[106070]: pgmap v46: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-09T04:12:53.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:53 smithi177 ceph-mon[106070]: mgrmap e26: x(active, since 86s), standbys: y 2023-10-09T04:12:54.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:54 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2410465999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:54.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:54 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2410465999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:54 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2410465999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:55.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:55 smithi029 ceph-mon[101043]: pgmap v47: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2023-10-09T04:12:55.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:55 smithi029 ceph-mon[119131]: pgmap v47: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2023-10-09T04:12:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:55 smithi177 ceph-mon[106070]: pgmap v47: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2023-10-09T04:12:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:56 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:56.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:56 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:56.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:56 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:57 smithi029 ceph-mon[101043]: pgmap v48: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:12:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:57 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1177095382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:57.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:57 smithi029 ceph-mon[119131]: pgmap v48: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:12:57.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:57 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1177095382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:57 smithi177 ceph-mon[106070]: pgmap v48: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:12:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:57 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1177095382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:12:57.529 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:57 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:12:57] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2023-10-09T04:12:58.016 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:57 smithi177 conmon[160950]: 2023-10-09T04:12:57.760+0000 7efe0a24b700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-09T04:12:58.016 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:57 smithi177 conmon[160950]: ignoring --setuser ceph since I am not root 2023-10-09T04:12:58.016 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:57 smithi177 conmon[160950]: ignoring --setgroup ceph since I am not root 2023-10-09T04:12:58.016 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:57 smithi177 ceph-mgr[160975]: -- 172.21.15.177:0/3163138995 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e9d2f671e0 con 0x55e9d2f3e800 2023-10-09T04:12:58.017 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:58 smithi177 conmon[160950]: 2023-10-09T04:12:58.015+0000 7f452e776200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:12:58.114 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:57 smithi029 conmon[202582]: [09/Oct/2023:04:12:57] ENGINE Bus STOPPING 2023-10-09T04:12:58.115 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:57 smithi029 conmon[202582]: [09/Oct/2023:04:12:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T04:12:58.115 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:57 smithi029 conmon[202582]: [09/Oct/2023:04:12:57] ENGINE Bus STOPPED 2023-10-09T04:12:58.115 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:57 smithi029 conmon[202582]: [09/Oct/2023:04:12:57] ENGINE Bus STARTING 2023-10-09T04:12:58.115 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: [09/Oct/2023:04:12:58] ENGINE Serving on http://:::9283 2023-10-09T04:12:58.115 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: [09/Oct/2023:04:12:58] ENGINE Bus STARTED 2023-10-09T04:12:58.268 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:58 smithi177 conmon[160950]: 2023-10-09T04:12:58.139+0000 7f452e776200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:12:58.268 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:58 smithi177 conmon[160950]: 2023-10-09T04:12:58.267+0000 7f452e776200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:12:58.423 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.129+0000 7fd4629dc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.423 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.129+0000 7fd4629dc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.129+0000 7fd4629dc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.129+0000 7fd4629dc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.129+0000 7fd4629dc700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.133+0000 7fd45f1d5700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.133+0000 7fd45f1d5700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.133+0000 7fd45f1d5700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.133+0000 7fd45f1d5700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:12:58 smithi029 conmon[202582]: 2023-10-09T04:12:58.133+0000 7fd45f1d5700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:12:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.24769 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: mgrmap e27: y(active, starting, since 0.0257492s) 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:12:58.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: Manager daemon y is now available 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:12:58.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:12:58.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:12:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.24769 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: mgrmap e27: y(active, starting, since 0.0257492s) 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:12:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: Manager daemon y is now available 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:58.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:12:58.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:12:58.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:12:58.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:58 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:12:58.531 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:58 smithi177 conmon[160950]: 2023-10-09T04:12:58.405+0000 7f452e776200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:12:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:12:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 172.21.15.177:0/262194997' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: osdmap e76: 8 total, 8 up, 8 in 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.24769 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: mgrmap e27: y(active, starting, since 0.0257492s) 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:12:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:12:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: Manager daemon y is now available 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:12:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:58 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:12:59.030 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:58 smithi177 conmon[160950]: 2023-10-09T04:12:58.670+0000 7f452e776200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:12:59.031 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:12:58 smithi177 conmon[173579]: ts=2023-10-09T04:12:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:12:59.661 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:59 smithi177 conmon[160950]: 2023-10-09T04:12:59.305+0000 7f452e776200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:12:59.661 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:59 smithi177 conmon[160950]: 2023-10-09T04:12:59.511+0000 7f452e776200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:12:59.661 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:59 smithi177 conmon[160950]: 2023-10-09T04:12:59.660+0000 7f452e776200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:13:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: [09/Oct/2023:04:12:58] ENGINE Bus STARTING 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: [09/Oct/2023:04:12:58] ENGINE Serving on http://172.21.15.29:8765 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: mgrmap e28: y(active, since 1.02867s) 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: pgmap v3: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: [09/Oct/2023:04:12:58] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: [09/Oct/2023:04:12:58] ENGINE Bus STARTED 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:12:59 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1553159953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:00.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:12:59 smithi177 conmon[160950]: 2023-10-09T04:12:59.817+0000 7f452e776200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: [09/Oct/2023:04:12:58] ENGINE Bus STARTING 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: [09/Oct/2023:04:12:58] ENGINE Serving on http://172.21.15.29:8765 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: mgrmap e28: y(active, since 1.02867s) 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: pgmap v3: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: [09/Oct/2023:04:12:58] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: [09/Oct/2023:04:12:58] ENGINE Bus STARTED 2023-10-09T04:13:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1553159953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: [09/Oct/2023:04:12:58] ENGINE Bus STARTING 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: [09/Oct/2023:04:12:58] ENGINE Serving on http://172.21.15.29:8765 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: mgrmap e28: y(active, since 1.02867s) 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: pgmap v3: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: [09/Oct/2023:04:12:58] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: [09/Oct/2023:04:12:58] ENGINE Bus STARTED 2023-10-09T04:13:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:12:59 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1553159953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:00.801 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:00 smithi177 conmon[160950]: 2023-10-09T04:13:00.536+0000 7f452e776200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:13:01.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:00 smithi029 ceph-mon[101043]: mgrmap e29: y(active, since 2s) 2023-10-09T04:13:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:00 smithi029 ceph-mon[101043]: pgmap v4: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:00 smithi029 ceph-mon[119131]: mgrmap e29: y(active, since 2s) 2023-10-09T04:13:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:00 smithi029 ceph-mon[119131]: pgmap v4: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:00 smithi177 ceph-mon[106070]: mgrmap e29: y(active, since 2s) 2023-10-09T04:13:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:00 smithi177 ceph-mon[106070]: pgmap v4: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:02.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:01 smithi177 conmon[160950]: 2023-10-09T04:13:01.621+0000 7f452e776200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:13:02.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:01 smithi177 conmon[160950]: 2023-10-09T04:13:01.773+0000 7f452e776200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:13:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:01 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2356316438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:02.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:01 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2356316438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:01 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2356316438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:03.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:02 smithi177 ceph-mon[106070]: pgmap v5: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:03.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:02 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:03.083 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:02 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:03.083 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:03 smithi177 conmon[160950]: 2023-10-09T04:13:03.081+0000 7f452e776200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:03.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:02 smithi029 ceph-mon[101043]: pgmap v5: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:03.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:02 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:03.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:02 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:03.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:02 smithi029 ceph-mon[119131]: pgmap v5: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:13:03.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:02 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:03.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:02 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:03.538 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:03 smithi177 conmon[160950]: 2023-10-09T04:13:03.231+0000 7f452e776200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:13:03.539 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:03 smithi177 conmon[160950]: 2023-10-09T04:13:03.527+0000 7f452e776200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:03.999 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:03 smithi177 conmon[160950]: 2023-10-09T04:13:03.989+0000 7f452e776200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:13:04.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:04 smithi177 conmon[160950]: 2023-10-09T04:13:04.354+0000 7f452e776200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:13:04.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:04 smithi177 conmon[160950]: 2023-10-09T04:13:04.587+0000 7f452e776200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:13:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[101043]: pgmap v6: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 42 op/s 2023-10-09T04:13:05.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:05.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:05.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1667414964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:05.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[119131]: pgmap v6: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 42 op/s 2023-10-09T04:13:05.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:04 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1667414964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:05.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:04 smithi177 ceph-mon[106070]: pgmap v6: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 42 op/s 2023-10-09T04:13:05.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:04 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:05.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:04 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:05.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:04 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1667414964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:05.501 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:05 smithi177 conmon[160950]: 2023-10-09T04:13:05.040+0000 7f452e776200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:13:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:05 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:06.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:05 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:06.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:06 smithi177 conmon[160950]: 2023-10-09T04:13:06.163+0000 7f452e776200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:13:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:05 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:06.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:05 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:06.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:05 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:06.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:05 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:06.716 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:06 smithi177 conmon[160950]: 2023-10-09T04:13:06.381+0000 7f452e776200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:13:06.863 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:06 smithi177 conmon[160950]: 2023-10-09T04:13:06.713+0000 7f452e776200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:13:06.986 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:06 smithi177 ceph-mon[106070]: pgmap v7: 89 pgs: 89 active+clean; 1003 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 459 op/s 2023-10-09T04:13:07.818 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:06 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:07.818 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:06 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:07.818 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:06 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/797973544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:07.819 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:06 smithi177 conmon[160950]: 2023-10-09T04:13:06.900+0000 7f452e776200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:07.819 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:07 smithi177 conmon[160950]: 2023-10-09T04:13:07.251+0000 7f452e776200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:13:07.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[101043]: pgmap v7: 89 pgs: 89 active+clean; 1003 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 459 op/s 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/797973544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[119131]: pgmap v7: 89 pgs: 89 active+clean; 1003 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 459 op/s 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:07.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:06 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/797973544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:08.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: 2023-10-09T04:13:08.466+0000 7f452e776200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:13:08.817 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: [09/Oct/2023:04:13:08] ENGINE Bus STARTING 2023-10-09T04:13:08.817 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: CherryPy Checker: 2023-10-09T04:13:08.817 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: The Application mounted at '' has an empty config. 2023-10-09T04:13:08.817 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: 2023-10-09T04:13:08.817 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: [09/Oct/2023:04:13:08] ENGINE Serving on http://:::9283 2023-10-09T04:13:08.817 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:08 smithi177 conmon[160950]: [09/Oct/2023:04:13:08] ENGINE Bus STARTED 2023-10-09T04:13:09.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:09 smithi177 ceph-mon[106070]: pgmap v8: 89 pgs: 89 active+clean; 1003 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-10-09T04:13:09.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:09 smithi177 ceph-mon[106070]: Standby manager daemon x started 2023-10-09T04:13:09.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:09 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:09.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:09 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:09.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:09 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:09.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:09 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:09.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[101043]: pgmap v8: 89 pgs: 89 active+clean; 1003 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-10-09T04:13:09.475 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[101043]: Standby manager daemon x started 2023-10-09T04:13:09.475 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:09.475 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:09.475 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[119131]: pgmap v8: 89 pgs: 89 active+clean; 1003 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[119131]: Standby manager daemon x started 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:09.476 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:09 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/4146826709' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:10 smithi029 ceph-mon[101043]: mgrmap e30: y(active, since 11s), standbys: x 2023-10-09T04:13:10.485 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:10 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:10 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3489663136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:10 smithi029 ceph-mon[119131]: mgrmap e30: y(active, since 11s), standbys: x 2023-10-09T04:13:10.486 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:10 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:13:10.487 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:10 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3489663136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:10.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:10 smithi177 ceph-mon[106070]: mgrmap e30: y(active, since 11s), standbys: x 2023-10-09T04:13:10.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:10 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:13:10.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:10 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3489663136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:11 smithi029 ceph-mon[101043]: pgmap v9: 89 pgs: 89 active+clean; 799 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:11 smithi029 ceph-mon[119131]: pgmap v9: 89 pgs: 89 active+clean; 799 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:12 smithi029 ceph-mon[119131]: pgmap v10: 89 pgs: 89 active+clean; 799 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:12 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2865108997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:12 smithi029 ceph-mon[101043]: pgmap v10: 89 pgs: 89 active+clean; 799 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:13:13.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:12 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2865108997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:11 smithi177 ceph-mon[106070]: pgmap v9: 89 pgs: 89 active+clean; 799 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 590 op/s 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:12 smithi177 ceph-mon[106070]: pgmap v10: 89 pgs: 89 active+clean; 799 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:13:13.383 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:12 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2865108997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:14.600 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:14.601 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:14.601 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:14.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:14.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:13:14.617 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:13 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:13:14.618 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:14.618 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:14.618 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:13:14.619 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:13:14.619 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:14 smithi177 conmon[173579]: ts=2023-10-09T04:13:14.144Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.177:8765: connect: connection refused" 2023-10-09T04:13:14.619 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:14 smithi177 conmon[173579]: ts=2023-10-09T04:13:14.144Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.177:8765: connect: connection refused" 2023-10-09T04:13:14.619 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:14 smithi177 conmon[173579]: ts=2023-10-09T04:13:14.152Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.177:8765: connect: connection refused" 2023-10-09T04:13:14.619 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:14 smithi177 conmon[173579]: ts=2023-10-09T04:13:14.159Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.177:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.177:8765: connect: connection refused" 2023-10-09T04:13:14.620 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:14.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:14.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:14.621 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:14.622 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:14.622 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:13:14.623 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:13 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:13:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:13:15.237 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:13:15.237 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[101043]: pgmap v11: 89 pgs: 89 active+clean; 784 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-09T04:13:15.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[101043]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:13:15.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[101043]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:13:15.238 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:13:15.238 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:13:15.238 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[119131]: pgmap v11: 89 pgs: 89 active+clean; 784 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-09T04:13:15.241 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:13:15.241 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:14 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:13:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:14 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:13:15.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:14 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:13:15.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:14 smithi177 ceph-mon[106070]: pgmap v11: 89 pgs: 89 active+clean; 784 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-09T04:13:15.374 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:14 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:13:15.374 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:14 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2722167898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2722167898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:15 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:13:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:13:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2722167898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:15 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:17 smithi177 ceph-mon[106070]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:13:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:17 smithi177 ceph-mon[106070]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:13:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:17 smithi177 ceph-mon[106070]: pgmap v12: 89 pgs: 89 active+clean; 581 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T04:13:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:17 smithi029 ceph-mon[101043]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:13:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:17 smithi029 ceph-mon[101043]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:13:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:17 smithi029 ceph-mon[101043]: pgmap v12: 89 pgs: 89 active+clean; 581 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T04:13:17.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:17 smithi029 ceph-mon[119131]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-09T04:13:17.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:17 smithi029 ceph-mon[119131]: Reconfiguring daemon prometheus.a on smithi177 2023-10-09T04:13:17.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:17 smithi029 ceph-mon[119131]: pgmap v12: 89 pgs: 89 active+clean; 581 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2023-10-09T04:13:17.780 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:17 smithi177 conmon[160950]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:13:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-10-09T04:13:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:18 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1064894242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:18 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1064894242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:18.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:18 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1064894242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:19.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:19 smithi177 ceph-mon[106070]: pgmap v13: 89 pgs: 89 active+clean; 581 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-10-09T04:13:19.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:19 smithi029 ceph-mon[101043]: pgmap v13: 89 pgs: 89 active+clean; 581 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-10-09T04:13:19.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:19 smithi029 ceph-mon[119131]: pgmap v13: 89 pgs: 89 active+clean; 581 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s 2023-10-09T04:13:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:20 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/756864275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:20.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:20 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/756864275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:20 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/756864275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:21.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:21 smithi029 ceph-mon[101043]: pgmap v14: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-10-09T04:13:21.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:21 smithi029 ceph-mon[119131]: pgmap v14: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-10-09T04:13:21.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:21 smithi177 ceph-mon[106070]: pgmap v14: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2023-10-09T04:13:22.527 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 systemd[1]: Stopping Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:13:22.950 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T04:13:22.950 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T04:13:22.950 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T04:13:22.950 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T04:13:22.950 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T04:13:22.950 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.612Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.613Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.613Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.613Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 conmon[173579]: ts=2023-10-09T04:13:22.613Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T04:13:22.951 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:22 smithi177 bash[178967]: bb8553476ffeb1498124fb6d8f506e330816564ef5518e65fbeeda7bbe94d55d 2023-10-09T04:13:22.954 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:22 smithi177 ceph-mon[106070]: pgmap v15: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:13:23.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:22 smithi029 ceph-mon[119131]: pgmap v15: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:13:23.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:22 smithi029 ceph-mon[101043]: pgmap v15: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-09T04:13:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:23 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1040082899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:24.033 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:23 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-09T04:13:24.033 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:23 smithi177 systemd[1]: Stopped Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:13:24.033 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:23 smithi177 systemd[1]: Starting Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:13:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:23 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1040082899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:23 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1040082899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:24 smithi177 ceph-mon[106070]: pgmap v16: 89 pgs: 89 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-09T04:13:25.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 bash[179268]: 2d31282ff47711aef35a18382a5fed0ea070324812aab38ccb1a6cc7f2726c7c 2023-10-09T04:13:25.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.719Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-10-09T04:13:25.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.719Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-10-09T04:13:25.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.719Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi177 (none))" 2023-10-09T04:13:25.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.719Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-09T04:13:25.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.719Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-09T04:13:25.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.721Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.722Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 systemd[1]: Started Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.742Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.742Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.458µs 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.742Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.742Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.742Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-10-09T04:13:25.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.742Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-10-09T04:13:25.031 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.812Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-09T04:13:25.031 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.817Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-09T04:13:25.031 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.818Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-09T04:13:25.031 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.818Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.41µs wal_replay_duration=76.282957ms wbl_replay_duration=110ns total_replay_duration=76.351663ms 2023-10-09T04:13:25.034 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.822Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-10-09T04:13:25.034 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.822Z caller=main.go:1029 level=info msg="TSDB started" 2023-10-09T04:13:25.035 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.822Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-09T04:13:25.035 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.844Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.500982ms db_storage=954ns remote_storage=1.641µs web_handler=549ns query_engine=782ns scrape=246.049µs scrape_sd=118.06µs notify=16.073µs notify_sd=14.759µs rules=20.782616ms tracing=7.61µs 2023-10-09T04:13:25.035 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.844Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-10-09T04:13:25.035 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:24 smithi177 conmon[179305]: ts=2023-10-09T04:13:24.844Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-10-09T04:13:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:24 smithi029 ceph-mon[119131]: pgmap v16: 89 pgs: 89 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-09T04:13:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:24 smithi029 ceph-mon[101043]: pgmap v16: 89 pgs: 89 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-09T04:13:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:25 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:25 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:25 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:13:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:25 smithi177 ceph-mon[106070]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:13:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:25 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:25 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3981360996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:26.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:26.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:26.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:13:26.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[101043]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:13:26.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3981360996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[119131]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:25 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3981360996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:26.560 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 124101 objects 2023-10-09T04:13:26.561 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :171.66 2023-10-09T04:13:27.150 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-09T04:13:27.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-09T04:13:27.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:26 smithi029 ceph-mon[101043]: pgmap v17: 89 pgs: 89 active+clean; 126 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T04:13:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:26 smithi029 ceph-mon[119131]: pgmap v17: 89 pgs: 89 active+clean; 126 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T04:13:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:26 smithi177 ceph-mon[106070]: pgmap v17: 89 pgs: 89 active+clean; 126 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-09T04:13:27.533 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:28.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:27 smithi177 ceph-mon[106070]: Upgrade: Updating mgr.x 2023-10-09T04:13:28.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:27 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:13:28.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:27 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:13:28.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:27 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:13:28.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:27 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:28.052 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:27 smithi177 ceph-mon[106070]: Deploying daemon mgr.x on smithi177 2023-10-09T04:13:28.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[101043]: Upgrade: Updating mgr.x 2023-10-09T04:13:28.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:13:28.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:13:28.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:13:28.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:28.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[101043]: Deploying daemon mgr.x on smithi177 2023-10-09T04:13:28.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[119131]: Upgrade: Updating mgr.x 2023-10-09T04:13:28.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:13:28.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:13:28.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:13:28.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:28.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:27 smithi029 ceph-mon[119131]: Deploying daemon mgr.x on smithi177 2023-10-09T04:13:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:28 smithi177 ceph-mon[106070]: pgmap v18: 89 pgs: 89 active+clean; 126 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:13:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:28 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:28 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2143149480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:29.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:28 smithi177 systemd[1]: Stopping Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:13:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:28 smithi029 ceph-mon[119131]: pgmap v18: 89 pgs: 89 active+clean; 126 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:13:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:28 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:28 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2143149480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:28 smithi029 ceph-mon[101043]: pgmap v18: 89 pgs: 89 active+clean; 126 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:13:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:28 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:28 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2143149480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:29.912 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:29 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-09T04:13:29.913 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:29 smithi177 bash[179929]: ffa2fda2ab93a2500a750459d9999143fb156bfd62cb7734ff4e3fc3a47eb8f2 2023-10-09T04:13:29.913 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:29 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-09T04:13:29.913 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:29 smithi177 systemd[1]: Stopped Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:13:29.913 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:29 smithi177 systemd[1]: Starting Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:13:30.646 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 ceph-mgr[180358]: -- 172.21.15.177:0/3140237981 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557d770971e0 con 0x557d7706f000 2023-10-09T04:13:30.646 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 bash[180293]: c43f5ec5cd804743305f3be1f9163d3ed7cc0f086bd62c87104a59343cc9f20a 2023-10-09T04:13:30.647 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 systemd[1]: Started Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:13:30.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:30 smithi177 ceph-mon[106070]: pgmap v19: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-10-09T04:13:30.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:30 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/575737575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:30.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:30 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:30.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:30 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:30.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:30 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:30.938 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 conmon[180334]: 2023-10-09T04:13:30.644+0000 7f1429b37200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:13:30.938 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 conmon[180334]: 2023-10-09T04:13:30.727+0000 7f1429b37200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:30.938 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 conmon[180334]: 2023-10-09T04:13:30.826+0000 7f1429b37200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[101043]: pgmap v19: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/575737575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[119131]: pgmap v19: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-10-09T04:13:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/575737575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:30 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:31.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:30 smithi177 conmon[180334]: 2023-10-09T04:13:30.935+0000 7f1429b37200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:13:31.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:31 smithi177 conmon[180334]: 2023-10-09T04:13:31.131+0000 7f1429b37200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:13:31.867 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:31 smithi177 conmon[180334]: 2023-10-09T04:13:31.528+0000 7f1429b37200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:13:31.868 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:31 smithi177 conmon[180334]: 2023-10-09T04:13:31.672+0000 7f1429b37200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:13:31.868 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:31 smithi177 conmon[180334]: 2023-10-09T04:13:31.766+0000 7f1429b37200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:13:31.868 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:31 smithi177 conmon[180334]: 2023-10-09T04:13:31.866+0000 7f1429b37200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:13:32.748 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:32 smithi177 conmon[180334]: 2023-10-09T04:13:32.356+0000 7f1429b37200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:13:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:32 smithi177 ceph-mon[106070]: pgmap v20: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 467 op/s 2023-10-09T04:13:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:32 smithi029 ceph-mon[101043]: pgmap v20: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 467 op/s 2023-10-09T04:13:33.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:13:32] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:13:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:32 smithi029 ceph-mon[119131]: pgmap v20: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 467 op/s 2023-10-09T04:13:33.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:33 smithi177 conmon[180334]: 2023-10-09T04:13:33.096+0000 7f1429b37200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:13:33.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:33 smithi177 conmon[180334]: 2023-10-09T04:13:33.190+0000 7f1429b37200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:13:33.767 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_0' removed 2023-10-09T04:13:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:33 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3938495222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:33 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2826432161' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:33 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:33 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/3938495222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2826432161' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/3938495222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2826432161' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:33 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.385 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:34 smithi177 conmon[180334]: 2023-10-09T04:13:34.077+0000 7f1429b37200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:34.385 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:34 smithi177 conmon[180334]: 2023-10-09T04:13:34.182+0000 7f1429b37200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:13:34.590 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:13:34.591 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:13:34.591 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-09T04:13:34.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-09T04:13:34.702 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:34 smithi177 conmon[180334]: 2023-10-09T04:13:34.384+0000 7f1429b37200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/2826432161' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:13:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[101043]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:13:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[101043]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2826432161' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:13:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[119131]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:13:34.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[119131]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:34.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:34 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:34 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2826432161' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:34 smithi177 ceph-mon[106070]: osdmap e77: 8 total, 8 up, 8 in 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:34 smithi177 ceph-mon[106070]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:34 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:34 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:34 smithi177 conmon[180334]: 2023-10-09T04:13:34.701+0000 7f1429b37200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:13:34.983 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:34 smithi177 conmon[180334]: 2023-10-09T04:13:34.981+0000 7f1429b37200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:13:35.056 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:35.238 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:35 smithi177 conmon[180334]: 2023-10-09T04:13:35.155+0000 7f1429b37200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:13:35.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:35 smithi177 conmon[180334]: 2023-10-09T04:13:35.445+0000 7f1429b37200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:13:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:35 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1778808333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:35 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1778808333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:35 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1778808333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:36.330 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:36 smithi177 conmon[180334]: 2023-10-09T04:13:36.188+0000 7f1429b37200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:13:36.330 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:36 smithi177 conmon[180334]: 2023-10-09T04:13:36.329+0000 7f1429b37200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:13:36.669 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:36 smithi177 conmon[180334]: 2023-10-09T04:13:36.545+0000 7f1429b37200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:13:36.669 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:36 smithi177 conmon[180334]: 2023-10-09T04:13:36.668+0000 7f1429b37200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:36.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:36 smithi029 ceph-mon[101043]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:36 smithi029 ceph-mon[119131]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:36 smithi177 ceph-mon[106070]: pgmap v23: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:37.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:36 smithi177 conmon[180334]: 2023-10-09T04:13:36.908+0000 7f1429b37200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:13:37.780 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_1' created 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:37 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/1385546163' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:37 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:37 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/990293657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: 2023-10-09T04:13:37.870+0000 7f1429b37200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: [09/Oct/2023:04:13:37] ENGINE Bus STARTING 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: CherryPy Checker: 2023-10-09T04:13:38.028 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: The Application mounted at '' has an empty config. 2023-10-09T04:13:38.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: 2023-10-09T04:13:38.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: [09/Oct/2023:04:13:37] ENGINE Serving on http://:::9283 2023-10-09T04:13:38.029 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:37 smithi177 conmon[180334]: [09/Oct/2023:04:13:37] ENGINE Bus STARTED 2023-10-09T04:13:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:37 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/1385546163' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:13:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:37 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:13:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:37 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/990293657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:37 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/1385546163' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:13:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:37 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-09T04:13:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:37 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/990293657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:38.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-09T04:13:38.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:13:38.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:13:38.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: pgmap v25: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:38.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: Standby manager daemon x restarted 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: Standby manager daemon x started 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:38.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: Upgrade: Setting container_image for all mgr 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:38.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:13:38.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[119131]: Upgrade: It appears safe to stop mon.a 2023-10-09T04:13:39.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:38 smithi177 conmon[179305]: ts=2023-10-09T04:13:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: pgmap v25: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: Standby manager daemon x restarted 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: Standby manager daemon x started 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: Upgrade: Setting container_image for all mgr 2023-10-09T04:13:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:13:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:38 smithi177 ceph-mon[106070]: Upgrade: It appears safe to stop mon.a 2023-10-09T04:13:39.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-09T04:13:39.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: osdmap e78: 8 total, 8 up, 8 in 2023-10-09T04:13:39.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: pgmap v25: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: Standby manager daemon x restarted 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: Standby manager daemon x started 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.? 172.21.15.177:0/2439046781' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: Upgrade: Setting container_image for all mgr 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-09T04:13:39.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:38 smithi029 ceph-mon[101043]: Upgrade: It appears safe to stop mon.a 2023-10-09T04:13:39.824 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:40.043 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:39 smithi029 ceph-mon[101043]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:13:40.043 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:39 smithi029 ceph-mon[101043]: mgrmap e31: y(active, since 41s), standbys: x 2023-10-09T04:13:40.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:39 smithi029 ceph-mon[119131]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:13:40.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:39 smithi029 ceph-mon[119131]: mgrmap e31: y(active, since 41s), standbys: x 2023-10-09T04:13:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:39 smithi177 ceph-mon[106070]: osdmap e79: 8 total, 8 up, 8 in 2023-10-09T04:13:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:39 smithi177 ceph-mon[106070]: mgrmap e31: y(active, since 41s), standbys: x 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: pgmap v28: 89 pgs: 2 creating+activating, 87 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: Upgrade: Updating mon.a 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: Deploying daemon mon.a on smithi029 2023-10-09T04:13:41.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/416142851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: pgmap v28: 89 pgs: 2 creating+activating, 87 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: Upgrade: Updating mon.a 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: Deploying daemon mon.a on smithi029 2023-10-09T04:13:41.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:40 smithi029 ceph-mon[101043]: from='client.? 172.21.15.29:0/416142851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: pgmap v28: 89 pgs: 2 creating+activating, 87 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: Upgrade: Updating mon.a 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='mgr.y' 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: Deploying daemon mon.a on smithi029 2023-10-09T04:13:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:40 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/416142851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:42.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:42 smithi029 systemd[1]: Stopping Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:13:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:13:42] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:13:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:42 smithi029 conmon[101019]: 2023-10-09T04:13:42.666+0000 7f371bd14700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:13:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:42 smithi029 conmon[101019]: 2023-10-09T04:13:42.666+0000 7f371bd14700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-09T04:13:43.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:43 smithi029 bash[216477]: 5d294109024f4fa65129e3df83107a7f046f2279dc3f346e6bcbc496bcf26123 2023-10-09T04:13:43.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:43 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-09T04:13:43.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:43 smithi029 systemd[1]: Stopped Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:13:44.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:43 smithi029 systemd[1]: Starting Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 bash[216837]: 8369665f49b3cc675032cad34677648ac99f89b23ff42ade039e7af12221fa98 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 systemd[1]: Started Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: pidfile_write: ignore empty --pid-file 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: load: jerasure load: lrc 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: RocksDB version: 7.9.2 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Git sha 0 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: DB SUMMARY 2023-10-09T04:13:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: DB Session ID: W2R0OOD4GEFAAT5CSB5P 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1178 Bytes 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 3318820 ; 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:13:44.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.env: 0x560b8a0e1720 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.info_log: 0x560b8cd33420 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.statistics: (nil) 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.use_fsync: 0 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:13:44.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.db_log_dir: 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.wal_dir: 2023-10-09T04:13:44.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.write_buffer_manager: 0x560b8bfc2c80 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:13:44.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.unordered_write: 0 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.row_cache: None 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.wal_filter: None 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:13:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.wal_compression: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:13:44.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:13:44.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_open_files: -1 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Compression algorithms supported: 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kZSTD supported: 0 2023-10-09T04:13:44.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.merge_operator: 2023-10-09T04:13:44.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_filter: None 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560b8cd33520) 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_type: 0 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_index_type: 0 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_shortening: 1 2023-10-09T04:13:44.933 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: checksum: 4 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: no_block_cache: 0 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache: 0x560b8c045350 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_options: 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: capacity : 536870912 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_shard_bits : 4 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: strict_capacity_limit : 0 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_compressed: (nil) 2023-10-09T04:13:44.934 INFO:journalctl@ceph.mon.a.smithi029.stdout: persistent_cache: (nil) 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size: 4096 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size_deviation: 10 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_restart_interval: 16 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_block_restart_interval: 1 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_block_size: 4096 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: partition_filters: 0 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: use_delta_encoding: 1 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: filter_policy: bloomfilter 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: whole_key_filtering: 1 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: verify_compression: 0 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: format_version: 5 2023-10-09T04:13:44.935 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_index_compression: 1 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_align: 0 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_auto_readahead_size: 262144 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout: prepopulate_block_cache: 0 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout: initial_auto_readahead_size: 8192 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression: NoCompression 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.num_levels: 7 2023-10-09T04:13:44.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:13:44.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:13:44.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:13:44.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:13:44.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:13:44.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.ttl: 2592000 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:13:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12383, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7d007fbd-c8a8-4b8e-9007-3978688ecf5f 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824824461711, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-10-09T04:13:44.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824824476777, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 3030656, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12384, "largest_seqno": 14654, "table_properties": {"data_size": 3021318, "index_size": 5565, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2757, "raw_key_size": 22862, "raw_average_key_size": 21, "raw_value_size": 3000520, "raw_average_value_size": 2783, "num_data_blocks": 254, "num_entries": 1078, "num_filter_entries": 1078, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696824824, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7d007fbd-c8a8-4b8e-9007-3978688ecf5f", "db_session_id": "W2R0OOD4GEFAAT5CSB5P", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824824476887, "job": 1, "event": "recovery_finished"} 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560b8c0e0000 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: DB pointer 0x560b8c0ce000 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** DB Stats ** 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:13:44.944 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 1/0 2.89 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 484.5 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: L6 1/0 5.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 8.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 484.5 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 484.5 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-09T04:13:44.945 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 484.5 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:13:44.946 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 160.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 160.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache BinnedLRUCache@0x560b8c045350#7 capacity: 512.00 MB usage: 8.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.78 KB,0.000530481%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: starting mon.a rank 0 at public addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] at bind addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???) e3 preinit fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).mds e8 new map 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).mds e8 print_map 2023-10-09T04:13:44.947 INFO:journalctl@ceph.mon.a.smithi029.stdout: e8 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: legacy client fscid: 1 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: Filesystem 'foo' (1) 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: fs_name foo 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: epoch 7 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: created 2023-10-09T04:07:34.635169+0000 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: modified 2023-10-09T04:07:47.848728+0000 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: tableserver 0 2023-10-09T04:13:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout: root 0 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: session_timeout 60 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: session_autoclose 300 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_file_size 1099511627776 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: required_client_features {} 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: last_failure 0 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: last_failure_osd_epoch 0 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_mds 1 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: in 0 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: up {0=14649} 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: failed 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: damaged 2023-10-09T04:13:44.949 INFO:journalctl@ceph.mon.a.smithi029.stdout: stopped 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_pools [4] 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_pool 3 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: inline_data disabled 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: balancer 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: bal_rank_mask -1 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: standby_count_wanted 1 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: [mds.foo.smithi029.mozfhx{0:14649} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: Standby daemons: 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2023-10-09T04:13:44.950 INFO:journalctl@ceph.mon.a.smithi029.stdout: [mds.foo.smithi177.pbovgs{-1:24547} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:13:44.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).osd e80 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:13:44.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:13:44.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:13:44.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).osd e80 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:13:44.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:44 smithi029 ceph-mon[216903]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-10-09T04:13:45.984 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-09T04:13:46.243 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:45 smithi029 conmon[202582]: ignoring --setuser ceph since I am not root 2023-10-09T04:13:46.243 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:45 smithi029 conmon[202582]: ignoring --setgroup ceph since I am not root 2023-10-09T04:13:46.243 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mgr[202628]: -- 172.21.15.29:0/2483189959 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d1caf431e0 con 0x55d1caf1ac00 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:46 smithi029 conmon[202582]: 2023-10-09T04:13:46.143+0000 7f6077e0d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:46 smithi029 conmon[202582]: 2023-10-09T04:13:46.242+0000 7f6077e0d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/391418947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: mon.a calling monitor election 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: from='client.? ' entity='' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:46.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: mgrmap e31: y(active, since 48s), standbys: x 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: mgrmap e32: y(active, since 48s), standbys: x 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: overall HEALTH_OK 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[216903]: from='mgr.25019 ' entity='' 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/391418947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: mon.a calling monitor election 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: from='client.? ' entity='' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: mgrmap e31: y(active, since 48s), standbys: x 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: mgrmap e32: y(active, since 48s), standbys: x 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: overall HEALTH_OK 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 ceph-mon[119131]: from='mgr.25019 ' entity='' 2023-10-09T04:13:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:45 smithi029 conmon[119107]: 2023-10-09T04:13:45.970+0000 7fca17373700 -1 mon.c@2(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:13:46.267 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:45 smithi177 conmon[180334]: ignoring --setuser ceph since I am not root 2023-10-09T04:13:46.267 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:45 smithi177 conmon[180334]: ignoring --setgroup ceph since I am not root 2023-10-09T04:13:46.267 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mgr[180358]: -- 172.21.15.177:0/1576342891 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e275e911e0 con 0x55e275e68800 2023-10-09T04:13:46.267 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:46 smithi177 conmon[180334]: 2023-10-09T04:13:46.096+0000 7fbccfd35200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-09T04:13:46.267 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:46 smithi177 conmon[180334]: 2023-10-09T04:13:46.177+0000 7fbccfd35200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:46.267 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:46 smithi177 conmon[180334]: 2023-10-09T04:13:46.266+0000 7fbccfd35200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: from='mgr.25019 172.21.15.29:0/2508157897' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/391418947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: mon.a calling monitor election 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: from='client.? ' entity='' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: osdmap e80: 8 total, 8 up, 8 in 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: mgrmap e31: y(active, since 48s), standbys: x 2023-10-09T04:13:46.268 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: mgrmap e32: y(active, since 48s), standbys: x 2023-10-09T04:13:46.269 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: overall HEALTH_OK 2023-10-09T04:13:46.269 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 ceph-mon[106070]: from='mgr.25019 ' entity='' 2023-10-09T04:13:46.269 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:45 smithi177 conmon[106046]: 2023-10-09T04:13:45.971+0000 7f373e8d3700 -1 mon.b@1(peon).osd e81 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:13:46.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:46 smithi177 conmon[180334]: 2023-10-09T04:13:46.360+0000 7fbccfd35200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:13:46.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:46 smithi029 conmon[202582]: 2023-10-09T04:13:46.351+0000 7f6077e0d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-09T04:13:46.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:46 smithi029 conmon[202582]: 2023-10-09T04:13:46.464+0000 7f6077e0d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-09T04:13:46.891 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:46 smithi177 conmon[180334]: 2023-10-09T04:13:46.528+0000 7fbccfd35200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:13:46.973 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:46 smithi029 conmon[202582]: 2023-10-09T04:13:46.672+0000 7f6077e0d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-09T04:13:47.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:46 smithi177 ceph-mon[106070]: from='client.? ' entity='' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:13:47.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:46 smithi177 ceph-mon[106070]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:13:47.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:46 smithi177 ceph-mon[106070]: failed to encode map e81 with expected crc 2023-10-09T04:13:47.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:46 smithi177 ceph-mon[106070]: failed to encode map e81 with expected crc 2023-10-09T04:13:47.210 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:46 smithi177 conmon[180334]: 2023-10-09T04:13:46.891+0000 7fbccfd35200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:13:47.210 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:47 smithi177 conmon[180334]: 2023-10-09T04:13:47.024+0000 7fbccfd35200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:13:47.210 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:47 smithi177 conmon[180334]: 2023-10-09T04:13:47.114+0000 7fbccfd35200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:13:47.210 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:47 smithi177 conmon[180334]: 2023-10-09T04:13:47.208+0000 7fbccfd35200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:13:47.317 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:47 smithi029 conmon[202582]: 2023-10-09T04:13:47.161+0000 7f6077e0d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-09T04:13:47.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[216903]: from='client.? ' entity='' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:13:47.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[216903]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:13:47.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[216903]: failed to encode map e81 with expected crc 2023-10-09T04:13:47.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[216903]: failed to encode map e81 with expected crc 2023-10-09T04:13:47.317 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[119131]: from='client.? ' entity='' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:13:47.318 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[119131]: osdmap e81: 8 total, 8 up, 8 in 2023-10-09T04:13:47.318 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[119131]: failed to encode map e81 with expected crc 2023-10-09T04:13:47.318 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:46 smithi029 ceph-mon[119131]: failed to encode map e81 with expected crc 2023-10-09T04:13:47.360 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 1 2023-10-09T04:13:47.583 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:47 smithi029 conmon[202582]: 2023-10-09T04:13:47.316+0000 7f6077e0d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-09T04:13:47.583 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:47 smithi029 conmon[202582]: 2023-10-09T04:13:47.431+0000 7f6077e0d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-09T04:13:47.583 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:47 smithi029 conmon[202582]: 2023-10-09T04:13:47.549+0000 7f6077e0d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-09T04:13:47.784 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-09T04:13:47.786 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:13:47.786 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:13:47.836 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:13:47.837 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:13:47.837 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_217131 2023-10-09T04:13:47.837 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:13:47.837 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:13:47.975 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:47 smithi177 conmon[180334]: 2023-10-09T04:13:47.632+0000 7fbccfd35200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:13:48.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:13:48.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[216903]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[216903]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[216903]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[216903]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2062256078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:48.256 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:48 smithi029 conmon[202582]: 2023-10-09T04:13:48.253+0000 7f6077e0d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-09T04:13:48.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[119131]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[119131]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[119131]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.257 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[119131]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.257 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:47 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/2062256078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:47 smithi177 ceph-mon[106070]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:47 smithi177 ceph-mon[106070]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:47 smithi177 ceph-mon[106070]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:47 smithi177 ceph-mon[106070]: failed to encode map e81 with expected crc 2023-10-09T04:13:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:47 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2062256078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:13:48.777 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:13:48 smithi177 conmon[179305]: ts=2023-10-09T04:13:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:13:48.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:48 smithi177 conmon[180334]: 2023-10-09T04:13:48.524+0000 7fbccfd35200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:13:48.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:48 smithi177 conmon[180334]: 2023-10-09T04:13:48.661+0000 7fbccfd35200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:13:48.838 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3042 3026 188.882 189.125 0.00829042 0.00527373 2023-10-09T04:13:48.885 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:13:49.600 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:49 smithi029 conmon[202582]: 2023-10-09T04:13:49.596+0000 7f6077e0d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-09T04:13:49.838 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 5841 5825 181.906 174.938 0.00250059 0.00548528 2023-10-09T04:13:49.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:49 smithi029 conmon[202582]: 2023-10-09T04:13:49.779+0000 7f6077e0d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-09T04:13:50.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:49 smithi177 conmon[180334]: 2023-10-09T04:13:49.869+0000 7fbccfd35200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:50.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:50 smithi177 conmon[180334]: 2023-10-09T04:13:50.020+0000 7fbccfd35200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:13:50.777 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:50 smithi177 conmon[180334]: 2023-10-09T04:13:50.318+0000 7fbccfd35200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:50.838 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 8875 8859 184.46 189.625 0.0115899 0.00541108 2023-10-09T04:13:51.145 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:50 smithi177 conmon[180334]: 2023-10-09T04:13:50.806+0000 7fbccfd35200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:13:51.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:51 smithi177 conmon[180334]: 2023-10-09T04:13:51.144+0000 7fbccfd35200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:13:51.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:51 smithi177 conmon[180334]: 2023-10-09T04:13:51.383+0000 7fbccfd35200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:13:51.838 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 11664 11648 181.92 174.312 0.00545098 0.00549124 2023-10-09T04:13:51.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:51 smithi029 conmon[202582]: 2023-10-09T04:13:51.451+0000 7f6077e0d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:51.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:51 smithi029 conmon[202582]: 2023-10-09T04:13:51.653+0000 7f6077e0d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-09T04:13:52.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:51 smithi177 conmon[180334]: 2023-10-09T04:13:51.809+0000 7fbccfd35200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:13:52.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:52 smithi029 conmon[202582]: 2023-10-09T04:13:52.027+0000 7f6077e0d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-09T04:13:52.839 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 14325 14309 178.786 166.312 0.00602399 0.0055855 2023-10-09T04:13:52.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:52 smithi029 conmon[202582]: 2023-10-09T04:13:52.627+0000 7f6077e0d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-09T04:13:53.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:52 smithi177 conmon[180334]: 2023-10-09T04:13:52.933+0000 7fbccfd35200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:13:53.278 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:53 smithi177 conmon[180334]: 2023-10-09T04:13:53.148+0000 7fbccfd35200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:13:53.327 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:53 smithi029 conmon[202582]: 2023-10-09T04:13:53.023+0000 7f6077e0d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-09T04:13:53.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:53 smithi177 conmon[180334]: 2023-10-09T04:13:53.470+0000 7fbccfd35200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:13:53.778 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:53 smithi177 conmon[180334]: 2023-10-09T04:13:53.655+0000 7fbccfd35200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:53.839 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 16844 16828 175.217 157.438 0.00329503 0.00569711 2023-10-09T04:13:54.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:53 smithi029 conmon[202582]: 2023-10-09T04:13:53.326+0000 7f6077e0d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-09T04:13:54.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:53 smithi029 conmon[202582]: 2023-10-09T04:13:53.887+0000 7f6077e0d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-09T04:13:54.277 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:53 smithi177 conmon[180334]: 2023-10-09T04:13:53.994+0000 7fbccfd35200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:13:54.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 18911 18895 168.642 129.188 0.013476 0.00592391 2023-10-09T04:13:55.498 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[216903]: Standby manager daemon x restarted 2023-10-09T04:13:55.498 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[216903]: Standby manager daemon x started 2023-10-09T04:13:55.498 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[216903]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:55.498 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[216903]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:55.498 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[216903]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:55.498 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[216903]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:55.500 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:55 smithi029 conmon[202582]: 2023-10-09T04:13:55.238+0000 7f6077e0d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-09T04:13:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:55 smithi177 ceph-mon[106070]: Standby manager daemon x restarted 2023-10-09T04:13:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:55 smithi177 ceph-mon[106070]: Standby manager daemon x started 2023-10-09T04:13:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:55 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:55 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:55 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:55 smithi177 ceph-mon[106070]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: 2023-10-09T04:13:55.271+0000 7fbccfd35200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: [09/Oct/2023:04:13:55] ENGINE Bus STARTING 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: CherryPy Checker: 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: The Application mounted at '' has an empty config. 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: [09/Oct/2023:04:13:55] ENGINE Serving on http://:::9283 2023-10-09T04:13:55.528 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 04:13:55 smithi177 conmon[180334]: [09/Oct/2023:04:13:55] ENGINE Bus STARTED 2023-10-09T04:13:55.841 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 21107 21091 164.691 137.25 0.0059508 0.00606667 2023-10-09T04:13:55.914 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:55 smithi029 conmon[202582]: 2023-10-09T04:13:55.498+0000 7f6077e0d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-09T04:13:55.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[119131]: Standby manager daemon x restarted 2023-10-09T04:13:55.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[119131]: Standby manager daemon x started 2023-10-09T04:13:55.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-09T04:13:55.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-09T04:13:55.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-09T04:13:55.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:55 smithi029 ceph-mon[119131]: from='mgr.? 172.21.15.177:0/3816269953' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-09T04:13:56.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:55 smithi029 conmon[202582]: 2023-10-09T04:13:55.913+0000 7f6077e0d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-09T04:13:56.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:56 smithi029 conmon[202582]: 2023-10-09T04:13:56.142+0000 7f6077e0d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-09T04:13:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:56 smithi177 ceph-mon[106070]: mgrmap e33: y(active, since 57s), standbys: x 2023-10-09T04:13:56.841 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 23317 23301 161.738 138.125 0.00368704 0.0061789 2023-10-09T04:13:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:56 smithi029 ceph-mon[216903]: mgrmap e33: y(active, since 57s), standbys: x 2023-10-09T04:13:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:56 smithi029 ceph-mon[119131]: mgrmap e33: y(active, since 57s), standbys: x 2023-10-09T04:13:56.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:56 smithi029 conmon[202582]: 2023-10-09T04:13:56.596+0000 7f6077e0d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-09T04:13:57.841 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 25447 25431 158.877 133.125 0.00791702 0.0062879 2023-10-09T04:13:58.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 conmon[106046]: 2023-10-09T04:13:58.304+0000 7f373e8d3700 -1 mon.b@1(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:13:58.651 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: Active manager daemon y restarted 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: Activating manager daemon y 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: mgrmap e34: y(active, starting, since 0.011437s), standbys: x 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:13:58.652 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:13:58.653 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:13:58.654 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.283+0000 7f6077e0d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-09T04:13:58.654 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: [09/Oct/2023:04:13:58] ENGINE Bus STARTING 2023-10-09T04:13:58.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: Active manager daemon y restarted 2023-10-09T04:13:58.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: Activating manager daemon y 2023-10-09T04:13:58.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: mgrmap e34: y(active, starting, since 0.011437s), standbys: x 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:13:58.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[119107]: 2023-10-09T04:13:58.305+0000 7fca17373700 -1 mon.c@2(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: Active manager daemon y restarted 2023-10-09T04:13:58.656 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: Activating manager daemon y 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: mgrmap e34: y(active, starting, since 0.011437s), standbys: x 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.mozfhx"}]: dispatch 2023-10-09T04:13:58.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi177.pbovgs"}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-09T04:13:58.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-09T04:13:58.659 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-09T04:13:58.659 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-09T04:13:58.659 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:58 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-09T04:13:58.843 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 27259 27243 154.697 113.25 0.00918553 0.00645763 2023-10-09T04:13:58.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: CherryPy Checker: 2023-10-09T04:13:58.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: The Application mounted at '' has an empty config. 2023-10-09T04:13:58.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: [09/Oct/2023:04:13:58] ENGINE Serving on http://:::9283 2023-10-09T04:13:58.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: [09/Oct/2023:04:13:58] ENGINE Bus STARTED 2023-10-09T04:13:58.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.689+0000 7f5fd107d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.689+0000 7f5fd107d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.689+0000 7f5fd107d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.689+0000 7f5fd107d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.689+0000 7f5fd107d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.692+0000 7f5fd988e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.692+0000 7f5fd988e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.692+0000 7f5fd988e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.692+0000 7f5fd988e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:58.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:58 smithi029 conmon[202582]: 2023-10-09T04:13:58.692+0000 7f5fd988e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (119s) - 7m 27.6M - 0.25.0 c8568f914cd2 7ab788eff7be 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi177 *:3000 running (99s) - 6m 85.9M - 9.4.7 2c41d148cca3 dbe79334ca87 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.mozfhx smithi029 running (6m) - 6m 18.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 e346a0e9dcdd 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi177.pbovgs smithi177 running (6m) - 6m 15.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 a20b9123e852 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi177 *:8443,9283,8765 running (29s) - 11m 45.4M - 18.2.0-598-gb4fa663f 3060897fdf97 c43f5ec5cd80 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (80s) - 13m 483M - 18.2.0-598-gb4fa663f 3060897fdf97 239eea6ccc4c 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (13m) - 13m 109M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 5d294109024f 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi177 running (11m) - 11m 76.4M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 f19800b51616 2023-10-09T04:13:59.406 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (11m) - 11m 70.1M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 c8851494e8fd 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (114s) - 6m 9357k - 1.5.0 0da6a335fe13 2be0e618a0f3 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi177 *:9100 running (95s) - 6m 9.80M - 1.5.0 0da6a335fe13 2adf0c28b454 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (10m) - 10m 2066M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 2c8a6318fb47 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (10m) - 10m 1229M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 951ea50b247d 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (10m) - 10m 2716M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 7d2e4b8000f3 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (9m) - 9m 877M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 e4ab91bdb38b 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi177 running (9m) - 9m 2194M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 d4b592bd1322 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi177 running (8m) - 8m 2369M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 28d4ab6e8bfe 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi177 running (8m) - 8m 1149M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 03ede6ceccc0 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi177 running (8m) - 8m 1674M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0f3217cc8fa1 2023-10-09T04:13:59.407 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi177 *:9095 running (35s) - 7m 44.2M - 2.43.0 a07b618ecd1d 2d31282ff477 2023-10-09T04:13:59.671 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:13:59 smithi029 conmon[202582]: 2023-10-09T04:13:59.379+0000 7f6049533700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-09T04:13:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: Manager daemon y is now available 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: Manager daemon y is now available 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:13:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:13:59 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: failed to encode map e82 with expected crc 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: Manager daemon y is now available 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:59.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:13:59.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:13:59.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-09T04:13:59.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:13:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-09T04:13:59.843 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 28968 28952 150.708 106.812 0.00774215 0.00662975 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: [09/Oct/2023:04:13:59] ENGINE Bus STARTING 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: [09/Oct/2023:04:13:59] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: mgrmap e35: y(active, since 1.07549s), standbys: x 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: from='client.34122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: pgmap v3: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: [09/Oct/2023:04:13:59] ENGINE Serving on http://172.21.15.29:8765 2023-10-09T04:14:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[119131]: [09/Oct/2023:04:13:59] ENGINE Bus STARTED 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: [09/Oct/2023:04:13:59] ENGINE Bus STARTING 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: [09/Oct/2023:04:13:59] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: mgrmap e35: y(active, since 1.07549s), standbys: x 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: from='client.34122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: pgmap v3: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: [09/Oct/2023:04:13:59] ENGINE Serving on http://172.21.15.29:8765 2023-10-09T04:14:00.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:00 smithi029 ceph-mon[216903]: [09/Oct/2023:04:13:59] ENGINE Bus STARTED 2023-10-09T04:14:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: [09/Oct/2023:04:13:59] ENGINE Bus STARTING 2023-10-09T04:14:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: [09/Oct/2023:04:13:59] ENGINE Serving on https://172.21.15.29:7150 2023-10-09T04:14:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: mgrmap e35: y(active, since 1.07549s), standbys: x 2023-10-09T04:14:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: from='client.34122 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:14:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: pgmap v3: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: [09/Oct/2023:04:13:59] ENGINE Serving on http://172.21.15.29:8765 2023-10-09T04:14:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:00 smithi177 ceph-mon[106070]: [09/Oct/2023:04:13:59] ENGINE Bus STARTED 2023-10-09T04:14:00.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 30794 30778 147.894 114.125 0.00443926 0.00675686 2023-10-09T04:14:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:01 smithi177 ceph-mon[106070]: pgmap v4: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:01 smithi177 ceph-mon[106070]: mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T04:14:01.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 32878 32862 146.634 130.25 0.00285723 0.00681444 2023-10-09T04:14:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:01 smithi029 ceph-mon[119131]: pgmap v4: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:01 smithi029 ceph-mon[119131]: mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T04:14:01.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:01 smithi029 ceph-mon[216903]: pgmap v4: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:01.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:01 smithi029 ceph-mon[216903]: mgrmap e36: y(active, since 2s), standbys: x 2023-10-09T04:14:02.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 34885 34869 145.221 125.438 0.00271381 0.00688193 2023-10-09T04:14:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:14:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:14:02] "GET /metrics HTTP/1.1" 200 33495 "" "Prometheus/2.43.0" 2023-10-09T04:14:03.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:03 smithi177 ceph-mon[106070]: pgmap v5: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:03.844 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 36832 36816 143.749 121.688 0.00598295 0.00695085 2023-10-09T04:14:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:03 smithi029 ceph-mon[216903]: pgmap v5: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:03 smithi029 ceph-mon[119131]: pgmap v5: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:04.845 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 38721 38705 142.229 118.062 0.0041477 0.00702586 2023-10-09T04:14:05.845 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 40374 40358 140.068 103.312 0.00852425 0.00713383 2023-10-09T04:14:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:05 smithi029 ceph-mon[216903]: pgmap v6: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:05 smithi029 ceph-mon[119131]: pgmap v6: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:05 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:05 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:06.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:05 smithi177 ceph-mon[106070]: pgmap v6: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail 2023-10-09T04:14:06.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:05 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:06.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:05 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:06.845 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 42180 42164 138.636 112.875 0.00292848 0.00720813 2023-10-09T04:14:06.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:06 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:06.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:06 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:07.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:06 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:06 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:07.849 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:14:07.848351+0000 min lat: 0.00126737 max lat: 0.130188 avg lat: 0.00727745 2023-10-09T04:14:07.849 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:14:07.849 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 43980 43964 137.313 112.5 0.00543578 0.00727745 2023-10-09T04:14:08.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:07 smithi177 ceph-mon[106070]: pgmap v7: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:14:08.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:07 smithi029 ceph-mon[216903]: pgmap v7: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:14:08.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:07 smithi029 ceph-mon[119131]: pgmap v7: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:14:08.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 45735 45719 135.997 109.688 0.00903833 0.00734737 2023-10-09T04:14:09.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:14:08 smithi177 conmon[179305]: ts=2023-10-09T04:14:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:14:09.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 47431 47415 134.634 106 0.00983843 0.00742254 2023-10-09T04:14:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:09 smithi177 ceph-mon[106070]: pgmap v8: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:14:10.060 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:09 smithi029 ceph-mon[216903]: pgmap v8: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:14:10.061 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:09 smithi029 ceph-mon[119131]: pgmap v8: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:14:10.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:14:10.713 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:10.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 48508 48492 131.708 67.3125 0.0178816 0.00758682 2023-10-09T04:14:11.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:11 smithi029 ceph-mon[216903]: pgmap v9: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:14:11.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 49999 49983 130.103 93.1875 0.00969948 0.00768114 2023-10-09T04:14:12.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:11 smithi177 ceph-mon[106070]: pgmap v9: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:14:12.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:11 smithi029 ceph-mon[119131]: pgmap v9: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:14:12.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 51603 51587 128.909 100.25 0.0141618 0.00775171 2023-10-09T04:14:13.177 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:14:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:14:12] "GET /metrics HTTP/1.1" 200 33495 "" "Prometheus/2.43.0" 2023-10-09T04:14:13.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 53125 53109 127.61 95.125 0.00202881 0.00781912 2023-10-09T04:14:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:13 smithi177 ceph-mon[106070]: pgmap v10: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:13 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:13 smithi029 ceph-mon[216903]: pgmap v10: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:13 smithi029 ceph-mon[119131]: pgmap v10: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:13 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:14.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 53742 53726 124.313 38.5625 0.0115621 0.0080396 2023-10-09T04:14:15.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 55084 55068 122.847 83.875 0.0102536 0.0081339 2023-10-09T04:14:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:15 smithi029 ceph-mon[119131]: pgmap v11: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:15 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:15 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:15 smithi029 ceph-mon[216903]: pgmap v11: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:15 smithi177 ceph-mon[106070]: pgmap v11: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:15 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:15 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:16.389 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:14:16.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 56334 56318 121.305 78.125 0.00680448 0.00823935 2023-10-09T04:14:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[216903]: pgmap v12: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-10-09T04:14:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[216903]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:14:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[216903]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:14:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:17.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[119131]: pgmap v12: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-10-09T04:14:17.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[119131]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:14:17.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[119131]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:14:17.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:17 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:17 smithi177 ceph-mon[106070]: pgmap v12: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-10-09T04:14:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:17 smithi177 ceph-mon[106070]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:14:17.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:17 smithi177 ceph-mon[106070]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:14:17.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:17 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:17.856 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 57818 57802 120.344 92.75 0.00750655 0.00830398 2023-10-09T04:14:18.856 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 58983 58967 118.811 72.8125 0.00684521 0.00841193 2023-10-09T04:14:18.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:14:19.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:14:18 smithi177 conmon[179305]: ts=2023-10-09T04:14:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:14:19.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:19 smithi029 ceph-mon[119131]: pgmap v13: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:14:19.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:19 smithi029 ceph-mon[216903]: pgmap v13: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:14:19.747 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:14:19.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:19 smithi177 ceph-mon[106070]: pgmap v13: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:14:19.856 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 60197 60181 117.47 75.875 0.00822383 0.00850784 2023-10-09T04:14:20.857 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 61469 61453 116.319 79.5 0.0155373 0.00859282 2023-10-09T04:14:21.857 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 62641 62625 115.053 73.25 0.00269754 0.00865252 2023-10-09T04:14:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:21 smithi029 ceph-mon[216903]: pgmap v14: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T04:14:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:21 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:21 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:21 smithi029 ceph-mon[119131]: pgmap v14: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T04:14:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:21 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:21 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:21 smithi177 ceph-mon[106070]: pgmap v14: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-09T04:14:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:21 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:21 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:22.857 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 63564 63548 113.414 57.6875 0.0160484 0.00881163 2023-10-09T04:14:23.125 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:14:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:14:22] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-10-09T04:14:23.857 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 64722 64706 112.275 72.375 0.0118248 0.00890099 2023-10-09T04:14:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[119131]: pgmap v15: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:14:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[216903]: pgmap v15: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:23.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:23 smithi177 ceph-mon[106070]: pgmap v15: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:14:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:23 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:23 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:23 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:23 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:24.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 65889 65873 111.21 72.9375 0.016392 0.00898646 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[216903]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[216903]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[216903]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[216903]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:14:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:14:24.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:24 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:14:25.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:24 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.conf 2023-10-09T04:14:25.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:24 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.conf 2023-10-09T04:14:25.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:24 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:14:25.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:24 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-09T04:14:25.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 67094 67078 110.265 75.3125 0.00957767 0.00906492 2023-10-09T04:14:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[216903]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:14:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[216903]: pgmap v16: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:14:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[216903]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:14:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[216903]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:14:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[119131]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[119131]: pgmap v16: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[119131]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[119131]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:25 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:26.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:25 smithi177 ceph-mon[106070]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:14:26.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:25 smithi177 ceph-mon[106070]: pgmap v16: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-09T04:14:26.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:25 smithi177 ceph-mon[106070]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2023-10-09T04:14:26.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:25 smithi177 ceph-mon[106070]: Updating smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:14:26.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:25 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:26.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:25 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:26.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 68371 68355 109.484 79.8125 0.0191794 0.0091286 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:27.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: Updating smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:14:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:14:27.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:14:27.858601+0000 min lat: 0.00126737 max lat: 0.424506 avg lat: 0.00918466 2023-10-09T04:14:27.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:14:27.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 69691 69675 108.81 82.5 0.0116144 0.00918466 2023-10-09T04:14:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:27 smithi029 ceph-mon[119131]: pgmap v17: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:27 smithi029 ceph-mon[216903]: pgmap v17: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:27 smithi177 ceph-mon[106070]: pgmap v17: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-09T04:14:28.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 70977 70961 108.116 80.375 0.00968272 0.0092442 2023-10-09T04:14:29.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:14:28 smithi177 conmon[179305]: ts=2023-10-09T04:14:28.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:14:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:28 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:29.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:29.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 72239 72223 107.42 78.875 0.00850233 0.00930388 2023-10-09T04:14:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:29 smithi029 ceph-mon[216903]: pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:29 smithi029 ceph-mon[119131]: pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:29 smithi177 ceph-mon[106070]: pgmap v18: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:30.858 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 73497 73481 106.751 78.625 0.00689044 0.00936261 2023-10-09T04:14:31.860 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 74765 74749 106.121 79.25 0.00738769 0.00941876 2023-10-09T04:14:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:31 smithi029 ceph-mon[216903]: pgmap v19: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T04:14:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:31 smithi029 ceph-mon[119131]: pgmap v19: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T04:14:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:31 smithi177 ceph-mon[106070]: pgmap v19: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T04:14:32.860 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 75970 75954 105.436 75.3125 0.0124604 0.0094795 2023-10-09T04:14:33.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:14:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:14:32] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.43.0" 2023-10-09T04:14:33.861 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 77210 77194 104.828 77.5 0.0122526 0.00953501 2023-10-09T04:14:34.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:33 smithi029 ceph-mon[216903]: pgmap v20: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:34.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:33 smithi029 ceph-mon[119131]: pgmap v20: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:34.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:33 smithi177 ceph-mon[106070]: pgmap v20: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:34.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 78407 78391 104.184 74.8125 0.00900309 0.0095939 2023-10-09T04:14:35.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 79580 79564 103.541 73.3125 0.012983 0.00965277 2023-10-09T04:14:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:35 smithi177 ceph-mon[106070]: pgmap v21: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:36.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:35 smithi029 ceph-mon[216903]: pgmap v21: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:36.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:35 smithi029 ceph-mon[119131]: pgmap v21: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:14:36.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 80769 80753 102.944 74.3125 0.0203141 0.00970923 2023-10-09T04:14:37.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 81947 81931 102.358 73.625 0.00916556 0.00976509 2023-10-09T04:14:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:37 smithi177 ceph-mon[106070]: pgmap v22: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:14:38.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:37 smithi029 ceph-mon[216903]: pgmap v22: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:14:38.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:37 smithi029 ceph-mon[119131]: pgmap v22: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:14:38.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 83111 83095 101.778 72.75 0.0134687 0.00982061 2023-10-09T04:14:39.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:14:38 smithi177 conmon[179305]: ts=2023-10-09T04:14:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:14:39.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 84260 84244 101.201 71.8125 0.00855493 0.00987588 2023-10-09T04:14:40.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:39 smithi177 ceph-mon[106070]: pgmap v23: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:14:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:39 smithi029 ceph-mon[119131]: pgmap v23: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:14:40.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:39 smithi029 ceph-mon[216903]: pgmap v23: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:14:40.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 85388 85372 100.622 70.5 0.00769435 0.00993315 2023-10-09T04:14:41.864 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 86514 86498 100.062 70.375 0.0136277 0.00998926 2023-10-09T04:14:42.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:41 smithi177 ceph-mon[106070]: pgmap v24: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:14:42.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:41 smithi029 ceph-mon[216903]: pgmap v24: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:14:42.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:41 smithi029 ceph-mon[119131]: pgmap v24: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-09T04:14:42.865 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 87630 87614 99.5112 69.75 0.0107852 0.0100444 2023-10-09T04:14:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:14:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:14:42] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.43.0" 2023-10-09T04:14:43.865 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 88661 88645 98.885 64.4375 0.00430871 0.0101081 2023-10-09T04:14:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:43 smithi177 ceph-mon[106070]: pgmap v25: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:14:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:43 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:43 smithi029 ceph-mon[216903]: pgmap v25: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:14:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:43 smithi029 ceph-mon[119131]: pgmap v25: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:14:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:43 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:44.868 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 89736 89720 98.3231 67.1875 0.00673767 0.0101654 2023-10-09T04:14:45.868 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 90797 90781 97.7713 66.3125 0.0153019 0.010223 2023-10-09T04:14:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:45 smithi177 ceph-mon[106070]: pgmap v26: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:14:46.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:45 smithi029 ceph-mon[216903]: pgmap v26: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:14:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:45 smithi029 ceph-mon[119131]: pgmap v26: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:14:46.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 91868 91852 97.2429 66.9375 0.0159374 0.0102785 2023-10-09T04:14:47.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:14:47.872707+0000 min lat: 0.00126737 max lat: 0.424506 avg lat: 0.0103322 2023-10-09T04:14:47.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:14:47.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 92937 92921 96.7359 66.8125 0.0152727 0.0103322 2023-10-09T04:14:48.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:48 smithi177 ceph-mon[106070]: pgmap v27: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:14:48.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:48 smithi029 ceph-mon[216903]: pgmap v27: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:14:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:48 smithi029 ceph-mon[119131]: pgmap v27: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-09T04:14:48.872 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 93995 93979 96.2342 66.125 0.0125808 0.0103863 2023-10-09T04:14:49.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:14:48 smithi177 conmon[179305]: ts=2023-10-09T04:14:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:14:49.875 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 95053 95037 95.7443 66.125 0.0187306 0.0104398 2023-10-09T04:14:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:50 smithi177 ceph-mon[106070]: pgmap v28: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T04:14:50.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:50 smithi029 ceph-mon[119131]: pgmap v28: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T04:14:50.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:50 smithi029 ceph-mon[216903]: pgmap v28: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-09T04:14:50.875 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 95879 95863 95.0442 51.625 0.00498381 0.0104823 2023-10-09T04:14:51.877 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 96678 96662 94.3397 49.9375 0.0132767 0.0105956 2023-10-09T04:14:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:52 smithi177 ceph-mon[106070]: pgmap v29: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:14:52.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:52 smithi029 ceph-mon[216903]: pgmap v29: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:14:52.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:52 smithi029 ceph-mon[119131]: pgmap v29: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-09T04:14:52.875 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 97673 97657 93.8452 62.1875 0.0142603 0.0106513 2023-10-09T04:14:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:14:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:14:52] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2023-10-09T04:14:53.876 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 98679 98663 93.376 62.875 0.0105438 0.010706 2023-10-09T04:14:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:54 smithi177 ceph-mon[106070]: pgmap v30: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:14:54.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:54 smithi029 ceph-mon[216903]: pgmap v30: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:14:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:54 smithi029 ceph-mon[119131]: pgmap v30: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:14:54.876 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 99678 99662 92.9143 62.4375 0.00946243 0.0107581 2023-10-09T04:14:55.878 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 100523 100507 92.322 52.8125 0.0137687 0.0108276 2023-10-09T04:14:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:56 smithi177 ceph-mon[106070]: pgmap v31: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:14:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:56 smithi029 ceph-mon[216903]: pgmap v31: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:14:56.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:56 smithi029 ceph-mon[119131]: pgmap v31: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:14:56.884 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 101529 101513 91.8871 62.875 0.0122439 0.0108782 2023-10-09T04:14:57.884 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 102519 102503 91.4585 61.875 0.0116929 0.0109294 2023-10-09T04:14:58.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:58 smithi029 ceph-mon[216903]: pgmap v32: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:14:58.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:58 smithi029 ceph-mon[119131]: pgmap v32: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:14:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:58 smithi177 ceph-mon[106070]: pgmap v32: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:14:58.886 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 103496 103480 91.0285 61.0625 0.0239999 0.0109802 2023-10-09T04:14:59.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:14:58 smithi177 conmon[179305]: ts=2023-10-09T04:14:58.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:14:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:14:59 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:59.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:14:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:14:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:14:59.886 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 104470 104454 90.6098 60.875 0.0112673 0.0110314 2023-10-09T04:15:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:00 smithi029 ceph-mon[216903]: pgmap v33: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T04:15:00.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:00 smithi029 ceph-mon[119131]: pgmap v33: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T04:15:00.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:00 smithi177 ceph-mon[106070]: pgmap v33: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T04:15:00.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 105438 105422 90.1964 60.5 0.0160041 0.011082 2023-10-09T04:15:01.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:01 smithi029 ceph-mon[216903]: pgmap v34: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:15:01.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:01 smithi029 ceph-mon[119131]: pgmap v34: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:15:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:01 smithi177 ceph-mon[106070]: pgmap v34: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-09T04:15:01.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 106403 106387 89.7927 60.3125 0.00901018 0.0111311 2023-10-09T04:15:02.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 107362 107346 89.3948 59.9375 0.013301 0.0111817 2023-10-09T04:15:03.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:15:02] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-10-09T04:15:03.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:03 smithi029 ceph-mon[216903]: pgmap v35: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T04:15:03.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:03 smithi029 ceph-mon[119131]: pgmap v35: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T04:15:03.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:03 smithi177 ceph-mon[106070]: pgmap v35: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T04:15:03.887 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 108317 108301 89.0041 59.6875 0.0148673 0.0112306 2023-10-09T04:15:04.891 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 109252 109236 88.6031 58.4375 0.011663 0.0112815 2023-10-09T04:15:05.891 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 110183 110167 88.2133 58.1875 0.0140934 0.0113311 2023-10-09T04:15:05.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:05 smithi029 ceph-mon[216903]: pgmap v36: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T04:15:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:05 smithi029 ceph-mon[119131]: pgmap v36: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T04:15:06.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:05 smithi177 ceph-mon[106070]: pgmap v36: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-09T04:15:06.891 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 111113 111097 87.8326 58.125 0.0196169 0.0113797 2023-10-09T04:15:07.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:15:07.895021+0000 min lat: 0.00126737 max lat: 0.424506 avg lat: 0.0114287 2023-10-09T04:15:07.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:15:07.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 112045 112029 87.4598 58.25 0.0203039 0.0114287 2023-10-09T04:15:07.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:07 smithi029 ceph-mon[119131]: pgmap v37: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T04:15:07.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:07 smithi029 ceph-mon[216903]: pgmap v37: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T04:15:08.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:07 smithi177 ceph-mon[106070]: pgmap v37: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T04:15:08.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 112965 112949 87.0891 57.5 0.0206946 0.0114771 2023-10-09T04:15:09.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:15:08 smithi177 conmon[179305]: ts=2023-10-09T04:15:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:09.895 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 113886 113870 86.7292 57.5625 0.0130772 0.0115259 2023-10-09T04:15:09.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:09 smithi029 ceph-mon[216903]: pgmap v38: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2023-10-09T04:15:09.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:09 smithi029 ceph-mon[119131]: pgmap v38: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2023-10-09T04:15:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:09 smithi177 ceph-mon[106070]: pgmap v38: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s 2023-10-09T04:15:10.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 114783 114767 86.3593 56.0625 0.0314161 0.0115754 2023-10-09T04:15:11.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 115685 115669 86.0025 56.375 0.0166666 0.011623 2023-10-09T04:15:11.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:11 smithi029 ceph-mon[216903]: pgmap v39: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T04:15:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:11 smithi029 ceph-mon[119131]: pgmap v39: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T04:15:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:11 smithi177 ceph-mon[106070]: pgmap v39: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-09T04:15:12.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 116550 116534 85.6269 54.0625 0.0229844 0.0116741 2023-10-09T04:15:13.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:15:12] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-10-09T04:15:13.896 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 117443 117427 85.2803 55.8125 0.00505549 0.0117209 2023-10-09T04:15:13.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:13 smithi029 ceph-mon[216903]: pgmap v40: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:15:13.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:13.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:13 smithi029 ceph-mon[119131]: pgmap v40: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:15:13.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:13 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:13 smithi177 ceph-mon[106070]: pgmap v40: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:15:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:13 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:14.897 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 118332 118316 84.9389 55.5625 0.0137597 0.0117684 2023-10-09T04:15:15.897 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 119215 119199 84.601 55.1875 0.0212513 0.0118158 2023-10-09T04:15:15.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:15 smithi029 ceph-mon[119131]: pgmap v41: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:15:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:15 smithi029 ceph-mon[216903]: pgmap v41: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:15:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:15 smithi177 ceph-mon[106070]: pgmap v41: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:15:16.897 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 120116 120100 84.2832 56.3125 0.00850494 0.0118607 2023-10-09T04:15:17.897 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0102 2023-10-09T04:15:17.897 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 120944 2023-10-09T04:15:17.897 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 83.9794 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 33.6077 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 189.625 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 38.5625 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1343 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 537.723 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3034 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 617 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0119039 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0107613 2023-10-09T04:15:17.898 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 0.424506 2023-10-09T04:15:17.899 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00126737 2023-10-09T04:15:17.901 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:15:17.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:17 smithi029 ceph-mon[216903]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:15:17.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:17 smithi029 ceph-mon[119131]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:15:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:17 smithi177 ceph-mon[106070]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-09T04:15:19.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:15:18 smithi177 conmon[179305]: ts=2023-10-09T04:15:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:19.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:19 smithi029 ceph-mon[119131]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s 2023-10-09T04:15:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:19 smithi029 ceph-mon[216903]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s 2023-10-09T04:15:20.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:19 smithi177 ceph-mon[106070]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 757 op/s 2023-10-09T04:15:21.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:21 smithi029 ceph-mon[119131]: pgmap v44: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.13k op/s 2023-10-09T04:15:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:21 smithi029 ceph-mon[216903]: pgmap v44: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.13k op/s 2023-10-09T04:15:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:21 smithi177 ceph-mon[106070]: pgmap v44: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.13k op/s 2023-10-09T04:15:23.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:15:22] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:15:23.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:23 smithi029 ceph-mon[216903]: pgmap v45: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 740 op/s 2023-10-09T04:15:23.667 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:23 smithi029 ceph-mon[119131]: pgmap v45: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 740 op/s 2023-10-09T04:15:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:23 smithi177 ceph-mon[106070]: pgmap v45: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 740 op/s 2023-10-09T04:15:25.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:25 smithi029 ceph-mon[216903]: pgmap v46: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 740 op/s 2023-10-09T04:15:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:25 smithi029 ceph-mon[119131]: pgmap v46: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 740 op/s 2023-10-09T04:15:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:25 smithi177 ceph-mon[106070]: pgmap v46: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 740 op/s 2023-10-09T04:15:27.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:26 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:26 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:27.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:28.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:27 smithi177 ceph-mon[106070]: pgmap v47: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 1.11k op/s 2023-10-09T04:15:28.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:27 smithi029 ceph-mon[216903]: pgmap v47: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 1.11k op/s 2023-10-09T04:15:28.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:27 smithi029 ceph-mon[119131]: pgmap v47: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 39 MiB/s wr, 1.11k op/s 2023-10-09T04:15:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:29.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:15:28 smithi177 conmon[179305]: ts=2023-10-09T04:15:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:29.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:28 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:30.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:29 smithi177 ceph-mon[106070]: pgmap v48: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 741 op/s 2023-10-09T04:15:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:29 smithi029 ceph-mon[216903]: pgmap v48: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 741 op/s 2023-10-09T04:15:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:29 smithi029 ceph-mon[119131]: pgmap v48: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 741 op/s 2023-10-09T04:15:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:30 smithi177 ceph-mon[106070]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T04:15:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:30 smithi029 ceph-mon[216903]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T04:15:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:30 smithi029 ceph-mon[119131]: mgrmap e37: y(active, since 91s), standbys: x 2023-10-09T04:15:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:31 smithi029 ceph-mon[119131]: pgmap v49: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 1.08k op/s 2023-10-09T04:15:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:31 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:31 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:32.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:31 smithi029 ceph-mon[216903]: pgmap v49: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 1.08k op/s 2023-10-09T04:15:32.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:32.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:31 smithi177 ceph-mon[106070]: pgmap v49: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 1.08k op/s 2023-10-09T04:15:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:31 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:31 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:32.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:15:32.830 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:15:32] "GET /metrics HTTP/1.1" 200 36215 "" "Prometheus/2.43.0" 2023-10-09T04:15:33.495 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:34.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:33 smithi029 ceph-mon[216903]: pgmap v50: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 255 B/s wr, 715 op/s 2023-10-09T04:15:34.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:33 smithi029 ceph-mon[119131]: pgmap v50: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 255 B/s wr, 715 op/s 2023-10-09T04:15:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:33 smithi177 ceph-mon[106070]: pgmap v50: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 255 B/s wr, 715 op/s 2023-10-09T04:15:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:35 smithi029 ceph-mon[216903]: pgmap v51: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 255 B/s wr, 715 op/s 2023-10-09T04:15:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:35 smithi029 ceph-mon[119131]: pgmap v51: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 255 B/s wr, 715 op/s 2023-10-09T04:15:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:35 smithi177 ceph-mon[106070]: pgmap v51: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 255 B/s wr, 715 op/s 2023-10-09T04:15:37.271 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:15:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:37 smithi029 ceph-mon[216903]: pgmap v52: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 255 B/s wr, 1.02k op/s 2023-10-09T04:15:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:37 smithi029 ceph-mon[119131]: pgmap v52: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 255 B/s wr, 1.02k op/s 2023-10-09T04:15:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:37 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:37 smithi177 ceph-mon[106070]: pgmap v52: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 255 B/s wr, 1.02k op/s 2023-10-09T04:15:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:37 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:38.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-09T04:15:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:38 smithi177 ceph-mon[106070]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:38 smithi177 ceph-mon[106070]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:15:39.030 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:15:38 smithi177 conmon[179305]: ts=2023-10-09T04:15:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:38 smithi029 ceph-mon[216903]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:38 smithi029 ceph-mon[216903]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:15:39.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:38 smithi029 ceph-mon[119131]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:15:39.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:38 smithi029 ceph-mon[119131]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:15:39.746 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:39 smithi029 ceph-mon[119131]: pgmap v53: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 255 B/s wr, 647 op/s 2023-10-09T04:15:40.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:39 smithi029 ceph-mon[216903]: pgmap v53: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 255 B/s wr, 647 op/s 2023-10-09T04:15:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:39 smithi177 ceph-mon[106070]: pgmap v53: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 255 B/s wr, 647 op/s 2023-10-09T04:15:41.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:41 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:41.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:41 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:41.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:41 smithi177 ceph-mon[106070]: pgmap v54: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 929 op/s 2023-10-09T04:15:41.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:41 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:41.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:41 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:41.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:41 smithi029 ceph-mon[119131]: pgmap v54: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 929 op/s 2023-10-09T04:15:41.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:41.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:41.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:41 smithi029 ceph-mon[216903]: pgmap v54: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 929 op/s 2023-10-09T04:15:42.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:15:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:15:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:42 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:15:42.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:42.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:42.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:42.548 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:42.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:15:42.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:15:42.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:42 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-09T04:15:42.552 INFO:teuthology.orchestra.run.smithi029.stdout:wait for majority of mons to upgrade 2023-10-09T04:15:43.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:15:42] "GET /metrics HTTP/1.1" 200 36215 "" "Prometheus/2.43.0" 2023-10-09T04:15:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:43 smithi177 ceph-mon[106070]: Upgrade: It appears safe to stop mon.c 2023-10-09T04:15:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:43 smithi177 ceph-mon[106070]: pgmap v55: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:15:43.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:43 smithi029 ceph-mon[119131]: Upgrade: It appears safe to stop mon.c 2023-10-09T04:15:43.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:43 smithi029 ceph-mon[119131]: pgmap v55: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:15:43.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:43 smithi029 ceph-mon[216903]: Upgrade: It appears safe to stop mon.c 2023-10-09T04:15:43.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:43 smithi029 ceph-mon[216903]: pgmap v55: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:15:43.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-09T04:15:44.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:44 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:44 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:44.833 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:45 smithi029 ceph-mon[216903]: pgmap v56: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:15:45.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:45 smithi029 ceph-mon[119131]: pgmap v56: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:15:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:45 smithi177 ceph-mon[106070]: pgmap v56: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-09T04:15:47.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: Upgrade: Updating mon.c 2023-10-09T04:15:47.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:47.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:15:47.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:15:47.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:47.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: Deploying daemon mon.c on smithi029 2023-10-09T04:15:47.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:47 smithi177 ceph-mon[106070]: pgmap v57: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T04:15:47.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: Upgrade: Updating mon.c 2023-10-09T04:15:47.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:47.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:15:47.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:15:47.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:47.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: Deploying daemon mon.c on smithi029 2023-10-09T04:15:47.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[216903]: pgmap v57: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T04:15:47.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: Upgrade: Updating mon.c 2023-10-09T04:15:47.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:47.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:15:47.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:15:47.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:15:47.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: Deploying daemon mon.c on smithi029 2023-10-09T04:15:47.636 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:47 smithi029 ceph-mon[119131]: pgmap v57: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2023-10-09T04:15:49.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:15:48 smithi177 conmon[179305]: ts=2023-10-09T04:15:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:49.764 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:49 smithi029 ceph-mon[119131]: pgmap v58: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:15:49.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:49 smithi029 ceph-mon[119131]: from='client.? 172.21.15.29:0/827231624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:49.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:49 smithi029 ceph-mon[216903]: pgmap v58: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:15:49.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/827231624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:49 smithi177 ceph-mon[106070]: pgmap v58: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 547 op/s 2023-10-09T04:15:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:49 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/827231624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:49 smithi029 systemd[1]: Stopping Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:15:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:50 smithi029 conmon[119107]: 2023-10-09T04:15:50.281+0000 7fca1cb7e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:15:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:50 smithi029 conmon[119107]: 2023-10-09T04:15:50.281+0000 7fca1cb7e700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:15:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:50 smithi029 bash[223431]: c8851494e8fd5b2bfa8ff6afa1915574d9118a787c60627e7612bfccb7e86d37 2023-10-09T04:15:52.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:52 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-09T04:15:52.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:52 smithi029 systemd[1]: Stopped Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:52.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:52 smithi029 systemd[1]: Starting Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:15:53.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:15:52] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2023-10-09T04:15:53.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:15:53.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 6 2023-10-09T04:15:53.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: pidfile_write: ignore empty --pid-file 2023-10-09T04:15:53.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 bash[223797]: 7e2b5984a98662edae48f925c20edae8e844bd6a4e67395360eca375d54d773f 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: load: jerasure load: lrc 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: RocksDB version: 7.9.2 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Git sha 0 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: DB SUMMARY 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: DB Session ID: EHB9LSJPQY8QT1DODA9S 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1103 Bytes 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-10-09T04:15:53.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 9079426 ; 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.env: 0x555d10bee720 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.info_log: 0x555d1220d420 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.statistics: (nil) 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.use_fsync: 0 2023-10-09T04:15:53.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.db_log_dir: 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.wal_dir: 2023-10-09T04:15:53.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 systemd[1]: Started Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.write_buffer_manager: 0x555d1149cc80 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:15:53.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.unordered_write: 0 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.row_cache: None 2023-10-09T04:15:53.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.wal_filter: None 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.wal_compression: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:15:53.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:15:53.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_open_files: -1 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Compression algorithms supported: 2023-10-09T04:15:53.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kZSTD supported: 0 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:15:53.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.merge_operator: 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_filter: None 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555d1220d520) 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_type: 0 2023-10-09T04:15:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_index_type: 0 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_shortening: 1 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: checksum: 4 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: no_block_cache: 0 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache: 0x555d1151f350 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_options: 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: capacity : 536870912 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_shard_bits : 4 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: strict_capacity_limit : 0 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_compressed: (nil) 2023-10-09T04:15:53.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: persistent_cache: (nil) 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size: 4096 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size_deviation: 10 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_restart_interval: 16 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_block_restart_interval: 1 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_block_size: 4096 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: partition_filters: 0 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: use_delta_encoding: 1 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: filter_policy: bloomfilter 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: whole_key_filtering: 1 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: verify_compression: 0 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:15:53.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: format_version: 5 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_index_compression: 1 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_align: 0 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_auto_readahead_size: 262144 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: prepopulate_block_cache: 0 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: initial_auto_readahead_size: 8192 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression: NoCompression 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:15:53.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.num_levels: 7 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:15:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:15:53.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:15:53.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:15:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:15:53.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.ttl: 2592000 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:15:53.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12948, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-10-09T04:15:53.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a33e7a1c-39b8-4df7-9c56-991955364e33 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824953346146, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824953380513, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5534203, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12953, "largest_seqno": 17153, "table_properties": {"data_size": 5519421, "index_size": 9665, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4101, "raw_key_size": 34606, "raw_average_key_size": 21, "raw_value_size": 5487099, "raw_average_value_size": 3372, "num_data_blocks": 436, "num_entries": 1627, "num_filter_entries": 1627, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696824953, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a33e7a1c-39b8-4df7-9c56-991955364e33", "db_session_id": "EHB9LSJPQY8QT1DODA9S", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696824953380680, "job": 1, "event": "recovery_finished"} 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555d115ba000 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: DB pointer 0x555d115a8000 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:15:53.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** DB Stats ** 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 5.28 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 369.6 0.01 0.00 1 0.014 0 0 0.0 0.0 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: L6 1/0 5.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 2/0 10.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 369.6 0.01 0.00 1 0.014 0 0 0.0 0.0 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 369.6 0.01 0.00 1 0.014 0 0 0.0 0.0 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:15:53.701 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 369.6 0.01 0.00 1 0.014 0 0 0.0 0.0 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:15:53.702 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.01 GB write, 97.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.01 GB write, 97.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout: Block cache BinnedLRUCache@0x555d1151f350#6 capacity: 512.00 MB usage: 58.53 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout: Block cache entry stats(count,size,portion): DataBlock(4,22.81 KB,0.00435114%) FilterBlock(2,12.56 KB,0.00239611%) IndexBlock(2,23.16 KB,0.0044167%) Misc(1,0.00 KB,0%) 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: starting mon.c rank 2 at public addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] at bind addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???) e3 preinit fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).mds e8 new map 2023-10-09T04:15:53.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).mds e8 print_map 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: e8 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: legacy client fscid: 1 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: Filesystem 'foo' (1) 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: fs_name foo 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: epoch 7 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: created 2023-10-09T04:07:34.635169+0000 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: modified 2023-10-09T04:07:47.848728+0000 2023-10-09T04:15:53.704 INFO:journalctl@ceph.mon.c.smithi029.stdout: tableserver 0 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: root 0 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: session_timeout 60 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: session_autoclose 300 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_file_size 1099511627776 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: required_client_features {} 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: last_failure 0 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: last_failure_osd_epoch 0 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_mds 1 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: in 0 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: up {0=14649} 2023-10-09T04:15:53.705 INFO:journalctl@ceph.mon.c.smithi029.stdout: failed 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: damaged 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: stopped 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_pools [4] 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_pool 3 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: inline_data disabled 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: balancer 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: bal_rank_mask -1 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: standby_count_wanted 1 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: [mds.foo.smithi029.mozfhx{0:14649} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.706 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout: Standby daemons: 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout: [mds.foo.smithi177.pbovgs{-1:24547} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expand map: {default=false} 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta from 'false' to 'false' 2023-10-09T04:15:53.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expanded map: {default=false} 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expand map: {default=info} 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta from 'info' to 'info' 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expanded map: {default=info} 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expand map: {default=daemon} 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expanded map: {default=daemon} 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expand map: {default=debug} 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta from 'debug' to 'debug' 2023-10-09T04:15:53.708 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:53 smithi029 ceph-mon[223858]: expand_channel_meta expanded map: {default=debug} 2023-10-09T04:15:54.258 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3805607722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: pgmap v60: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: mon.c calling monitor election 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: mon.a calling monitor election 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:15:54.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: mgrmap e37: y(active, since 115s), standbys: x 2023-10-09T04:15:54.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: overall HEALTH_OK 2023-10-09T04:15:54.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:54.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:54.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:54.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3486129270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3805607722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: pgmap v60: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: mon.c calling monitor election 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: mon.a calling monitor election 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: mgrmap e37: y(active, since 115s), standbys: x 2023-10-09T04:15:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: overall HEALTH_OK 2023-10-09T04:15:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3486129270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3805607722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: pgmap v60: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: mon.c calling monitor election 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: mon.a calling monitor election 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: mgrmap e37: y(active, since 115s), standbys: x 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: overall HEALTH_OK 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:15:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:15:55.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:54 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/3486129270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:15:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:55 smithi029 ceph-mon[216903]: pgmap v61: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-09T04:15:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:55 smithi029 ceph-mon[223858]: pgmap v61: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-09T04:15:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:55 smithi177 ceph-mon[106070]: pgmap v61: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-09T04:15:56.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:15:57.255 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:15:57.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:57 smithi029 ceph-mon[216903]: pgmap v62: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T04:15:57.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:57 smithi029 ceph-mon[223858]: pgmap v62: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T04:15:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:57 smithi177 ceph-mon[106070]: pgmap v62: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-10-09T04:15:58.675 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:15:58 smithi029 conmon[202582]: 2023-10-09T04:15:58.398+0000 7f6049533700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-09T04:15:59.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:15:58 smithi177 conmon[179305]: ts=2023-10-09T04:15:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:15:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:59 smithi029 ceph-mon[223858]: pgmap v63: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-10-09T04:15:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:15:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:59.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:59 smithi029 ceph-mon[216903]: pgmap v63: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-10-09T04:15:59.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:15:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:15:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:59 smithi177 ceph-mon[106070]: pgmap v63: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 550 op/s 2023-10-09T04:15:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:15:59 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:00.929 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (4m) - 9m 29.8M - 0.25.0 c8568f914cd2 7ab788eff7be 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi177 *:3000 running (3m) - 8m 95.7M - 9.4.7 2c41d148cca3 dbe79334ca87 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.mozfhx smithi029 running (8m) - 8m 19.7M - 17.2.6-1303-gb2903cb0 5517396c50d4 e346a0e9dcdd 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi177.pbovgs smithi177 running (8m) - 8m 15.6M - 17.2.6-1303-gb2903cb0 5517396c50d4 a20b9123e852 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi177 *:8443,9283,8765 running (2m) - 13m 447M - 18.2.0-598-gb4fa663f 3060897fdf97 c43f5ec5cd80 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (3m) - 15m 502M - 18.2.0-598-gb4fa663f 3060897fdf97 239eea6ccc4c 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (2m) - 15m 54.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8369665f49b3 2023-10-09T04:16:00.930 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi177 running (13m) - 13m 91.7M 2048M 17.2.6-1303-gb2903cb0 5517396c50d4 f19800b51616 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 starting - - - 2048M 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (3m) - 8m 17.2M - 1.5.0 0da6a335fe13 2be0e618a0f3 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi177 *:9100 running (3m) - 8m 12.3M - 1.5.0 0da6a335fe13 2adf0c28b454 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (12m) - 12m 2888M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 2c8a6318fb47 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (12m) - 12m 2143M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 951ea50b247d 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (12m) - 12m 3833M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 7d2e4b8000f3 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (11m) - 11m 2565M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 e4ab91bdb38b 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi177 running (11m) - 11m 2966M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 d4b592bd1322 2023-10-09T04:16:00.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi177 running (11m) - 10m 3309M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 28d4ab6e8bfe 2023-10-09T04:16:00.932 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi177 running (10m) - 10m 1592M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 03ede6ceccc0 2023-10-09T04:16:00.932 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi177 running (10m) - 10m 2778M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0f3217cc8fa1 2023-10-09T04:16:00.932 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi177 *:9095 running (2m) - 9m 53.0M - 2.43.0 a07b618ecd1d 2d31282ff477 2023-10-09T04:16:01.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:01 smithi029 ceph-mon[216903]: pgmap v64: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-09T04:16:01.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:01 smithi029 ceph-mon[216903]: from='client.25141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:01 smithi029 ceph-mon[223858]: pgmap v64: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-09T04:16:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:01 smithi029 ceph-mon[223858]: from='client.25141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:01 smithi177 ceph-mon[106070]: pgmap v64: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-09T04:16:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:01 smithi177 ceph-mon[106070]: from='client.25141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:02.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:16:02.824 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:16:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:16:02] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2023-10-09T04:16:03.419 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:03.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:03 smithi029 ceph-mon[216903]: pgmap v65: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T04:16:03.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:03 smithi029 ceph-mon[223858]: pgmap v65: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T04:16:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:03 smithi177 ceph-mon[106070]: pgmap v65: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T04:16:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:05.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:05.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:05.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:04 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:04 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:06.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:05 smithi029 ceph-mon[223858]: pgmap v66: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T04:16:06.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:06.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:06.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:05 smithi029 ceph-mon[216903]: pgmap v66: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T04:16:06.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:06.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:05 smithi177 ceph-mon[106070]: pgmap v66: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-09T04:16:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:05 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:06.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:05 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:07.075 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:16:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[216903]: pgmap v67: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[216903]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[216903]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[223858]: pgmap v67: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[223858]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[223858]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:16:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:08.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:08 smithi177 ceph-mon[106070]: pgmap v67: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-09T04:16:08.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:08 smithi177 ceph-mon[106070]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:16:08.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:08 smithi177 ceph-mon[106070]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:16:08.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:08 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:09.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:16:08 smithi177 conmon[179305]: ts=2023-10-09T04:16:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:09.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:16:10.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:10 smithi029 ceph-mon[216903]: pgmap v68: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T04:16:10.372 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:10 smithi029 ceph-mon[223858]: pgmap v68: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T04:16:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:10 smithi177 ceph-mon[106070]: pgmap v68: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T04:16:10.676 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:16:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:12 smithi029 ceph-mon[216903]: pgmap v69: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:16:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:12 smithi029 ceph-mon[223858]: pgmap v69: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:16:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:12 smithi177 ceph-mon[106070]: pgmap v69: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:16:12.823 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:16:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:16:12] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2023-10-09T04:16:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: pgmap v70: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: pgmap v70: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: pgmap v70: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:16:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:16:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:16 smithi029 ceph-mon[216903]: pgmap v71: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:16 smithi029 ceph-mon[223858]: pgmap v71: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:16.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:16 smithi177 ceph-mon[106070]: pgmap v71: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:18 smithi029 ceph-mon[216903]: pgmap v72: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-10-09T04:16:18.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:18 smithi029 ceph-mon[223858]: pgmap v72: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-10-09T04:16:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:18 smithi177 ceph-mon[106070]: pgmap v72: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-10-09T04:16:19.029 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:16:18 smithi177 conmon[179305]: ts=2023-10-09T04:16:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:20 smithi029 ceph-mon[216903]: pgmap v73: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:20 smithi029 ceph-mon[223858]: pgmap v73: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:20.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:20 smithi177 ceph-mon[106070]: pgmap v73: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-09T04:16:21.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:21 smithi029 ceph-mon[223858]: pgmap v74: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-10-09T04:16:21.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:21 smithi029 ceph-mon[216903]: pgmap v74: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-10-09T04:16:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:21 smithi177 ceph-mon[106070]: pgmap v74: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-10-09T04:16:23.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:16:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:16:22] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.43.0" 2023-10-09T04:16:23.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:23 smithi177 ceph-mon[106070]: pgmap v75: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:16:23.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:23 smithi029 ceph-mon[216903]: pgmap v75: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:16:23.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:23 smithi029 ceph-mon[223858]: pgmap v75: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:16:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:25 smithi029 ceph-mon[216903]: pgmap v76: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:16:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:25 smithi029 ceph-mon[223858]: pgmap v76: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:16:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:25 smithi177 ceph-mon[106070]: pgmap v76: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:16:27.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:27 smithi029 ceph-mon[223858]: pgmap v77: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T04:16:27.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:27 smithi029 ceph-mon[216903]: pgmap v77: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T04:16:28.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:27 smithi177 ceph-mon[106070]: pgmap v77: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-09T04:16:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:28.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:29.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:16:28 smithi177 conmon[179305]: ts=2023-10-09T04:16:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:29.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:29 smithi029 ceph-mon[223858]: pgmap v78: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T04:16:29.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:29 smithi029 ceph-mon[216903]: pgmap v78: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T04:16:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:29 smithi177 ceph-mon[106070]: pgmap v78: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T04:16:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:31 smithi029 ceph-mon[223858]: pgmap v79: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 994 op/s 2023-10-09T04:16:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:31 smithi029 ceph-mon[216903]: pgmap v79: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 994 op/s 2023-10-09T04:16:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:31 smithi177 ceph-mon[106070]: pgmap v79: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 994 op/s 2023-10-09T04:16:33.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:16:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:16:32] "GET /metrics HTTP/1.1" 200 36221 "" "Prometheus/2.43.0" 2023-10-09T04:16:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:33 smithi029 ceph-mon[216903]: pgmap v80: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:33 smithi029 ceph-mon[223858]: pgmap v80: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:33 smithi177 ceph-mon[106070]: pgmap v80: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:35 smithi029 ceph-mon[216903]: pgmap v81: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:35.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:35 smithi029 ceph-mon[223858]: pgmap v81: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:35 smithi177 ceph-mon[106070]: pgmap v81: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:37 smithi029 ceph-mon[223858]: pgmap v82: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2023-10-09T04:16:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:37 smithi029 ceph-mon[216903]: pgmap v82: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2023-10-09T04:16:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:37 smithi177 ceph-mon[106070]: pgmap v82: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2023-10-09T04:16:39.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:16:38 smithi177 conmon[179305]: ts=2023-10-09T04:16:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:39 smithi029 ceph-mon[216903]: pgmap v83: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:39 smithi029 ceph-mon[223858]: pgmap v83: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:39 smithi177 ceph-mon[106070]: pgmap v83: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-09T04:16:42.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:41 smithi177 ceph-mon[106070]: pgmap v84: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-10-09T04:16:42.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:41 smithi029 ceph-mon[216903]: pgmap v84: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-10-09T04:16:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:41 smithi029 ceph-mon[223858]: pgmap v84: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2023-10-09T04:16:43.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:16:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:16:42] "GET /metrics HTTP/1.1" 200 36221 "" "Prometheus/2.43.0" 2023-10-09T04:16:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:43 smithi177 ceph-mon[106070]: pgmap v85: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:16:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:43 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:44.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:43 smithi029 ceph-mon[216903]: pgmap v85: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:16:44.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:44.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:43 smithi029 ceph-mon[223858]: pgmap v85: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:16:44.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:45 smithi177 ceph-mon[106070]: pgmap v86: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:16:46.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:45 smithi029 ceph-mon[216903]: pgmap v86: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:16:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:45 smithi029 ceph-mon[223858]: pgmap v86: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:16:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:47 smithi177 ceph-mon[106070]: pgmap v87: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2023-10-09T04:16:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:47 smithi029 ceph-mon[216903]: pgmap v87: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2023-10-09T04:16:48.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:47 smithi029 ceph-mon[223858]: pgmap v87: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2023-10-09T04:16:49.027 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:16:48 smithi177 conmon[179305]: ts=2023-10-09T04:16:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:50.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:49 smithi177 ceph-mon[106070]: pgmap v88: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T04:16:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:49 smithi029 ceph-mon[223858]: pgmap v88: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T04:16:50.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:49 smithi029 ceph-mon[216903]: pgmap v88: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T04:16:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:51 smithi177 ceph-mon[106070]: pgmap v89: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T04:16:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:51 smithi029 ceph-mon[223858]: pgmap v89: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T04:16:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:51 smithi029 ceph-mon[216903]: pgmap v89: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T04:16:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:16:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:16:52] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2023-10-09T04:16:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:53 smithi177 ceph-mon[106070]: pgmap v90: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T04:16:54.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:53 smithi029 ceph-mon[216903]: pgmap v90: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T04:16:54.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:53 smithi029 ceph-mon[223858]: pgmap v90: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T04:16:56.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:55 smithi177 ceph-mon[106070]: pgmap v91: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T04:16:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:55 smithi029 ceph-mon[216903]: pgmap v91: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T04:16:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:55 smithi029 ceph-mon[223858]: pgmap v91: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-09T04:16:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:57 smithi177 ceph-mon[106070]: pgmap v92: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T04:16:58.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:57 smithi029 ceph-mon[216903]: pgmap v92: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T04:16:58.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:57 smithi029 ceph-mon[223858]: pgmap v92: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-10-09T04:16:59.028 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 04:16:58 smithi177 conmon[179305]: ts=2023-10-09T04:16:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi177.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-09T04:16:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:58 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:16:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:16:59 smithi029 ceph-mon[216903]: pgmap v93: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:17:00.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:16:59 smithi029 ceph-mon[223858]: pgmap v93: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:17:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:16:59 smithi177 ceph-mon[106070]: pgmap v93: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:17:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:01 smithi029 ceph-mon[216903]: pgmap v94: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2023-10-09T04:17:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:01 smithi029 ceph-mon[223858]: pgmap v94: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2023-10-09T04:17:02.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:01 smithi177 ceph-mon[106070]: pgmap v94: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2023-10-09T04:17:03.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:17:02] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-10-09T04:17:04.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:03 smithi029 ceph-mon[216903]: pgmap v95: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:03 smithi029 ceph-mon[223858]: pgmap v95: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:03 smithi177 ceph-mon[106070]: pgmap v95: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:06.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:05 smithi029 ceph-mon[216903]: pgmap v96: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:06.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:05 smithi029 ceph-mon[223858]: pgmap v96: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:06.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:05 smithi177 ceph-mon[106070]: pgmap v96: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:08.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:07 smithi029 ceph-mon[216903]: pgmap v97: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T04:17:08.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:07 smithi029 ceph-mon[223858]: pgmap v97: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T04:17:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:07 smithi177 ceph-mon[106070]: pgmap v97: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2023-10-09T04:17:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:09 smithi029 ceph-mon[216903]: pgmap v98: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:09 smithi029 ceph-mon[223858]: pgmap v98: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:10.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:09 smithi177 ceph-mon[106070]: pgmap v98: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-09T04:17:12.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:11 smithi029 ceph-mon[216903]: pgmap v99: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2023-10-09T04:17:12.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:11 smithi029 ceph-mon[223858]: pgmap v99: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2023-10-09T04:17:12.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:11 smithi177 ceph-mon[106070]: pgmap v99: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2023-10-09T04:17:13.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:17:12] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-10-09T04:17:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:13 smithi029 ceph-mon[216903]: pgmap v100: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:17:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:13 smithi029 ceph-mon[223858]: pgmap v100: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:17:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:13 smithi177 ceph-mon[106070]: pgmap v100: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:17:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:13 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:15.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:14 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:15 smithi029 ceph-mon[223858]: pgmap v101: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:17:16.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:15 smithi029 ceph-mon[216903]: pgmap v101: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:17:16.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:15 smithi177 ceph-mon[106070]: pgmap v101: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-09T04:17:17.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:17:18.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:17 smithi029 ceph-mon[223858]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-10-09T04:17:18.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:17 smithi029 ceph-mon[216903]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-10-09T04:17:18.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:17 smithi177 ceph-mon[106070]: pgmap v102: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-10-09T04:17:18.861 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:19.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:19 smithi029 ceph-mon[223858]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:17:19.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:19.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:19.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:19 smithi029 ceph-mon[216903]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:17:19.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:19.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:19.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:19 smithi177 ceph-mon[106070]: pgmap v103: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-09T04:17:19.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:19 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:19.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:19 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:21 smithi029 ceph-mon[216903]: pgmap v104: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T04:17:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:21 smithi029 ceph-mon[223858]: pgmap v104: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T04:17:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:21 smithi177 ceph-mon[106070]: pgmap v104: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2023-10-09T04:17:22.401 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:23.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:17:22] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:17:23.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[223858]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:23.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[223858]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:23.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:23.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[223858]: pgmap v105: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T04:17:23.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[216903]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:23.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[216903]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:23.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:23.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:23 smithi029 ceph-mon[216903]: pgmap v105: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T04:17:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:23 smithi177 ceph-mon[106070]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:23 smithi177 ceph-mon[106070]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:23.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:23 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:23.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:23 smithi177 ceph-mon[106070]: pgmap v105: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-09T04:17:23.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for all mons to upgrade' 2023-10-09T04:17:24.862 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:25.752 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:25 smithi029 ceph-mon[223858]: pgmap v106: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:17:25.756 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:25 smithi029 ceph-mon[216903]: pgmap v106: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:17:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:25 smithi177 ceph-mon[106070]: pgmap v106: 89 pgs: 89 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-09T04:17:27.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:27 smithi029 ceph-mon[216903]: pgmap v107: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-10-09T04:17:27.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:27 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:27.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:27 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:27.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:27 smithi029 ceph-mon[223858]: pgmap v107: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-10-09T04:17:27.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:27 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:27.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:27 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:27.852 INFO:teuthology.orchestra.run.smithi029.stdout:wait for all mons to upgrade 2023-10-09T04:17:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:27 smithi177 ceph-mon[106070]: pgmap v107: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2023-10-09T04:17:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:27 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:28.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:27 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:17:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:17:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:17:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:17:28.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-09T04:17:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:28 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:29.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-09T04:17:29.851 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:29 smithi029 ceph-mon[223858]: Upgrade: It appears safe to stop mon.b 2023-10-09T04:17:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:29 smithi029 ceph-mon[223858]: pgmap v108: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-10-09T04:17:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:29 smithi029 ceph-mon[216903]: Upgrade: It appears safe to stop mon.b 2023-10-09T04:17:29.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:29 smithi029 ceph-mon[216903]: pgmap v108: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-10-09T04:17:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:29 smithi177 ceph-mon[106070]: Upgrade: It appears safe to stop mon.b 2023-10-09T04:17:30.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:29 smithi177 ceph-mon[106070]: pgmap v108: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-10-09T04:17:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:31 smithi177 ceph-mon[106070]: pgmap v109: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T04:17:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:31 smithi029 ceph-mon[216903]: pgmap v109: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T04:17:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:31 smithi029 ceph-mon[223858]: pgmap v109: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2023-10-09T04:17:33.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:17:32] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-10-09T04:17:33.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: pgmap v110: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-10-09T04:17:33.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: Upgrade: Updating mon.b 2023-10-09T04:17:33.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:33.513 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: Deploying daemon mon.b on smithi177 2023-10-09T04:17:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:33 smithi177 ceph-mon[106070]: from='client.? 172.21.15.29:0/2273294976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: pgmap v110: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: Upgrade: Updating mon.b 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: Deploying daemon mon.b on smithi177 2023-10-09T04:17:33.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2273294976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: pgmap v110: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: Upgrade: Updating mon.b 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: Deploying daemon mon.b on smithi177 2023-10-09T04:17:33.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2273294976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:34.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:34 smithi177 systemd[1]: Stopping Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:17:34.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:34 smithi177 conmon[106046]: 2023-10-09T04:17:34.918+0000 7f37440de700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:17:34.921 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:34 smithi177 conmon[106046]: 2023-10-09T04:17:34.918+0000 7f37440de700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-09T04:17:35.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:35 smithi177 bash[195393]: f19800b5161693708d00ecf019d8edf8491f3fc2e39448428cef4ee2678bc212 2023-10-09T04:17:36.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:36 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-09T04:17:36.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:36 smithi177 systemd[1]: Stopped Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:17:36.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:36 smithi177 systemd[1]: Starting Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:17:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: set uid:gid to 167:167 (ceph:ceph) 2023-10-09T04:17:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable), process ceph-mon, pid 7 2023-10-09T04:17:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: pidfile_write: ignore empty --pid-file 2023-10-09T04:17:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: load: jerasure load: lrc 2023-10-09T04:17:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 bash[195734]: 8b10005e656535485ab06d18167ec102424da0a6387a643fabbddcb1142c2299 2023-10-09T04:17:37.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: RocksDB version: 7.9.2 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Git sha 0 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Compile date 2023-10-06 16:19:06 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: DB SUMMARY 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: DB Session ID: YZ59ZPSW1SJS1YNNE2H2 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: CURRENT file: CURRENT 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: IDENTITY file: IDENTITY 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1293 Bytes 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000030.sst 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000028.log size: 1053018 ; 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.error_if_exists: 0 2023-10-09T04:17:37.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.create_if_missing: 0 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.paranoid_checks: 1 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.env: 0x55cc8ff65720 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.fs: PosixFileSystem 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.info_log: 0x55cc929d5420 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_file_opening_threads: 16 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.statistics: (nil) 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.use_fsync: 0 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_log_file_size: 0 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-09T04:17:37.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.keep_log_file_num: 1000 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.recycle_log_file_num: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.allow_fallocate: 1 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.allow_mmap_reads: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.allow_mmap_writes: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.use_direct_reads: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.create_missing_column_families: 0 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.db_log_dir: 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.wal_dir: 2023-10-09T04:17:37.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.advise_random_on_open: 1 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.db_write_buffer_size: 0 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.write_buffer_manager: 0x55cc91c64c80 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-09T04:17:37.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.rate_limiter: (nil) 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.wal_recovery_mode: 2 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.enable_thread_tracking: 0 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.enable_pipelined_write: 0 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.unordered_write: 0 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.row_cache: None 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.wal_filter: None 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-09T04:17:37.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.allow_ingest_behind: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.two_write_queues: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.manual_wal_flush: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.wal_compression: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.atomic_flush: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.log_readahead_size: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.best_efforts_recovery: 0 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-09T04:17:37.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-09T04:17:37.538 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.allow_data_in_errors: 0 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.db_host_id: __hostname__ 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_background_jobs: 2 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_background_compactions: -1 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_subcompactions: 1 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-09T04:17:37.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_total_wal_size: 0 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_open_files: -1 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bytes_per_sync: 0 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_readahead_size: 0 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_background_flushes: -1 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Compression algorithms supported: 2023-10-09T04:17:37.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kZSTD supported: 0 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kXpressCompression supported: 0 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kLZ4HCCompression supported: 1 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kZlibCompression supported: 1 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kSnappyCompression supported: 1 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kLZ4Compression supported: 1 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: kBZip2Compression supported: 0 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-09T04:17:37.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.merge_operator: 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_filter: None 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_filter_factory: None 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.sst_partitioner_factory: None 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cc929d5520) 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout: cache_index_and_filter_blocks: 1 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout: pin_top_level_index_and_filter: 1 2023-10-09T04:17:37.542 INFO:journalctl@ceph.mon.b.smithi177.stdout: index_type: 0 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: data_block_index_type: 0 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: index_shortening: 1 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: checksum: 4 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: no_block_cache: 0 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache: 0x55cc91ce7350 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache_name: BinnedLRUCache 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache_options: 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: capacity : 536870912 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: num_shard_bits : 4 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: strict_capacity_limit : 0 2023-10-09T04:17:37.543 INFO:journalctl@ceph.mon.b.smithi177.stdout: high_pri_pool_ratio: 0.000 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_cache_compressed: (nil) 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: persistent_cache: (nil) 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_size: 4096 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_size_deviation: 10 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_restart_interval: 16 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: index_block_restart_interval: 1 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: metadata_block_size: 4096 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: partition_filters: 0 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: use_delta_encoding: 1 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: filter_policy: bloomfilter 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: whole_key_filtering: 1 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: verify_compression: 0 2023-10-09T04:17:37.544 INFO:journalctl@ceph.mon.b.smithi177.stdout: read_amp_bytes_per_bit: 0 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: format_version: 5 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: enable_index_compression: 1 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: block_align: 0 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: max_auto_readahead_size: 262144 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: prepopulate_block_cache: 0 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: initial_auto_readahead_size: 8192 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout: num_file_reads_for_auto_readahead: 2 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.write_buffer_size: 33554432 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_write_buffer_number: 2 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression: NoCompression 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression: Disabled 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.prefix_extractor: nullptr 2023-10-09T04:17:37.545 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.num_levels: 7 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-09T04:17:37.546 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.level: 32767 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.strategy: 0 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.enabled: false 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-09T04:17:37.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.target_file_size_base: 67108864 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-09T04:17:37.548 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.arena_block_size: 1048576 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.disable_auto_compactions: 0 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-09T04:17:37.549 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.table_properties_collectors: 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.inplace_update_support: 0 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-09T04:17:37.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.bloom_locality: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.max_successive_merges: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.paranoid_file_checks: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.force_consistency_checks: 1 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.report_bg_io_stats: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.ttl: 2592000 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.enable_blob_files: false 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.min_blob_size: 0 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.blob_file_size: 268435456 2023-10-09T04:17:37.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.blob_file_starting_level: 0 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 32, last_sequence is 17953, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a48596dd-d9f3-4dbf-a9b0-e1b5bb0abe2f 2023-10-09T04:17:37.552 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696825057102408, "job": 1, "event": "recovery_started", "wal_files": [28]} 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696825057107441, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 687595, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17958, "largest_seqno": 18830, "table_properties": {"data_size": 683777, "index_size": 2100, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 6051, "raw_average_key_size": 21, "raw_value_size": 677494, "raw_average_value_size": 2428, "num_data_blocks": 94, "num_entries": 279, "num_filter_entries": 279, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696825057, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a48596dd-d9f3-4dbf-a9b0-e1b5bb0abe2f", "db_session_id": "YZ59ZPSW1SJS1YNNE2H2", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696825057107555, "job": 1, "event": "recovery_finished"} 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 systemd[1]: Started Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55cc91d82000 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: DB pointer 0x55cc91d70000 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-09T04:17:37.553 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** DB Stats ** 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** Compaction Stats [default] ** 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-09T04:17:37.554 INFO:journalctl@ceph.mon.b.smithi177.stdout: L0 1/0 671.48 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 272.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: L6 1/0 8.75 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: Sum 2/0 9.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 272.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 272.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** Compaction Stats [default] ** 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 272.4 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.555 INFO:journalctl@ceph.mon.b.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: Cumulative compaction: 0.00 GB write, 44.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: Interval compaction: 0.00 GB write, 44.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: Block cache BinnedLRUCache@0x55cc91ce7350#7 capacity: 512.00 MB usage: 3.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.556 INFO:journalctl@ceph.mon.b.smithi177.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: starting mon.b rank 1 at public addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] at bind addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???) e3 preinit fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).mds e8 new map 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).mds e8 print_map 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: e8 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: legacy client fscid: 1 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: Filesystem 'foo' (1) 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: fs_name foo 2023-10-09T04:17:37.557 INFO:journalctl@ceph.mon.b.smithi177.stdout: epoch 7 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: created 2023-10-09T04:07:34.635169+0000 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: modified 2023-10-09T04:07:47.848728+0000 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: tableserver 0 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: root 0 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: session_timeout 60 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: session_autoclose 300 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: max_file_size 1099511627776 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: required_client_features {} 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: last_failure 0 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: last_failure_osd_epoch 0 2023-10-09T04:17:37.558 INFO:journalctl@ceph.mon.b.smithi177.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: max_mds 1 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: in 0 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: up {0=14649} 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: failed 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: damaged 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: stopped 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: data_pools [4] 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: metadata_pool 3 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: inline_data disabled 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: balancer 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: bal_rank_mask -1 2023-10-09T04:17:37.559 INFO:journalctl@ceph.mon.b.smithi177.stdout: standby_count_wanted 1 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout: [mds.foo.smithi029.mozfhx{0:14649} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.29:6834/3379855837,v1:172.21.15.29:6835/3379855837] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout: Standby daemons: 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout: 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout: [mds.foo.smithi177.pbovgs{-1:24547} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.177:6832/2076758848,v1:172.21.15.177:6833/2076758848] compat {c=[1],r=[1],i=[7ff]}] 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2023-10-09T04:17:37.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:37 smithi177 ceph-mon[195797]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2023-10-09T04:17:38.437 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:17:38.666 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:38 smithi029 ceph-mon[223858]: pgmap v111: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-10-09T04:17:38.667 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:38 smithi029 ceph-mon[216903]: pgmap v111: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-10-09T04:17:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:38 smithi177 ceph-mon[195797]: pgmap v111: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 522 op/s 2023-10-09T04:17:41.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:17:42.275 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:42.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:42 smithi029 conmon[202582]: 2023-10-09T04:17:42.137+0000 7f6049533700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-09T04:17:43.011 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:17:42] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2023-10-09T04:17:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: overall HEALTH_OK 2023-10-09T04:17:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mon.a calling monitor election 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mon.c calling monitor election 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mon.b calling monitor election 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/833795618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: pgmap v113: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 548 op/s 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: pgmap v114: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: pgmap v115: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: overall HEALTH_OK 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mon.a calling monitor election 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:17:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: monmap e4: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: overall HEALTH_OK 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: overall HEALTH_OK 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:43.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mon.a calling monitor election 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mon.c calling monitor election 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mon.b calling monitor election 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/833795618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: pgmap v113: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 548 op/s 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: pgmap v114: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: pgmap v115: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-10-09T04:17:43.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: overall HEALTH_OK 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mon.a calling monitor election 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: monmap e4: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: overall HEALTH_OK 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:43.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:43.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: overall HEALTH_OK 2023-10-09T04:17:43.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mon.a calling monitor election 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mon.c calling monitor election 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mon.b calling monitor election 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/833795618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: pgmap v113: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 548 op/s 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: pgmap v114: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: pgmap v115: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:17:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: overall HEALTH_OK 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mon.a calling monitor election 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: monmap e4: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: fsmap foo:1 {0=foo.smithi029.mozfhx=up:active} 1 up:standby 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: osdmap e82: 8 total, 8 up, 8 in 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: mgrmap e37: y(active, since 3m), standbys: x 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: overall HEALTH_OK 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:43.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:44.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:44.788 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:17:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:45.754 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (5m) - 11m 29.8M - 0.25.0 c8568f914cd2 7ab788eff7be 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi177 *:3000 running (5m) - 10m 97.5M - 9.4.7 2c41d148cca3 dbe79334ca87 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.mozfhx smithi029 running (10m) - 10m 20.1M - 17.2.6-1303-gb2903cb0 5517396c50d4 e346a0e9dcdd 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi177.pbovgs smithi177 running (10m) - 9m 16.0M - 17.2.6-1303-gb2903cb0 5517396c50d4 a20b9123e852 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi177 *:8443,9283,8765 running (4m) - 15m 448M - 18.2.0-598-gb4fa663f 3060897fdf97 c43f5ec5cd80 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (5m) - 16m 506M - 18.2.0-598-gb4fa663f 3060897fdf97 239eea6ccc4c 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (4m) - 16m 66.7M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8369665f49b3 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi177 starting - - - 2048M 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (112s) - 15m 49.0M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7e2b5984a986 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (5m) - 10m 17.4M - 1.5.0 0da6a335fe13 2be0e618a0f3 2023-10-09T04:17:45.755 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi177 *:9100 running (5m) - 10m 18.9M - 1.5.0 0da6a335fe13 2adf0c28b454 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (14m) - 14m 3286M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 2c8a6318fb47 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (14m) - 14m 2588M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 951ea50b247d 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (13m) - 13m 4302M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 7d2e4b8000f3 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (13m) - 13m 3340M 2501M 17.2.6-1303-gb2903cb0 5517396c50d4 e4ab91bdb38b 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi177 running (13m) - 13m 3432M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 d4b592bd1322 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi177 running (12m) - 12m 3870M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 28d4ab6e8bfe 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi177 running (12m) - 12m 1831M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 03ede6ceccc0 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi177 running (12m) - 12m 3398M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0f3217cc8fa1 2023-10-09T04:17:45.756 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi177 *:9095 running (4m) - 11m 63.9M - 2.43.0 a07b618ecd1d 2d31282ff477 2023-10-09T04:17:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:45 smithi029 ceph-mon[216903]: pgmap v116: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:17:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:45 smithi029 ceph-mon[223858]: pgmap v116: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:17:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:45 smithi177 ceph-mon[195797]: pgmap v116: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s 2023-10-09T04:17:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:46 smithi029 ceph-mon[216903]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:46 smithi029 ceph-mon[223858]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:46 smithi177 ceph-mon[195797]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:47.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:17:48.182 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:48 smithi029 ceph-mon[223858]: pgmap v117: 89 pgs: 89 active+clean; 842 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-09T04:17:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:48.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:48 smithi029 ceph-mon[216903]: pgmap v117: 89 pgs: 89 active+clean; 842 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-09T04:17:48.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:48.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:48 smithi177 ceph-mon[195797]: pgmap v117: 89 pgs: 89 active+clean; 842 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-09T04:17:48.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:48.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:49.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:49 smithi177 ceph-mon[195797]: pgmap v118: 89 pgs: 89 active+clean; 842 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:17:49.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:49 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:49.933 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:49 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:50.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:49 smithi029 ceph-mon[223858]: pgmap v118: 89 pgs: 89 active+clean; 842 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:17:50.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:49 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:50.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:49 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:50.265 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:49 smithi029 ceph-mon[216903]: pgmap v118: 89 pgs: 89 active+clean; 842 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-09T04:17:50.265 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:49 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:50.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:49 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:51.414 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:52.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:51 smithi177 ceph-mon[195797]: pgmap v119: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-10-09T04:17:52.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:51 smithi029 ceph-mon[216903]: pgmap v119: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-10-09T04:17:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:51 smithi029 ceph-mon[223858]: pgmap v119: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-10-09T04:17:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:52.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'sleep 60' 2023-10-09T04:17:53.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:52 smithi029 ceph-mon[216903]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:53.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:52 smithi029 ceph-mon[216903]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:53.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:17:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:17:52] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.43.0" 2023-10-09T04:17:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:52 smithi029 ceph-mon[223858]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:52 smithi029 ceph-mon[223858]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:52 smithi177 ceph-mon[195797]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:17:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:52 smithi177 ceph-mon[195797]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:17:53.586 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:17:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:53 smithi029 ceph-mon[216903]: pgmap v120: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:17:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:53 smithi029 ceph-mon[223858]: pgmap v120: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:17:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:53 smithi177 ceph-mon[195797]: pgmap v120: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:17:56.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: pgmap v121: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: Reconfiguring mon.a (monmap changed)... 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:56.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[216903]: Reconfiguring daemon mon.a on smithi029 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: pgmap v121: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: Reconfiguring mon.a (monmap changed)... 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:56.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:56.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:56.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:55 smithi029 ceph-mon[223858]: Reconfiguring daemon mon.a on smithi029 2023-10-09T04:17:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: pgmap v121: 89 pgs: 89 active+clean; 644 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: Reconfiguring mon.a (monmap changed)... 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:55 smithi177 ceph-mon[195797]: Reconfiguring daemon mon.a on smithi029 2023-10-09T04:17:58.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:57 smithi029 ceph-mon[216903]: pgmap v122: 89 pgs: 89 active+clean; 451 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T04:17:58.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:57 smithi029 ceph-mon[223858]: pgmap v122: 89 pgs: 89 active+clean; 451 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T04:17:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:57 smithi177 ceph-mon[195797]: pgmap v122: 89 pgs: 89 active+clean; 451 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 814 op/s 2023-10-09T04:17:59.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:59.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:59.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:17:59.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: Reconfiguring daemon mgr.y on smithi029 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:17:59.202 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:17:59.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:59.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: Reconfiguring daemon mgr.y on smithi029 2023-10-09T04:17:59.203 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:17:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:17:59.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:59.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:17:59.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: Reconfiguring mgr.y (monmap changed)... 2023-10-09T04:17:59.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:17:59.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:17:59.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:17:59.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: Reconfiguring daemon mgr.y on smithi029 2023-10-09T04:17:59.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:17:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:00.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:00 smithi177 ceph-mon[195797]: pgmap v123: 89 pgs: 89 active+clean; 451 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s 2023-10-09T04:18:00.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:00 smithi029 ceph-mon[223858]: pgmap v123: 89 pgs: 89 active+clean; 451 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s 2023-10-09T04:18:00.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:00 smithi029 ceph-mon[216903]: pgmap v123: 89 pgs: 89 active+clean; 451 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 520 op/s 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: pgmap v124: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:01.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[216903]: Reconfiguring daemon mon.c on smithi029 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: pgmap v124: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:01.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:01 smithi029 ceph-mon[223858]: Reconfiguring daemon mon.c on smithi029 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: pgmap v124: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: Reconfiguring mon.c (monmap changed)... 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:01 smithi177 ceph-mon[195797]: Reconfiguring daemon mon.c on smithi029 2023-10-09T04:18:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:18:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:18:02] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.43.0" 2023-10-09T04:18:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:03 smithi177 ceph-mon[195797]: pgmap v125: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:18:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:18:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:04.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[216903]: pgmap v125: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[223858]: pgmap v125: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:18:04.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:04.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:04 smithi029 ceph-mon[223858]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:18:04.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:04 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.0 on smithi029 2023-10-09T04:18:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:04 smithi177 ceph-mon[195797]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:18:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:04 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.0 on smithi029 2023-10-09T04:18:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:04 smithi029 ceph-mon[216903]: Reconfiguring osd.0 (monmap changed)... 2023-10-09T04:18:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:04 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.0 on smithi029 2023-10-09T04:18:05.285 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 120944 objects 2023-10-09T04:18:05.285 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :167.383 2023-10-09T04:18:05.991 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:05 smithi029 ceph-mon[216903]: pgmap v126: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:18:05.992 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:05 smithi029 ceph-mon[223858]: pgmap v126: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:18:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:05 smithi177 ceph-mon[195797]: pgmap v126: 89 pgs: 89 active+clean; 239 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s 2023-10-09T04:18:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[216903]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:18:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.1 on smithi029 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[223858]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:07 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.1 on smithi029 2023-10-09T04:18:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:07 smithi177 ceph-mon[195797]: Reconfiguring osd.1 (monmap changed)... 2023-10-09T04:18:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:18:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:07 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.1 on smithi029 2023-10-09T04:18:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[216903]: pgmap v127: 89 pgs: 89 active+clean; 26 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T04:18:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:18:08.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[223858]: pgmap v127: 89 pgs: 89 active+clean; 26 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T04:18:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:18:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:08 smithi177 ceph-mon[195797]: pgmap v127: 89 pgs: 89 active+clean; 26 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-10-09T04:18:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:18:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:09 smithi029 ceph-mon[223858]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:18:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:09 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.2 on smithi029 2023-10-09T04:18:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:09 smithi029 ceph-mon[216903]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:18:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:09 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.2 on smithi029 2023-10-09T04:18:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:09 smithi177 ceph-mon[195797]: Reconfiguring osd.2 (monmap changed)... 2023-10-09T04:18:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:09 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.2 on smithi029 2023-10-09T04:18:10.144 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-09T04:18:10.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-09T04:18:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: pgmap v128: 89 pgs: 89 active+clean; 26 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:18:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.3 on smithi029 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: pgmap v128: 89 pgs: 89 active+clean; 26 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:18:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:10.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:10 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.3 on smithi029 2023-10-09T04:18:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: pgmap v128: 89 pgs: 89 active+clean; 26 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:18:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: Reconfiguring osd.3 (monmap changed)... 2023-10-09T04:18:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:18:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:10 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.3 on smithi029 2023-10-09T04:18:10.648 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: pgmap v129: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:18:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: Reconfiguring mds.foo.smithi029.mozfhx (monmap changed)... 2023-10-09T04:18:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:18:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:11 smithi177 ceph-mon[195797]: Reconfiguring daemon mds.foo.smithi029.mozfhx on smithi029 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: pgmap v129: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: Reconfiguring mds.foo.smithi029.mozfhx (monmap changed)... 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:12.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[216903]: Reconfiguring daemon mds.foo.smithi029.mozfhx on smithi029 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: pgmap v129: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: Reconfiguring mds.foo.smithi029.mozfhx (monmap changed)... 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.mozfhx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:12.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:11 smithi029 ceph-mon[223858]: Reconfiguring daemon mds.foo.smithi029.mozfhx on smithi029 2023-10-09T04:18:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:18:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:18:12] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.43.0" 2023-10-09T04:18:13.804 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_1' removed 2023-10-09T04:18:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 317 op/s 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: Reconfiguring daemon mon.b on smithi177 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/247840144' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:18:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:14.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 317 op/s 2023-10-09T04:18:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:18:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:14.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: Reconfiguring daemon mon.b on smithi177 2023-10-09T04:18:14.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/247840144' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:18:14.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:18:14.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:14.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 317 op/s 2023-10-09T04:18:14.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:14.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:14.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: Reconfiguring mon.b (monmap changed)... 2023-10-09T04:18:14.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-09T04:18:14.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-09T04:18:14.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:14.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: Reconfiguring daemon mon.b on smithi177 2023-10-09T04:18:14.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/247840144' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:18:14.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:18:14.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:14.641 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:18:14.641 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:18:14.642 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-09T04:18:14.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-09T04:18:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[216903]: failed to encode map e83 with expected crc 2023-10-09T04:18:14.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[216903]: failed to encode map e83 with expected crc 2023-10-09T04:18:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:18:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[216903]: osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:18:14.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[223858]: failed to encode map e83 with expected crc 2023-10-09T04:18:14.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[223858]: failed to encode map e83 with expected crc 2023-10-09T04:18:14.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:18:14.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:14 smithi029 ceph-mon[223858]: osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:18:15.077 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:15.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:14 smithi177 ceph-mon[195797]: failed to encode map e83 with expected crc 2023-10-09T04:18:15.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:14 smithi177 ceph-mon[195797]: failed to encode map e83 with expected crc 2023-10-09T04:18:15.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:18:15.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:14 smithi177 ceph-mon[195797]: osdmap e83: 8 total, 8 up, 8 in 2023-10-09T04:18:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: failed to encode map e83 with expected crc 2023-10-09T04:18:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[216903]: Reconfiguring daemon mgr.x on smithi177 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: failed to encode map e83 with expected crc 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:18:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:18:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:18:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:15 smithi029 ceph-mon[223858]: Reconfiguring daemon mgr.x on smithi177 2023-10-09T04:18:15.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: failed to encode map e83 with expected crc 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: Reconfiguring mgr.x (monmap changed)... 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:15.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:15 smithi177 ceph-mon[195797]: Reconfiguring daemon mgr.x on smithi177 2023-10-09T04:18:17.719 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:17.719 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:17.719 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:17.719 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:18:17.719 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:18:17.719 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:17.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.4 on smithi177 2023-10-09T04:18:17.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1414025227' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:18:17.812 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_2' created 2023-10-09T04:18:17.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:17.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:17.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:17.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:18:17.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:18:17.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:17.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.4 on smithi177 2023-10-09T04:18:17.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1414025227' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:18:17.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: pgmap v133: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:17.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:17.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:17.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: Reconfiguring osd.4 (monmap changed)... 2023-10-09T04:18:17.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-09T04:18:17.847 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:17.847 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.4 on smithi177 2023-10-09T04:18:17.847 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1414025227' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-09T04:18:18.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-09T04:18:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: failed to encode map e84 with expected crc 2023-10-09T04:18:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: failed to encode map e84 with expected crc 2023-10-09T04:18:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1414025227' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:18:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.5 on smithi177 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: failed to encode map e84 with expected crc 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: failed to encode map e84 with expected crc 2023-10-09T04:18:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1414025227' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:18 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.5 on smithi177 2023-10-09T04:18:19.004 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:18:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: failed to encode map e84 with expected crc 2023-10-09T04:18:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: failed to encode map e84 with expected crc 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1414025227' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: osdmap e84: 8 total, 8 up, 8 in 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: Reconfiguring osd.5 (monmap changed)... 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:18 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.5 on smithi177 2023-10-09T04:18:19.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: pgmap v135: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:19.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: failed to encode map e85 with expected crc 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:19.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:18:19.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:18:19.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:19.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:19 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.6 on smithi177 2023-10-09T04:18:20.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: pgmap v135: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.6 on smithi177 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: pgmap v135: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: osdmap e85: 8 total, 8 up, 8 in 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: failed to encode map e85 with expected crc 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:20.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: Reconfiguring osd.6 (monmap changed)... 2023-10-09T04:18:20.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-09T04:18:20.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:20.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:19 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.6 on smithi177 2023-10-09T04:18:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: failed to encode map e86 with expected crc 2023-10-09T04:18:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: failed to encode map e86 with expected crc 2023-10-09T04:18:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T04:18:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:21.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:21.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:18:21.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: failed to encode map e86 with expected crc 2023-10-09T04:18:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: failed to encode map e86 with expected crc 2023-10-09T04:18:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T04:18:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:18:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:21.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: failed to encode map e86 with expected crc 2023-10-09T04:18:21.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: failed to encode map e86 with expected crc 2023-10-09T04:18:21.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: osdmap e86: 8 total, 8 up, 8 in 2023-10-09T04:18:21.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:21.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:21.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-09T04:18:21.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:21.830 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-09T04:18:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[216903]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[216903]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[216903]: Reconfiguring daemon osd.7 on smithi177 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/198982032' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[216903]: failed to encode map e86 with expected crc 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[223858]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[223858]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[223858]: Reconfiguring daemon osd.7 on smithi177 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/198982032' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:21 smithi029 ceph-mon[223858]: failed to encode map e86 with expected crc 2023-10-09T04:18:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:21 smithi177 ceph-mon[195797]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:21 smithi177 ceph-mon[195797]: Reconfiguring osd.7 (monmap changed)... 2023-10-09T04:18:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:21 smithi177 ceph-mon[195797]: Reconfiguring daemon osd.7 on smithi177 2023-10-09T04:18:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/198982032' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:18:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:21 smithi177 ceph-mon[195797]: failed to encode map e86 with expected crc 2023-10-09T04:18:23.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: failed to encode map e86 with expected crc 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: failed to encode map e87 with expected crc 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: failed to encode map e87 with expected crc 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/198982032' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: osdmap e87: 8 total, 8 up, 8 in 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: Reconfiguring mds.foo.smithi177.pbovgs (monmap changed)... 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[216903]: Reconfiguring daemon mds.foo.smithi177.pbovgs on smithi177 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:18:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:18:22] "GET /metrics HTTP/1.1" 200 36162 "" "Prometheus/2.43.0" 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: failed to encode map e86 with expected crc 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: failed to encode map e87 with expected crc 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: failed to encode map e87 with expected crc 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/198982032' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: osdmap e87: 8 total, 8 up, 8 in 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:23.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: Reconfiguring mds.foo.smithi177.pbovgs (monmap changed)... 2023-10-09T04:18:23.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:18:23.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:23.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:22 smithi029 ceph-mon[223858]: Reconfiguring daemon mds.foo.smithi177.pbovgs on smithi177 2023-10-09T04:18:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: failed to encode map e86 with expected crc 2023-10-09T04:18:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: failed to encode map e87 with expected crc 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: failed to encode map e87 with expected crc 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/198982032' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: osdmap e87: 8 total, 8 up, 8 in 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: Reconfiguring mds.foo.smithi177.pbovgs (monmap changed)... 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi177.pbovgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:22 smithi177 ceph-mon[195797]: Reconfiguring daemon mds.foo.smithi177.pbovgs on smithi177 2023-10-09T04:18:23.723 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-09T04:18:23.725 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:18:23.725 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:18:23.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:18:23.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:18:23.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_237125 2023-10-09T04:18:23.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:18:23.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: pgmap v140: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:18:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: pgmap v140: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:18:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: pgmap v140: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:18:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:18:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:18:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:24.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3641 3625 226.55 226.562 0.00614394 0.00439498 2023-10-09T04:18:25.781 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 7035 7019 219.328 212.125 0.00383611 0.00454844 2023-10-09T04:18:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:25 smithi029 ceph-mon[216903]: pgmap v141: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:25 smithi029 ceph-mon[223858]: pgmap v141: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:25 smithi177 ceph-mon[195797]: pgmap v141: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:18:26.782 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10170 10154 211.462 195.938 0.00753896 0.00471886 2023-10-09T04:18:27.784 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 15 13121 13106 204.646 184.5 0.00804541 0.00488086 2023-10-09T04:18:28.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:27 smithi029 ceph-mon[216903]: pgmap v142: 89 pgs: 89 active+clean; 194 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 403 op/s 2023-10-09T04:18:28.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:27 smithi029 ceph-mon[223858]: pgmap v142: 89 pgs: 89 active+clean; 194 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 403 op/s 2023-10-09T04:18:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:27 smithi177 ceph-mon[195797]: pgmap v142: 89 pgs: 89 active+clean; 194 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 403 op/s 2023-10-09T04:18:28.784 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 15836 15820 197.639 169.625 0.00538853 0.00505164 2023-10-09T04:18:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:29.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:29.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:18:29.786 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 18428 18412 191.631 162 0.00514308 0.005211 2023-10-09T04:18:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:29 smithi177 ceph-mon[195797]: pgmap v143: 89 pgs: 89 active+clean; 194 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s wr, 357 op/s 2023-10-09T04:18:30.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:29 smithi029 ceph-mon[223858]: pgmap v143: 89 pgs: 89 active+clean; 194 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s wr, 357 op/s 2023-10-09T04:18:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:29 smithi029 ceph-mon[216903]: pgmap v143: 89 pgs: 89 active+clean; 194 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s wr, 357 op/s 2023-10-09T04:18:30.786 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 20888 20872 186.22 153.75 0.00533225 0.00536493 2023-10-09T04:18:31.786 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 23231 23215 181.248 146.438 0.00495751 0.00551082 2023-10-09T04:18:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:31 smithi177 ceph-mon[195797]: pgmap v144: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:18:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:31 smithi029 ceph-mon[216903]: pgmap v144: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:18:32.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:31 smithi029 ceph-mon[223858]: pgmap v144: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-10-09T04:18:32.787 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 25495 25479 176.828 141.5 0.0112254 0.00564898 2023-10-09T04:18:33.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:18:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:18:32] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.43.0" 2023-10-09T04:18:33.787 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 27651 27635 172.62 134.75 0.00327972 0.0057883 2023-10-09T04:18:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:33 smithi177 ceph-mon[195797]: pgmap v145: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:18:34.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:33 smithi029 ceph-mon[216903]: pgmap v145: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:18:34.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:33 smithi029 ceph-mon[223858]: pgmap v145: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-09T04:18:34.787 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 29692 29676 168.525 127.562 0.00246605 0.00592834 2023-10-09T04:18:35.787 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 31701 31685 164.945 125.562 0.00271182 0.00605713 2023-10-09T04:18:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:35 smithi177 ceph-mon[195797]: pgmap v146: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:18:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:35 smithi029 ceph-mon[216903]: pgmap v146: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:18:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:35 smithi029 ceph-mon[223858]: pgmap v146: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:18:36.787 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 33639 33623 161.573 121.125 0.010559 0.00618449 2023-10-09T04:18:37.787 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 35454 35438 158.135 113.438 0.00405006 0.00631793 2023-10-09T04:18:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:37 smithi177 ceph-mon[195797]: pgmap v147: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2023-10-09T04:18:38.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:37 smithi029 ceph-mon[216903]: pgmap v147: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2023-10-09T04:18:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:37 smithi029 ceph-mon[223858]: pgmap v147: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s wr, 2.44k op/s 2023-10-09T04:18:38.788 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 37284 37268 155.218 114.375 0.00628413 0.00643665 2023-10-09T04:18:39.791 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 39058 39042 152.417 110.875 0.0116918 0.0065556 2023-10-09T04:18:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:39 smithi177 ceph-mon[195797]: pgmap v148: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T04:18:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:39 smithi029 ceph-mon[216903]: pgmap v148: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T04:18:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:39 smithi029 ceph-mon[223858]: pgmap v148: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-09T04:18:40.791 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 40773 40757 149.757 107.188 0.00789217 0.00667279 2023-10-09T04:18:41.791 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 42469 42453 147.326 106 0.00485013 0.00678249 2023-10-09T04:18:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:41 smithi177 ceph-mon[195797]: pgmap v149: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2023-10-09T04:18:42.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:41 smithi029 ceph-mon[216903]: pgmap v149: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2023-10-09T04:18:42.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:41 smithi029 ceph-mon[223858]: pgmap v149: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2023-10-09T04:18:42.791 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 43654 43638 143.472 74.0625 0.00225488 0.00693305 2023-10-09T04:18:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:18:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:18:42] "GET /metrics HTTP/1.1" 200 36201 "" "Prometheus/2.43.0" 2023-10-09T04:18:43.794 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:18:43.792628+0000 min lat: 0.00115347 max lat: 0.187975 avg lat: 0.00707332 2023-10-09T04:18:43.794 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:18:43.794 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 45250 45234 141.28 99.75 0.0170397 0.00707332 2023-10-09T04:18:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:43 smithi177 ceph-mon[195797]: pgmap v150: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:18:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:43 smithi029 ceph-mon[216903]: pgmap v150: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:18:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:43 smithi029 ceph-mon[223858]: pgmap v150: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:18:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:44.792 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 46832 46816 139.261 98.875 0.00451305 0.00717439 2023-10-09T04:18:45.792 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 48392 48376 137.363 97.5 0.00599543 0.00727527 2023-10-09T04:18:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:45 smithi177 ceph-mon[195797]: pgmap v151: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:18:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:45 smithi029 ceph-mon[216903]: pgmap v151: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:18:46.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:45 smithi029 ceph-mon[223858]: pgmap v151: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:18:46.792 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 49902 49886 135.494 94.375 0.0149381 0.00737536 2023-10-09T04:18:47.792 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 51408 51392 133.771 94.125 0.00247783 0.00746934 2023-10-09T04:18:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:48 smithi177 ceph-mon[195797]: pgmap v152: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T04:18:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:48 smithi029 ceph-mon[216903]: pgmap v152: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T04:18:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:48 smithi029 ceph-mon[223858]: pgmap v152: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-10-09T04:18:48.793 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 52851 52835 132.027 90.1875 0.0130428 0.00756941 2023-10-09T04:18:49.793 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 54296 54280 130.423 90.3125 0.0108001 0.00766208 2023-10-09T04:18:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:50 smithi177 ceph-mon[195797]: pgmap v153: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:18:50.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:50 smithi029 ceph-mon[216903]: pgmap v153: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:18:50.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:50 smithi029 ceph-mon[223858]: pgmap v153: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2023-10-09T04:18:50.796 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 55720 55704 128.872 89 0.0169659 0.00775413 2023-10-09T04:18:51.796 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 15 57126 57111 127.411 87.9375 0.0126708 0.00784393 2023-10-09T04:18:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:52 smithi177 ceph-mon[195797]: pgmap v154: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T04:18:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:52 smithi029 ceph-mon[216903]: pgmap v154: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T04:18:52.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:52 smithi029 ceph-mon[223858]: pgmap v154: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-09T04:18:52.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 58504 58488 125.985 86.0625 0.0108521 0.00793255 2023-10-09T04:18:53.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:18:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:18:52] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2023-10-09T04:18:53.798 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 59861 59845 124.613 84.8125 0.00832318 0.00802051 2023-10-09T04:18:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:54 smithi177 ceph-mon[195797]: pgmap v155: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:18:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:54 smithi029 ceph-mon[216903]: pgmap v155: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:18:54.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:54 smithi029 ceph-mon[223858]: pgmap v155: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:18:54.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 61154 61138 123.201 80.8125 0.0161387 0.00811178 2023-10-09T04:18:55.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 62477 62461 121.935 82.6875 0.0171859 0.00819607 2023-10-09T04:18:56.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:56 smithi029 ceph-mon[216903]: pgmap v156: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:18:56.378 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:56 smithi029 ceph-mon[223858]: pgmap v156: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:18:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:56 smithi177 ceph-mon[195797]: pgmap v156: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-09T04:18:56.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 63755 63739 120.66 79.875 0.00685937 0.00828192 2023-10-09T04:18:57.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 65039 65023 119.472 80.25 0.0152754 0.00836422 2023-10-09T04:18:58.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:58 smithi029 ceph-mon[216903]: pgmap v157: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-10-09T04:18:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:58 smithi029 ceph-mon[223858]: pgmap v157: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-10-09T04:18:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:58 smithi177 ceph-mon[195797]: pgmap v157: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-10-09T04:18:58.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 66093 66077 117.941 65.875 0.00276173 0.00844706 2023-10-09T04:18:59.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:18:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:59.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:18:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:18:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:18:59.797 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 67237 67221 116.651 71.5 0.00904623 0.00856687 2023-10-09T04:19:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:00 smithi029 ceph-mon[216903]: pgmap v158: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T04:19:00.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:00 smithi029 ceph-mon[223858]: pgmap v158: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T04:19:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:00 smithi177 ceph-mon[195797]: pgmap v158: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-09T04:19:00.799 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 68233 68217 115.176 62.25 0.0140912 0.00867761 2023-10-09T04:19:01.801 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 69453 69437 114.146 76.25 0.0106927 0.00875545 2023-10-09T04:19:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:02 smithi029 ceph-mon[223858]: pgmap v159: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:19:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:02 smithi029 ceph-mon[216903]: pgmap v159: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:19:02.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:02 smithi177 ceph-mon[195797]: pgmap v159: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:19:02.802 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 70651 70635 113.139 74.875 0.00962356 0.00883365 2023-10-09T04:19:03.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:19:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:19:02] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-10-09T04:19:03.804 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:19:03.803033+0000 min lat: 0.00115347 max lat: 0.228138 avg lat: 0.00891103 2023-10-09T04:19:03.805 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:19:03.805 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 71838 71822 112.162 74.1875 0.0085022 0.00891103 2023-10-09T04:19:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:04 smithi029 ceph-mon[216903]: pgmap v160: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:19:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:04 smithi029 ceph-mon[223858]: pgmap v160: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:19:04.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:04 smithi177 ceph-mon[195797]: pgmap v160: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:19:04.803 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 73005 72989 111.206 72.9375 0.0200811 0.00898723 2023-10-09T04:19:05.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade resume' 2023-10-09T04:19:05.803 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 74165 74149 110.285 72.5 0.0109394 0.00906229 2023-10-09T04:19:06.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:06 smithi029 ceph-mon[216903]: pgmap v161: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:19:06.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:06 smithi029 ceph-mon[223858]: pgmap v161: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:19:06.488 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:06 smithi177 ceph-mon[195797]: pgmap v161: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:19:06.803 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 75193 75177 109.213 64.25 0.0231682 0.00915086 2023-10-09T04:19:07.804 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 76288 76272 108.286 68.4375 0.00932451 0.00922998 2023-10-09T04:19:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:08 smithi029 ceph-mon[223858]: pgmap v162: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T04:19:08.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:08 smithi029 ceph-mon[216903]: pgmap v162: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T04:19:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:08 smithi177 ceph-mon[195797]: pgmap v162: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-09T04:19:08.804 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 77169 77153 107.104 55.0625 0.00563063 0.00928514 2023-10-09T04:19:09.804 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 77952 77936 105.839 48.9375 0.015007 0.00944301 2023-10-09T04:19:10.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:10 smithi029 ceph-mon[216903]: pgmap v163: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T04:19:10.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:10 smithi029 ceph-mon[223858]: pgmap v163: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T04:19:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:10 smithi177 ceph-mon[195797]: pgmap v163: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-10-09T04:19:10.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 78960 78944 104.922 63 0.0127871 0.00952537 2023-10-09T04:19:10.840 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:19:11.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 79918 79902 103.983 59.875 0.00339543 0.00959885 2023-10-09T04:19:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[216903]: pgmap v164: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:19:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[216903]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:19:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[216903]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:19:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:12.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[223858]: pgmap v164: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:19:12.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[223858]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:19:12.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[223858]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:19:12.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:12.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:11 smithi177 ceph-mon[195797]: pgmap v164: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:19:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:11 smithi177 ceph-mon[195797]: from='client.54133 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:19:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:11 smithi177 ceph-mon[195797]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:19:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:12.809 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 80564 80548 102.682 40.375 0.00696592 0.00973402 2023-10-09T04:19:12.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'echo wait for half of osds to upgrade' 2023-10-09T04:19:13.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:19:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:19:12] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.43.0" 2023-10-09T04:19:13.810 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 81433 81417 101.712 54.3125 0.0116876 0.00982547 2023-10-09T04:19:13.836 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:14.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:13 smithi029 ceph-mon[216903]: pgmap v165: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-10-09T04:19:14.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:14.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:13 smithi029 ceph-mon[223858]: pgmap v165: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-10-09T04:19:14.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:13 smithi177 ceph-mon[195797]: pgmap v165: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-10-09T04:19:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:14.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 82375 82359 100.872 58.875 0.0191945 0.00990921 2023-10-09T04:19:15.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 83294 83278 100.037 57.4375 0.0148183 0.00996617 2023-10-09T04:19:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:15 smithi029 ceph-mon[223858]: pgmap v166: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T04:19:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:15 smithi029 ceph-mon[216903]: pgmap v166: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T04:19:16.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:16.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:15 smithi177 ceph-mon[195797]: pgmap v166: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-09T04:19:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:16.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 83985 83969 98.9652 43.1875 0.00556905 0.0101 2023-10-09T04:19:17.434 INFO:teuthology.orchestra.run.smithi029.stdout:wait for half of osds to upgrade 2023-10-09T04:19:17.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 84944 84928 98.2427 59.9375 0.0188762 0.0101738 2023-10-09T04:19:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:17 smithi029 ceph-mon[216903]: pgmap v167: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:19:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:17 smithi029 ceph-mon[223858]: pgmap v167: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:19:18.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:17 smithi177 ceph-mon[195797]: pgmap v167: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:19:18.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 85662 85646 97.2727 44.875 0.0199066 0.0102744 2023-10-09T04:19:18.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-09T04:19:19.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 86537 86521 96.5125 54.6875 0.0182263 0.0103565 2023-10-09T04:19:19.950 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:19:20.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:19 smithi029 ceph-mon[216903]: pgmap v168: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2023-10-09T04:19:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:19 smithi029 ceph-mon[223858]: pgmap v168: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2023-10-09T04:19:20.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:19 smithi177 ceph-mon[195797]: pgmap v168: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2023-10-09T04:19:20.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 87425 87409 95.7932 55.5 0.00455465 0.0104299 2023-10-09T04:19:21.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 88279 88263 95.062 53.375 0.00498662 0.0104885 2023-10-09T04:19:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:21 smithi029 ceph-mon[223858]: pgmap v169: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:19:22.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:21 smithi029 ceph-mon[216903]: pgmap v169: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:19:22.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:21 smithi177 ceph-mon[195797]: pgmap v169: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:19:22.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 89076 89060 94.2953 49.8125 0.00857145 0.0106 2023-10-09T04:19:22.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:19:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:19:22] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.43.0" 2023-10-09T04:19:23.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:19:23.811899+0000 min lat: 0.00115347 max lat: 0.53106 avg lat: 0.010682 2023-10-09T04:19:23.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:19:23.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 89886 89870 93.5677 50.625 0.0211856 0.010682 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: pgmap v170: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4269540022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: pgmap v170: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4269540022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: pgmap v170: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T04:19:24.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4269540022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:24.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:24.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:19:24.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:24.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 90764 90748 92.9296 54.875 0.00864969 0.0107557 2023-10-09T04:19:25.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:25.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: Upgrade: Setting container_image for all mon 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: Upgrade: Setting container_image for all crash 2023-10-09T04:19:25.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:25.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:19:25.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:25.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: Upgrade: Setting container_image for all mon 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: Upgrade: Setting container_image for all crash 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:19:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: Upgrade: Setting container_image for all mon 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: Upgrade: Setting container_image for all crash 2023-10-09T04:19:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:25.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:19:25.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 91675 91659 92.3423 56.9375 0.0171637 0.0108241 2023-10-09T04:19:26.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:26 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:19:26.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:26 smithi177 ceph-mon[195797]: Upgrade: osd.0 is safe to restart 2023-10-09T04:19:26.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:26 smithi177 ceph-mon[195797]: pgmap v171: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T04:19:26.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:26 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:19:26.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:26 smithi029 ceph-mon[216903]: Upgrade: osd.0 is safe to restart 2023-10-09T04:19:26.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:26 smithi029 ceph-mon[216903]: pgmap v171: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T04:19:26.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:26 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-09T04:19:26.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:26 smithi029 ceph-mon[223858]: Upgrade: osd.0 is safe to restart 2023-10-09T04:19:26.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:26 smithi029 ceph-mon[223858]: pgmap v171: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-10-09T04:19:26.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 92590 92574 91.7845 57.1875 0.0117942 0.0108903 2023-10-09T04:19:27.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1519203570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:27.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1519203570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:27.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1519203570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:27.822 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 93030 93014 90.7763 27.5 0.0105992 0.011011 2023-10-09T04:19:28.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[223858]: pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:19:28.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[223858]: Upgrade: Updating osd.0 2023-10-09T04:19:28.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:28.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:19:28.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:28.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[223858]: Deploying daemon osd.0 on smithi029 2023-10-09T04:19:28.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[216903]: pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:19:28.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[216903]: Upgrade: Updating osd.0 2023-10-09T04:19:28.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:28.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:19:28.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:28.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:28 smithi029 ceph-mon[216903]: Deploying daemon osd.0 on smithi029 2023-10-09T04:19:28.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:28 smithi177 ceph-mon[195797]: pgmap v172: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:19:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:28 smithi177 ceph-mon[195797]: Upgrade: Updating osd.0 2023-10-09T04:19:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-09T04:19:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:19:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:28 smithi177 ceph-mon[195797]: Deploying daemon osd.0 on smithi029 2023-10-09T04:19:28.825 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 93880 93864 90.193 53.125 0.0157158 0.0110811 2023-10-09T04:19:29.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:29.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:29 smithi029 ceph-mon[216903]: pgmap v173: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2023-10-09T04:19:29.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2420144845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:29.347 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:29.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:29 smithi029 ceph-mon[223858]: pgmap v173: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2023-10-09T04:19:29.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2420144845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:29.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:29.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:29 smithi177 ceph-mon[195797]: pgmap v173: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2023-10-09T04:19:29.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2420144845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:29.827 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 94776 94760 89.6731 56 0.016819 0.0111463 2023-10-09T04:19:30.827 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 95606 95590 89.1092 51.875 0.0186218 0.0112173 2023-10-09T04:19:31.827 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 96401 96385 88.5297 49.6875 0.01643 0.0112902 2023-10-09T04:19:31.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:31 smithi029 ceph-mon[216903]: pgmap v174: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:19:31.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1123871874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:31.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:31 smithi029 ceph-mon[223858]: pgmap v174: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:19:31.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1123871874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:31 smithi177 ceph-mon[195797]: pgmap v174: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-10-09T04:19:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1123871874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:32.827 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 97300 97284 88.0612 56.1875 0.0140342 0.0113503 2023-10-09T04:19:33.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:19:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:19:32] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:19:33.827 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 97924 97908 87.3606 39 0.0312786 0.0114413 2023-10-09T04:19:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:33 smithi177 ceph-mon[195797]: pgmap v175: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T04:19:34.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:33 smithi029 ceph-mon[223858]: pgmap v175: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T04:19:34.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:33 smithi029 ceph-mon[216903]: pgmap v175: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T04:19:34.423 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:34 smithi029 systemd[1]: Stopping Ceph osd.0 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:19:34.833 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 98697 98681 86.8061 48.3125 0.0103598 0.0115152 2023-10-09T04:19:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3771338656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:35.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3771338656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:35.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3771338656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:35.174 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:34 smithi029 conmon[125444]: 2023-10-09T04:19:34.792+0000 7fafc2de6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:19:35.174 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:34 smithi029 conmon[125444]: 2023-10-09T04:19:34.792+0000 7fafc2de6700 -1 osd.0 87 *** Got signal Terminated *** 2023-10-09T04:19:35.174 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:34 smithi029 conmon[125444]: 2023-10-09T04:19:34.792+0000 7fafc2de6700 -1 osd.0 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:19:35.831 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 99565 99549 86.354 54.25 0.00334278 0.0115635 2023-10-09T04:19:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:35 smithi177 ceph-mon[195797]: pgmap v176: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T04:19:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:35 smithi177 ceph-mon[195797]: osd.0 marked itself down and dead 2023-10-09T04:19:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:35 smithi029 ceph-mon[216903]: pgmap v176: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T04:19:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:35 smithi029 ceph-mon[216903]: osd.0 marked itself down and dead 2023-10-09T04:19:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:35 smithi029 ceph-mon[223858]: pgmap v176: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 686 op/s 2023-10-09T04:19:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:35 smithi029 ceph-mon[223858]: osd.0 marked itself down and dead 2023-10-09T04:19:36.831 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 99644 99628 85.2394 4.9375 0.015286 0.0117272 2023-10-09T04:19:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:19:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[216903]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T04:19:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[216903]: failed to encode map e88 with expected crc 2023-10-09T04:19:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[216903]: failed to encode map e88 with expected crc 2023-10-09T04:19:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/92866298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:36.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:19:36.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[223858]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T04:19:36.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[223858]: failed to encode map e88 with expected crc 2023-10-09T04:19:36.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[223858]: failed to encode map e88 with expected crc 2023-10-09T04:19:36.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/92866298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:37.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:36 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:19:37.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:36 smithi177 ceph-mon[195797]: osdmap e88: 8 total, 7 up, 8 in 2023-10-09T04:19:37.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:36 smithi177 ceph-mon[195797]: failed to encode map e88 with expected crc 2023-10-09T04:19:37.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:36 smithi177 ceph-mon[195797]: failed to encode map e88 with expected crc 2023-10-09T04:19:37.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/92866298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:37.833 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 100631 100615 84.9195 61.6875 0.0130409 0.0117716 2023-10-09T04:19:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: pgmap v178: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:37 smithi177 ceph-mon[195797]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: pgmap v178: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T04:19:38.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T04:19:38.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[216903]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: pgmap v178: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: osdmap e89: 8 total, 7 up, 8 in 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:37 smithi029 ceph-mon[223858]: failed to encode map e89 with expected crc 2023-10-09T04:19:38.422 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:38 smithi029 bash[240231]: 2c8a6318fb476faa3b784b85184bd00ff26a462b77c6eb0408e7a4712c5ff08c 2023-10-09T04:19:38.836 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 101700 101684 84.6748 66.8125 0.0213499 0.0118035 2023-10-09T04:19:39.837 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 102729 102713 84.4069 64.3125 0.00920906 0.0118428 2023-10-09T04:19:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:39 smithi177 ceph-mon[195797]: pgmap v180: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 31 MiB/s wr, 491 op/s 2023-10-09T04:19:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3968038188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:40.043 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:39 smithi029 ceph-mon[216903]: pgmap v180: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 31 MiB/s wr, 491 op/s 2023-10-09T04:19:40.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3968038188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:40.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:39 smithi029 ceph-mon[223858]: pgmap v180: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 31 MiB/s wr, 491 op/s 2023-10-09T04:19:40.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3968038188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:40.838 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 103849 103833 84.2199 70 0.0105478 0.0118695 2023-10-09T04:19:41.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:40 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 51395/307713 objects degraded (16.702%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:40 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 51395/307713 objects degraded (16.702%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:40 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 51395/307713 objects degraded (16.702%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:41.836 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 104937 104921 84.012 68 0.0113066 0.0118987 2023-10-09T04:19:42.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:41 smithi029 ceph-mon[216903]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:42.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1745457979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:41 smithi029 ceph-mon[223858]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1745457979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:42.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:41 smithi177 ceph-mon[195797]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:42.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1745457979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:42.842 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 106088 106072 83.8559 71.9375 0.0190433 0.0119205 2023-10-09T04:19:43.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:19:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:19:42] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2023-10-09T04:19:43.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:19:43.840125+0000 min lat: 0.00115347 max lat: 1.03698 avg lat: 0.011946 2023-10-09T04:19:43.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:19:43.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 107202 107186 83.6781 69.625 0.00607595 0.011946 2023-10-09T04:19:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:43 smithi029 ceph-mon[216903]: pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:44.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:43 smithi029 ceph-mon[223858]: pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:44.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:44.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:44.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:43 smithi177 ceph-mon[195797]: pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:44.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:44.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:44.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 108275 108259 83.473 67.0625 0.0060082 0.0119756 2023-10-09T04:19:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/641210400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/641210400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/641210400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:45.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 109356 109340 83.279 67.5625 0.011802 0.0120032 2023-10-09T04:19:46.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:45 smithi029 ceph-mon[216903]: pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 458 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:46.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:45 smithi029 ceph-mon[223858]: pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 458 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:45 smithi177 ceph-mon[195797]: pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 29 MiB/s wr, 458 op/s; 51395/307713 objects degraded (16.702%) 2023-10-09T04:19:46.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 110388 110372 83.0528 64.5 0.0096366 0.0120358 2023-10-09T04:19:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3123309505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3123309505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3123309505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:47.674 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:47 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-09T04:19:47.674 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:47 smithi029 systemd[1]: Stopped Ceph osd.0 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:19:47.674 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:47 smithi029 systemd[1]: Starting Ceph osd.0 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:19:47.840 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 111439 111423 82.8462 65.6875 0.0158348 0.0120662 2023-10-09T04:19:48.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:47 smithi029 ceph-mon[216903]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 54115/324237 objects degraded (16.690%) 2023-10-09T04:19:48.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:47 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 54115/324237 objects degraded (16.690%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:48.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:47 smithi029 ceph-mon[223858]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 54115/324237 objects degraded (16.690%) 2023-10-09T04:19:48.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:47 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 54115/324237 objects degraded (16.690%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:47 smithi177 ceph-mon[195797]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 54115/324237 objects degraded (16.690%) 2023-10-09T04:19:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:47 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 54115/324237 objects degraded (16.690%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:48.843 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 112435 112419 82.6006 62.25 0.0128906 0.012101 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-09T04:19:49.335 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:49 smithi029 bash[240948]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-09T04:19:49.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 113437 113421 82.3642 62.625 0.0115889 0.0121366 2023-10-09T04:19:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:49 smithi029 ceph-mon[216903]: pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 54115/324237 objects degraded (16.690%) 2023-10-09T04:19:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1965077674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:49 smithi029 ceph-mon[223858]: pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 54115/324237 objects degraded (16.690%) 2023-10-09T04:19:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1965077674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:49 smithi177 ceph-mon[195797]: pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 54115/324237 objects degraded (16.690%) 2023-10-09T04:19:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1965077674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:50.848 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 114523 114507 82.1977 67.875 0.0124868 0.0121609 2023-10-09T04:19:51.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 115511 115495 81.9616 61.75 0.0172133 0.0121955 2023-10-09T04:19:52.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:51 smithi029 ceph-mon[216903]: pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:51 smithi029 ceph-mon[223858]: pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:51 smithi177 ceph-mon[195797]: pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:52.852 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 116477 116461 81.7191 60.375 0.0157895 0.0122328 2023-10-09T04:19:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1759011430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:53.172 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:52 smithi029 bash[241408]: e385ec644294308898589b59f72e0463224682fc177c5c9a09f117e57fcd1b3f 2023-10-09T04:19:53.172 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:52 smithi029 systemd[1]: Started Ceph osd.0 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:19:53.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:19:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:19:52] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-10-09T04:19:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1759011430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1759011430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:19:53.672 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:19:53 smithi029 conmon[241447]: 2023-10-09T04:19:53.199+0000 7f1238e10640 -1 Falling back to public interface 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.009 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 117345 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 81.4814 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 39.9955 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 226.562 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 4.9375 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1303 2023-10-09T04:19:53.853 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 639.928 2023-10-09T04:19:53.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3625 2023-10-09T04:19:53.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 79 2023-10-09T04:19:53.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.012269 2023-10-09T04:19:53.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0184906 2023-10-09T04:19:53.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 1.03698 2023-10-09T04:19:53.854 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00115347 2023-10-09T04:19:53.861 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:19:54.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:53 smithi029 ceph-mon[216903]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:53 smithi029 ceph-mon[223858]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:54.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:53 smithi177 ceph-mon[195797]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1398097524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:54 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 56689/339678 objects degraded (16.689%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:55.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1398097524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:55.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:54 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 56689/339678 objects degraded (16.689%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:55.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1398097524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:55.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:54 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 56689/339678 objects degraded (16.689%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:19:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:55 smithi177 ceph-mon[195797]: pgmap v188: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:55 smithi029 ceph-mon[216903]: pgmap v188: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:56.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:55 smithi029 ceph-mon[223858]: pgmap v188: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s; 56689/339678 objects degraded (16.689%) 2023-10-09T04:19:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:57 smithi177 ceph-mon[195797]: pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 77 MiB/s wr, 1.29k op/s; 58443/349995 objects degraded (16.698%) 2023-10-09T04:19:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1844695115' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:58.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[216903]: pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 77 MiB/s wr, 1.29k op/s; 58443/349995 objects degraded (16.698%) 2023-10-09T04:19:58.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1844695115' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:58.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:58.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:58.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[223858]: pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 77 MiB/s wr, 1.29k op/s; 58443/349995 objects degraded (16.698%) 2023-10-09T04:19:58.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1844695115' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:19:58.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:58.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:19:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:19:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:00.231 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[216903]: pgmap v190: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 832 op/s; 58443/349995 objects degraded (16.698%) 2023-10-09T04:20:00.231 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3384673583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[223858]: pgmap v190: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 832 op/s; 58443/349995 objects degraded (16.698%) 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:00.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:19:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3384673583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:00.236 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:20:00 smithi029 conmon[241447]: 2023-10-09T04:20:00.222+0000 7f1238e10640 -1 osd.0 87 log_to_monitors true 2023-10-09T04:20:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:59 smithi177 ceph-mon[195797]: pgmap v190: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 832 op/s; 58443/349995 objects degraded (16.698%) 2023-10-09T04:20:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:59 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:19:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3384673583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: [WRN] OSD_DOWN: 1 osds down 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: osd.0 (root=default,host=smithi029) is down 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: [WRN] PG_DEGRADED: Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 3.1f is active+undersized+degraded, acting [5,2] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.4 is active+undersized+degraded, acting [4,1] 2023-10-09T04:20:01.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.5 is active+undersized+degraded, acting [7,2] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.6 is active+undersized+degraded, acting [7,3] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.7 is active+undersized+degraded, acting [6,3] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.9 is active+undersized+degraded, acting [1,4] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.d is active+undersized+degraded, acting [6,1] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.e is active+undersized+degraded, acting [2,5] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: pg 7.f is active+undersized+degraded, acting [3,7] 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: from='osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:20:01.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:00 smithi177 ceph-mon[195797]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:20:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded 2023-10-09T04:20:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: [WRN] OSD_DOWN: 1 osds down 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: osd.0 (root=default,host=smithi029) is down 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: [WRN] PG_DEGRADED: Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 3.1f is active+undersized+degraded, acting [5,2] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.4 is active+undersized+degraded, acting [4,1] 2023-10-09T04:20:01.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.5 is active+undersized+degraded, acting [7,2] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.6 is active+undersized+degraded, acting [7,3] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.7 is active+undersized+degraded, acting [6,3] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.9 is active+undersized+degraded, acting [1,4] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.d is active+undersized+degraded, acting [6,1] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.e is active+undersized+degraded, acting [2,5] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: pg 7.f is active+undersized+degraded, acting [3,7] 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: from='osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:20:01.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[216903]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: [WRN] OSD_DOWN: 1 osds down 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: osd.0 (root=default,host=smithi029) is down 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: [WRN] PG_DEGRADED: Degraded data redundancy: 58443/349995 objects degraded (16.698%), 15 pgs degraded 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 1.0 is active+undersized+degraded, acting [7,6] 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 3.6 is active+undersized+degraded, acting [1,4] 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 3.7 is active+undersized+degraded, acting [3,7] 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 3.b is active+undersized+degraded, acting [3,4] 2023-10-09T04:20:01.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 3.f is active+undersized+degraded, acting [7,4] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 3.10 is active+undersized+degraded, acting [6,5] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 3.1f is active+undersized+degraded, acting [5,2] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.4 is active+undersized+degraded, acting [4,1] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.5 is active+undersized+degraded, acting [7,2] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.6 is active+undersized+degraded, acting [7,3] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.7 is active+undersized+degraded, acting [6,3] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.9 is active+undersized+degraded, acting [1,4] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.d is active+undersized+degraded, acting [6,1] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.e is active+undersized+degraded, acting [2,5] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: pg 7.f is active+undersized+degraded, acting [3,7] 2023-10-09T04:20:01.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: from='osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:20:01.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:00 smithi029 ceph-mon[223858]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-09T04:20:01.426 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 04:20:01 smithi029 conmon[241447]: 2023-10-09T04:20:01.031+0000 7f122a27c700 -1 osd.0 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: pgmap v191: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 1.18k op/s; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: from='osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:20:02.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[223858]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: pgmap v191: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 1.18k op/s; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: from='osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:20:02.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:02 smithi029 ceph-mon[216903]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:20:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: pgmap v191: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 48 MiB/s wr, 1.18k op/s; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-09T04:20:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: osdmap e90: 8 total, 7 up, 8 in 2023-10-09T04:20:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: failed to encode map e90 with expected crc 2023-10-09T04:20:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: from='osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:20:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:02 smithi177 ceph-mon[195797]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:20:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:20:02] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-10-09T04:20:03.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: failed to encode map e91 with expected crc 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403] boot 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: failed to encode map e91 with expected crc 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4080430221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[223858]: osdmap e92: 8 total, 8 up, 8 in 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: failed to encode map e91 with expected crc 2023-10-09T04:20:03.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403] boot 2023-10-09T04:20:03.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T04:20:03.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: failed to encode map e91 with expected crc 2023-10-09T04:20:03.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:20:03.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4080430221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:03.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:03 smithi029 ceph-mon[216903]: osdmap e92: 8 total, 8 up, 8 in 2023-10-09T04:20:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: failed to encode map e91 with expected crc 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: osd.0 [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403] boot 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: osdmap e91: 8 total, 8 up, 8 in 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: failed to encode map e91 with expected crc 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4080430221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:03 smithi177 ceph-mon[195797]: osdmap e92: 8 total, 8 up, 8 in 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: pgmap v194: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 32 MiB/s wr, 1.12k op/s; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[216903]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: pgmap v194: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 32 MiB/s wr, 1.12k op/s; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:04 smithi029 ceph-mon[223858]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T04:20:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: pgmap v194: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 32 MiB/s wr, 1.12k op/s; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: failed to encode map e92 with expected crc 2023-10-09T04:20:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:04 smithi177 ceph-mon[195797]: osdmap e93: 8 total, 8 up, 8 in 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2056242879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:05.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 56385/337515 objects degraded (16.706%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:05.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.353 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2056242879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:05.353 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 56385/337515 objects degraded (16.706%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: failed to encode map e93 with expected crc 2023-10-09T04:20:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2056242879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 56385/337515 objects degraded (16.706%), 15 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:06 smithi029 ceph-mon[216903]: pgmap v197: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:06.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:06 smithi029 ceph-mon[223858]: pgmap v197: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:06 smithi177 ceph-mon[195797]: pgmap v197: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56385/337515 objects degraded (16.706%) 2023-10-09T04:20:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:08 smithi029 ceph-mon[216903]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 11935/330468 objects degraded (3.612%); 650 KiB/s, 10 objects/s recovering 2023-10-09T04:20:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3323813677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:08.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:08 smithi029 ceph-mon[223858]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 11935/330468 objects degraded (3.612%); 650 KiB/s, 10 objects/s recovering 2023-10-09T04:20:08.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3323813677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:08 smithi177 ceph-mon[195797]: pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 11935/330468 objects degraded (3.612%); 650 KiB/s, 10 objects/s recovering 2023-10-09T04:20:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3323813677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:10 smithi029 ceph-mon[216903]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 11935/330468 objects degraded (3.612%); 552 KiB/s, 9 objects/s recovering 2023-10-09T04:20:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11935/330468 objects degraded (3.612%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:10.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2858838142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:10.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:10 smithi029 ceph-mon[223858]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 11935/330468 objects degraded (3.612%); 552 KiB/s, 9 objects/s recovering 2023-10-09T04:20:10.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11935/330468 objects degraded (3.612%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:10.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2858838142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:10 smithi177 ceph-mon[195797]: pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 11935/330468 objects degraded (3.612%); 552 KiB/s, 9 objects/s recovering 2023-10-09T04:20:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11935/330468 objects degraded (3.612%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2858838142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:12.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:12 smithi029 ceph-mon[216903]: pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 11708/320523 objects degraded (3.653%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:20:12.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:12 smithi029 ceph-mon[223858]: pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 11708/320523 objects degraded (3.653%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:20:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:12 smithi177 ceph-mon[195797]: pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 11708/320523 objects degraded (3.653%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:20:13.116 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:20:12] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-10-09T04:20:13.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3194574666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3194574666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3194574666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:14 smithi029 ceph-mon[223858]: pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11708/320523 objects degraded (3.653%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:20:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:14.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:14 smithi029 ceph-mon[216903]: pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11708/320523 objects degraded (3.653%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:20:14.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:14.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:14 smithi177 ceph-mon[195797]: pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11708/320523 objects degraded (3.653%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:20:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:15.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11708/320523 objects degraded (3.653%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:15.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/906476674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:15.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11708/320523 objects degraded (3.653%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:15.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/906476674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:15.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11708/320523 objects degraded (3.653%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:15.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/906476674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:16 smithi029 ceph-mon[216903]: pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 11708/320523 objects degraded (3.653%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:20:16.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:16 smithi029 ceph-mon[223858]: pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 11708/320523 objects degraded (3.653%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:20:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:16 smithi177 ceph-mon[195797]: pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 11708/320523 objects degraded (3.653%); 1.7 MiB/s, 27 objects/s recovering 2023-10-09T04:20:18.157 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:17 smithi029 conmon[202582]: 2023-10-09T04:20:17.694+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:18.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 11454/310116 objects degraded (3.693%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:20:18.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:18.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:18.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1040106594' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:18.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:18.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 11454/310116 objects degraded (3.693%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1040106594' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: pgmap v203: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 11454/310116 objects degraded (3.693%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1040106594' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:20:18.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:19 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:19 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:19.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:19 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:19 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:19.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:19 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:19.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:19 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:20.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:20 smithi029 ceph-mon[223858]: pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11454/310116 objects degraded (3.693%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:20:20.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11454/310116 objects degraded (3.693%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:20.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1324757553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:20.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:20 smithi029 ceph-mon[216903]: pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11454/310116 objects degraded (3.693%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:20:20.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11454/310116 objects degraded (3.693%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:20.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1324757553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:20 smithi177 ceph-mon[195797]: pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 11454/310116 objects degraded (3.693%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T04:20:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11454/310116 objects degraded (3.693%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:20.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1324757553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:22 smithi177 ceph-mon[195797]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 11180/299913 objects degraded (3.728%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:20:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:22 smithi029 ceph-mon[216903]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 11180/299913 objects degraded (3.728%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:20:22.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:22 smithi029 ceph-mon[223858]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 11180/299913 objects degraded (3.728%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:20:23.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:20:22] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-10-09T04:20:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1871305125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:23.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1871305125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:23.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1871305125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:24.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:24 smithi177 ceph-mon[195797]: pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11180/299913 objects degraded (3.728%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:20:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:24 smithi029 ceph-mon[216903]: pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11180/299913 objects degraded (3.728%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:20:24.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:24 smithi029 ceph-mon[223858]: pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11180/299913 objects degraded (3.728%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:20:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11180/299913 objects degraded (3.728%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:25.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11180/299913 objects degraded (3.728%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:25.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11180/299913 objects degraded (3.728%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:26 smithi177 ceph-mon[195797]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11180/299913 objects degraded (3.728%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:20:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/717975943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:26 smithi029 ceph-mon[223858]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11180/299913 objects degraded (3.728%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:20:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/717975943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:26.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:26 smithi029 ceph-mon[216903]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 11180/299913 objects degraded (3.728%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:20:26.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/717975943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:28.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:28 smithi177 ceph-mon[195797]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 10895/289398 objects degraded (3.765%); 4.2 MiB/s, 69 objects/s recovering 2023-10-09T04:20:28.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1985344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:28.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:28 smithi029 ceph-mon[216903]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 10895/289398 objects degraded (3.765%); 4.2 MiB/s, 69 objects/s recovering 2023-10-09T04:20:28.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1985344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:28.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:28 smithi029 ceph-mon[223858]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 10895/289398 objects degraded (3.765%); 4.2 MiB/s, 69 objects/s recovering 2023-10-09T04:20:28.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1985344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:29.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:29.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:29.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:30.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:30 smithi177 ceph-mon[195797]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 10895/289398 objects degraded (3.765%); 2.9 MiB/s, 48 objects/s recovering 2023-10-09T04:20:30.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10895/289398 objects degraded (3.765%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:30 smithi029 ceph-mon[216903]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 10895/289398 objects degraded (3.765%); 2.9 MiB/s, 48 objects/s recovering 2023-10-09T04:20:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10895/289398 objects degraded (3.765%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:30.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:30 smithi029 ceph-mon[223858]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 10895/289398 objects degraded (3.765%); 2.9 MiB/s, 48 objects/s recovering 2023-10-09T04:20:30.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10895/289398 objects degraded (3.765%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:31.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/513780568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:31.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/513780568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/513780568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[223858]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 10895/278766 objects degraded (3.908%); 2.9 MiB/s, 72 objects/s recovering 2023-10-09T04:20:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[223858]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[223858]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[223858]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:20:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[223858]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[223858]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[216903]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 10895/278766 objects degraded (3.908%); 2.9 MiB/s, 72 objects/s recovering 2023-10-09T04:20:32.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[216903]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[216903]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[216903]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:20:32.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[216903]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:32 smithi029 ceph-mon[216903]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:32 smithi177 ceph-mon[195797]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 10895/278766 objects degraded (3.908%); 2.9 MiB/s, 72 objects/s recovering 2023-10-09T04:20:32.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:32 smithi177 ceph-mon[195797]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:32 smithi177 ceph-mon[195797]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:32 smithi177 ceph-mon[195797]: osdmap e94: 8 total, 8 up, 8 in 2023-10-09T04:20:32.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:32 smithi177 ceph-mon[195797]: failed to encode map e94 with expected crc 2023-10-09T04:20:32.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:32 smithi177 ceph-mon[195797]: failed to encode map e94 with expected crc 2023-10-09T04:20:33.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:32 smithi029 conmon[202582]: 2023-10-09T04:20:32.718+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:33.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:20:32] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.43.0" 2023-10-09T04:20:33.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T04:20:33.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 10895/278766 objects degraded (3.908%); 2.2 MiB/s, 75 objects/s recovering 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3988357263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T04:20:33.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 10895/278766 objects degraded (3.908%); 2.2 MiB/s, 75 objects/s recovering 2023-10-09T04:20:33.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:33.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:33.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:33.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3988357263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: osdmap e95: 8 total, 8 up, 8 in 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: failed to encode map e95 with expected crc 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 10895/278766 objects degraded (3.908%); 2.2 MiB/s, 75 objects/s recovering 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:33.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3988357263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:35.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:35 smithi029 ceph-mon[216903]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 10895/278766 objects degraded (3.908%); 0 B/s, 37 objects/s recovering 2023-10-09T04:20:35.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10895/278766 objects degraded (3.908%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:35.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:35 smithi029 ceph-mon[223858]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 10895/278766 objects degraded (3.908%); 0 B/s, 37 objects/s recovering 2023-10-09T04:20:35.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10895/278766 objects degraded (3.908%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:35 smithi177 ceph-mon[195797]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 10895/278766 objects degraded (3.908%); 0 B/s, 37 objects/s recovering 2023-10-09T04:20:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10895/278766 objects degraded (3.908%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/404203869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:36.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/404203869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/404203869' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:37 smithi029 ceph-mon[216903]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 10179/269769 objects degraded (3.773%); 2.2 MiB/s, 84 objects/s recovering 2023-10-09T04:20:37.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:37 smithi029 ceph-mon[223858]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 10179/269769 objects degraded (3.773%); 2.2 MiB/s, 84 objects/s recovering 2023-10-09T04:20:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:37 smithi177 ceph-mon[195797]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 10179/269769 objects degraded (3.773%); 2.2 MiB/s, 84 objects/s recovering 2023-10-09T04:20:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1615989212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1615989212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1615989212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:39 smithi029 ceph-mon[216903]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 10179/269769 objects degraded (3.773%); 2.2 MiB/s, 47 objects/s recovering 2023-10-09T04:20:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:39 smithi029 ceph-mon[223858]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 10179/269769 objects degraded (3.773%); 2.2 MiB/s, 47 objects/s recovering 2023-10-09T04:20:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:39 smithi177 ceph-mon[195797]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 374 op/s; 10179/269769 objects degraded (3.773%); 2.2 MiB/s, 47 objects/s recovering 2023-10-09T04:20:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10179/269769 objects degraded (3.773%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10179/269769 objects degraded (3.773%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10179/269769 objects degraded (3.773%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:41 smithi177 ceph-mon[195797]: pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 9896/259302 objects degraded (3.816%); 3.8 MiB/s, 71 objects/s recovering 2023-10-09T04:20:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3036238944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:41 smithi029 ceph-mon[223858]: pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 9896/259302 objects degraded (3.816%); 3.8 MiB/s, 71 objects/s recovering 2023-10-09T04:20:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3036238944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:42.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:41 smithi029 ceph-mon[216903]: pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 9896/259302 objects degraded (3.816%); 3.8 MiB/s, 71 objects/s recovering 2023-10-09T04:20:42.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3036238944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:43.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:20:42] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.43.0" 2023-10-09T04:20:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:43 smithi177 ceph-mon[195797]: pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9896/259302 objects degraded (3.816%); 3.5 MiB/s, 64 objects/s recovering 2023-10-09T04:20:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2306674675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:44.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:43 smithi029 ceph-mon[223858]: pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9896/259302 objects degraded (3.816%); 3.5 MiB/s, 64 objects/s recovering 2023-10-09T04:20:44.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:44.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2306674675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:44.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:43 smithi029 ceph-mon[216903]: pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9896/259302 objects degraded (3.816%); 3.5 MiB/s, 64 objects/s recovering 2023-10-09T04:20:44.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:44.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2306674675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:45 smithi177 ceph-mon[195797]: pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9896/259302 objects degraded (3.816%); 3.0 MiB/s, 55 objects/s recovering 2023-10-09T04:20:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9896/259302 objects degraded (3.816%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:46.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:45 smithi029 ceph-mon[216903]: pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9896/259302 objects degraded (3.816%); 3.0 MiB/s, 55 objects/s recovering 2023-10-09T04:20:46.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9896/259302 objects degraded (3.816%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:46.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:45 smithi029 ceph-mon[223858]: pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9896/259302 objects degraded (3.816%); 3.0 MiB/s, 55 objects/s recovering 2023-10-09T04:20:46.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9896/259302 objects degraded (3.816%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1921057558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:47.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1921057558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:47.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1921057558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:48.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:47 smithi177 ceph-mon[195797]: pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 9610/248844 objects degraded (3.862%); 4.4 MiB/s, 79 objects/s recovering 2023-10-09T04:20:48.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:47 smithi029 ceph-mon[216903]: pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 9610/248844 objects degraded (3.862%); 4.4 MiB/s, 79 objects/s recovering 2023-10-09T04:20:48.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:47 smithi029 conmon[202582]: 2023-10-09T04:20:47.738+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:48.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:47 smithi029 ceph-mon[223858]: pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 9610/248844 objects degraded (3.862%); 4.4 MiB/s, 79 objects/s recovering 2023-10-09T04:20:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:48 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:48 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1435736267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1435736267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:20:49.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:20:49.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1435736267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:49 smithi177 ceph-mon[195797]: pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 9610/248844 objects degraded (3.862%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:20:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:49 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9610/248844 objects degraded (3.862%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:49 smithi029 ceph-mon[216903]: pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 9610/248844 objects degraded (3.862%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:20:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:49 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9610/248844 objects degraded (3.862%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:50.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:49 smithi029 ceph-mon[223858]: pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 9610/248844 objects degraded (3.862%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:20:50.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:49 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9610/248844 objects degraded (3.862%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:52.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:51 smithi177 ceph-mon[195797]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 9310/238728 objects degraded (3.900%); 4.5 MiB/s, 74 objects/s recovering 2023-10-09T04:20:52.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3861298187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:52.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:51 smithi029 ceph-mon[216903]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 9310/238728 objects degraded (3.900%); 4.5 MiB/s, 74 objects/s recovering 2023-10-09T04:20:52.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3861298187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:51 smithi029 ceph-mon[223858]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 9310/238728 objects degraded (3.900%); 4.5 MiB/s, 74 objects/s recovering 2023-10-09T04:20:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3861298187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:20:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:20:52] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2023-10-09T04:20:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:53 smithi177 ceph-mon[195797]: pgmap v223: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 9310/238728 objects degraded (3.900%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:20:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1541892898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:53 smithi029 ceph-mon[223858]: pgmap v223: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 9310/238728 objects degraded (3.900%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:20:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1541892898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:54.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:53 smithi029 ceph-mon[216903]: pgmap v223: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 9310/238728 objects degraded (3.900%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:20:54.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1541892898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:54 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9310/238728 objects degraded (3.900%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:54 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9310/238728 objects degraded (3.900%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:54 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9310/238728 objects degraded (3.900%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:20:56.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:55 smithi029 ceph-mon[216903]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 9310/238728 objects degraded (3.900%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:20:56.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:55 smithi029 ceph-mon[216903]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T04:20:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:55 smithi029 ceph-mon[223858]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 9310/238728 objects degraded (3.900%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:20:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:55 smithi029 ceph-mon[223858]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T04:20:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:55 smithi177 ceph-mon[195797]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 9310/238728 objects degraded (3.900%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:20:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:55 smithi177 ceph-mon[195797]: osdmap e96: 8 total, 8 up, 8 in 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[216903]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[216903]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[216903]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[216903]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1751670510' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[216903]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[223858]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[223858]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[223858]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[223858]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1751670510' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:56 smithi029 ceph-mon[223858]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T04:20:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:56 smithi177 ceph-mon[195797]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:56 smithi177 ceph-mon[195797]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:56 smithi177 ceph-mon[195797]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:56 smithi177 ceph-mon[195797]: failed to encode map e96 with expected crc 2023-10-09T04:20:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1751670510' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:56 smithi177 ceph-mon[195797]: osdmap e97: 8 total, 8 up, 8 in 2023-10-09T04:20:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[216903]: pgmap v226: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 9310/230838 objects degraded (4.033%); 1.9 MiB/s, 71 objects/s recovering 2023-10-09T04:20:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[216903]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[216903]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[216903]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[223858]: pgmap v226: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 9310/230838 objects degraded (4.033%); 1.9 MiB/s, 71 objects/s recovering 2023-10-09T04:20:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[223858]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[223858]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:57 smithi029 ceph-mon[223858]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:57 smithi177 ceph-mon[195797]: pgmap v226: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 9310/230838 objects degraded (4.033%); 1.9 MiB/s, 71 objects/s recovering 2023-10-09T04:20:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:57 smithi177 ceph-mon[195797]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:57 smithi177 ceph-mon[195797]: failed to encode map e97 with expected crc 2023-10-09T04:20:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:57 smithi177 ceph-mon[195797]: failed to encode map e97 with expected crc 2023-10-09T04:20:59.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:59.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2291599911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:59.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:59.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2291599911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:20:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:20:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2291599911' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:59 smithi029 ceph-mon[216903]: pgmap v228: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 9310/230838 objects degraded (4.033%); 0 B/s, 48 objects/s recovering 2023-10-09T04:21:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:20:59 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9310/230838 objects degraded (4.033%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:21:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:59 smithi029 ceph-mon[223858]: pgmap v228: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 9310/230838 objects degraded (4.033%); 0 B/s, 48 objects/s recovering 2023-10-09T04:21:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:20:59 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9310/230838 objects degraded (4.033%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:21:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:59 smithi177 ceph-mon[195797]: pgmap v228: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 9310/230838 objects degraded (4.033%); 0 B/s, 48 objects/s recovering 2023-10-09T04:21:00.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:20:59 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9310/230838 objects degraded (4.033%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:21:02.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:01 smithi029 ceph-mon[223858]: pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 8440/222729 objects degraded (3.789%); 3.3 MiB/s, 103 objects/s recovering 2023-10-09T04:21:02.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1209007904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:02.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:01 smithi029 ceph-mon[216903]: pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 8440/222729 objects degraded (3.789%); 3.3 MiB/s, 103 objects/s recovering 2023-10-09T04:21:02.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1209007904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:01 smithi177 ceph-mon[195797]: pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 8440/222729 objects degraded (3.789%); 3.3 MiB/s, 103 objects/s recovering 2023-10-09T04:21:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1209007904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:03.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:02 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:03.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:02 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:03.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:02 smithi029 conmon[202582]: 2023-10-09T04:21:02.756+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:03.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:21:02] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2023-10-09T04:21:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:02 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:04.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[216903]: pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 8440/222729 objects degraded (3.789%); 3.3 MiB/s, 103 objects/s recovering 2023-10-09T04:21:04.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:04.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:04.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3178716162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:04.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[223858]: pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 8440/222729 objects degraded (3.789%); 3.3 MiB/s, 103 objects/s recovering 2023-10-09T04:21:04.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:04.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:04.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3178716162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:03 smithi177 ceph-mon[195797]: pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 8440/222729 objects degraded (3.789%); 3.3 MiB/s, 103 objects/s recovering 2023-10-09T04:21:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:03 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:03 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3178716162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:04 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8440/222729 objects degraded (3.789%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:21:05.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:04 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8440/222729 objects degraded (3.789%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:21:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:04 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8440/222729 objects degraded (3.789%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:21:06.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:05 smithi029 ceph-mon[216903]: pgmap v231: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 8440/222729 objects degraded (3.789%); 3.0 MiB/s, 49 objects/s recovering 2023-10-09T04:21:06.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:05 smithi029 ceph-mon[223858]: pgmap v231: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 8440/222729 objects degraded (3.789%); 3.0 MiB/s, 49 objects/s recovering 2023-10-09T04:21:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:05 smithi177 ceph-mon[195797]: pgmap v231: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 306 op/s; 8440/222729 objects degraded (3.789%); 3.0 MiB/s, 49 objects/s recovering 2023-10-09T04:21:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2338144660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2338144660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2338144660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:08.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:07 smithi177 ceph-mon[195797]: pgmap v232: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 8153/212283 objects degraded (3.841%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T04:21:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:07 smithi029 ceph-mon[216903]: pgmap v232: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 8153/212283 objects degraded (3.841%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T04:21:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:07 smithi029 ceph-mon[223858]: pgmap v232: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 8153/212283 objects degraded (3.841%); 4.4 MiB/s, 72 objects/s recovering 2023-10-09T04:21:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:09 smithi177 ceph-mon[195797]: pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 8153/212283 objects degraded (3.841%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:21:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/774308552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:09 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8153/212283 objects degraded (3.841%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:09 smithi029 ceph-mon[216903]: pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 8153/212283 objects degraded (3.841%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:21:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/774308552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:09 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8153/212283 objects degraded (3.841%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:09 smithi029 ceph-mon[223858]: pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 8153/212283 objects degraded (3.841%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:21:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/774308552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:09 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8153/212283 objects degraded (3.841%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:12.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:11 smithi177 ceph-mon[195797]: pgmap v234: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 7856/201966 objects degraded (3.890%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:21:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2681356627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:11 smithi029 ceph-mon[216903]: pgmap v234: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 7856/201966 objects degraded (3.890%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:21:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2681356627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:12.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:11 smithi029 ceph-mon[223858]: pgmap v234: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 7856/201966 objects degraded (3.890%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:21:12.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2681356627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:13.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:21:12] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2023-10-09T04:21:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:13 smithi177 ceph-mon[195797]: pgmap v235: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 7856/201966 objects degraded (3.890%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:21:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:13 smithi029 ceph-mon[216903]: pgmap v235: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 7856/201966 objects degraded (3.890%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:21:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:13 smithi029 ceph-mon[223858]: pgmap v235: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 7856/201966 objects degraded (3.890%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:21:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4288817323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:14 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 7856/201966 objects degraded (3.890%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4288817323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:14 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 7856/201966 objects degraded (3.890%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4288817323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:14 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 7856/201966 objects degraded (3.890%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:16 smithi177 ceph-mon[195797]: pgmap v236: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 7856/201966 objects degraded (3.890%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:21:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:16 smithi029 ceph-mon[216903]: pgmap v236: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 7856/201966 objects degraded (3.890%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:21:16.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:16 smithi029 ceph-mon[223858]: pgmap v236: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 7856/201966 objects degraded (3.890%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:21:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2019843188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2019843188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:17.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2019843188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:18 smithi029 ceph-mon[216903]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7748/191559 objects degraded (4.045%); 3.6 MiB/s, 73 objects/s recovering 2023-10-09T04:21:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:18 smithi029 ceph-mon[223858]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7748/191559 objects degraded (4.045%); 3.6 MiB/s, 73 objects/s recovering 2023-10-09T04:21:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:18 smithi177 ceph-mon[195797]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7748/191559 objects degraded (4.045%); 3.6 MiB/s, 73 objects/s recovering 2023-10-09T04:21:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:20.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: pgmap v238: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7748/191559 objects degraded (4.045%); 2.1 MiB/s, 49 objects/s recovering 2023-10-09T04:21:20.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2968980378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 7748/191559 objects degraded (4.045%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: pgmap v238: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7748/191559 objects degraded (4.045%); 2.1 MiB/s, 49 objects/s recovering 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:21:20.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2968980378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:20.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 7748/191559 objects degraded (4.045%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:20.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: pgmap v238: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 7748/191559 objects degraded (4.045%); 2.1 MiB/s, 49 objects/s recovering 2023-10-09T04:21:20.526 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: osdmap e98: 8 total, 8 up, 8 in 2023-10-09T04:21:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: failed to encode map e98 with expected crc 2023-10-09T04:21:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2968980378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 7748/191559 objects degraded (4.045%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[216903]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[216903]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:21:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[216903]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[216903]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[216903]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[223858]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[223858]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:21:21.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[223858]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[223858]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:21 smithi029 ceph-mon[223858]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:21 smithi177 ceph-mon[195797]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:21 smithi177 ceph-mon[195797]: osdmap e99: 8 total, 8 up, 8 in 2023-10-09T04:21:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:21 smithi177 ceph-mon[195797]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:21 smithi177 ceph-mon[195797]: failed to encode map e99 with expected crc 2023-10-09T04:21:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:21 smithi177 ceph-mon[195797]: failed to encode map e99 with expected crc 2023-10-09T04:21:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[223858]: pgmap v241: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 7185/182493 objects degraded (3.937%); 1.6 MiB/s, 81 objects/s recovering 2023-10-09T04:21:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:22.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1483632977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:22.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[216903]: pgmap v241: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 7185/182493 objects degraded (3.937%); 1.6 MiB/s, 81 objects/s recovering 2023-10-09T04:21:22.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:22.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:22.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1483632977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:22 smithi177 ceph-mon[195797]: pgmap v241: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 7185/182493 objects degraded (3.937%); 1.6 MiB/s, 81 objects/s recovering 2023-10-09T04:21:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1483632977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:23.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:21:22] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2023-10-09T04:21:24.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:24 smithi029 conmon[202582]: 2023-10-09T04:21:24.114+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[223858]: pgmap v242: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 7185/182493 objects degraded (3.937%); 1.6 MiB/s, 81 objects/s recovering 2023-10-09T04:21:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[216903]: pgmap v242: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 7185/182493 objects degraded (3.937%); 1.6 MiB/s, 81 objects/s recovering 2023-10-09T04:21:24.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:24.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:24.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:23 smithi177 ceph-mon[195797]: pgmap v242: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 7185/182493 objects degraded (3.937%); 1.6 MiB/s, 81 objects/s recovering 2023-10-09T04:21:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:21:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:21:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:21:25.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:25.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2831946423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 7185/182493 objects degraded (3.937%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:25.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:25.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:25.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2831946423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:25.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:24 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 7185/182493 objects degraded (3.937%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:21:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:21:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2831946423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:24 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 7185/182493 objects degraded (3.937%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:25 smithi029 ceph-mon[216903]: pgmap v243: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 7185/180420 objects degraded (3.982%); 823 KiB/s, 44 objects/s recovering 2023-10-09T04:21:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:25 smithi029 ceph-mon[223858]: pgmap v243: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 7185/180420 objects degraded (3.982%); 823 KiB/s, 44 objects/s recovering 2023-10-09T04:21:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:25 smithi177 ceph-mon[195797]: pgmap v243: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 7185/180420 objects degraded (3.982%); 823 KiB/s, 44 objects/s recovering 2023-10-09T04:21:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:27 smithi029 ceph-mon[216903]: pgmap v244: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6910/173850 objects degraded (3.975%); 2.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/858519647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:27 smithi029 ceph-mon[223858]: pgmap v244: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6910/173850 objects degraded (3.975%); 2.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/858519647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:27 smithi177 ceph-mon[195797]: pgmap v244: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6910/173850 objects degraded (3.975%); 2.9 MiB/s, 78 objects/s recovering 2023-10-09T04:21:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/858519647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:29.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:29 smithi177 ceph-mon[195797]: pgmap v245: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 6910/173850 objects degraded (3.975%); 2.5 MiB/s, 66 objects/s recovering 2023-10-09T04:21:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:29 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6910/173850 objects degraded (3.975%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/680848708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:30.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:29 smithi029 ceph-mon[216903]: pgmap v245: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 6910/173850 objects degraded (3.975%); 2.5 MiB/s, 66 objects/s recovering 2023-10-09T04:21:30.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:29 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6910/173850 objects degraded (3.975%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:30.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/680848708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:30.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:29 smithi029 ceph-mon[223858]: pgmap v245: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 6910/173850 objects degraded (3.975%); 2.5 MiB/s, 66 objects/s recovering 2023-10-09T04:21:30.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:29 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6910/173850 objects degraded (3.975%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:30.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/680848708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:31 smithi177 ceph-mon[195797]: pgmap v246: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 6633/163848 objects degraded (4.048%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:21:32.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:31 smithi029 ceph-mon[216903]: pgmap v246: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 6633/163848 objects degraded (4.048%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:21:32.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:31 smithi029 ceph-mon[223858]: pgmap v246: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 6633/163848 objects degraded (4.048%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:21:33.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1769881237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:33.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:21:32] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2023-10-09T04:21:33.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1769881237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1769881237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:33 smithi177 ceph-mon[195797]: pgmap v247: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 6633/163848 objects degraded (4.048%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:21:34.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:33 smithi029 ceph-mon[216903]: pgmap v247: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 6633/163848 objects degraded (4.048%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:21:34.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:33 smithi029 ceph-mon[223858]: pgmap v247: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 6633/163848 objects degraded (4.048%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:21:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6633/163848 objects degraded (4.048%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1836713134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6633/163848 objects degraded (4.048%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1836713134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:35.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6633/163848 objects degraded (4.048%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:35.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1836713134' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:36 smithi177 ceph-mon[195797]: pgmap v248: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 6633/163848 objects degraded (4.048%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:21:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:36 smithi029 ceph-mon[216903]: pgmap v248: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 6633/163848 objects degraded (4.048%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:21:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:36 smithi029 ceph-mon[223858]: pgmap v248: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 6633/163848 objects degraded (4.048%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:21:38.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:38 smithi029 ceph-mon[216903]: pgmap v249: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6346/154059 objects degraded (4.119%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:21:38.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3995356725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:38 smithi029 ceph-mon[223858]: pgmap v249: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6346/154059 objects degraded (4.119%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:21:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3995356725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:38 smithi177 ceph-mon[195797]: pgmap v249: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 6346/154059 objects degraded (4.119%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:21:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3995356725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:39.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:39 smithi029 conmon[202582]: 2023-10-09T04:21:39.138+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[216903]: pgmap v250: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 6346/154059 objects degraded (4.119%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:21:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6346/154059 objects degraded (4.119%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3115826206' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:40.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[223858]: pgmap v250: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 6346/154059 objects degraded (4.119%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:21:40.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:40.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:40.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:40.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6346/154059 objects degraded (4.119%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:40.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3115826206' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:40 smithi177 ceph-mon[195797]: pgmap v250: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 6346/154059 objects degraded (4.119%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:21:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:40 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:40 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:21:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6346/154059 objects degraded (4.119%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3115826206' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:42.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:42 smithi029 ceph-mon[223858]: pgmap v251: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 6199/144216 objects degraded (4.298%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T04:21:42.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:42 smithi029 ceph-mon[216903]: pgmap v251: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 6199/144216 objects degraded (4.298%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T04:21:42.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:42 smithi177 ceph-mon[195797]: pgmap v251: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 6199/144216 objects degraded (4.298%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T04:21:43.084 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:21:42] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2023-10-09T04:21:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/900063308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/900063308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:43.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/900063308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:44.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:44 smithi029 ceph-mon[216903]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 6199/144216 objects degraded (4.298%); 2.3 MiB/s, 52 objects/s recovering 2023-10-09T04:21:44.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:44 smithi029 ceph-mon[223858]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 6199/144216 objects degraded (4.298%); 2.3 MiB/s, 52 objects/s recovering 2023-10-09T04:21:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:44 smithi177 ceph-mon[195797]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 6199/144216 objects degraded (4.298%); 2.3 MiB/s, 52 objects/s recovering 2023-10-09T04:21:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[223858]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:21:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[223858]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[223858]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[223858]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6199/144216 objects degraded (4.298%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3388584322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[216903]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:21:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[216903]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[216903]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[216903]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6199/144216 objects degraded (4.298%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3388584322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:45 smithi177 ceph-mon[195797]: osdmap e100: 8 total, 8 up, 8 in 2023-10-09T04:21:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:45 smithi177 ceph-mon[195797]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:45 smithi177 ceph-mon[195797]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:45 smithi177 ceph-mon[195797]: failed to encode map e100 with expected crc 2023-10-09T04:21:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6199/144216 objects degraded (4.298%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3388584322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:46.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[216903]: pgmap v254: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 6199/141054 objects degraded (4.395%); 2.7 MiB/s, 62 objects/s recovering 2023-10-09T04:21:46.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[216903]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[216903]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[216903]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T04:21:46.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[216903]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[216903]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[223858]: pgmap v254: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 6199/141054 objects degraded (4.395%); 2.7 MiB/s, 62 objects/s recovering 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[223858]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[223858]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[223858]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[223858]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:46 smithi029 ceph-mon[223858]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:46 smithi177 ceph-mon[195797]: pgmap v254: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 6199/141054 objects degraded (4.395%); 2.7 MiB/s, 62 objects/s recovering 2023-10-09T04:21:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:46 smithi177 ceph-mon[195797]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:46 smithi177 ceph-mon[195797]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:46 smithi177 ceph-mon[195797]: osdmap e101: 8 total, 8 up, 8 in 2023-10-09T04:21:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:46 smithi177 ceph-mon[195797]: failed to encode map e101 with expected crc 2023-10-09T04:21:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:46 smithi177 ceph-mon[195797]: failed to encode map e101 with expected crc 2023-10-09T04:21:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:48 smithi029 ceph-mon[216903]: pgmap v256: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 5630/134121 objects degraded (4.198%); 2.0 MiB/s, 88 objects/s recovering 2023-10-09T04:21:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/115973200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:48.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:48 smithi029 ceph-mon[223858]: pgmap v256: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 5630/134121 objects degraded (4.198%); 2.0 MiB/s, 88 objects/s recovering 2023-10-09T04:21:48.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/115973200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:48 smithi177 ceph-mon[195797]: pgmap v256: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 5630/134121 objects degraded (4.198%); 2.0 MiB/s, 88 objects/s recovering 2023-10-09T04:21:48.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/115973200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:50.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:50 smithi029 ceph-mon[216903]: pgmap v257: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 5630/134121 objects degraded (4.198%); 824 KiB/s, 46 objects/s recovering 2023-10-09T04:21:50.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5630/134121 objects degraded (4.198%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:50.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2543123349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:50.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:50 smithi029 ceph-mon[223858]: pgmap v257: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 5630/134121 objects degraded (4.198%); 824 KiB/s, 46 objects/s recovering 2023-10-09T04:21:50.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5630/134121 objects degraded (4.198%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:50.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2543123349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:50 smithi177 ceph-mon[195797]: pgmap v257: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 5630/134121 objects degraded (4.198%); 824 KiB/s, 46 objects/s recovering 2023-10-09T04:21:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5630/134121 objects degraded (4.198%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2543123349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:52 smithi029 ceph-mon[216903]: pgmap v258: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 5384/124932 objects degraded (4.310%); 2.7 MiB/s, 77 objects/s recovering 2023-10-09T04:21:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:52 smithi029 ceph-mon[223858]: pgmap v258: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 5384/124932 objects degraded (4.310%); 2.7 MiB/s, 77 objects/s recovering 2023-10-09T04:21:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:52 smithi177 ceph-mon[195797]: pgmap v258: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 5384/124932 objects degraded (4.310%); 2.7 MiB/s, 77 objects/s recovering 2023-10-09T04:21:53.163 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:21:52] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.43.0" 2023-10-09T04:21:53.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2311881797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2311881797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2311881797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:54.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:54 smithi029 ceph-mon[216903]: pgmap v259: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 5384/124278 objects degraded (4.332%); 2.6 MiB/s, 73 objects/s recovering 2023-10-09T04:21:54.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:54.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:21:54 smithi029 conmon[202582]: 2023-10-09T04:21:54.160+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:21:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:54 smithi029 ceph-mon[223858]: pgmap v259: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 5384/124278 objects degraded (4.332%); 2.6 MiB/s, 73 objects/s recovering 2023-10-09T04:21:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:54 smithi177 ceph-mon[195797]: pgmap v259: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 5384/124278 objects degraded (4.332%); 2.6 MiB/s, 73 objects/s recovering 2023-10-09T04:21:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:54 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:55 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:55.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:55 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:21:55.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5384/124278 objects degraded (4.332%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:55.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:55 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:55.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:55 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:21:55.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5384/124278 objects degraded (4.332%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:55.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:55 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:21:55.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:55 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:21:55.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5384/124278 objects degraded (4.332%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:21:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:56 smithi177 ceph-mon[195797]: pgmap v260: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5384/121146 objects degraded (4.444%); 2.2 MiB/s, 61 objects/s recovering 2023-10-09T04:21:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3181698771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:56 smithi029 ceph-mon[216903]: pgmap v260: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5384/121146 objects degraded (4.444%); 2.2 MiB/s, 61 objects/s recovering 2023-10-09T04:21:56.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3181698771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:56.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:56 smithi029 ceph-mon[223858]: pgmap v260: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 5384/121146 objects degraded (4.444%); 2.2 MiB/s, 61 objects/s recovering 2023-10-09T04:21:56.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3181698771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:58 smithi177 ceph-mon[195797]: pgmap v261: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5161/113931 objects degraded (4.530%); 2.6 MiB/s, 64 objects/s recovering 2023-10-09T04:21:58.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1468150333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:58 smithi029 ceph-mon[216903]: pgmap v261: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5161/113931 objects degraded (4.530%); 2.6 MiB/s, 64 objects/s recovering 2023-10-09T04:21:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1468150333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:58.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:58 smithi029 ceph-mon[223858]: pgmap v261: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 5161/113931 objects degraded (4.530%); 2.6 MiB/s, 64 objects/s recovering 2023-10-09T04:21:58.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1468150333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:21:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:21:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:21:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:21:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:21:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:00 smithi177 ceph-mon[195797]: pgmap v262: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5161/113931 objects degraded (4.530%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T04:22:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5161/113931 objects degraded (4.530%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:00.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:00 smithi029 ceph-mon[223858]: pgmap v262: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5161/113931 objects degraded (4.530%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T04:22:01.058 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5161/113931 objects degraded (4.530%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:01.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:00 smithi029 ceph-mon[216903]: pgmap v262: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5161/113931 objects degraded (4.530%); 2.4 MiB/s, 39 objects/s recovering 2023-10-09T04:22:01.059 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5161/113931 objects degraded (4.530%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:01.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3606744496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:03.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3606744496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:03.087 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:02 smithi177 ceph-mon[195797]: pgmap v263: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 4929/104301 objects degraded (4.726%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:22:03.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3606744496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:03.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:02 smithi029 ceph-mon[216903]: pgmap v263: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 4929/104301 objects degraded (4.726%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:22:03.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:02 smithi029 ceph-mon[223858]: pgmap v263: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 4929/104301 objects degraded (4.726%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:22:03.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:22:02] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-10-09T04:22:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/610092724' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:03.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/610092724' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:03.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/610092724' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:04 smithi029 ceph-mon[223858]: pgmap v264: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4929/103689 objects degraded (4.754%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:22:04.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:04 smithi029 ceph-mon[216903]: pgmap v264: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4929/103689 objects degraded (4.754%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:22:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:04 smithi177 ceph-mon[195797]: pgmap v264: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 4929/103689 objects degraded (4.754%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:22:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4929/103689 objects degraded (4.754%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:05.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4929/103689 objects degraded (4.754%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4929/103689 objects degraded (4.754%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:06.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:06 smithi029 ceph-mon[216903]: pgmap v265: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4929/100356 objects degraded (4.912%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:22:06.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3657968508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:06 smithi029 ceph-mon[223858]: pgmap v265: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4929/100356 objects degraded (4.912%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:22:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3657968508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:06 smithi177 ceph-mon[195797]: pgmap v265: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4929/100356 objects degraded (4.912%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:22:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3657968508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:08.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:08 smithi029 ceph-mon[216903]: pgmap v266: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 4668/93267 objects degraded (5.005%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:22:08.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:08 smithi029 ceph-mon[223858]: pgmap v266: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 4668/93267 objects degraded (5.005%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:22:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:08 smithi177 ceph-mon[195797]: pgmap v266: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 4668/93267 objects degraded (5.005%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:22:09.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3277303798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:09.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:09.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3277303798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:09.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:09.674 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:09 smithi029 conmon[202582]: 2023-10-09T04:22:09.181+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3277303798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:10.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[216903]: pgmap v267: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 4668/93267 objects degraded (5.005%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:10.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:10.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:10.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4668/93267 objects degraded (5.005%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:10.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[223858]: pgmap v267: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 4668/93267 objects degraded (5.005%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:10.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:10.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:10.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4668/93267 objects degraded (5.005%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:10.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:10 smithi177 ceph-mon[195797]: pgmap v267: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 4668/93267 objects degraded (5.005%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:10 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:10 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4668/93267 objects degraded (5.005%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:11.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/374995416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:11.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/374995416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:11.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/374995416' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:12.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:12 smithi029 ceph-mon[223858]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 4628/84018 objects degraded (5.508%); 2.8 MiB/s, 60 objects/s recovering 2023-10-09T04:22:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:12 smithi029 ceph-mon[216903]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 4628/84018 objects degraded (5.508%); 2.8 MiB/s, 60 objects/s recovering 2023-10-09T04:22:12.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:12 smithi177 ceph-mon[195797]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 4628/84018 objects degraded (5.508%); 2.8 MiB/s, 60 objects/s recovering 2023-10-09T04:22:13.177 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:22:12] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-10-09T04:22:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:13 smithi177 ceph-mon[195797]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 4628/83319 objects degraded (5.555%); 1.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:13.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:13 smithi029 ceph-mon[216903]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 4628/83319 objects degraded (5.555%); 1.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:13.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:13 smithi029 ceph-mon[223858]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 4628/83319 objects degraded (5.555%); 1.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2445030299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2445030299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2445030299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:15.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:15 smithi177 ceph-mon[195797]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4628/80211 objects degraded (5.770%); 1.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:15.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4628/83319 objects degraded (5.555%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:15.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:15 smithi029 ceph-mon[216903]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4628/80211 objects degraded (5.770%); 1.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4628/83319 objects degraded (5.555%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:15.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:15 smithi029 ceph-mon[223858]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4628/80211 objects degraded (5.770%); 1.6 MiB/s, 41 objects/s recovering 2023-10-09T04:22:15.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4628/83319 objects degraded (5.555%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[223858]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T04:22:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[223858]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[223858]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[223858]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/160013649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:17.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[216903]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T04:22:17.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[216903]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[216903]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[216903]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/160013649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:16 smithi177 ceph-mon[195797]: osdmap e102: 8 total, 8 up, 8 in 2023-10-09T04:22:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:16 smithi177 ceph-mon[195797]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:16 smithi177 ceph-mon[195797]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:16 smithi177 ceph-mon[195797]: failed to encode map e102 with expected crc 2023-10-09T04:22:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/160013649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[216903]: pgmap v272: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 4628/73320 objects degraded (6.312%); 256 KiB/s, 50 objects/s recovering 2023-10-09T04:22:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[216903]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[216903]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[216903]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:22:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[216903]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[216903]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[223858]: pgmap v272: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 4628/73320 objects degraded (6.312%); 256 KiB/s, 50 objects/s recovering 2023-10-09T04:22:18.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[223858]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[223858]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[223858]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:22:18.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[223858]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:17 smithi029 ceph-mon[223858]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:17 smithi177 ceph-mon[195797]: pgmap v272: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 4628/73320 objects degraded (6.312%); 256 KiB/s, 50 objects/s recovering 2023-10-09T04:22:18.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:17 smithi177 ceph-mon[195797]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:17 smithi177 ceph-mon[195797]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:17 smithi177 ceph-mon[195797]: osdmap e103: 8 total, 8 up, 8 in 2023-10-09T04:22:18.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:17 smithi177 ceph-mon[195797]: failed to encode map e103 with expected crc 2023-10-09T04:22:18.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:17 smithi177 ceph-mon[195797]: failed to encode map e103 with expected crc 2023-10-09T04:22:19.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3751028444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:19.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3751028444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:19.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3751028444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:20.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:19 smithi029 ceph-mon[216903]: pgmap v274: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 4628/73320 objects degraded (6.312%); 0 B/s, 33 objects/s recovering 2023-10-09T04:22:20.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:19 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4628/73320 objects degraded (6.312%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:20.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:19 smithi029 ceph-mon[223858]: pgmap v274: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 4628/73320 objects degraded (6.312%); 0 B/s, 33 objects/s recovering 2023-10-09T04:22:20.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:19 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4628/73320 objects degraded (6.312%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:20.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:19 smithi177 ceph-mon[195797]: pgmap v274: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 4628/73320 objects degraded (6.312%); 0 B/s, 33 objects/s recovering 2023-10-09T04:22:20.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:19 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4628/73320 objects degraded (6.312%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:22.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:21 smithi029 ceph-mon[223858]: pgmap v275: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3852/65532 objects degraded (5.878%); 2.7 MiB/s, 76 objects/s recovering 2023-10-09T04:22:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4035239192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:22.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:21 smithi029 ceph-mon[216903]: pgmap v275: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3852/65532 objects degraded (5.878%); 2.7 MiB/s, 76 objects/s recovering 2023-10-09T04:22:22.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4035239192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:21 smithi177 ceph-mon[195797]: pgmap v275: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 3852/65532 objects degraded (5.878%); 2.7 MiB/s, 76 objects/s recovering 2023-10-09T04:22:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4035239192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:23.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:22:22] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-10-09T04:22:23.777 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:22:23 smithi177 conmon[174387]: logger=cleanup t=2023-10-09T04:22:23.324193204Z level=info msg="Completed cleanup jobs" duration=140.459579ms 2023-10-09T04:22:24.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:23 smithi029 ceph-mon[216903]: pgmap v276: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3852/64527 objects degraded (5.970%); 2.7 MiB/s, 76 objects/s recovering 2023-10-09T04:22:24.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/808265738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:24.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:23 smithi029 ceph-mon[223858]: pgmap v276: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3852/64527 objects degraded (5.970%); 2.7 MiB/s, 76 objects/s recovering 2023-10-09T04:22:24.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/808265738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:23 smithi177 ceph-mon[195797]: pgmap v276: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 3852/64527 objects degraded (5.970%); 2.7 MiB/s, 76 objects/s recovering 2023-10-09T04:22:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/808265738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:25.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:25.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:24 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3852/64527 objects degraded (5.970%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:25.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:25.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:24 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3852/64527 objects degraded (5.970%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:25.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:25.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:24 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3852/64527 objects degraded (5.970%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:26.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:25 smithi029 ceph-mon[216903]: pgmap v277: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 3591/61296 objects degraded (5.858%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:22:26.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:25 smithi029 ceph-mon[223858]: pgmap v277: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 3591/61296 objects degraded (5.858%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:22:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:25 smithi177 ceph-mon[195797]: pgmap v277: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 3591/61296 objects degraded (5.858%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:22:26.904 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/840644367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:27.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/840644367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:27.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/840644367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:28.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:27 smithi029 ceph-mon[216903]: pgmap v278: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 3591/54258 objects degraded (6.618%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:22:28.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:27 smithi029 ceph-mon[223858]: pgmap v278: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 3591/54258 objects degraded (6.618%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:22:28.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:27 smithi177 ceph-mon[195797]: pgmap v278: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 3591/54258 objects degraded (6.618%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:22:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/952494002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:29.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:29.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:29.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:29.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/952494002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/952494002' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:30.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:30 smithi029 ceph-mon[223858]: pgmap v279: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 3591/54258 objects degraded (6.618%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:22:30.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3591/54258 objects degraded (6.618%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:30.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:30 smithi029 ceph-mon[216903]: pgmap v279: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 3591/54258 objects degraded (6.618%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:22:30.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3591/54258 objects degraded (6.618%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:30 smithi177 ceph-mon[195797]: pgmap v279: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 3591/54258 objects degraded (6.618%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:22:30.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3591/54258 objects degraded (6.618%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:31.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:31.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:31 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:31 smithi029 conmon[202582]: 2023-10-09T04:22:31.169+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:31 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:31 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:31 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:22:31.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:31 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:22:31.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:31 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:22:32.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: pgmap v280: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 958 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 3341/46023 objects degraded (7.259%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:22:32.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:32.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:32.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:32.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:32.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2407727064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:32.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: pgmap v280: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 958 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 3341/46023 objects degraded (7.259%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:22:32.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:32.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:32.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:32.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:32.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2407727064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:32.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: pgmap v280: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 958 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 3341/46023 objects degraded (7.259%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:22:32.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:22:32.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2407727064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:33.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:22:32] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-10-09T04:22:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:34 smithi029 ceph-mon[216903]: pgmap v281: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 934 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3341/44862 objects degraded (7.447%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:22:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:34 smithi029 ceph-mon[223858]: pgmap v281: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 934 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3341/44862 objects degraded (7.447%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:22:34.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:34 smithi177 ceph-mon[195797]: pgmap v281: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 934 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 3341/44862 objects degraded (7.447%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:22:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2998802810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3341/44862 objects degraded (7.447%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2998802810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3341/44862 objects degraded (7.447%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2998802810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3341/44862 objects degraded (7.447%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:36 smithi029 ceph-mon[216903]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 870 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 3098/41814 objects degraded (7.409%); 3.9 MiB/s, 66 objects/s recovering 2023-10-09T04:22:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:36 smithi029 ceph-mon[223858]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 870 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 3098/41814 objects degraded (7.409%); 3.9 MiB/s, 66 objects/s recovering 2023-10-09T04:22:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:36 smithi177 ceph-mon[195797]: pgmap v282: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 870 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 3098/41814 objects degraded (7.409%); 3.9 MiB/s, 66 objects/s recovering 2023-10-09T04:22:37.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3389034369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:37.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3389034369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3389034369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:38.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:38 smithi177 ceph-mon[195797]: pgmap v283: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 724 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3098/34782 objects degraded (8.907%); 2.6 MiB/s, 44 objects/s recovering 2023-10-09T04:22:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:38 smithi029 ceph-mon[216903]: pgmap v283: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 724 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3098/34782 objects degraded (8.907%); 2.6 MiB/s, 44 objects/s recovering 2023-10-09T04:22:38.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:38 smithi029 ceph-mon[223858]: pgmap v283: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 724 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3098/34782 objects degraded (8.907%); 2.6 MiB/s, 44 objects/s recovering 2023-10-09T04:22:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:40 smithi177 ceph-mon[195797]: pgmap v284: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 724 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 3098/34782 objects degraded (8.907%); 2.6 MiB/s, 44 objects/s recovering 2023-10-09T04:22:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2956109524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3098/34782 objects degraded (8.907%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:40.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:40 smithi029 ceph-mon[216903]: pgmap v284: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 724 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 3098/34782 objects degraded (8.907%); 2.6 MiB/s, 44 objects/s recovering 2023-10-09T04:22:40.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2956109524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:40.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3098/34782 objects degraded (8.907%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:40.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:40 smithi029 ceph-mon[223858]: pgmap v284: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 724 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 3098/34782 objects degraded (8.907%); 2.6 MiB/s, 44 objects/s recovering 2023-10-09T04:22:40.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2956109524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:40.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3098/34782 objects degraded (8.907%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:42.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:42 smithi177 ceph-mon[195797]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 533 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 3098/25638 objects degraded (12.084%); 2.6 MiB/s, 68 objects/s recovering 2023-10-09T04:22:42.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1509853294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:42 smithi029 ceph-mon[216903]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 533 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 3098/25638 objects degraded (12.084%); 2.6 MiB/s, 68 objects/s recovering 2023-10-09T04:22:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1509853294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:42 smithi029 ceph-mon[223858]: pgmap v285: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 533 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 3098/25638 objects degraded (12.084%); 2.6 MiB/s, 68 objects/s recovering 2023-10-09T04:22:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1509853294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:22:42] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-10-09T04:22:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:43 smithi177 ceph-mon[195797]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[223858]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: osdmap e104: 8 total, 8 up, 8 in 2023-10-09T04:22:43.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:43.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:43 smithi029 ceph-mon[216903]: failed to encode map e104 with expected crc 2023-10-09T04:22:44.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:44 smithi177 ceph-mon[195797]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 504 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3098/24258 objects degraded (12.771%); 1.5 MiB/s, 56 objects/s recovering 2023-10-09T04:22:44.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:44 smithi177 ceph-mon[195797]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:44 smithi177 ceph-mon[195797]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:44 smithi177 ceph-mon[195797]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:22:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:44 smithi177 ceph-mon[195797]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[216903]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 504 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3098/24258 objects degraded (12.771%); 1.5 MiB/s, 56 objects/s recovering 2023-10-09T04:22:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[216903]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[216903]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[216903]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:22:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[216903]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:44.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[223858]: pgmap v287: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 504 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 3098/24258 objects degraded (12.771%); 1.5 MiB/s, 56 objects/s recovering 2023-10-09T04:22:44.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[223858]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[223858]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[223858]: osdmap e105: 8 total, 8 up, 8 in 2023-10-09T04:22:44.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[223858]: failed to encode map e105 with expected crc 2023-10-09T04:22:44.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2011775294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3098/24258 objects degraded (12.771%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2011775294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3098/24258 objects degraded (12.771%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:45.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2011775294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:45.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3098/24258 objects degraded (12.771%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:46 smithi177 ceph-mon[195797]: pgmap v289: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 451 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 3098/21675 objects degraded (14.293%); 0 B/s, 47 objects/s recovering 2023-10-09T04:22:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:46.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:46 smithi029 ceph-mon[216903]: pgmap v289: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 451 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 3098/21675 objects degraded (14.293%); 0 B/s, 47 objects/s recovering 2023-10-09T04:22:46.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:46.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:46 smithi029 ceph-mon[223858]: pgmap v289: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 451 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 3098/21675 objects degraded (14.293%); 0 B/s, 47 objects/s recovering 2023-10-09T04:22:46.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:46.678 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:46 smithi029 conmon[202582]: 2023-10-09T04:22:46.186+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:22:47.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:47 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:47.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:47 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:22:47.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/475443342' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:47.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:47 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:47.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:47 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:22:47.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/475443342' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:47 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:22:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:47 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:22:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/475443342' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:48.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:48 smithi029 ceph-mon[216903]: pgmap v290: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 2376/15642 objects degraded (15.190%); 2.2 MiB/s, 83 objects/s recovering 2023-10-09T04:22:48.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:48 smithi029 ceph-mon[223858]: pgmap v290: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 2376/15642 objects degraded (15.190%); 2.2 MiB/s, 83 objects/s recovering 2023-10-09T04:22:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:48 smithi177 ceph-mon[195797]: pgmap v290: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 2376/15642 objects degraded (15.190%); 2.2 MiB/s, 83 objects/s recovering 2023-10-09T04:22:50.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:50 smithi029 ceph-mon[216903]: pgmap v291: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 2376/15642 objects degraded (15.190%); 2.2 MiB/s, 48 objects/s recovering 2023-10-09T04:22:50.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/856000776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:50.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2376/15642 objects degraded (15.190%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:50.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:50 smithi029 ceph-mon[223858]: pgmap v291: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 2376/15642 objects degraded (15.190%); 2.2 MiB/s, 48 objects/s recovering 2023-10-09T04:22:50.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/856000776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:50.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2376/15642 objects degraded (15.190%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:50.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:50 smithi177 ceph-mon[195797]: pgmap v291: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 325 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 2376/15642 objects degraded (15.190%); 2.2 MiB/s, 48 objects/s recovering 2023-10-09T04:22:50.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/856000776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:50.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2376/15642 objects degraded (15.190%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:52 smithi029 ceph-mon[216903]: pgmap v292: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 140 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 2243/6765 objects degraded (33.156%); 3.1 MiB/s, 79 objects/s recovering 2023-10-09T04:22:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:52 smithi029 ceph-mon[223858]: pgmap v292: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 140 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 2243/6765 objects degraded (33.156%); 3.1 MiB/s, 79 objects/s recovering 2023-10-09T04:22:52.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:52 smithi177 ceph-mon[195797]: pgmap v292: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 140 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 2243/6765 objects degraded (33.156%); 3.1 MiB/s, 79 objects/s recovering 2023-10-09T04:22:52.935 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 117345 objects 2023-10-09T04:22:52.935 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :179.074 2023-10-09T04:22:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:22:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:22:52] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2023-10-09T04:22:53.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2191115384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:53.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2191115384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2191115384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:54.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:54 smithi029 ceph-mon[216903]: pgmap v293: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 115 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 2243/5544 objects degraded (40.458%); 2.6 MiB/s, 66 objects/s recovering 2023-10-09T04:22:54.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:54 smithi029 ceph-mon[223858]: pgmap v293: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 115 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 2243/5544 objects degraded (40.458%); 2.6 MiB/s, 66 objects/s recovering 2023-10-09T04:22:54.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:54 smithi177 ceph-mon[195797]: pgmap v293: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 115 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 2243/5544 objects degraded (40.458%); 2.6 MiB/s, 66 objects/s recovering 2023-10-09T04:22:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3618687038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2243/5544 objects degraded (40.458%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3618687038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2243/5544 objects degraded (40.458%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:55.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3618687038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:55.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2243/5544 objects degraded (40.458%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[223858]: pgmap v294: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 74 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 2243/3579 objects degraded (62.671%); 2.3 MiB/s, 58 objects/s recovering 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[223858]: failed to encode map e106 with expected crc 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[223858]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[223858]: failed to encode map e106 with expected crc 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[216903]: pgmap v294: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 74 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 2243/3579 objects degraded (62.671%); 2.3 MiB/s, 58 objects/s recovering 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[216903]: failed to encode map e106 with expected crc 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[216903]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:22:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:56 smithi029 ceph-mon[216903]: failed to encode map e106 with expected crc 2023-10-09T04:22:56.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:56 smithi177 ceph-mon[195797]: pgmap v294: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 74 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 2243/3579 objects degraded (62.671%); 2.3 MiB/s, 58 objects/s recovering 2023-10-09T04:22:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:56 smithi177 ceph-mon[195797]: failed to encode map e106 with expected crc 2023-10-09T04:22:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:56 smithi177 ceph-mon[195797]: osdmap e106: 8 total, 8 up, 8 in 2023-10-09T04:22:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:56 smithi177 ceph-mon[195797]: failed to encode map e106 with expected crc 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[216903]: failed to encode map e106 with expected crc 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[216903]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[216903]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[216903]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[216903]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3638096052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:57.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[223858]: failed to encode map e106 with expected crc 2023-10-09T04:22:57.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[223858]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:22:57.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[223858]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[223858]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[223858]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3638096052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:57 smithi177 ceph-mon[195797]: failed to encode map e106 with expected crc 2023-10-09T04:22:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:57 smithi177 ceph-mon[195797]: osdmap e107: 8 total, 8 up, 8 in 2023-10-09T04:22:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:57 smithi177 ceph-mon[195797]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:57 smithi177 ceph-mon[195797]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:57 smithi177 ceph-mon[195797]: failed to encode map e107 with expected crc 2023-10-09T04:22:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3638096052' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:22:58.285 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-09T04:22:58.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-09T04:22:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:58 smithi029 ceph-mon[216903]: pgmap v296: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 1120/72 objects degraded (1555.556%); 850 KiB/s, 128 objects/s recovering 2023-10-09T04:22:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:58 smithi029 ceph-mon[216903]: failed to encode map e107 with expected crc 2023-10-09T04:22:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:58 smithi029 ceph-mon[216903]: failed to encode map e107 with expected crc 2023-10-09T04:22:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:58 smithi029 ceph-mon[223858]: pgmap v296: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 1120/72 objects degraded (1555.556%); 850 KiB/s, 128 objects/s recovering 2023-10-09T04:22:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:58 smithi029 ceph-mon[223858]: failed to encode map e107 with expected crc 2023-10-09T04:22:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:58 smithi029 ceph-mon[223858]: failed to encode map e107 with expected crc 2023-10-09T04:22:58.690 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:22:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:58 smithi177 ceph-mon[195797]: pgmap v296: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 1120/72 objects degraded (1555.556%); 850 KiB/s, 128 objects/s recovering 2023-10-09T04:22:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:58 smithi177 ceph-mon[195797]: failed to encode map e107 with expected crc 2023-10-09T04:22:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:58 smithi177 ceph-mon[195797]: failed to encode map e107 with expected crc 2023-10-09T04:22:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:22:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:22:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:22:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:22:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: pgmap v298: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 1120/72 objects degraded (1555.556%); 0 B/s, 126 objects/s recovering 2023-10-09T04:23:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3242177137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1120/72 objects degraded (1555.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: pgmap v298: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 1120/72 objects degraded (1555.556%); 0 B/s, 126 objects/s recovering 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3242177137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1120/72 objects degraded (1555.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:23:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: pgmap v298: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 278 op/s; 1120/72 objects degraded (1555.556%); 0 B/s, 126 objects/s recovering 2023-10-09T04:23:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: osdmap e108: 8 total, 8 up, 8 in 2023-10-09T04:23:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: failed to encode map e108 with expected crc 2023-10-09T04:23:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3242177137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1120/72 objects degraded (1555.556%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:23:01.384 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:01 smithi029 conmon[202582]: 2023-10-09T04:23:01.192+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: osdmap e109: 8 total, 8 up, 8 in 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: failed to encode map e109 with expected crc 2023-10-09T04:23:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[216903]: pgmap v301: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 1120/72 objects degraded (1555.556%); 0 B/s, 6 objects/s recovering 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2371010566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[223858]: pgmap v301: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 1120/72 objects degraded (1555.556%); 0 B/s, 6 objects/s recovering 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:02.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:02.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2371010566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:02 smithi177 ceph-mon[195797]: pgmap v301: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 1120/72 objects degraded (1555.556%); 0 B/s, 6 objects/s recovering 2023-10-09T04:23:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:02 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:02 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:23:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2371010566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:02.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:23:02] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2023-10-09T04:23:03.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:03 smithi177 ceph-mon[195797]: pgmap v302: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 254 objects/s recovering 2023-10-09T04:23:03.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:03 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1120/72 objects degraded (1555.556%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:23:03.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:03 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:23:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:03 smithi029 ceph-mon[216903]: pgmap v302: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 254 objects/s recovering 2023-10-09T04:23:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:03 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1120/72 objects degraded (1555.556%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:23:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:03 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:23:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:03 smithi029 ceph-mon[223858]: pgmap v302: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 254 objects/s recovering 2023-10-09T04:23:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:03 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1120/72 objects degraded (1555.556%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:23:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:03 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:23:04.450 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_2' removed 2023-10-09T04:23:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3737542020' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:23:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2123810359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3737542020' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:23:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2123810359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3737542020' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:23:04.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2123810359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:05.405 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:23:05.406 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:23:05.406 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-09T04:23:05.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-09T04:23:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[216903]: failed to encode map e110 with expected crc 2023-10-09T04:23:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[216903]: failed to encode map e110 with expected crc 2023-10-09T04:23:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3737542020' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:23:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[216903]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:23:05.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[216903]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:05.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[223858]: failed to encode map e110 with expected crc 2023-10-09T04:23:05.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[223858]: failed to encode map e110 with expected crc 2023-10-09T04:23:05.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3737542020' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:23:05.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[223858]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:23:05.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:05 smithi029 ceph-mon[223858]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:05.767 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:05 smithi177 ceph-mon[195797]: failed to encode map e110 with expected crc 2023-10-09T04:23:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:05 smithi177 ceph-mon[195797]: failed to encode map e110 with expected crc 2023-10-09T04:23:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3737542020' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:23:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:05 smithi177 ceph-mon[195797]: osdmap e110: 8 total, 8 up, 8 in 2023-10-09T04:23:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:05 smithi177 ceph-mon[195797]: pgmap v304: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2756619959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2756619959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2756619959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:07 smithi029 ceph-mon[216903]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:07 smithi029 ceph-mon[223858]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:07.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:07 smithi177 ceph-mon[195797]: pgmap v305: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:08.545 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_3' created 2023-10-09T04:23:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/217018968' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:23:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:23:08.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/217018968' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:23:08.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:23:08.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/217018968' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:23:08.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-09T04:23:09.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-09T04:23:09.857 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:23:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: failed to encode map e111 with expected crc 2023-10-09T04:23:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: failed to encode map e111 with expected crc 2023-10-09T04:23:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:23:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:23:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: pgmap v307: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2061336307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: failed to encode map e111 with expected crc 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: failed to encode map e111 with expected crc 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: pgmap v307: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2061336307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: failed to encode map e111 with expected crc 2023-10-09T04:23:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: failed to encode map e111 with expected crc 2023-10-09T04:23:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-09T04:23:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: osdmap e111: 8 total, 8 up, 8 in 2023-10-09T04:23:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: pgmap v307: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:10.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2061336307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:23:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:10.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:10 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: osdmap e112: 8 total, 8 up, 8 in 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:10 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: failed to encode map e113 with expected crc 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: failed to encode map e113 with expected crc 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: pgmap v310: 89 pgs: 2 creating+activating, 3 unknown, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1110360909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[216903]: failed to encode map e113 with expected crc 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: failed to encode map e112 with expected crc 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: failed to encode map e113 with expected crc 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: failed to encode map e113 with expected crc 2023-10-09T04:23:11.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:23:11.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: pgmap v310: 89 pgs: 2 creating+activating, 3 unknown, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:11.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1110360909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:11.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:11 smithi029 ceph-mon[223858]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: failed to encode map e112 with expected crc 2023-10-09T04:23:12.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: osdmap e113: 8 total, 8 up, 8 in 2023-10-09T04:23:12.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: pgmap v310: 89 pgs: 2 creating+activating, 3 unknown, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:12.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1110360909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:12.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:11 smithi177 ceph-mon[195797]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.558 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-09T04:23:12.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:12 smithi029 ceph-mon[216903]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:12 smithi029 ceph-mon[216903]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/269273647' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:23:12.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:12 smithi029 ceph-mon[223858]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:12 smithi029 ceph-mon[223858]: failed to encode map e113 with expected crc 2023-10-09T04:23:12.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/269273647' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:23:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:12 smithi177 ceph-mon[195797]: failed to encode map e113 with expected crc 2023-10-09T04:23:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:12 smithi177 ceph-mon[195797]: failed to encode map e113 with expected crc 2023-10-09T04:23:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/269273647' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:23:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:23:12] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2023-10-09T04:23:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: failed to encode map e113 with expected crc 2023-10-09T04:23:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: failed to encode map e114 with expected crc 2023-10-09T04:23:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: failed to encode map e114 with expected crc 2023-10-09T04:23:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/269273647' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:23:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T04:23:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: pgmap v312: 89 pgs: 2 creating+activating, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/470714832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: failed to encode map e113 with expected crc 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: failed to encode map e114 with expected crc 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: failed to encode map e114 with expected crc 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/269273647' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: pgmap v312: 89 pgs: 2 creating+activating, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/470714832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:13.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:13.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: failed to encode map e113 with expected crc 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: failed to encode map e114 with expected crc 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: failed to encode map e114 with expected crc 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/269273647' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: osdmap e114: 8 total, 8 up, 8 in 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: pgmap v312: 89 pgs: 2 creating+activating, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/470714832' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:14.567 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-09T04:23:14.570 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:23:14.570 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:23:14.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:23:14.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:23:14.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_252925 2023-10-09T04:23:14.626 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:23:14.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:23:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:14 smithi029 ceph-mon[223858]: failed to encode map e114 with expected crc 2023-10-09T04:23:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:14 smithi029 ceph-mon[216903]: failed to encode map e114 with expected crc 2023-10-09T04:23:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:14 smithi177 ceph-mon[195797]: failed to encode map e114 with expected crc 2023-10-09T04:23:15.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3837 3821 238.728 238.812 0.00310686 0.00416799 2023-10-09T04:23:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:15 smithi029 ceph-mon[216903]: pgmap v313: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:15 smithi029 ceph-mon[223858]: pgmap v313: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:15 smithi177 ceph-mon[195797]: pgmap v313: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:23:16.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6994 6978 218.014 197.312 0.00546103 0.00457378 2023-10-09T04:23:17.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2803154656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:17.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:17.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:17.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[216903]: Upgrade: osd.1 is safe to restart 2023-10-09T04:23:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2803154656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:17.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:17.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:17.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:16 smithi029 ceph-mon[223858]: Upgrade: osd.1 is safe to restart 2023-10-09T04:23:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2803154656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:16 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:16 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-09T04:23:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:16 smithi177 ceph-mon[195797]: Upgrade: osd.1 is safe to restart 2023-10-09T04:23:17.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 9985 9969 207.651 186.938 0.00581305 0.00480323 2023-10-09T04:23:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:17 smithi029 ceph-mon[223858]: pgmap v314: 89 pgs: 89 active+clean; 123 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 275 op/s 2023-10-09T04:23:17.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:17 smithi029 ceph-mon[216903]: pgmap v314: 89 pgs: 89 active+clean; 123 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 275 op/s 2023-10-09T04:23:18.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:17 smithi177 ceph-mon[195797]: pgmap v314: 89 pgs: 89 active+clean; 123 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 275 op/s 2023-10-09T04:23:18.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 12841 12825 200.36 178.5 0.00579861 0.00498519 2023-10-09T04:23:19.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 15700 15684 196.022 178.688 0.00526569 0.00509778 2023-10-09T04:23:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:19 smithi029 ceph-mon[216903]: pgmap v315: 89 pgs: 89 active+clean; 123 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 241 op/s 2023-10-09T04:23:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/798885805' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:19.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:19 smithi029 ceph-mon[223858]: pgmap v315: 89 pgs: 89 active+clean; 123 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 241 op/s 2023-10-09T04:23:19.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:19.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/798885805' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:19 smithi177 ceph-mon[195797]: pgmap v315: 89 pgs: 89 active+clean; 123 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s wr, 241 op/s 2023-10-09T04:23:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/798885805' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:20.628 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 18445 18429 191.943 171.562 0.00759809 0.0052045 2023-10-09T04:23:21.628 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 20871 20855 186.182 151.625 0.00638605 0.00536683 2023-10-09T04:23:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:21 smithi029 ceph-mon[223858]: pgmap v316: 89 pgs: 89 active+clean; 902 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:23:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/461112803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:21 smithi029 ceph-mon[216903]: pgmap v316: 89 pgs: 89 active+clean; 902 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:23:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/461112803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:22.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:21 smithi177 ceph-mon[195797]: pgmap v316: 89 pgs: 89 active+clean; 902 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-09T04:23:22.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/461112803' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:22.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 23313 23297 181.986 152.625 0.00302525 0.00548598 2023-10-09T04:23:23.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:23:22] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2023-10-09T04:23:23.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 25372 25356 176.062 128.688 0.00800177 0.00567325 2023-10-09T04:23:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:23 smithi029 ceph-mon[216903]: pgmap v317: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:23:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:23 smithi029 ceph-mon[223858]: pgmap v317: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:23:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:23 smithi177 ceph-mon[195797]: pgmap v317: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-10-09T04:23:24.627 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 27587 27571 172.298 138.438 0.00615789 0.00579898 2023-10-09T04:23:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/812372696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/812372696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/812372696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:25.628 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 29802 29786 169.206 138.438 0.00943986 0.00590437 2023-10-09T04:23:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:25 smithi029 ceph-mon[216903]: pgmap v318: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T04:23:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:25 smithi029 ceph-mon[223858]: pgmap v318: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T04:23:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:25 smithi177 ceph-mon[195797]: pgmap v318: 89 pgs: 89 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-10-09T04:23:26.630 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 31818 31802 165.591 126 0.00316347 0.00603416 2023-10-09T04:23:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/881358888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/881358888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/881358888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:27.630 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 33857 33841 162.656 127.438 0.00290966 0.00614247 2023-10-09T04:23:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:27 smithi029 ceph-mon[216903]: pgmap v319: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-10-09T04:23:28.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:27 smithi029 ceph-mon[223858]: pgmap v319: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-10-09T04:23:28.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:27 smithi177 ceph-mon[195797]: pgmap v319: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-10-09T04:23:28.635 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 35768 35752 159.508 119.438 0.0101117 0.00626368 2023-10-09T04:23:29.638 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 37427 37411 155.788 103.688 0.0117346 0.00641398 2023-10-09T04:23:29.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: Upgrade: Updating osd.1 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: Deploying daemon osd.1 on smithi029 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1029011264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: Upgrade: Updating osd.1 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:29.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:23:29.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:29.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: Deploying daemon osd.1 on smithi029 2023-10-09T04:23:29.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:29.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1029011264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: Upgrade: Updating osd.1 2023-10-09T04:23:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-09T04:23:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:23:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: Deploying daemon osd.1 on smithi029 2023-10-09T04:23:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1029011264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:30.635 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 39275 39259 153.27 115.5 0.00931442 0.00651985 2023-10-09T04:23:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:30 smithi029 ceph-mon[216903]: pgmap v320: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-10-09T04:23:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:30 smithi029 ceph-mon[223858]: pgmap v320: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-10-09T04:23:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:30 smithi177 ceph-mon[195797]: pgmap v320: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-10-09T04:23:31.635 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 41028 41012 150.7 109.562 0.0160157 0.00663174 2023-10-09T04:23:32.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 42837 42821 148.608 113.062 0.0104747 0.00672288 2023-10-09T04:23:32.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:32 smithi029 ceph-mon[216903]: pgmap v321: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 180 MiB/s wr, 2.88k op/s 2023-10-09T04:23:32.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3807121038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:32.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:32 smithi029 ceph-mon[223858]: pgmap v321: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 180 MiB/s wr, 2.88k op/s 2023-10-09T04:23:32.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3807121038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:32.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:32 smithi177 ceph-mon[195797]: pgmap v321: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 180 MiB/s wr, 2.88k op/s 2023-10-09T04:23:32.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3807121038' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:33.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:23:32] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2023-10-09T04:23:33.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 44612 44596 146.625 110.938 0.00723142 0.00681527 2023-10-09T04:23:34.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:23:34.636373+0000 min lat: 0.00133368 max lat: 0.136982 avg lat: 0.00692909 2023-10-09T04:23:34.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:23:34.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 46184 46168 144.207 98.25 0.0053627 0.00692909 2023-10-09T04:23:34.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:34 smithi029 ceph-mon[216903]: pgmap v322: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:23:34.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3989875361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:34 smithi029 ceph-mon[223858]: pgmap v322: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:23:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3989875361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:34 smithi177 ceph-mon[195797]: pgmap v322: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:23:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3989875361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:35.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 47868 47852 142.352 105.25 0.00589195 0.00702042 2023-10-09T04:23:36.636 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 49469 49453 140.43 100.062 0.0146398 0.00711543 2023-10-09T04:23:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:36 smithi029 ceph-mon[216903]: pgmap v323: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T04:23:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:36 smithi029 ceph-mon[223858]: pgmap v323: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T04:23:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:36 smithi177 ceph-mon[195797]: pgmap v323: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T04:23:37.639 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 50965 50949 138.37 93.5 0.00750107 0.00722206 2023-10-09T04:23:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1542679177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1542679177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1542679177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:38.640 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 52402 52386 136.347 89.8125 0.0112736 0.00732899 2023-10-09T04:23:38.755 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:38 smithi029 ceph-mon[216903]: pgmap v324: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T04:23:38.756 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:38 smithi029 ceph-mon[223858]: pgmap v324: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T04:23:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:38 smithi177 ceph-mon[195797]: pgmap v324: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T04:23:39.642 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 53830 53814 134.452 89.25 0.00906384 0.00743248 2023-10-09T04:23:39.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:39 smithi177 ceph-mon[195797]: pgmap v325: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T04:23:39.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:39 smithi029 ceph-mon[216903]: pgmap v325: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T04:23:39.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:39 smithi029 ceph-mon[223858]: pgmap v325: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-10-09T04:23:40.642 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 55364 55348 132.968 95.875 0.00784172 0.00751638 2023-10-09T04:23:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2666927733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2666927733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:40.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2666927733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:41.642 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 56832 56816 131.441 91.75 0.00970372 0.00760193 2023-10-09T04:23:41.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:41 smithi177 ceph-mon[195797]: pgmap v326: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T04:23:41.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:41 smithi029 ceph-mon[216903]: pgmap v326: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T04:23:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:41 smithi029 ceph-mon[223858]: pgmap v326: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-10-09T04:23:42.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 58239 58223 129.883 87.9375 0.01556 0.00769454 2023-10-09T04:23:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3146241722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:42.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3146241722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:42.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3146241722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:43.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:23:42] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2023-10-09T04:23:43.173 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:43 smithi029 systemd[1]: Stopping Ceph osd.1 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:23:43.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 59538 59522 128.204 81.1875 0.0106068 0.00779569 2023-10-09T04:23:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:43 smithi177 ceph-mon[195797]: pgmap v327: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:23:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:43 smithi029 ceph-mon[216903]: pgmap v327: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:23:43.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:43 smithi029 ceph-mon[223858]: pgmap v327: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-09T04:23:43.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:43 smithi029 conmon[134427]: 2023-10-09T04:23:43.531+0000 7fc8dac40700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:23:43.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:43 smithi029 conmon[134427]: 2023-10-09T04:23:43.531+0000 7fc8dac40700 -1 osd.1 114 *** Got signal Terminated *** 2023-10-09T04:23:43.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:43 smithi029 conmon[134427]: 2023-10-09T04:23:43.531+0000 7fc8dac40700 -1 osd.1 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:23:44.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 60703 60687 126.359 72.8125 0.00231067 0.00787155 2023-10-09T04:23:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:44 smithi177 ceph-mon[195797]: osd.1 marked itself down and dead 2023-10-09T04:23:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:44 smithi029 ceph-mon[216903]: osd.1 marked itself down and dead 2023-10-09T04:23:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:44 smithi029 ceph-mon[223858]: osd.1 marked itself down and dead 2023-10-09T04:23:45.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 60915 60899 122.712 13.25 0.0150199 0.00814491 2023-10-09T04:23:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[216903]: osdmap e115: 8 total, 7 up, 8 in 2023-10-09T04:23:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[216903]: failed to encode map e115 with expected crc 2023-10-09T04:23:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[216903]: failed to encode map e115 with expected crc 2023-10-09T04:23:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1543194340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:45.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[216903]: pgmap v329: 89 pgs: 16 peering, 9 stale+active+clean, 64 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T04:23:45.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:45.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[223858]: osdmap e115: 8 total, 7 up, 8 in 2023-10-09T04:23:45.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[223858]: failed to encode map e115 with expected crc 2023-10-09T04:23:45.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[223858]: failed to encode map e115 with expected crc 2023-10-09T04:23:45.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1543194340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:45.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:45 smithi029 ceph-mon[223858]: pgmap v329: 89 pgs: 16 peering, 9 stale+active+clean, 64 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T04:23:45.673 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:45 smithi029 bash[253969]: 951ea50b247d6df5335ddc24f8382fe83d5a0d3e74a9bee70a6d134bb93a6322 2023-10-09T04:23:45.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:45 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:23:45.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:45 smithi177 ceph-mon[195797]: osdmap e115: 8 total, 7 up, 8 in 2023-10-09T04:23:45.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:45 smithi177 ceph-mon[195797]: failed to encode map e115 with expected crc 2023-10-09T04:23:45.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:45 smithi177 ceph-mon[195797]: failed to encode map e115 with expected crc 2023-10-09T04:23:45.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1543194340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:45.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:45 smithi177 ceph-mon[195797]: pgmap v329: 89 pgs: 16 peering, 9 stale+active+clean, 64 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-10-09T04:23:46.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 62561 62545 122.092 102.875 0.00566437 0.00818554 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: osdmap e116: 8 total, 7 up, 8 in 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:46 smithi177 ceph-mon[195797]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: osdmap e116: 8 total, 7 up, 8 in 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[216903]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: osdmap e116: 8 total, 7 up, 8 in 2023-10-09T04:23:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:46 smithi029 ceph-mon[223858]: failed to encode map e116 with expected crc 2023-10-09T04:23:47.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 64142 64126 121.386 98.8125 0.0156182 0.00823407 2023-10-09T04:23:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:47 smithi177 ceph-mon[195797]: pgmap v331: 89 pgs: 2 active+undersized+degraded, 35 peering, 2 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 3889/183213 objects degraded (2.123%) 2023-10-09T04:23:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4134843570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:47 smithi029 ceph-mon[216903]: pgmap v331: 89 pgs: 2 active+undersized+degraded, 35 peering, 2 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 3889/183213 objects degraded (2.123%) 2023-10-09T04:23:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4134843570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:47 smithi029 ceph-mon[223858]: pgmap v331: 89 pgs: 2 active+undersized+degraded, 35 peering, 2 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s; 3889/183213 objects degraded (2.123%) 2023-10-09T04:23:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4134843570' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:48.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 65756 65740 120.783 100.875 0.0153299 0.00827465 2023-10-09T04:23:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:48 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 3889/183213 objects degraded (2.123%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:48 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 3889/183213 objects degraded (2.123%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:48 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 3889/183213 objects degraded (2.123%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:49.644 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 67150 67134 119.822 87.125 0.00660264 0.00834102 2023-10-09T04:23:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:49 smithi029 ceph-mon[216903]: pgmap v332: 89 pgs: 2 active+undersized+degraded, 35 peering, 2 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 3889/183213 objects degraded (2.123%) 2023-10-09T04:23:49.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:49 smithi029 ceph-mon[223858]: pgmap v332: 89 pgs: 2 active+undersized+degraded, 35 peering, 2 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 3889/183213 objects degraded (2.123%) 2023-10-09T04:23:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:49 smithi177 ceph-mon[195797]: pgmap v332: 89 pgs: 2 active+undersized+degraded, 35 peering, 2 active+undersized, 50 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 3889/183213 objects degraded (2.123%) 2023-10-09T04:23:50.645 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 68664 68648 119.121 94.625 0.0107788 0.0083904 2023-10-09T04:23:50.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/469733014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/469733014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/469733014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:51.645 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 70190 70174 118.479 95.375 0.00685726 0.00843567 2023-10-09T04:23:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:51 smithi029 ceph-mon[223858]: pgmap v333: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 29478/198372 objects degraded (14.860%) 2023-10-09T04:23:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:51 smithi029 ceph-mon[216903]: pgmap v333: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 29478/198372 objects degraded (14.860%) 2023-10-09T04:23:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:51 smithi177 ceph-mon[195797]: pgmap v333: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s; 29478/198372 objects degraded (14.860%) 2023-10-09T04:23:52.645 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 71644 71628 117.753 90.875 0.0147068 0.00848792 2023-10-09T04:23:52.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:52 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2023-10-09T04:23:52.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2540938144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:52.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:52 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2023-10-09T04:23:52.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2540938144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:52.820 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:23:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:23:52] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.43.0" 2023-10-09T04:23:53.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:52 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2023-10-09T04:23:53.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2540938144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:53.172 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:52 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-09T04:23:53.173 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:52 smithi029 systemd[1]: Stopped Ceph osd.1 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:23:53.173 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:53 smithi029 systemd[1]: Starting Ceph osd.1 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:23:53.645 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 73012 72996 116.926 85.5 0.00752703 0.00854729 2023-10-09T04:23:53.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:53 smithi029 ceph-mon[216903]: pgmap v334: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s; 29971/204042 objects degraded (14.689%) 2023-10-09T04:23:53.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:53 smithi029 ceph-mon[223858]: pgmap v334: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s; 29971/204042 objects degraded (14.689%) 2023-10-09T04:23:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:53 smithi177 ceph-mon[195797]: pgmap v334: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s; 29971/204042 objects degraded (14.689%) 2023-10-09T04:23:54.647 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:23:54.647071+0000 min lat: 0.00133368 max lat: 1.03507 avg lat: 0.00861137 2023-10-09T04:23:54.647 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:23:54.647 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 74330 74314 116.057 82.375 0.0166172 0.00861137 2023-10-09T04:23:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:54 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 29971/204042 objects degraded (14.689%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:54.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:23:54.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:23:54.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-09T04:23:54.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:23:54.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-09T04:23:54.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-09T04:23:54.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-09T04:23:54.922 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:54 smithi029 bash[254694]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-09T04:23:54.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:54 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 29971/204042 objects degraded (14.689%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:54 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 29971/204042 objects degraded (14.689%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:23:55.646 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 15 75755 75740 115.4 89.125 0.00992597 0.00866111 2023-10-09T04:23:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:55 smithi029 ceph-mon[223858]: pgmap v335: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 30380/213384 objects degraded (14.237%) 2023-10-09T04:23:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/98863125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:55.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:55 smithi029 ceph-mon[216903]: pgmap v335: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 30380/213384 objects degraded (14.237%) 2023-10-09T04:23:55.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/98863125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:56.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:55 smithi177 ceph-mon[195797]: pgmap v335: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 30380/213384 objects degraded (14.237%) 2023-10-09T04:23:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/98863125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:56.647 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 76864 76848 114.302 69.25 0.00945756 0.00874382 2023-10-09T04:23:57.647 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 77914 77898 113.168 65.625 0.0177209 0.0088316 2023-10-09T04:23:58.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:57 smithi029 ceph-mon[216903]: pgmap v336: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 33026/225390 objects degraded (14.653%) 2023-10-09T04:23:58.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2464165831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:58.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:57 smithi029 ceph-mon[223858]: pgmap v336: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 33026/225390 objects degraded (14.653%) 2023-10-09T04:23:58.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2464165831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:57 smithi177 ceph-mon[195797]: pgmap v336: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 33026/225390 objects degraded (14.653%) 2023-10-09T04:23:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2464165831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:23:58.307 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:58 smithi029 ceph-osd[255203]: -- 172.21.15.29:0/2857530713 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b0b36284e0 con 0x55b0b35fac00 2023-10-09T04:23:58.307 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:58 smithi029 bash[255138]: f8791635683fcef5dca76defb5eec05aac07d62920d3feffa1553d299f973932 2023-10-09T04:23:58.307 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:58 smithi029 systemd[1]: Started Ceph osd.1 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:23:58.307 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:23:58 smithi029 conmon[255177]: 2023-10-09T04:23:58.303+0000 7f8ae3983640 -1 Falling back to public interface 2023-10-09T04:23:58.650 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 79190 79174 112.401 79.75 0.00844879 0.00889186 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:23:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:23:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:23:59.651 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 80350 80334 111.515 72.5 0.0170827 0.00896239 2023-10-09T04:24:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:00 smithi029 ceph-mon[216903]: pgmap v337: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 33026/225390 objects degraded (14.653%) 2023-10-09T04:24:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2074287936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 33026/225390 objects degraded (14.653%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:00.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:00 smithi029 ceph-mon[223858]: pgmap v337: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 33026/225390 objects degraded (14.653%) 2023-10-09T04:24:00.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2074287936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:00.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 33026/225390 objects degraded (14.653%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:00 smithi177 ceph-mon[195797]: pgmap v337: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 33026/225390 objects degraded (14.653%) 2023-10-09T04:24:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2074287936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 33026/225390 objects degraded (14.653%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:00.652 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 81433 81417 110.559 67.6875 0.0148468 0.00903982 2023-10-09T04:24:01.652 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 82567 82551 109.715 70.875 0.018411 0.00911052 2023-10-09T04:24:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:02 smithi029 ceph-mon[216903]: pgmap v338: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s; 35232/238821 objects degraded (14.752%) 2023-10-09T04:24:02.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:02 smithi029 ceph-mon[223858]: pgmap v338: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s; 35232/238821 objects degraded (14.752%) 2023-10-09T04:24:02.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:02 smithi177 ceph-mon[195797]: pgmap v338: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s; 35232/238821 objects degraded (14.752%) 2023-10-09T04:24:02.652 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 83636 83620 108.821 66.8125 0.00815133 0.0091844 2023-10-09T04:24:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:24:02] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2023-10-09T04:24:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/394052572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:03.652 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 84712 84696 107.973 67.25 0.00547366 0.00925655 2023-10-09T04:24:03.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/394052572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:03.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:03.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:03.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/394052572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:03.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:03.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:04 smithi177 ceph-mon[195797]: pgmap v339: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 35554/242874 objects degraded (14.639%) 2023-10-09T04:24:04.653 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 85776 85760 107.142 66.5 0.0324071 0.00932999 2023-10-09T04:24:04.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:04 smithi029 ceph-mon[216903]: pgmap v339: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 35554/242874 objects degraded (14.639%) 2023-10-09T04:24:04.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:04 smithi029 ceph-mon[223858]: pgmap v339: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 35554/242874 objects degraded (14.639%) 2023-10-09T04:24:05.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35554/242874 objects degraded (14.639%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:05.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2544955653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[216903]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35554/242874 objects degraded (14.639%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2544955653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:05.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:05 smithi029 ceph-mon[223858]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:24:05.594 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:24:05 smithi029 conmon[255177]: 2023-10-09T04:24:05.332+0000 7f8ae3983640 -1 osd.1 114 log_to_monitors true 2023-10-09T04:24:05.653 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 86784 86768 106.277 63 0.0126865 0.0094047 2023-10-09T04:24:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35554/242874 objects degraded (14.639%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2544955653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:05 smithi177 ceph-mon[195797]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-09T04:24:05.921 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 04:24:05 smithi029 conmon[255177]: 2023-10-09T04:24:05.918+0000 7f8ad35ec700 -1 osd.1 114 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:24:06.654 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 88063 88047 105.77 79.9375 0.00683153 0.00945024 2023-10-09T04:24:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:06 smithi177 ceph-mon[195797]: pgmap v340: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 35886/250119 objects degraded (14.348%) 2023-10-09T04:24:06.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:06 smithi177 ceph-mon[195797]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:24:06.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:06 smithi177 ceph-mon[195797]: osdmap e117: 8 total, 7 up, 8 in 2023-10-09T04:24:06.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:06 smithi177 ceph-mon[195797]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:24:06.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[216903]: pgmap v340: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 35886/250119 objects degraded (14.348%) 2023-10-09T04:24:06.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[216903]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:24:06.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[216903]: osdmap e117: 8 total, 7 up, 8 in 2023-10-09T04:24:06.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[216903]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:24:06.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[223858]: pgmap v340: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 35886/250119 objects degraded (14.348%) 2023-10-09T04:24:06.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[223858]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-09T04:24:06.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[223858]: osdmap e117: 8 total, 7 up, 8 in 2023-10-09T04:24:06.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:06 smithi029 ceph-mon[223858]: from='osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:24:07.654 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 88406 88390 104.18 21.4375 0.0028547 0.00946252 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: failed to encode map e118 with expected crc 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802] boot 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:24:07.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3849096656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: failed to encode map e118 with expected crc 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802] boot 2023-10-09T04:24:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3849096656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: failed to encode map e117 with expected crc 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: failed to encode map e118 with expected crc 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: osd.1 [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802] boot 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: osdmap e118: 8 total, 8 up, 8 in 2023-10-09T04:24:07.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-09T04:24:07.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3849096656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:08.654 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 88406 88390 102.251 0 - 0.00946252 2023-10-09T04:24:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: pgmap v342: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 37894/259191 objects degraded (14.620%) 2023-10-09T04:24:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: failed to encode map e118 with expected crc 2023-10-09T04:24:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T04:24:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:08 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: pgmap v342: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 37894/259191 objects degraded (14.620%) 2023-10-09T04:24:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: failed to encode map e118 with expected crc 2023-10-09T04:24:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T04:24:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: pgmap v342: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 37894/259191 objects degraded (14.620%) 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: failed to encode map e118 with expected crc 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: osdmap e119: 8 total, 8 up, 8 in 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:08 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.659 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 88926 88910 100.973 16.25 0.0101894 0.00989829 2023-10-09T04:24:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:09 smithi177 ceph-mon[195797]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T04:24:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[216903]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T04:24:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: failed to encode map e119 with expected crc 2023-10-09T04:24:09.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:09 smithi029 ceph-mon[223858]: osdmap e120: 8 total, 8 up, 8 in 2023-10-09T04:24:10.663 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 89829 89813 100.173 56.4375 0.0113992 0.00997816 2023-10-09T04:24:10.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: pgmap v345: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 37894/259191 objects degraded (14.620%) 2023-10-09T04:24:10.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37894/259191 objects degraded (14.620%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:10.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3915046274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: pgmap v345: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 37894/259191 objects degraded (14.620%) 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37894/259191 objects degraded (14.620%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3915046274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: pgmap v345: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 37894/259191 objects degraded (14.620%) 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: failed to encode map e120 with expected crc 2023-10-09T04:24:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37894/259191 objects degraded (14.620%), 10 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:10.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3915046274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:11.664 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 90682 90666 99.3488 53.3125 0.00985919 0.0100607 2023-10-09T04:24:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:11 smithi177 ceph-mon[195797]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 11 active+undersized, 70 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 15776/266169 objects degraded (5.927%) 2023-10-09T04:24:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:11 smithi029 ceph-mon[223858]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 11 active+undersized, 70 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 15776/266169 objects degraded (5.927%) 2023-10-09T04:24:11.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:11 smithi029 ceph-mon[216903]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 11 active+undersized, 70 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 15776/266169 objects degraded (5.927%) 2023-10-09T04:24:12.664 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 91445 91429 98.4585 47.6875 0.0156477 0.0101512 2023-10-09T04:24:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2422421700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:12.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2422421700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:12.819 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:24:12] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.43.0" 2023-10-09T04:24:12.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2422421700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:13.667 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 92326 92310 97.7182 55.0625 0.0171917 0.0102288 2023-10-09T04:24:13.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:13 smithi177 ceph-mon[195797]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 32 MiB/s wr, 505 op/s; 11885/268290 objects degraded (4.430%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:24:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:13 smithi029 ceph-mon[223858]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 32 MiB/s wr, 505 op/s; 11885/268290 objects degraded (4.430%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:24:13.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:13 smithi029 ceph-mon[216903]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 32 MiB/s wr, 505 op/s; 11885/268290 objects degraded (4.430%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:24:14.667 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:24:14.667981+0000 min lat: 0.00133368 max lat: 2.15664 avg lat: 0.0103068 2023-10-09T04:24:14.668 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:24:14.668 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 93174 93158 96.9733 53 0.0294442 0.0103068 2023-10-09T04:24:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:14.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:14.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:15.667 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 94017 94001 96.2476 52.6875 0.0135537 0.0103846 2023-10-09T04:24:15.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:15 smithi177 ceph-mon[195797]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 11885/273765 objects degraded (4.341%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:24:15.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11885/268290 objects degraded (4.430%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:15.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2376705266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:15 smithi029 ceph-mon[223858]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 11885/273765 objects degraded (4.341%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:24:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11885/268290 objects degraded (4.430%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2376705266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:15.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:15 smithi029 ceph-mon[216903]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s; 11885/273765 objects degraded (4.341%); 1.0 MiB/s, 16 objects/s recovering 2023-10-09T04:24:15.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11885/268290 objects degraded (4.430%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:15.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2376705266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:16.668 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 94894 94878 95.5796 54.8125 0.00788544 0.0104577 2023-10-09T04:24:17.668 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 95637 95621 94.7999 46.4375 0.0178837 0.010544 2023-10-09T04:24:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:17 smithi177 ceph-mon[195797]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 11639/280956 objects degraded (4.143%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:24:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2805481882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:17 smithi029 ceph-mon[216903]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 11639/280956 objects degraded (4.143%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:24:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2805481882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:17 smithi029 ceph-mon[223858]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 11639/280956 objects degraded (4.143%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:24:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2805481882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:18.669 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 15 96566 96551 94.2246 58.125 0.0168644 0.0106081 2023-10-09T04:24:19.669 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 97468 97452 93.6415 56.3125 0.0264628 0.0106743 2023-10-09T04:24:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:19 smithi177 ceph-mon[195797]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 11639/280956 objects degraded (4.143%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:24:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:19 smithi029 ceph-mon[216903]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 11639/280956 objects degraded (4.143%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:24:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:19 smithi029 ceph-mon[223858]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 725 op/s; 11639/280956 objects degraded (4.143%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:24:20.671 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 98310 98294 93.0193 52.625 0.0129036 0.0107452 2023-10-09T04:24:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11639/280956 objects degraded (4.143%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/822974781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:21.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11639/280956 objects degraded (4.143%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:21.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/822974781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11639/280956 objects degraded (4.143%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:21.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/822974781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:21.671 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 99112 99096 92.3793 50.125 0.0065018 0.010808 2023-10-09T04:24:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:21 smithi029 ceph-mon[216903]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 11639/290526 objects degraded (4.006%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:24:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:21 smithi029 ceph-mon[223858]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 11639/290526 objects degraded (4.006%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:24:22.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:21 smithi177 ceph-mon[195797]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s; 11639/290526 objects degraded (4.006%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T04:24:22.672 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 99207 99191 91.1066 5.9375 0.0188641 0.0109708 2023-10-09T04:24:23.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3942576814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:23.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:24:22] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.43.0" 2023-10-09T04:24:23.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3942576814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3942576814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:23.673 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 100085 100069 90.5817 54.875 0.00712649 0.0110336 2023-10-09T04:24:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:23 smithi029 ceph-mon[216903]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 11330/293844 objects degraded (3.856%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T04:24:24.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:23 smithi029 ceph-mon[223858]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 11330/293844 objects degraded (3.856%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T04:24:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:23 smithi177 ceph-mon[195797]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 11330/293844 objects degraded (3.856%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T04:24:24.673 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 100958 100942 90.0674 54.5625 0.00761676 0.0110971 2023-10-09T04:24:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:24 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11330/293844 objects degraded (3.856%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:24 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11330/293844 objects degraded (3.856%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:24 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11330/293844 objects degraded (3.856%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:25.676 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 101776 101760 89.5143 51.125 0.0167698 0.0111668 2023-10-09T04:24:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:25 smithi177 ceph-mon[195797]: pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s; 11330/298377 objects degraded (3.797%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2413253108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:25 smithi029 ceph-mon[216903]: pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s; 11330/298377 objects degraded (3.797%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2413253108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:25 smithi029 ceph-mon[223858]: pgmap v354: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s; 11330/298377 objects degraded (3.797%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2413253108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:26.676 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 102648 102632 89.0283 54.5 0.0221656 0.0112275 2023-10-09T04:24:27.678 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 103458 103442 88.5008 50.625 0.0228242 0.0112935 2023-10-09T04:24:28.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:27 smithi177 ceph-mon[195797]: pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 11086/304242 objects degraded (3.644%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:24:28.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3032275977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:27 smithi029 ceph-mon[216903]: pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 11086/304242 objects degraded (3.644%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:24:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3032275977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:27 smithi029 ceph-mon[223858]: pgmap v355: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 11086/304242 objects degraded (3.644%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:24:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3032275977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:28.682 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 104301 104285 88.0118 52.6875 0.0151561 0.0113568 2023-10-09T04:24:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:29.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:29.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:29.683 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 105141 105125 87.5374 52.5 0.0360604 0.0114181 2023-10-09T04:24:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 646 op/s; 11086/304242 objects degraded (3.644%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11086/304242 objects degraded (3.644%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:30.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 646 op/s; 11086/304242 objects degraded (3.644%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11086/304242 objects degraded (3.644%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:30.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:29 smithi029 conmon[202582]: 2023-10-09T04:24:29.893+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:24:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: pgmap v356: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 646 op/s; 11086/304242 objects degraded (3.644%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11086/304242 objects degraded (3.644%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:24:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:30.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 105909 105893 87.0175 48 0.012621 0.0114868 2023-10-09T04:24:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:30 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:30 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:24:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4232051106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:31.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:30 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:31.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:30 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:24:31.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4232051106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:31.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:30 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:31.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:30 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-09T04:24:31.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4232051106' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:31.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 106784 106768 86.5978 54.6875 0.0108556 0.0115433 2023-10-09T04:24:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:31 smithi177 ceph-mon[195797]: pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 11086/313614 objects degraded (3.535%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:32.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:31 smithi029 ceph-mon[216903]: pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 11086/313614 objects degraded (3.535%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:32.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:31 smithi029 ceph-mon[223858]: pgmap v357: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 11086/313614 objects degraded (3.535%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:32.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 107595 107579 86.1377 50.6875 0.0256677 0.0116041 2023-10-09T04:24:33.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2015866256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:33.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:24:32] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-10-09T04:24:33.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2015866256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:33.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2015866256' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:33.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 108402 108386 85.686 50.4375 0.0128196 0.0116655 2023-10-09T04:24:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:33 smithi177 ceph-mon[195797]: pgmap v358: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s; 10767/316806 objects degraded (3.399%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:24:34.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:33 smithi029 ceph-mon[216903]: pgmap v358: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s; 10767/316806 objects degraded (3.399%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:24:34.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:33 smithi029 ceph-mon[223858]: pgmap v358: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s; 10767/316806 objects degraded (3.399%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:24:34.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:24:34.684589+0000 min lat: 0.00133368 max lat: 2.15664 avg lat: 0.0117233 2023-10-09T04:24:34.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:24:34.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 109235 109219 85.2659 52.0625 0.024539 0.0117233 2023-10-09T04:24:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10767/316806 objects degraded (3.399%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:35.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10767/316806 objects degraded (3.399%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:35.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10767/316806 objects degraded (3.399%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:35.684 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 110018 110002 84.8176 48.9375 0.0219923 0.0117847 2023-10-09T04:24:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:36 smithi029 ceph-mon[216903]: pgmap v359: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 10767/322092 objects degraded (3.343%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/246587932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:36 smithi029 ceph-mon[223858]: pgmap v359: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 10767/322092 objects degraded (3.343%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/246587932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:36 smithi177 ceph-mon[195797]: pgmap v359: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 10767/322092 objects degraded (3.343%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:24:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/246587932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:36.685 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 110865 110849 84.4285 52.9375 0.00613364 0.0118393 2023-10-09T04:24:37.687 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 111708 111692 84.0443 52.6875 0.0293167 0.0118933 2023-10-09T04:24:38.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:38 smithi029 ceph-mon[216903]: pgmap v360: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 10405/329085 objects degraded (3.162%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:24:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:38 smithi029 ceph-mon[223858]: pgmap v360: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 10405/329085 objects degraded (3.162%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:24:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:38 smithi177 ceph-mon[195797]: pgmap v360: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 10405/329085 objects degraded (3.162%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:24:38.687 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 112273 112257 83.4645 35.3125 0.00406835 0.0119162 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1129479794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[216903]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T04:24:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1129479794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:39 smithi029 ceph-mon[223858]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: osdmap e121: 8 total, 8 up, 8 in 2023-10-09T04:24:39.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1129479794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:39.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:39 smithi177 ceph-mon[195797]: failed to encode map e121 with expected crc 2023-10-09T04:24:39.687 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 112751 112735 82.8342 29.875 0.0184987 0.0120673 2023-10-09T04:24:40.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[216903]: pgmap v362: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10405/329085 objects degraded (3.162%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:24:40.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[216903]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[216903]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[216903]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T04:24:40.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[216903]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10405/329085 objects degraded (3.162%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[223858]: pgmap v362: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10405/329085 objects degraded (3.162%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[223858]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[223858]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[223858]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[223858]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10405/329085 objects degraded (3.162%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:40 smithi177 ceph-mon[195797]: pgmap v362: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s; 10405/329085 objects degraded (3.162%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:24:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:40 smithi177 ceph-mon[195797]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:40 smithi177 ceph-mon[195797]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:40 smithi177 ceph-mon[195797]: osdmap e122: 8 total, 8 up, 8 in 2023-10-09T04:24:40.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:40 smithi177 ceph-mon[195797]: failed to encode map e122 with expected crc 2023-10-09T04:24:40.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10405/329085 objects degraded (3.162%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:40.687 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 113516 113500 82.4272 47.8125 0.0170422 0.0121263 2023-10-09T04:24:41.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/294061653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:41.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/294061653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/294061653' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:41.687 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 114337 114321 82.0697 51.3125 0.0267236 0.0121796 2023-10-09T04:24:42.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:42 smithi029 ceph-mon[216903]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 10200/337035 objects degraded (3.026%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T04:24:42.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:42 smithi029 ceph-mon[223858]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 10200/337035 objects degraded (3.026%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T04:24:42.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:42 smithi177 ceph-mon[195797]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 10200/337035 objects degraded (3.026%); 4.4 MiB/s, 70 objects/s recovering 2023-10-09T04:24:42.687 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 15 115140 115125 81.7083 50.25 0.0238971 0.0122337 2023-10-09T04:24:43.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:24:42] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-10-09T04:24:43.689 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 115877 115861 81.3055 46 0.0182263 0.0122939 2023-10-09T04:24:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:44 smithi029 ceph-mon[216903]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s; 10072/339441 objects degraded (2.967%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:24:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1192574179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:44 smithi029 ceph-mon[223858]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s; 10072/339441 objects degraded (2.967%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:24:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1192574179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:44 smithi177 ceph-mon[195797]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s; 10072/339441 objects degraded (2.967%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:24:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1192574179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:44.689 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0092 2023-10-09T04:24:44.689 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 116621 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 80.9785 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 43.5567 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 238.812 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1295 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 696.907 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3821 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2023-10-09T04:24:44.690 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0123453 2023-10-09T04:24:44.691 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0324289 2023-10-09T04:24:44.691 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.15664 2023-10-09T04:24:44.691 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00133368 2023-10-09T04:24:44.695 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:24:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10072/339441 objects degraded (2.967%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10072/339441 objects degraded (2.967%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:45.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:44 smithi029 conmon[202582]: 2023-10-09T04:24:44.908+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:24:45.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10072/339441 objects degraded (2.967%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:45.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:46.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[216903]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 10072/344691 objects degraded (2.922%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:24:46.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:46.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:24:46.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3559730218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[223858]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 10072/344691 objects degraded (2.922%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:24:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:24:46.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3559730218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:46 smithi177 ceph-mon[195797]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 10072/344691 objects degraded (2.922%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:24:46.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:46 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:24:46.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:46 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:24:46.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3559730218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:48 smithi029 ceph-mon[223858]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 858 op/s; 9784/348558 objects degraded (2.807%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:24:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:48 smithi029 ceph-mon[216903]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 858 op/s; 9784/348558 objects degraded (2.807%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:24:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:48 smithi177 ceph-mon[195797]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 858 op/s; 9784/348558 objects degraded (2.807%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:24:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3077351090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:49.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3077351090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:49.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3077351090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:50 smithi029 ceph-mon[216903]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 739 op/s; 9784/348558 objects degraded (2.807%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:24:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9784/348558 objects degraded (2.807%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:50 smithi029 ceph-mon[223858]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 739 op/s; 9784/348558 objects degraded (2.807%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:24:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9784/348558 objects degraded (2.807%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:50.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:50 smithi177 ceph-mon[195797]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 739 op/s; 9784/348558 objects degraded (2.807%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:24:50.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9784/348558 objects degraded (2.807%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1122727771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:51.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1122727771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:51.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1122727771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:52 smithi177 ceph-mon[195797]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88 B/s rd, 37 MiB/s wr, 892 op/s; 9458/339702 objects degraded (2.784%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:24:52.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:52 smithi029 ceph-mon[216903]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88 B/s rd, 37 MiB/s wr, 892 op/s; 9458/339702 objects degraded (2.784%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:24:52.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:52 smithi029 ceph-mon[223858]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88 B/s rd, 37 MiB/s wr, 892 op/s; 9458/339702 objects degraded (2.784%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:24:53.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:24:52] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-10-09T04:24:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:54 smithi177 ceph-mon[195797]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 722 op/s; 9458/336816 objects degraded (2.808%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:24:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3111274491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:54.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:54 smithi029 ceph-mon[216903]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 722 op/s; 9458/336816 objects degraded (2.808%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:24:54.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3111274491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:54.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:54 smithi029 ceph-mon[223858]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 722 op/s; 9458/336816 objects degraded (2.808%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:24:54.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3111274491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:55.535 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9458/336816 objects degraded (2.808%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9458/336816 objects degraded (2.808%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9458/336816 objects degraded (2.808%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:24:56.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:56 smithi177 ceph-mon[195797]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 793 op/s; 9458/331845 objects degraded (2.850%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:24:56.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1161754693' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:56.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:56 smithi029 ceph-mon[216903]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 793 op/s; 9458/331845 objects degraded (2.850%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:24:56.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1161754693' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:56.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:56 smithi029 ceph-mon[223858]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 793 op/s; 9458/331845 objects degraded (2.850%); 3.2 MiB/s, 51 objects/s recovering 2023-10-09T04:24:56.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1161754693' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:58 smithi177 ceph-mon[195797]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.1 MiB/s wr, 836 op/s; 9120/325077 objects degraded (2.805%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:24:58.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:58 smithi029 ceph-mon[223858]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.1 MiB/s wr, 836 op/s; 9120/325077 objects degraded (2.805%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:24:58.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:58 smithi029 ceph-mon[216903]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 9.1 MiB/s wr, 836 op/s; 9120/325077 objects degraded (2.805%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:24:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:24:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2180924060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:59.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:59.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:24:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2180924060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:24:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:24:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:24:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2180924060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:00.266 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:24:59 smithi029 conmon[202582]: 2023-10-09T04:24:59.925+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:25:00.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:00 smithi029 ceph-mon[216903]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 KiB/s wr, 653 op/s; 9120/325077 objects degraded (2.805%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:25:00.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 9120/325077 objects degraded (2.805%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:00.267 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:00 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:00 smithi177 ceph-mon[195797]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 KiB/s wr, 653 op/s; 9120/325077 objects degraded (2.805%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:25:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 9120/325077 objects degraded (2.805%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:00.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:00 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:00.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:00 smithi029 ceph-mon[223858]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 KiB/s wr, 653 op/s; 9120/325077 objects degraded (2.805%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:25:00.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 9120/325077 objects degraded (2.805%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:00.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:00 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:01.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:01 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:01.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:01 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:25:01.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:01 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:01.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:01 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:25:01.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:01 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:01.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:01 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-09T04:25:02.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:02 smithi029 ceph-mon[216903]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 KiB/s wr, 892 op/s; 8767/316446 objects degraded (2.770%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:25:02.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2365751529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:02.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:02 smithi029 ceph-mon[223858]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 KiB/s wr, 892 op/s; 8767/316446 objects degraded (2.770%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:25:02.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2365751529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:02 smithi177 ceph-mon[195797]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 KiB/s wr, 892 op/s; 8767/316446 objects degraded (2.770%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:25:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2365751529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:03.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:25:02] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.43.0" 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[216903]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 8767/313605 objects degraded (2.796%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[216903]: failed to encode map e123 with expected crc 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[216903]: osdmap e123: 8 total, 8 up, 8 in 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[216903]: failed to encode map e123 with expected crc 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1188379403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[223858]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 8767/313605 objects degraded (2.796%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:25:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[223858]: failed to encode map e123 with expected crc 2023-10-09T04:25:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[223858]: osdmap e123: 8 total, 8 up, 8 in 2023-10-09T04:25:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[223858]: failed to encode map e123 with expected crc 2023-10-09T04:25:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1188379403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:04.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:04 smithi177 ceph-mon[195797]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 8767/313605 objects degraded (2.796%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:25:04.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:04 smithi177 ceph-mon[195797]: failed to encode map e123 with expected crc 2023-10-09T04:25:04.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:04 smithi177 ceph-mon[195797]: osdmap e123: 8 total, 8 up, 8 in 2023-10-09T04:25:04.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:04 smithi177 ceph-mon[195797]: failed to encode map e123 with expected crc 2023-10-09T04:25:04.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1188379403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:05.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[216903]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[216903]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[216903]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[216903]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T04:25:05.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8767/313605 objects degraded (2.796%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:05.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[223858]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[223858]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[223858]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[223858]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T04:25:05.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8767/313605 objects degraded (2.796%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:05 smithi177 ceph-mon[195797]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:05 smithi177 ceph-mon[195797]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:05 smithi177 ceph-mon[195797]: failed to encode map e124 with expected crc 2023-10-09T04:25:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:05 smithi177 ceph-mon[195797]: osdmap e124: 8 total, 8 up, 8 in 2023-10-09T04:25:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8767/313605 objects degraded (2.796%), 6 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:06.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:06 smithi029 ceph-mon[216903]: failed to encode map e124 with expected crc 2023-10-09T04:25:06.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:06 smithi029 ceph-mon[216903]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 8575/309333 objects degraded (2.772%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:25:06.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:06 smithi029 ceph-mon[223858]: failed to encode map e124 with expected crc 2023-10-09T04:25:06.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:06 smithi029 ceph-mon[223858]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 8575/309333 objects degraded (2.772%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:25:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:06 smithi177 ceph-mon[195797]: failed to encode map e124 with expected crc 2023-10-09T04:25:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:06 smithi177 ceph-mon[195797]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 8575/309333 objects degraded (2.772%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:25:07.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1609395202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:07.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1609395202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:07.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1609395202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:08.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:08 smithi029 ceph-mon[216903]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8343/303795 objects degraded (2.746%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:25:08.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:08 smithi029 ceph-mon[223858]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8343/303795 objects degraded (2.746%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:25:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:08 smithi177 ceph-mon[195797]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8343/303795 objects degraded (2.746%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3975686543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:09.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3975686543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3975686543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:10.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:10 smithi029 ceph-mon[216903]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 8343/303795 objects degraded (2.746%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:25:10.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8343/303795 objects degraded (2.746%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:10 smithi029 ceph-mon[223858]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 8343/303795 objects degraded (2.746%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:25:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8343/303795 objects degraded (2.746%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:10.787 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:10 smithi177 ceph-mon[195797]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 8343/303795 objects degraded (2.746%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:25:10.787 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8343/303795 objects degraded (2.746%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:25:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:12 smithi029 ceph-mon[223858]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8094/296085 objects degraded (2.734%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:25:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2386591287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:12.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:12 smithi029 ceph-mon[216903]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8094/296085 objects degraded (2.734%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:25:12.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2386591287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:12 smithi177 ceph-mon[195797]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 8094/296085 objects degraded (2.734%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:25:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2386591287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:13.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:25:12] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.43.0" 2023-10-09T04:25:14.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:14 smithi177 ceph-mon[195797]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8094/292674 objects degraded (2.766%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:25:14.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:14.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1406117522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:14.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:14 smithi029 ceph-mon[216903]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8094/292674 objects degraded (2.766%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:25:14.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:14.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1406117522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:14.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:14 smithi029 ceph-mon[223858]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8094/292674 objects degraded (2.766%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:25:14.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:14.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1406117522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:15.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:14 smithi029 conmon[202582]: 2023-10-09T04:25:14.940+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8094/292674 objects degraded (2.766%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8094/292674 objects degraded (2.766%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8094/292674 objects degraded (2.766%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:16.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:16 smithi177 ceph-mon[195797]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 7792/287967 objects degraded (2.706%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:25:16.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:16 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:16.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:16 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:16 smithi029 ceph-mon[223858]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 7792/287967 objects degraded (2.706%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:25:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:16 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:16 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:16 smithi029 ceph-mon[216903]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 7792/287967 objects degraded (2.706%); 5.9 MiB/s, 94 objects/s recovering 2023-10-09T04:25:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:16 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:16 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3968385336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3968385336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3968385336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:18.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:18 smithi177 ceph-mon[195797]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 7792/281679 objects degraded (2.766%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:25:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:18 smithi029 ceph-mon[216903]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 7792/281679 objects degraded (2.766%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:25:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:18 smithi029 ceph-mon[223858]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 7792/281679 objects degraded (2.766%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:25:19.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2224145824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2224145824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2224145824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:20 smithi177 ceph-mon[195797]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 7792/281679 objects degraded (2.766%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:25:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 7792/281679 objects degraded (2.766%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:20 smithi029 ceph-mon[223858]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 7792/281679 objects degraded (2.766%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:25:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 7792/281679 objects degraded (2.766%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:20.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:20 smithi029 ceph-mon[216903]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 7792/281679 objects degraded (2.766%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:25:20.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 7792/281679 objects degraded (2.766%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:22.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:22 smithi177 ceph-mon[195797]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 7504/273972 objects degraded (2.739%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:25:22.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1935206708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:22.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:22 smithi029 ceph-mon[216903]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 7504/273972 objects degraded (2.739%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:25:22.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1935206708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:22.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:22 smithi029 ceph-mon[223858]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 7504/273972 objects degraded (2.739%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:25:22.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1935206708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:23.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:25:22] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.43.0" 2023-10-09T04:25:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:23 smithi029 ceph-mon[216903]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7504/270501 objects degraded (2.774%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:23 smithi029 ceph-mon[223858]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7504/270501 objects degraded (2.774%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:23 smithi177 ceph-mon[195797]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7504/270501 objects degraded (2.774%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3090450299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:24.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3090450299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3090450299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:25 smithi029 ceph-mon[216903]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 7203/265404 objects degraded (2.714%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T04:25:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 7504/270501 objects degraded (2.774%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:25 smithi029 ceph-mon[223858]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 7203/265404 objects degraded (2.714%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T04:25:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 7504/270501 objects degraded (2.774%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:25 smithi177 ceph-mon[195797]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 7203/265404 objects degraded (2.714%); 4.6 MiB/s, 74 objects/s recovering 2023-10-09T04:25:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 7504/270501 objects degraded (2.774%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:28.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:27 smithi177 ceph-mon[195797]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7203/259029 objects degraded (2.781%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:28.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3232267819' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:27 smithi029 ceph-mon[216903]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7203/259029 objects degraded (2.781%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3232267819' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:27 smithi029 ceph-mon[223858]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7203/259029 objects degraded (2.781%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3232267819' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:29.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:29.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:29 smithi029 ceph-mon[223858]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 7203/259029 objects degraded (2.781%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4105495537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:29 smithi029 ceph-mon[216903]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 7203/259029 objects degraded (2.781%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4105495537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:29 smithi177 ceph-mon[195797]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 7203/259029 objects degraded (2.781%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:25:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4105495537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:31.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 7203/259029 objects degraded (2.781%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:31.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:31.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 7203/259029 objects degraded (2.781%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:31.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:31.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 7203/259029 objects degraded (2.781%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:31.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:31 smithi029 ceph-mon[216903]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 6893/251568 objects degraded (2.740%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:25:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:31 smithi029 ceph-mon[223858]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 6893/251568 objects degraded (2.740%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:25:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:31 smithi177 ceph-mon[195797]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 6893/251568 objects degraded (2.740%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:25:33.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T04:25:33.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2136614596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:25:32] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2136614596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: osdmap e125: 8 total, 8 up, 8 in 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: failed to encode map e125 with expected crc 2023-10-09T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2136614596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 6893/248061 objects degraded (2.779%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:25:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T04:25:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:33 smithi177 ceph-mon[195797]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 6893/248061 objects degraded (2.779%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:25:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[216903]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 6893/248061 objects degraded (2.779%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: osdmap e126: 8 total, 8 up, 8 in 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: failed to encode map e126 with expected crc 2023-10-09T04:25:34.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:33 smithi029 ceph-mon[223858]: failed to encode map e126 with expected crc 2023-10-09T04:25:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:35.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/760040146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:35.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6893/248061 objects degraded (2.779%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:35.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:35.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:35.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/760040146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:35.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6893/248061 objects degraded (2.779%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:35.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:35.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:35.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/760040146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:35.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6893/248061 objects degraded (2.779%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:36 smithi177 ceph-mon[195797]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 6893/244806 objects degraded (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:25:36.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:36 smithi029 ceph-mon[216903]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 6893/244806 objects degraded (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:25:36.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:36 smithi029 ceph-mon[223858]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 6893/244806 objects degraded (2.816%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:25:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:38 smithi177 ceph-mon[195797]: pgmap v396: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 6566/239241 objects degraded (2.745%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:25:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1804277747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:38.708 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:38 smithi029 ceph-mon[216903]: pgmap v396: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 6566/239241 objects degraded (2.745%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:25:38.708 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1804277747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:38.709 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:38 smithi029 ceph-mon[223858]: pgmap v396: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 6566/239241 objects degraded (2.745%); 5.0 MiB/s, 79 objects/s recovering 2023-10-09T04:25:38.709 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1804277747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[223858]: pgmap v397: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 6566/239241 objects degraded (2.745%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T04:25:40.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:40.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:40.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2971892748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:40.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6566/239241 objects degraded (2.745%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:40.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[216903]: pgmap v397: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 6566/239241 objects degraded (2.745%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T04:25:40.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:40.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:40.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2971892748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:40.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6566/239241 objects degraded (2.745%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:40 smithi177 ceph-mon[195797]: pgmap v397: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 6566/239241 objects degraded (2.745%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T04:25:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2971892748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6566/239241 objects degraded (2.745%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:40 smithi029 conmon[202582]: 2023-10-09T04:25:40.920+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.387 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.387 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:41.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:25:41.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:25:41.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:25:41.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:25:41.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:25:41.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:42 smithi029 ceph-mon[216903]: pgmap v398: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6390/233115 objects degraded (2.741%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:25:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:42 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:42 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:42 smithi029 ceph-mon[223858]: pgmap v398: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6390/233115 objects degraded (2.741%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:25:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:42 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:42 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:42.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:42 smithi177 ceph-mon[195797]: pgmap v398: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 6390/233115 objects degraded (2.741%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:25:42.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:42 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:42.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:42 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:43.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:25:42] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-10-09T04:25:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3519741659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:43.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3519741659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:43.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3519741659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:44 smithi029 ceph-mon[216903]: pgmap v399: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 6390/229311 objects degraded (2.787%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:25:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:44 smithi029 ceph-mon[223858]: pgmap v399: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 6390/229311 objects degraded (2.787%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:25:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:44 smithi177 ceph-mon[195797]: pgmap v399: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 6390/229311 objects degraded (2.787%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:25:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6390/229311 objects degraded (2.787%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/394801571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6390/229311 objects degraded (2.787%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:45.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/394801571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:45.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6390/229311 objects degraded (2.787%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:45.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/394801571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:46 smithi177 ceph-mon[195797]: pgmap v400: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 6390/225090 objects degraded (2.839%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:25:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:46 smithi029 ceph-mon[216903]: pgmap v400: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 6390/225090 objects degraded (2.839%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:25:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:46 smithi029 ceph-mon[223858]: pgmap v400: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 6390/225090 objects degraded (2.839%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T04:25:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:48 smithi177 ceph-mon[195797]: pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6162/218226 objects degraded (2.824%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:25:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/106329747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:48 smithi029 ceph-mon[216903]: pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6162/218226 objects degraded (2.824%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:25:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/106329747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:48 smithi029 ceph-mon[223858]: pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 6162/218226 objects degraded (2.824%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:25:48.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/106329747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:50.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:50 smithi177 ceph-mon[195797]: pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6162/218226 objects degraded (2.824%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:25:50.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6162/218226 objects degraded (2.824%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:50.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1847315646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:50 smithi029 ceph-mon[216903]: pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6162/218226 objects degraded (2.824%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:25:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6162/218226 objects degraded (2.824%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1847315646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:50 smithi029 ceph-mon[223858]: pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 6162/218226 objects degraded (2.824%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:25:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6162/218226 objects degraded (2.824%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1847315646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:52.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:52 smithi177 ceph-mon[195797]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 5914/211425 objects degraded (2.797%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:25:52.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:52 smithi029 ceph-mon[216903]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 5914/211425 objects degraded (2.797%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:25:52.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:52 smithi029 ceph-mon[223858]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 5914/211425 objects degraded (2.797%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:25:53.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:25:52] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-10-09T04:25:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1725837532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:53.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1725837532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:53.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1725837532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:54.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:54 smithi177 ceph-mon[195797]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5914/207324 objects degraded (2.853%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:25:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:54 smithi029 ceph-mon[216903]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5914/207324 objects degraded (2.853%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:25:54.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:54 smithi029 ceph-mon[223858]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5914/207324 objects degraded (2.853%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:25:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5914/207324 objects degraded (2.853%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/421061726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5914/207324 objects degraded (2.853%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/421061726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5914/207324 objects degraded (2.853%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:25:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/421061726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:56.424 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:25:55 smithi029 conmon[202582]: 2023-10-09T04:25:55.966+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[216903]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 5914/203376 objects degraded (2.908%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[223858]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 5914/203376 objects degraded (2.908%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:56.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:56 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:56 smithi177 ceph-mon[195797]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 5914/203376 objects degraded (2.908%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:25:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:56 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:56 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:25:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:56 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:25:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:57 smithi029 ceph-mon[223858]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 5636/196800 objects degraded (2.864%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:25:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:57 smithi029 ceph-mon[216903]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 5636/196800 objects degraded (2.864%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:25:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:57 smithi177 ceph-mon[195797]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 5636/196800 objects degraded (2.864%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:25:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2985690083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2985690083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2985690083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:25:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:25:59.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:25:59 smithi029 ceph-mon[216903]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 5636/196800 objects degraded (2.864%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:25:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:25:59 smithi029 ceph-mon[223858]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 5636/196800 objects degraded (2.864%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:26:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:25:59 smithi177 ceph-mon[195797]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 5636/196800 objects degraded (2.864%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:26:00.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5636/196800 objects degraded (2.864%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:00.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3364322398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:00.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5636/196800 objects degraded (2.864%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:00.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3364322398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5636/196800 objects degraded (2.864%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3364322398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:01.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:01 smithi029 ceph-mon[216903]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5384/189753 objects degraded (2.837%); 4.1 MiB/s, 64 objects/s recovering 2023-10-09T04:26:01.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:01 smithi029 ceph-mon[223858]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5384/189753 objects degraded (2.837%); 4.1 MiB/s, 64 objects/s recovering 2023-10-09T04:26:02.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:01 smithi177 ceph-mon[195797]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 5384/189753 objects degraded (2.837%); 4.1 MiB/s, 64 objects/s recovering 2023-10-09T04:26:03.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:26:02] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-10-09T04:26:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:03 smithi177 ceph-mon[195797]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 5384/186024 objects degraded (2.894%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:26:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3965879029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:04.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:03 smithi029 ceph-mon[216903]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 5384/186024 objects degraded (2.894%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:26:04.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3965879029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:04.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:03 smithi029 ceph-mon[223858]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 5384/186024 objects degraded (2.894%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:26:04.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3965879029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:05 smithi029 ceph-mon[216903]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 5384/182112 objects degraded (2.956%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:26:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5384/182112 objects degraded (2.956%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/487945520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:06.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:05 smithi029 ceph-mon[223858]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 5384/182112 objects degraded (2.956%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:26:06.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5384/182112 objects degraded (2.956%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:06.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/487945520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:05 smithi177 ceph-mon[195797]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 5384/182112 objects degraded (2.956%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:26:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5384/182112 objects degraded (2.956%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:06.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/487945520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T04:26:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:06 smithi177 ceph-mon[195797]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T04:26:07.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[216903]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: osdmap e127: 8 total, 8 up, 8 in 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: failed to encode map e127 with expected crc 2023-10-09T04:26:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:06 smithi029 ceph-mon[223858]: failed to encode map e127 with expected crc 2023-10-09T04:26:08.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: pgmap v412: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5082/175017 objects degraded (2.904%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:26:08.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T04:26:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:07 smithi177 ceph-mon[195797]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: pgmap v412: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5082/175017 objects degraded (2.904%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[216903]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: pgmap v412: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 5082/175017 objects degraded (2.904%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: osdmap e128: 8 total, 8 up, 8 in 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: failed to encode map e128 with expected crc 2023-10-09T04:26:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:07 smithi029 ceph-mon[223858]: failed to encode map e128 with expected crc 2023-10-09T04:26:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2349893981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:09.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2349893981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:09.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2349893981' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:10.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:09 smithi177 ceph-mon[195797]: pgmap v414: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5082/175017 objects degraded (2.904%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:26:10.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:09 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 5082/175017 objects degraded (2.904%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:09 smithi029 ceph-mon[223858]: pgmap v414: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5082/175017 objects degraded (2.904%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:26:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:09 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 5082/175017 objects degraded (2.904%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:10.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:09 smithi029 ceph-mon[216903]: pgmap v414: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 5082/175017 objects degraded (2.904%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:26:10.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:09 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 5082/175017 objects degraded (2.904%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2877565136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:11.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2877565136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:11.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:11 smithi029 conmon[202582]: 2023-10-09T04:26:10.998+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2877565136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:11 smithi177 ceph-mon[195797]: pgmap v415: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4724/170292 objects degraded (2.774%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:26:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:11 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:12.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:11 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:12.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[223858]: pgmap v415: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4724/170292 objects degraded (2.774%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:26:12.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:12.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:12.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:12.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[216903]: pgmap v415: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4724/170292 objects degraded (2.774%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:26:12.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:12.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:12.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:11 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:13.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:26:12] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-10-09T04:26:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:14 smithi177 ceph-mon[195797]: pgmap v416: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 4724/166425 objects degraded (2.839%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:26:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/820427220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:14 smithi029 ceph-mon[223858]: pgmap v416: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 4724/166425 objects degraded (2.839%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:26:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/820427220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:14.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:14 smithi029 ceph-mon[216903]: pgmap v416: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 4724/166425 objects degraded (2.839%); 5.2 MiB/s, 82 objects/s recovering 2023-10-09T04:26:14.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/820427220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:14.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:15.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4724/166425 objects degraded (2.839%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:15.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4724/166425 objects degraded (2.839%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4724/166425 objects degraded (2.839%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:16.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:16 smithi029 ceph-mon[216903]: pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 4724/162267 objects degraded (2.911%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:16.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1410709899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:16.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:16 smithi029 ceph-mon[223858]: pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 4724/162267 objects degraded (2.911%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:16.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1410709899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:16 smithi177 ceph-mon[195797]: pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 4724/162267 objects degraded (2.911%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1410709899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:18 smithi029 ceph-mon[216903]: pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4483/155478 objects degraded (2.883%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:26:18.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:18 smithi029 ceph-mon[223858]: pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4483/155478 objects degraded (2.883%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:26:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:18 smithi177 ceph-mon[195797]: pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 4483/155478 objects degraded (2.883%); 3.7 MiB/s, 59 objects/s recovering 2023-10-09T04:26:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/514952197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:19.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/514952197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:19.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/514952197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:20.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:20 smithi029 ceph-mon[216903]: pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 4483/155478 objects degraded (2.883%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:26:20.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4483/155478 objects degraded (2.883%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:20.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:20 smithi029 ceph-mon[223858]: pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 4483/155478 objects degraded (2.883%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:26:20.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4483/155478 objects degraded (2.883%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:20 smithi177 ceph-mon[195797]: pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 4483/155478 objects degraded (2.883%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:26:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4483/155478 objects degraded (2.883%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3379109057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3379109057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3379109057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:22.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:22 smithi029 ceph-mon[216903]: pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 4230/149295 objects degraded (2.833%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T04:26:22.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:22 smithi029 ceph-mon[223858]: pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 4230/149295 objects degraded (2.833%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T04:26:22.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:22 smithi177 ceph-mon[195797]: pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 4230/149295 objects degraded (2.833%); 4.4 MiB/s, 71 objects/s recovering 2023-10-09T04:26:23.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:26:22] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2023-10-09T04:26:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:24 smithi029 ceph-mon[216903]: pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 4230/144381 objects degraded (2.930%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/10624953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:24 smithi029 ceph-mon[223858]: pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 4230/144381 objects degraded (2.930%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/10624953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:24.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:24 smithi177 ceph-mon[195797]: pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 4230/144381 objects degraded (2.930%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:24.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/10624953' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:25.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4230/144381 objects degraded (2.930%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:25.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4230/144381 objects degraded (2.930%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4230/144381 objects degraded (2.930%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:26.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:26 smithi029 ceph-mon[216903]: pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 4230/140136 objects degraded (3.018%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:26.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:26 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:26.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2709536082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:26.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:26 smithi029 ceph-mon[223858]: pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 4230/140136 objects degraded (3.018%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:26.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:26 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:26.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2709536082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:26.427 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:26 smithi029 conmon[202582]: 2023-10-09T04:26:26.022+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:26.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:26 smithi177 ceph-mon[195797]: pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 4230/140136 objects degraded (3.018%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:26:26.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:26 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:26.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2709536082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:27.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:27 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:27.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:27 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:27.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:27 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:27.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:27 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:27.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:27 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:27.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:27 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-09T04:26:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:28 smithi177 ceph-mon[195797]: pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 3963/133251 objects degraded (2.974%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:26:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:28 smithi029 ceph-mon[216903]: pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 3963/133251 objects degraded (2.974%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:26:28.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:28 smithi029 ceph-mon[223858]: pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 3963/133251 objects degraded (2.974%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:26:29.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:29.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/497598364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:29.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:29.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/497598364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:29.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:29.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/497598364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:30 smithi177 ceph-mon[195797]: pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 3963/133251 objects degraded (2.974%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:26:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3963/133251 objects degraded (2.974%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:30 smithi029 ceph-mon[216903]: pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 3963/133251 objects degraded (2.974%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:26:30.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3963/133251 objects degraded (2.974%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:30 smithi029 ceph-mon[223858]: pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 3963/133251 objects degraded (2.974%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:26:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3963/133251 objects degraded (2.974%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:31.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1482351337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:31.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1482351337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:31.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1482351337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:32.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:32 smithi177 ceph-mon[195797]: pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 3684/127236 objects degraded (2.895%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:26:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:32 smithi029 ceph-mon[216903]: pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 3684/127236 objects degraded (2.895%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:26:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:32 smithi029 ceph-mon[223858]: pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 3684/127236 objects degraded (2.895%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:26:33.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:26:32] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-10-09T04:26:34.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:34 smithi029 ceph-mon[216903]: pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3684/122319 objects degraded (3.012%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:26:34.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2942673047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:34.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:34 smithi029 ceph-mon[223858]: pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3684/122319 objects degraded (3.012%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:26:34.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2942673047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:34 smithi177 ceph-mon[195797]: pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 3684/122319 objects degraded (3.012%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:26:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2942673047' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3684/122319 objects degraded (3.012%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:35.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3684/122319 objects degraded (3.012%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:35.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3684/122319 objects degraded (3.012%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:36 smithi029 ceph-mon[216903]: pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 3684/118032 objects degraded (3.121%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:26:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:36 smithi029 ceph-mon[216903]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T04:26:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:36 smithi029 ceph-mon[223858]: pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 3684/118032 objects degraded (3.121%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:26:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:36 smithi029 ceph-mon[223858]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T04:26:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:36 smithi177 ceph-mon[195797]: pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 3684/118032 objects degraded (3.121%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:26:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:36 smithi177 ceph-mon[195797]: osdmap e129: 8 total, 8 up, 8 in 2023-10-09T04:26:37.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[216903]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[216903]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[216903]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1644601578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:37.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[216903]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T04:26:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[223858]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[223858]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[223858]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1644601578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:37.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:37 smithi029 ceph-mon[223858]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T04:26:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:37 smithi177 ceph-mon[195797]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:37 smithi177 ceph-mon[195797]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:37 smithi177 ceph-mon[195797]: failed to encode map e129 with expected crc 2023-10-09T04:26:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1644601578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:37 smithi177 ceph-mon[195797]: osdmap e130: 8 total, 8 up, 8 in 2023-10-09T04:26:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[216903]: pgmap v429: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3384/111150 objects degraded (3.045%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:26:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[216903]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[216903]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[216903]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[216903]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[216903]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[223858]: pgmap v429: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3384/111150 objects degraded (3.045%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:26:38.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[223858]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[223858]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[223858]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[223858]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:38 smithi029 ceph-mon[223858]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:38 smithi177 ceph-mon[195797]: pgmap v429: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 3384/111150 objects degraded (3.045%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:26:38.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:38 smithi177 ceph-mon[195797]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:38 smithi177 ceph-mon[195797]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:38 smithi177 ceph-mon[195797]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:38 smithi177 ceph-mon[195797]: failed to encode map e130 with expected crc 2023-10-09T04:26:38.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:38 smithi177 ceph-mon[195797]: failed to encode map e130 with expected crc 2023-10-09T04:26:39.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2542077511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:39.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2542077511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:39.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2542077511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:40 smithi029 ceph-mon[223858]: pgmap v431: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 3384/111150 objects degraded (3.045%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:26:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3384/111150 objects degraded (3.045%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:40 smithi029 ceph-mon[216903]: pgmap v431: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 3384/111150 objects degraded (3.045%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:26:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3384/111150 objects degraded (3.045%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:40 smithi177 ceph-mon[195797]: pgmap v431: 89 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 3384/111150 objects degraded (3.045%); 2.3 MiB/s, 37 objects/s recovering 2023-10-09T04:26:40.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3384/111150 objects degraded (3.045%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:41.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:41.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:42 smithi177 ceph-mon[195797]: pgmap v432: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 3082/105723 objects degraded (2.915%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:26:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/290279563' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:42.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:42 smithi029 ceph-mon[216903]: pgmap v432: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 3082/105723 objects degraded (2.915%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:26:42.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/290279563' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:42.818 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:26:42] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-10-09T04:26:42.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:42 smithi029 ceph-mon[223858]: pgmap v432: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 3082/105723 objects degraded (2.915%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:26:42.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/290279563' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:44 smithi177 ceph-mon[195797]: pgmap v433: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3082/101628 objects degraded (3.033%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:26:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2869236623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:44 smithi029 ceph-mon[223858]: pgmap v433: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3082/101628 objects degraded (3.033%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:26:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2869236623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:44 smithi029 ceph-mon[216903]: pgmap v433: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3082/101628 objects degraded (3.033%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:26:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2869236623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:45.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3082/101628 objects degraded (3.033%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:45.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3082/101628 objects degraded (3.033%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3082/101628 objects degraded (3.033%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[216903]: pgmap v434: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 3082/97647 objects degraded (3.156%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:26:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[223858]: pgmap v434: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 3082/97647 objects degraded (3.156%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:26:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:46.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:46 smithi177 ceph-mon[195797]: pgmap v434: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 3082/97647 objects degraded (3.156%); 2.1 MiB/s, 34 objects/s recovering 2023-10-09T04:26:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1056712318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1056712318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:48.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1056712318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:48.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:48.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:26:48.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:48.828 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[216903]: pgmap v435: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 2838/91377 objects degraded (3.106%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[223858]: pgmap v435: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 2838/91377 objects degraded (3.106%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:26:48.829 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:48.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:48.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:48.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:48.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:48 smithi177 ceph-mon[195797]: pgmap v435: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 2838/91377 objects degraded (3.106%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:26:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:26:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:26:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:49 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:49 smithi029 ceph-mon[216903]: Upgrade: osd.2 is safe to restart 2023-10-09T04:26:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/382141472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:49.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:49 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:49.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:49 smithi029 ceph-mon[223858]: Upgrade: osd.2 is safe to restart 2023-10-09T04:26:49.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/382141472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:49 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-09T04:26:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:49 smithi177 ceph-mon[195797]: Upgrade: osd.2 is safe to restart 2023-10-09T04:26:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/382141472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:50.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:50 smithi029 ceph-mon[216903]: pgmap v436: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 2838/91377 objects degraded (3.106%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:26:50.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2838/91377 objects degraded (3.106%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:50 smithi029 ceph-mon[223858]: pgmap v436: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 2838/91377 objects degraded (3.106%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:26:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2838/91377 objects degraded (3.106%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:50 smithi177 ceph-mon[195797]: pgmap v436: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 2838/91377 objects degraded (3.106%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2838/91377 objects degraded (3.106%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:51 smithi029 ceph-mon[223858]: pgmap v437: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2583/85854 objects degraded (3.009%); 2.9 MiB/s, 66 objects/s recovering 2023-10-09T04:26:51.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:51 smithi029 ceph-mon[216903]: pgmap v437: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2583/85854 objects degraded (3.009%); 2.9 MiB/s, 66 objects/s recovering 2023-10-09T04:26:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:51 smithi177 ceph-mon[195797]: pgmap v437: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 2583/85854 objects degraded (3.009%); 2.9 MiB/s, 66 objects/s recovering 2023-10-09T04:26:52.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1417916918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:52.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:26:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:26:52] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-10-09T04:26:52.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1417916918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1417916918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:53.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:53 smithi029 ceph-mon[216903]: pgmap v438: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 2583/81207 objects degraded (3.181%); 1.3 MiB/s, 41 objects/s recovering 2023-10-09T04:26:53.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:53 smithi029 ceph-mon[223858]: pgmap v438: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 2583/81207 objects degraded (3.181%); 1.3 MiB/s, 41 objects/s recovering 2023-10-09T04:26:54.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:53 smithi177 ceph-mon[195797]: pgmap v438: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 2583/81207 objects degraded (3.181%); 1.3 MiB/s, 41 objects/s recovering 2023-10-09T04:26:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/282619320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:54.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/282619320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:55.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/282619320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:55 smithi029 ceph-mon[216903]: pgmap v439: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 2583/77163 objects degraded (3.347%); 1.3 MiB/s, 41 objects/s recovering 2023-10-09T04:26:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2583/81207 objects degraded (3.181%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:55 smithi029 ceph-mon[223858]: pgmap v439: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 2583/77163 objects degraded (3.347%); 1.3 MiB/s, 41 objects/s recovering 2023-10-09T04:26:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2583/81207 objects degraded (3.181%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:55 smithi177 ceph-mon[195797]: pgmap v439: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 2583/77163 objects degraded (3.347%); 1.3 MiB/s, 41 objects/s recovering 2023-10-09T04:26:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2583/81207 objects degraded (3.181%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:26:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:57 smithi029 ceph-mon[216903]: pgmap v440: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 2323/70170 objects degraded (3.311%); 1.3 MiB/s, 63 objects/s recovering 2023-10-09T04:26:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2853443431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:57 smithi029 ceph-mon[223858]: pgmap v440: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 2323/70170 objects degraded (3.311%); 1.3 MiB/s, 63 objects/s recovering 2023-10-09T04:26:58.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2853443431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:57 smithi177 ceph-mon[195797]: pgmap v440: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 2323/70170 objects degraded (3.311%); 1.3 MiB/s, 63 objects/s recovering 2023-10-09T04:26:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2853443431' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:26:59.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[216903]: Upgrade: Updating osd.2 2023-10-09T04:26:59.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:59.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:26:59.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:59.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[216903]: Deploying daemon osd.2 on smithi029 2023-10-09T04:26:59.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:59.097 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[223858]: Upgrade: Updating osd.2 2023-10-09T04:26:59.097 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:59.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:26:59.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:59.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[223858]: Deploying daemon osd.2 on smithi029 2023-10-09T04:26:59.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:26:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:58 smithi177 ceph-mon[195797]: Upgrade: Updating osd.2 2023-10-09T04:26:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:26:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-09T04:26:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:26:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:58 smithi177 ceph-mon[195797]: Deploying daemon osd.2 on smithi029 2023-10-09T04:26:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:00.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:59 smithi177 ceph-mon[195797]: pgmap v441: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2323/70170 objects degraded (3.311%); 21 KiB/s, 42 objects/s recovering 2023-10-09T04:27:00.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:26:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2724354263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:59 smithi029 ceph-mon[216903]: pgmap v441: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2323/70170 objects degraded (3.311%); 21 KiB/s, 42 objects/s recovering 2023-10-09T04:27:00.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:26:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2724354263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:00.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:59 smithi029 ceph-mon[223858]: pgmap v441: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 2323/70170 objects degraded (3.311%); 21 KiB/s, 42 objects/s recovering 2023-10-09T04:27:00.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:26:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2724354263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2323/70170 objects degraded (3.311%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:01.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2323/70170 objects degraded (3.311%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:01.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2323/70170 objects degraded (3.311%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:01.968 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:01 smithi029 ceph-mon[223858]: pgmap v442: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2053/64077 objects degraded (3.204%); 21 KiB/s, 65 objects/s recovering 2023-10-09T04:27:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:01 smithi177 ceph-mon[195797]: pgmap v442: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2053/64077 objects degraded (3.204%); 21 KiB/s, 65 objects/s recovering 2023-10-09T04:27:02.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:01 smithi029 ceph-mon[216903]: pgmap v442: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2053/64077 objects degraded (3.204%); 21 KiB/s, 65 objects/s recovering 2023-10-09T04:27:02.925 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:27:02] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-10-09T04:27:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2223010180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:03.397 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2223010180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:03.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2223010180' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:03 smithi177 ceph-mon[195797]: pgmap v443: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2053/59631 objects degraded (3.443%); 0 B/s, 44 objects/s recovering 2023-10-09T04:27:04.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:03 smithi029 ceph-mon[223858]: pgmap v443: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2053/59631 objects degraded (3.443%); 0 B/s, 44 objects/s recovering 2023-10-09T04:27:04.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:03 smithi029 ceph-mon[216903]: pgmap v443: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 2053/59631 objects degraded (3.443%); 0 B/s, 44 objects/s recovering 2023-10-09T04:27:05.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1112867684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:05.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2053/59631 objects degraded (3.443%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:05.172 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:05 smithi029 systemd[1]: Stopping Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:27:05.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1112867684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:05.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2053/59631 objects degraded (3.443%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1112867684' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2053/59631 objects degraded (3.443%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:05.923 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:05 smithi029 conmon[144116]: 2023-10-09T04:27:05.632+0000 7fb95a825700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:27:05.923 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:05 smithi029 conmon[144116]: 2023-10-09T04:27:05.632+0000 7fb95a825700 -1 osd.2 130 *** Got signal Terminated *** 2023-10-09T04:27:05.923 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:05 smithi029 conmon[144116]: 2023-10-09T04:27:05.632+0000 7fb95a825700 -1 osd.2 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:27:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:06 smithi177 ceph-mon[195797]: pgmap v444: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2053/56232 objects degraded (3.651%); 0 B/s, 44 objects/s recovering 2023-10-09T04:27:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:06 smithi177 ceph-mon[195797]: osd.2 marked itself down and dead 2023-10-09T04:27:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:06 smithi029 ceph-mon[216903]: pgmap v444: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2053/56232 objects degraded (3.651%); 0 B/s, 44 objects/s recovering 2023-10-09T04:27:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:06 smithi029 ceph-mon[216903]: osd.2 marked itself down and dead 2023-10-09T04:27:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:06 smithi029 ceph-mon[223858]: pgmap v444: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2053/56232 objects degraded (3.651%); 0 B/s, 44 objects/s recovering 2023-10-09T04:27:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:06 smithi029 ceph-mon[223858]: osd.2 marked itself down and dead 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[223858]: osdmap e131: 8 total, 7 up, 8 in 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[223858]: failed to encode map e131 with expected crc 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[223858]: failed to encode map e131 with expected crc 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[216903]: osdmap e131: 8 total, 7 up, 8 in 2023-10-09T04:27:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[216903]: failed to encode map e131 with expected crc 2023-10-09T04:27:07.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:07 smithi029 ceph-mon[216903]: failed to encode map e131 with expected crc 2023-10-09T04:27:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:07 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:27:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:07 smithi177 ceph-mon[195797]: osdmap e131: 8 total, 7 up, 8 in 2023-10-09T04:27:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:07 smithi177 ceph-mon[195797]: failed to encode map e131 with expected crc 2023-10-09T04:27:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:07 smithi177 ceph-mon[195797]: failed to encode map e131 with expected crc 2023-10-09T04:27:08.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: pgmap v446: 89 pgs: 10 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 1792/50484 objects degraded (3.550%); 0 B/s, 53 objects/s recovering 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: osdmap e132: 8 total, 7 up, 8 in 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3356987402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: pgmap v446: 89 pgs: 10 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 1792/50484 objects degraded (3.550%); 0 B/s, 53 objects/s recovering 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: osdmap e132: 8 total, 7 up, 8 in 2023-10-09T04:27:08.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3356987402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: pgmap v446: 89 pgs: 10 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 1792/50484 objects degraded (3.550%); 0 B/s, 53 objects/s recovering 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: osdmap e132: 8 total, 7 up, 8 in 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: failed to encode map e132 with expected crc 2023-10-09T04:27:08.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3356987402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:09.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[216903]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[216903]: osdmap e133: 8 total, 7 up, 8 in 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[216903]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[216903]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[216903]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[223858]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[223858]: osdmap e133: 8 total, 7 up, 8 in 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[223858]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[223858]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:09 smithi029 ceph-mon[223858]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.428 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:09 smithi029 bash[266198]: 7d2e4b8000f363d7a472ccf7c1a490a0def8e9f356752598513a9b204049aed0 2023-10-09T04:27:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:09 smithi177 ceph-mon[195797]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:09 smithi177 ceph-mon[195797]: osdmap e133: 8 total, 7 up, 8 in 2023-10-09T04:27:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:09 smithi177 ceph-mon[195797]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:09 smithi177 ceph-mon[195797]: failed to encode map e133 with expected crc 2023-10-09T04:27:09.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:09 smithi177 ceph-mon[195797]: failed to encode map e133 with expected crc 2023-10-09T04:27:10.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:10 smithi029 ceph-mon[216903]: pgmap v449: 89 pgs: 10 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 1792/50484 objects degraded (3.550%); 0 B/s, 43 objects/s recovering 2023-10-09T04:27:10.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1635781468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:10.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1792/50484 objects degraded (3.550%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:10.357 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:10 smithi029 ceph-mon[223858]: pgmap v449: 89 pgs: 10 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 1792/50484 objects degraded (3.550%); 0 B/s, 43 objects/s recovering 2023-10-09T04:27:10.357 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1635781468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:10.357 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1792/50484 objects degraded (3.550%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:10 smithi177 ceph-mon[195797]: pgmap v449: 89 pgs: 10 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 1792/50484 objects degraded (3.550%); 0 B/s, 43 objects/s recovering 2023-10-09T04:27:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1635781468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1792/50484 objects degraded (3.550%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:27:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:12 smithi029 ceph-mon[216903]: pgmap v450: 89 pgs: 4 active+undersized+degraded, 10 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 63 active+clean; 959 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 3870/46083 objects degraded (8.398%); 0 B/s, 136 objects/s recovering 2023-10-09T04:27:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:12 smithi029 ceph-mon[223858]: pgmap v450: 89 pgs: 4 active+undersized+degraded, 10 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 63 active+clean; 959 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 3870/46083 objects degraded (8.398%); 0 B/s, 136 objects/s recovering 2023-10-09T04:27:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:12 smithi177 ceph-mon[195797]: pgmap v450: 89 pgs: 4 active+undersized+degraded, 10 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 63 active+clean; 959 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 3870/46083 objects degraded (8.398%); 0 B/s, 136 objects/s recovering 2023-10-09T04:27:13.073 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:27:12] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.43.0" 2023-10-09T04:27:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1988939397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:13.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1988939397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1988939397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:14.349 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:14 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-09T04:27:14.350 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:14 smithi029 systemd[1]: Stopped Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:27:14.350 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:14 smithi029 systemd[1]: Starting Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:27:14.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[223858]: pgmap v451: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 877 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 6447/42135 objects degraded (15.301%); 0 B/s, 82 objects/s recovering 2023-10-09T04:27:14.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:27:14.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:14.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:14.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[216903]: pgmap v451: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 877 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 6447/42135 objects degraded (15.301%); 0 B/s, 82 objects/s recovering 2023-10-09T04:27:14.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:27:14.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:14.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:14.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:14 smithi177 ceph-mon[195797]: pgmap v451: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 877 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 6447/42135 objects degraded (15.301%); 0 B/s, 82 objects/s recovering 2023-10-09T04:27:14.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:14 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:27:14.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:14.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:15.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2824346537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:15.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6447/42135 objects degraded (15.301%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:15.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2824346537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:15.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6447/42135 objects degraded (15.301%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2824346537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6447/42135 objects degraded (15.301%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:16.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:16 smithi029 ceph-mon[216903]: pgmap v452: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 831 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 6447/39936 objects degraded (16.143%); 0 B/s, 69 objects/s recovering 2023-10-09T04:27:16.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:16 smithi029 ceph-mon[223858]: pgmap v452: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 831 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 6447/39936 objects degraded (16.143%); 0 B/s, 69 objects/s recovering 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T04:27:16.170 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:27:16.171 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:15 smithi029 bash[266897]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T04:27:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:16 smithi177 ceph-mon[195797]: pgmap v452: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 831 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 6447/39936 objects degraded (16.143%); 0 B/s, 69 objects/s recovering 2023-10-09T04:27:18.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:18 smithi029 ceph-mon[216903]: pgmap v453: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 626 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 4575/30102 objects degraded (15.198%); 0 B/s, 91 objects/s recovering 2023-10-09T04:27:18.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2032708987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:18 smithi029 ceph-mon[223858]: pgmap v453: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 626 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 4575/30102 objects degraded (15.198%); 0 B/s, 91 objects/s recovering 2023-10-09T04:27:18.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2032708987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:18 smithi177 ceph-mon[195797]: pgmap v453: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 626 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 4575/30102 objects degraded (15.198%); 0 B/s, 91 objects/s recovering 2023-10-09T04:27:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2032708987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:19.793 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:19 smithi029 bash[267331]: 0906042e125e7a8c588047bfe07393fee94dfeb9f3eed513575e6477fde04342 2023-10-09T04:27:19.793 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:19 smithi029 systemd[1]: Started Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:27:19.793 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:19 smithi029 ceph-osd[267393]: -- 172.21.15.29:0/1960868705 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558cc2bb24e0 con 0x558cc2b84c00 2023-10-09T04:27:19.794 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:19 smithi029 conmon[267369]: 2023-10-09T04:27:19.790+0000 7f0af81a7640 -1 Falling back to public interface 2023-10-09T04:27:20.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[223858]: pgmap v454: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 626 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 4575/30102 objects degraded (15.198%); 0 B/s, 83 objects/s recovering 2023-10-09T04:27:20.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:20.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:20.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:20.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4575/30102 objects degraded (15.198%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:20.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1831205011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[216903]: pgmap v454: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 626 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 4575/30102 objects degraded (15.198%); 0 B/s, 83 objects/s recovering 2023-10-09T04:27:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4575/30102 objects degraded (15.198%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:20.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1831205011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:20.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:20 smithi177 ceph-mon[195797]: pgmap v454: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 626 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 4575/30102 objects degraded (15.198%); 0 B/s, 83 objects/s recovering 2023-10-09T04:27:20.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:20.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4575/30102 objects degraded (15.198%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1831205011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:21.281 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:27:21 smithi177 conmon[174387]: logger=sqlstore.transactions t=2023-10-09T04:27:21.004860828Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-10-09T04:27:22.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:22 smithi029 ceph-mon[216903]: pgmap v455: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 493 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 3575/23691 objects degraded (15.090%); 0 B/s, 100 objects/s recovering 2023-10-09T04:27:22.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:22 smithi029 ceph-mon[223858]: pgmap v455: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 493 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 3575/23691 objects degraded (15.090%); 0 B/s, 100 objects/s recovering 2023-10-09T04:27:22.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:22 smithi177 ceph-mon[195797]: pgmap v455: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 493 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 3575/23691 objects degraded (15.090%); 0 B/s, 100 objects/s recovering 2023-10-09T04:27:23.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/753779336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:23.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/753779336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:23.145 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:27:22] "GET /metrics HTTP/1.1" 200 36332 "" "Prometheus/2.43.0" 2023-10-09T04:27:23.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/753779336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:24 smithi029 ceph-mon[216903]: pgmap v456: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 389 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2856/18711 objects degraded (15.264%); 0 B/s, 54 objects/s recovering 2023-10-09T04:27:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:24 smithi029 ceph-mon[223858]: pgmap v456: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 389 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2856/18711 objects degraded (15.264%); 0 B/s, 54 objects/s recovering 2023-10-09T04:27:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:24 smithi177 ceph-mon[195797]: pgmap v456: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 389 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 2856/18711 objects degraded (15.264%); 0 B/s, 54 objects/s recovering 2023-10-09T04:27:25.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2856/18711 objects degraded (15.264%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:25.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3587716795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:25.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2856/18711 objects degraded (15.264%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:25.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3587716795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2856/18711 objects degraded (15.264%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:25.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3587716795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:26.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:26 smithi029 ceph-mon[216903]: pgmap v457: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 343 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 2856/16530 objects degraded (17.278%); 0 B/s, 54 objects/s recovering 2023-10-09T04:27:26.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:26 smithi029 ceph-mon[223858]: pgmap v457: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 343 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 2856/16530 objects degraded (17.278%); 0 B/s, 54 objects/s recovering 2023-10-09T04:27:26.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:26 smithi177 ceph-mon[195797]: pgmap v457: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 343 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 2856/16530 objects degraded (17.278%); 0 B/s, 54 objects/s recovering 2023-10-09T04:27:28.172 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:27 smithi029 conmon[267369]: 2023-10-09T04:27:27.894+0000 7f0af81a7640 -1 osd.2 130 log_to_monitors true 2023-10-09T04:27:28.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:28 smithi177 ceph-mon[195797]: pgmap v458: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 165 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 1299/7983 objects degraded (16.272%); 0 B/s, 75 objects/s recovering 2023-10-09T04:27:28.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:28.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:28.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2398567467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:28.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:28 smithi177 ceph-mon[195797]: from='osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:27:28.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:28 smithi177 ceph-mon[195797]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:27:28.920 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:27:28 smithi029 conmon[267369]: 2023-10-09T04:27:28.497+0000 7f0ae9613700 -1 osd.2 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:27:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[216903]: pgmap v458: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 165 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 1299/7983 objects degraded (16.272%); 0 B/s, 75 objects/s recovering 2023-10-09T04:27:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2398567467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[216903]: from='osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:27:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[216903]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:27:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[223858]: pgmap v458: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 165 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 1299/7983 objects degraded (16.272%); 0 B/s, 75 objects/s recovering 2023-10-09T04:27:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2398567467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:28.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[223858]: from='osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:27:28.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:28 smithi029 ceph-mon[223858]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:27:29.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 116621 objects 2023-10-09T04:27:29.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :164.433 2023-10-09T04:27:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: osdmap e134: 8 total, 7 up, 8 in 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: failed to encode map e134 with expected crc 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: from='osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: failed to encode map e134 with expected crc 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:29.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:27:29.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: osdmap e134: 8 total, 7 up, 8 in 2023-10-09T04:27:29.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: failed to encode map e134 with expected crc 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: from='osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: failed to encode map e134 with expected crc 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: osdmap e134: 8 total, 7 up, 8 in 2023-10-09T04:27:29.882 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: failed to encode map e134 with expected crc 2023-10-09T04:27:29.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: from='osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:29.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:29.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: failed to encode map e134 with expected crc 2023-10-09T04:27:29.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:29.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:29.883 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: pgmap v460: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 165 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1299/7983 objects degraded (16.272%); 0 B/s, 57 objects/s recovering 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e135 with expected crc 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e135 with expected crc 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261] boot 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1299/7983 objects degraded (16.272%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/145376960' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:30.827 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-09T04:27:30.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: pgmap v460: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 165 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1299/7983 objects degraded (16.272%); 0 B/s, 57 objects/s recovering 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e135 with expected crc 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e135 with expected crc 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261] boot 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T04:27:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1299/7983 objects degraded (16.272%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/145376960' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: pgmap v460: 89 pgs: 8 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+undersized, 63 active+clean; 165 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 1299/7983 objects degraded (16.272%); 0 B/s, 57 objects/s recovering 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e135 with expected crc 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e135 with expected crc 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: osd.2 [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261] boot 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: osdmap e135: 8 total, 8 up, 8 in 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1299/7983 objects degraded (16.272%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: osdmap e136: 8 total, 8 up, 8 in 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/145376960' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: failed to encode map e137 with expected crc 2023-10-09T04:27:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:31 smithi177 ceph-mon[195797]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: failed to encode map e137 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[223858]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e136 with expected crc 2023-10-09T04:27:31.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: failed to encode map e137 with expected crc 2023-10-09T04:27:31.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:31 smithi029 ceph-mon[216903]: osdmap e137: 8 total, 8 up, 8 in 2023-10-09T04:27:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:32 smithi177 ceph-mon[195797]: pgmap v463: 89 pgs: 1 remapped+peering, 12 peering, 4 active+undersized+degraded, 7 active+undersized, 65 active+clean; 59 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 432/2862 objects degraded (15.094%); 0 B/s, 97 objects/s recovering 2023-10-09T04:27:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:32 smithi177 ceph-mon[195797]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:32 smithi177 ceph-mon[195797]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:32 smithi177 ceph-mon[195797]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1753036434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[216903]: pgmap v463: 89 pgs: 1 remapped+peering, 12 peering, 4 active+undersized+degraded, 7 active+undersized, 65 active+clean; 59 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 432/2862 objects degraded (15.094%); 0 B/s, 97 objects/s recovering 2023-10-09T04:27:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[216903]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[216903]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[216903]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1753036434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:32.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[223858]: pgmap v463: 89 pgs: 1 remapped+peering, 12 peering, 4 active+undersized+degraded, 7 active+undersized, 65 active+clean; 59 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 432/2862 objects degraded (15.094%); 0 B/s, 97 objects/s recovering 2023-10-09T04:27:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[223858]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[223858]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[223858]: failed to encode map e137 with expected crc 2023-10-09T04:27:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1753036434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:33.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:27:32] "GET /metrics HTTP/1.1" 200 36182 "" "Prometheus/2.43.0" 2023-10-09T04:27:33.351 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:34 smithi177 ceph-mon[195797]: pgmap v465: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 12 peering, 2 active+undersized, 71 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 3152/72 objects degraded (4377.778%); 0 B/s, 75 objects/s recovering 2023-10-09T04:27:34.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:34 smithi029 ceph-mon[216903]: pgmap v465: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 12 peering, 2 active+undersized, 71 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 3152/72 objects degraded (4377.778%); 0 B/s, 75 objects/s recovering 2023-10-09T04:27:34.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:34 smithi029 ceph-mon[223858]: pgmap v465: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 12 peering, 2 active+undersized, 71 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 3152/72 objects degraded (4377.778%); 0 B/s, 75 objects/s recovering 2023-10-09T04:27:35.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3190900959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:35.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3152/72 objects degraded (4377.778%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3190900959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3152/72 objects degraded (4377.778%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3190900959' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3152/72 objects degraded (4377.778%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:35.970 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_3' removed 2023-10-09T04:27:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:36 smithi177 ceph-mon[195797]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 peering, 82 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4152/72 objects degraded (5766.667%); 0 B/s, 200 objects/s recovering 2023-10-09T04:27:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2368732551' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:27:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2368732551' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:27:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:36 smithi177 ceph-mon[195797]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T04:27:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:36 smithi177 ceph-mon[195797]: failed to encode map e138 with expected crc 2023-10-09T04:27:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:36 smithi177 ceph-mon[195797]: failed to encode map e138 with expected crc 2023-10-09T04:27:36.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[216903]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 peering, 82 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4152/72 objects degraded (5766.667%); 0 B/s, 200 objects/s recovering 2023-10-09T04:27:36.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2368732551' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:27:36.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2368732551' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:27:36.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[216903]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T04:27:36.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[216903]: failed to encode map e138 with expected crc 2023-10-09T04:27:36.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[216903]: failed to encode map e138 with expected crc 2023-10-09T04:27:36.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[223858]: pgmap v466: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 peering, 82 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4152/72 objects degraded (5766.667%); 0 B/s, 200 objects/s recovering 2023-10-09T04:27:36.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2368732551' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:27:36.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2368732551' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:27:36.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[223858]: osdmap e138: 8 total, 8 up, 8 in 2023-10-09T04:27:36.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[223858]: failed to encode map e138 with expected crc 2023-10-09T04:27:36.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:36 smithi029 ceph-mon[223858]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.548 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T04:27:37.548 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T04:27:37.548 DEBUG:teuthology.parallel:result is None 2023-10-09T04:27:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2341194733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:27:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2341194733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: failed to encode map e138 with expected crc 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2341194733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:37.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:37.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:27:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[216903]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:27:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[216903]: Upgrade: osd.3 is safe to restart 2023-10-09T04:27:38.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4152/72 objects degraded (5766.667%), 4 pgs degraded) 2023-10-09T04:27:38.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:27:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[223858]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:27:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[223858]: Upgrade: osd.3 is safe to restart 2023-10-09T04:27:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4152/72 objects degraded (5766.667%), 4 pgs degraded) 2023-10-09T04:27:38.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:38 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:27:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:38 smithi177 ceph-mon[195797]: pgmap v468: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:38 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-09T04:27:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:38 smithi177 ceph-mon[195797]: Upgrade: osd.3 is safe to restart 2023-10-09T04:27:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:38 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4152/72 objects degraded (5766.667%), 4 pgs degraded) 2023-10-09T04:27:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:38 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:27:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: Upgrade: Updating osd.3 2023-10-09T04:27:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:27:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: Deploying daemon osd.3 on smithi029 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3121879730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: Upgrade: Updating osd.3 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: Deploying daemon osd.3 on smithi029 2023-10-09T04:27:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3121879730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: Upgrade: Updating osd.3 2023-10-09T04:27:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-09T04:27:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: Deploying daemon osd.3 on smithi029 2023-10-09T04:27:40.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3121879730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:42.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:42 smithi029 ceph-mon[216903]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:42.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1856823425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:42.222 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:42 smithi029 ceph-mon[223858]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:42.222 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1856823425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:42.505 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:42 smithi029 systemd[1]: Stopping Ceph osd.3 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:27:42.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:42 smithi177 ceph-mon[195797]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:42.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1856823425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:42.817 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:42 smithi029 conmon[153884]: 2023-10-09T04:27:42.503+0000 7fec0adb8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:27:42.817 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:42 smithi029 conmon[153884]: 2023-10-09T04:27:42.503+0000 7fec0adb8700 -1 osd.3 138 *** Got signal Terminated *** 2023-10-09T04:27:42.818 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:42 smithi029 conmon[153884]: 2023-10-09T04:27:42.503+0000 7fec0adb8700 -1 osd.3 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:27:43.153 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:27:42] "GET /metrics HTTP/1.1" 200 36182 "" "Prometheus/2.43.0" 2023-10-09T04:27:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:43 smithi029 ceph-mon[216903]: osd.3 marked itself down and dead 2023-10-09T04:27:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:43 smithi029 ceph-mon[223858]: osd.3 marked itself down and dead 2023-10-09T04:27:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:43 smithi177 ceph-mon[195797]: osd.3 marked itself down and dead 2023-10-09T04:27:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: osdmap e139: 8 total, 7 up, 8 in 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3840435532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: osdmap e139: 8 total, 7 up, 8 in 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3840435532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: osdmap e139: 8 total, 7 up, 8 in 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: failed to encode map e139 with expected crc 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3840435532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:44.921 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:44 smithi029 bash[274094]: e4ab91bdb38bda25d46216abdf96d662d83b84c2bb3b75f85a8fcc2547a62760 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: osdmap e140: 8 total, 7 up, 8 in 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[216903]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: osdmap e140: 8 total, 7 up, 8 in 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:45 smithi029 ceph-mon[223858]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: osdmap e140: 8 total, 7 up, 8 in 2023-10-09T04:27:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:45.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:45 smithi177 ceph-mon[195797]: failed to encode map e140 with expected crc 2023-10-09T04:27:46.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:46 smithi029 ceph-mon[223858]: pgmap v474: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 stale+active+clean, 12 active+undersized, 50 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:27:46.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:46 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:46 smithi029 ceph-mon[216903]: pgmap v474: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 stale+active+clean, 12 active+undersized, 50 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:27:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:46 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:46 smithi177 ceph-mon[195797]: pgmap v474: 73 pgs: 2 active+undersized+degraded, 3 peering, 6 stale+active+clean, 12 active+undersized, 50 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-09T04:27:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:46 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:46.921 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:46 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-09T04:27:46.921 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:46 smithi029 systemd[1]: Stopped Ceph osd.3 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:27:46.921 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:46 smithi029 systemd[1]: Starting Ceph osd.3 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:27:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1393609258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:47.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1393609258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:47.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1393609258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-09T04:27:48.007 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:47 smithi029 bash[274787]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-09T04:27:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:48 smithi029 ceph-mon[216903]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 3 peering, 25 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:48 smithi029 ceph-mon[223858]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 3 peering, 25 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:48 smithi177 ceph-mon[195797]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 3 peering, 25 active+undersized, 42 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:49.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1688873978' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1688873978' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:49.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1688873978' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:49.671 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:49 smithi029 ceph-osd[275279]: -- 172.21.15.29:0/1562053729 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b2dbe604e0 con 0x55b2dbe33400 2023-10-09T04:27:49.672 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:49 smithi029 bash[275213]: 8400b9c194003b5256e185328544a5c5c0e460efa9d14817ca5b9e1b2efadfcd 2023-10-09T04:27:49.672 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:49 smithi029 systemd[1]: Started Ceph osd.3 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:27:50.171 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:49 smithi029 conmon[275254]: 2023-10-09T04:27:49.721+0000 7fee1a1a5640 -1 Falling back to public interface 2023-10-09T04:27:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:50 smithi177 ceph-mon[195797]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:50 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:50 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:50 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[216903]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[223858]: pgmap v476: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:50 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/79899100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:51.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/79899100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:51.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/79899100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:52 smithi177 ceph-mon[195797]: pgmap v477: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:52 smithi029 ceph-mon[223858]: pgmap v477: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:52 smithi029 ceph-mon[216903]: pgmap v477: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:27:52] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.43.0" 2023-10-09T04:27:53.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.581 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:54.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:54 smithi029 ceph-mon[216903]: pgmap v478: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:54.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/747218136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:54.662 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:54 smithi029 ceph-mon[223858]: pgmap v478: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:54.662 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/747218136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:54.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:54 smithi177 ceph-mon[195797]: pgmap v478: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:54.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/747218136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:55.324 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:55 smithi029 conmon[275254]: 2023-10-09T04:27:55.097+0000 7fee1a1a5640 -1 osd.3 138 log_to_monitors true 2023-10-09T04:27:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:55 smithi029 ceph-mon[216903]: from='osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:27:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:55 smithi029 ceph-mon[216903]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:27:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:55 smithi029 ceph-mon[223858]: from='osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:27:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:55 smithi029 ceph-mon[223858]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:27:55.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:27:55.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:55 smithi177 ceph-mon[195797]: from='osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:27:55.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:55 smithi177 ceph-mon[195797]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-09T04:27:56.305 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:27:55 smithi029 conmon[275254]: 2023-10-09T04:27:55.988+0000 7fee08e0c700 -1 osd.3 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:27:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[216903]: pgmap v479: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3746463423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[216903]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[216903]: osdmap e141: 8 total, 7 up, 8 in 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[216903]: from='osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[216903]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[223858]: pgmap v479: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3746463423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[223858]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:27:56.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[223858]: osdmap e141: 8 total, 7 up, 8 in 2023-10-09T04:27:56.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[223858]: from='osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:56.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:56 smithi029 ceph-mon[223858]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:56 smithi177 ceph-mon[195797]: pgmap v479: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3746463423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:56 smithi177 ceph-mon[195797]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-09T04:27:56.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:56 smithi177 ceph-mon[195797]: osdmap e141: 8 total, 7 up, 8 in 2023-10-09T04:27:56.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:56 smithi177 ceph-mon[195797]: from='osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:56.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:56 smithi177 ceph-mon[195797]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:27:57.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:27:57 smithi029 conmon[202582]: 2023-10-09T04:27:57.044+0000 7f6049533700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:27:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: failed to encode map e141 with expected crc 2023-10-09T04:27:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922] boot 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: failed to encode map e141 with expected crc 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922] boot 2023-10-09T04:27:57.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T04:27:57.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:27:57.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:57.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: failed to encode map e141 with expected crc 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: osd.3 [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922] boot 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: osdmap e142: 8 total, 8 up, 8 in 2023-10-09T04:27:57.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-09T04:27:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:27:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:27:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:27:58.211 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: pgmap v481: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: failed to encode map e142 with expected crc 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: failed to encode map e141 with expected crc 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: failed to encode map e141 with expected crc 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T04:27:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2950620816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: pgmap v481: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: failed to encode map e142 with expected crc 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: failed to encode map e141 with expected crc 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: failed to encode map e141 with expected crc 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T04:27:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2950620816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: pgmap v481: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:27:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: failed to encode map e142 with expected crc 2023-10-09T04:27:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: failed to encode map e141 with expected crc 2023-10-09T04:27:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: failed to encode map e141 with expected crc 2023-10-09T04:27:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-09T04:27:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-09T04:27:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: osdmap e143: 8 total, 8 up, 8 in 2023-10-09T04:27:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2950620816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-09T04:27:59.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch upgrade pause' 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:27:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:27:59.558 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:27:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: failed to encode map e143 with expected crc 2023-10-09T04:27:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:27:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:27:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:00.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:00 smithi029 ceph-mon[216903]: pgmap v484: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:28:00.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:00 smithi029 ceph-mon[223858]: pgmap v484: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:28:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:00 smithi177 ceph-mon[195797]: pgmap v484: 73 pgs: 3 active+undersized+degraded, 26 active+undersized, 44 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-09T04:28:01.447 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:28:02.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:28:02.565 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:02 smithi177 ceph-mon[195797]: pgmap v485: 73 pgs: 2 active+undersized+degraded, 19 active+undersized, 52 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:28:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:02 smithi177 ceph-mon[195797]: from='client.45092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:02 smithi177 ceph-mon[195797]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:28:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:02 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:02 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:02.779 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[223858]: pgmap v485: 73 pgs: 2 active+undersized+degraded, 19 active+undersized, 52 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:28:02.779 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[223858]: from='client.45092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:02.779 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[223858]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:28:02.779 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:02.780 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:02.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[216903]: pgmap v485: 73 pgs: 2 active+undersized+degraded, 19 active+undersized, 52 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:28:02.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[216903]: from='client.45092 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:02.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[216903]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c 2023-10-09T04:28:02.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:02.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:02 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:28:03.138 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:28:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:28:02] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.43.0" 2023-10-09T04:28:03.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:03 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T04:28:03.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:03 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:28:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:03 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T04:28:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:03 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:28:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:03 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-09T04:28:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:03 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:28:04.425 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:28:04.425 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (16m) - 21m 32.4M - 0.25.0 c8568f914cd2 7ab788eff7be 2023-10-09T04:28:04.425 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi177 *:3000 running (15m) - 21m 95.9M - 9.4.7 2c41d148cca3 dbe79334ca87 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.mozfhx smithi029 running (20m) - 20m 23.3M - 17.2.6-1303-gb2903cb0 5517396c50d4 e346a0e9dcdd 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi177.pbovgs smithi177 running (20m) - 20m 18.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 a20b9123e852 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi177 *:8443,9283,8765 running (14m) - 25m 453M - 18.2.0-598-gb4fa663f 3060897fdf97 c43f5ec5cd80 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (15m) - 27m 524M - 18.2.0-598-gb4fa663f 3060897fdf97 239eea6ccc4c 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (14m) - 27m 125M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8369665f49b3 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi177 running (10m) - 25m 74.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8b10005e6565 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (12m) - 25m 82.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7e2b5984a986 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (15m) - 20m 20.0M - 1.5.0 0da6a335fe13 2be0e618a0f3 2023-10-09T04:28:04.426 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi177 *:9100 running (15m) - 20m 18.9M - 1.5.0 0da6a335fe13 2adf0c28b454 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (8m) - 25m 3524M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 e385ec644294 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (4m) - 24m 2009M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 f8791635683f 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (45s) - 24m 767M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 0906042e125e 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (15s) - 23m 12.3M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 8400b9c19400 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi177 running (23m) - 23m 5852M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 d4b592bd1322 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi177 running (23m) - 23m 8161M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 28d4ab6e8bfe 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi177 running (22m) - 22m 6060M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 03ede6ceccc0 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi177 running (22m) - 22m 5975M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0f3217cc8fa1 2023-10-09T04:28:04.427 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi177 *:9095 running (14m) - 21m 68.3M - 2.43.0 a07b618ecd1d 2d31282ff477 2023-10-09T04:28:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:04 smithi177 ceph-mon[195797]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:04 smithi029 ceph-mon[216903]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:04 smithi029 ceph-mon[223858]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:05.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph orch ps' 2023-10-09T04:28:05.549 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:05.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:05 smithi029 ceph-mon[216903]: from='client.35166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:05.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:05 smithi029 ceph-mon[223858]: from='client.35166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:05 smithi177 ceph-mon[195797]: from='client.35166 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:06 smithi177 ceph-mon[195797]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:06 smithi029 ceph-mon[216903]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:06 smithi029 ceph-mon[223858]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:07.417 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (16m) - 21m 32.4M - 0.25.0 c8568f914cd2 7ab788eff7be 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi177 *:3000 running (15m) - 21m 95.9M - 9.4.7 2c41d148cca3 dbe79334ca87 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.mozfhx smithi029 running (20m) - 20m 23.3M - 17.2.6-1303-gb2903cb0 5517396c50d4 e346a0e9dcdd 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi177.pbovgs smithi177 running (20m) - 20m 18.2M - 17.2.6-1303-gb2903cb0 5517396c50d4 a20b9123e852 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi177 *:8443,9283,8765 running (14m) - 25m 453M - 18.2.0-598-gb4fa663f 3060897fdf97 c43f5ec5cd80 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (15m) - 27m 524M - 18.2.0-598-gb4fa663f 3060897fdf97 239eea6ccc4c 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (14m) - 27m 125M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8369665f49b3 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi177 running (10m) - 25m 74.8M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 8b10005e6565 2023-10-09T04:28:07.418 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (12m) - 25m 82.2M 2048M 18.2.0-598-gb4fa663f 3060897fdf97 7e2b5984a986 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (16m) - 20m 20.0M - 1.5.0 0da6a335fe13 2be0e618a0f3 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi177 *:9100 running (15m) - 20m 18.9M - 1.5.0 0da6a335fe13 2adf0c28b454 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (8m) - 25m 3524M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 e385ec644294 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (4m) - 24m 2009M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 f8791635683f 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (48s) - 24m 767M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 0906042e125e 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (18s) - 23m 12.3M 2501M 18.2.0-598-gb4fa663f 3060897fdf97 8400b9c19400 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi177 running (23m) - 23m 5852M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 d4b592bd1322 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi177 running (23m) - 23m 8161M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 28d4ab6e8bfe 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi177 running (22m) - 22m 6060M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 03ede6ceccc0 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi177 running (22m) - 22m 5975M 2500M 17.2.6-1303-gb2903cb0 5517396c50d4 0f3217cc8fa1 2023-10-09T04:28:07.419 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi177 *:9095 running (14m) - 21m 68.3M - 2.43.0 a07b618ecd1d 2d31282ff477 2023-10-09T04:28:08.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -e sha1=b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0 -- bash -c 'ceph versions' 2023-10-09T04:28:08.594 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:08 smithi177 ceph-mon[195797]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:08 smithi177 ceph-mon[195797]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:08.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:08 smithi029 ceph-mon[216903]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:08.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:08 smithi029 ceph-mon[216903]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:08.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:08 smithi029 ceph-mon[223858]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:08.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:08 smithi029 ceph-mon[223858]: from='client.45104 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:28:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:09 smithi177 ceph-mon[195797]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:10.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:10.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:09 smithi029 ceph-mon[216903]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:10.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:10.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:09 smithi029 ceph-mon[223858]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:10.604 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:28:10.604 INFO:teuthology.orchestra.run.smithi029.stdout: "mon": { 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 3 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr": { 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 2 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "osd": { 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 4, 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 4 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "mds": { 2023-10-09T04:28:10.605 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 2 2023-10-09T04:28:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:28:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "overall": { 2023-10-09T04:28:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 17.2.6-1303-gb2903cb0 (b2903cb0842b1846a00474d4d51d49d1dc33bce1) quincy (stable)": 6, 2023-10-09T04:28:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 18.2.0-598-gb4fa663f (b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0) reef (stable)": 9 2023-10-09T04:28:10.606 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:28:10.606 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:28:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4098740551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:28:11.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4098740551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:28:11.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4098740551' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-09T04:28:11.296 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:11.296 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:28:11.305 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-09T04:28:11.305 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-09T04:28:11.340 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-09T04:28:11.341 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-09T04:28:11.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-09T04:28:11.701 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:11 smithi029 ceph-mon[216903]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:12.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:11 smithi029 ceph-mon[223858]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:12.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:11 smithi177 ceph-mon[195797]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:13.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:28:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:28:12] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.43.0" 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:13.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:28:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:13.810 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:28:14.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[216903]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:14.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:14.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:14.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4098091957' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:28:14.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[223858]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:14.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:14.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:14.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4098091957' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:28:14.746 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":37,"active_gid":25202,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":436570245},{"type":"v1","addr":"172.21.15.29:6801","nonce":436570245}]},"active_addr":"172.21.15.29:6801/436570245","active_change":"2023-10-09T04:13:58.298473+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25072,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":82,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3683873414}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":721269460}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":41816888}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2162930264}]}]} 2023-10-09T04:28:14.752 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-09T04:28:14.752 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T04:28:14.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:28:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:14 smithi177 ceph-mon[195797]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:28:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4098091957' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-09T04:28:15.116 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:16.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:16 smithi029 ceph-mon[216903]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:16.540 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:16 smithi029 ceph-mon[223858]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:16 smithi177 ceph-mon[195797]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:16.942 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:28:16.942 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":143,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:27:57.969171+0000","last_up_change":"2023-10-09T04:27:56.967421+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:07:33.266053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:07:34.056234+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":142,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6803","nonce":709667403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6805","nonce":709667403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6809","nonce":709667403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6807","nonce":709667403}]},"public_addr":"172.21.15.29:6803/709667403","cluster_addr":"172.21.15.29:6805/709667403","heartbeat_back_addr":"172.21.15.29:6809/709667403","heartbeat_front_addr":"172.21.15.29:6807/709667403","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":114,"up_from":118,"up_thru":142,"down_at":115,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6811","nonce":1230632802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6813","nonce":1230632802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6817","nonce":1230632802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6815","nonce":1230632802}]},"public_addr":"172.21.15.29:6811/1230632802","cluster_addr":"172.21.15.29:6813/1230632802","heartbeat_back_addr":"172.21.15.29:6817/1230632802","heartbeat_front_addr":"172.21.15.29:6815/1230632802","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":130,"up_from":135,"up_thru":142,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6819","nonce":193382261}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6821","nonce":193382261}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6825","nonce":193382261}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6823","nonce":193382261}]},"public_addr":"172.21.15.29:6819/193382261","cluster_addr":"172.21.15.29:6821/193382261","heartbeat_back_addr":"172.21.15.29:6825/193382261","heartbeat_front_addr":"172.21.15.29:6823/193382261","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":138,"up_from":142,"up_thru":142,"down_at":139,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6827","nonce":2977593922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6829","nonce":2977593922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6833","nonce":2977593922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6831","nonce":2977593922}]},"public_addr":"172.21.15.29:6827/2977593922","cluster_addr":"172.21.15.29:6829/2977593922","heartbeat_back_addr":"172.21.15.29:6833/2977593922","heartbeat_front_addr":"172.21.15.29:6831/2977593922","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":114},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":130},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":138},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/1371585021":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/3560371706":"2023-10-10T04:12:57.728727+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.177:0/2501999039":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/3682933143":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6801/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.177:6834/1319489652":"2023-10-10T04:12:57.728727+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/3925658916":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/3506139723":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/2321967128":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.177:6835/1319489652":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/1566968162":"2023-10-10T04:13:58.298189+0000","172.21.15.177:0/63096837":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/4232895263":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/1432692814":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1787157051":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/1665676424":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1007503695":"2023-10-10T04:13:58.298189+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:28:17.746 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/574410323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:28:17.746 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/574410323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:28:17.766 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T04:28:17.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T04:28:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/574410323' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:28:18.136 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:18 smithi029 ceph-mon[216903]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:18 smithi029 ceph-mon[223858]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:18.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:18 smithi177 ceph-mon[195797]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:20.052 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:28:20.052 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":143,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T04:27:57.969171+0000","last_up_change":"2023-10-09T04:27:56.967421+0000","last_in_change":"2023-10-09T04:05:34.237444+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:07:33.266053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:07:34.056234+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":142,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6803","nonce":709667403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6805","nonce":709667403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6809","nonce":709667403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6807","nonce":709667403}]},"public_addr":"172.21.15.29:6803/709667403","cluster_addr":"172.21.15.29:6805/709667403","heartbeat_back_addr":"172.21.15.29:6809/709667403","heartbeat_front_addr":"172.21.15.29:6807/709667403","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":114,"up_from":118,"up_thru":142,"down_at":115,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6811","nonce":1230632802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6813","nonce":1230632802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6817","nonce":1230632802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6815","nonce":1230632802}]},"public_addr":"172.21.15.29:6811/1230632802","cluster_addr":"172.21.15.29:6813/1230632802","heartbeat_back_addr":"172.21.15.29:6817/1230632802","heartbeat_front_addr":"172.21.15.29:6815/1230632802","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":130,"up_from":135,"up_thru":142,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6819","nonce":193382261}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6821","nonce":193382261}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6825","nonce":193382261}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":193382261},{"type":"v1","addr":"172.21.15.29:6823","nonce":193382261}]},"public_addr":"172.21.15.29:6819/193382261","cluster_addr":"172.21.15.29:6821/193382261","heartbeat_back_addr":"172.21.15.29:6825/193382261","heartbeat_front_addr":"172.21.15.29:6823/193382261","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":138,"up_from":142,"up_thru":142,"down_at":139,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6827","nonce":2977593922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6829","nonce":2977593922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6833","nonce":2977593922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6831","nonce":2977593922}]},"public_addr":"172.21.15.29:6827/2977593922","cluster_addr":"172.21.15.29:6829/2977593922","heartbeat_back_addr":"172.21.15.29:6833/2977593922","heartbeat_front_addr":"172.21.15.29:6831/2977593922","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6801","nonce":3399759139}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6803","nonce":3399759139}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6807","nonce":3399759139}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":3399759139},{"type":"v1","addr":"172.21.15.177:6805","nonce":3399759139}]},"public_addr":"172.21.15.177:6801/3399759139","cluster_addr":"172.21.15.177:6803/3399759139","heartbeat_back_addr":"172.21.15.177:6807/3399759139","heartbeat_front_addr":"172.21.15.177:6805/3399759139","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":142,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":114},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":130},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":138},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/3682933143":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6801/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1432692814":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1371585021":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/3560371706":"2023-10-10T04:12:57.728727+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/3925658916":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.177:0/2501999039":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/1007503695":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/3506139723":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.177:6834/1319489652":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/4232895263":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/2321967128":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1787157051":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/1665676424":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1566968162":"2023-10-10T04:13:58.298189+0000","172.21.15.177:0/63096837":"2023-10-10T04:12:57.728727+0000","172.21.15.177:6835/1319489652":"2023-10-10T04:12:57.728727+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T04:28:20.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:20 smithi177 ceph-mon[195797]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:20.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4057763022' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:28:20.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:20 smithi029 ceph-mon[216903]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:20.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4057763022' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:28:20.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:20 smithi029 ceph-mon[223858]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:20.827 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4057763022' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T04:28:20.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T04:28:20.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T04:28:20.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T04:28:20.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T04:28:20.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T04:28:20.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T04:28:20.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T04:28:20.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T04:28:22.105 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.156 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.252 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.289 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.398 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.428 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.510 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.561 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:22 smithi029 ceph-mon[216903]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:22.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:22 smithi029 ceph-mon[223858]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:22 smithi177 ceph-mon[195797]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:23.058 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:28:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:28:22] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.43.0" 2023-10-09T04:28:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:24 smithi177 ceph-mon[195797]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:24.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:24 smithi029 ceph-mon[216903]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:24.793 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:24 smithi029 ceph-mon[223858]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:26 smithi177 ceph-mon[195797]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:26.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:26 smithi029 ceph-mon[216903]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:26.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:26 smithi029 ceph-mon[223858]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:28.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:28 smithi029 ceph-mon[216903]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:28.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:28.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:28 smithi029 ceph-mon[223858]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:28.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:28.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:28 smithi177 ceph-mon[195797]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:28.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:30 smithi029 ceph-mon[216903]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:30.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:30 smithi029 ceph-mon[223858]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:30 smithi177 ceph-mon[195797]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:30.840 INFO:teuthology.orchestra.run.smithi029.stdout:128849019166 2023-10-09T04:28:30.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T04:28:32.606 INFO:teuthology.orchestra.run.smithi029.stdout:506806140982 2023-10-09T04:28:32.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T04:28:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:32 smithi177 ceph-mon[195797]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:32.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:32 smithi029 ceph-mon[216903]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:32.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:32 smithi029 ceph-mon[223858]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:33.043 INFO:teuthology.orchestra.run.smithi029.stdout:390842024040 2023-10-09T04:28:33.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T04:28:33.072 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:28:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:28:32] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.43.0" 2023-10-09T04:28:33.271 INFO:teuthology.orchestra.run.smithi029.stdout:163208757530 2023-10-09T04:28:33.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T04:28:34.707 INFO:teuthology.orchestra.run.smithi029.stdout:609885356041 2023-10-09T04:28:34.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T04:28:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:34 smithi177 ceph-mon[195797]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:34 smithi029 ceph-mon[216903]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:34.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:34 smithi029 ceph-mon[223858]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:35.072 INFO:teuthology.orchestra.run.smithi029.stdout:197568495893 2023-10-09T04:28:35.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T04:28:35.397 INFO:teuthology.orchestra.run.smithi029.stdout:579820584974 2023-10-09T04:28:35.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T04:28:35.547 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:35.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:35.897 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:36 smithi029 ceph-mon[216903]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:36 smithi029 ceph-mon[223858]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:36 smithi177 ceph-mon[195797]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:37.726 INFO:teuthology.orchestra.run.smithi029.stdout:231928234260 2023-10-09T04:28:37.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T04:28:37.938 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:38.193 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:38.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:38 smithi029 ceph-mon[216903]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:38.779 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:38 smithi029 ceph-mon[223858]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:39.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:38 smithi177 ceph-mon[195797]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:40.686 INFO:teuthology.orchestra.run.smithi029.stdout:197568495894 2023-10-09T04:28:40.879 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:40 smithi029 ceph-mon[216903]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:40.880 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:40 smithi029 ceph-mon[223858]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:40 smithi177 ceph-mon[195797]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:41.116 INFO:teuthology.orchestra.run.smithi029.stdout:506806140984 2023-10-09T04:28:41.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/810962980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:28:41.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1663146633' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:28:41.777 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/810962980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:28:41.777 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1663146633' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:28:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/810962980' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T04:28:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1663146633' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T04:28:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:42 smithi029 ceph-mon[216903]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:42.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:42 smithi029 ceph-mon[223858]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:42 smithi177 ceph-mon[195797]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:28:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:28:42] "GET /metrics HTTP/1.1" 200 33962 "" "Prometheus/2.43.0" 2023-10-09T04:28:43.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:43.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:43.851 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495893 got 197568495894 for osd.6 2023-10-09T04:28:43.851 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:43.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 506806140982 got 506806140984 for osd.1 2023-10-09T04:28:43.892 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:44.047 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:44.129 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:44.345 INFO:teuthology.orchestra.run.smithi029.stdout:390842024042 2023-10-09T04:28:44.427 INFO:teuthology.orchestra.run.smithi029.stdout:609885356043 2023-10-09T04:28:44.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:44 smithi029 ceph-mon[216903]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:44.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3119957999' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:28:44.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2354600111' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:28:44.653 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:44 smithi029 ceph-mon[223858]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:44.653 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3119957999' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:28:44.653 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2354600111' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:28:44.973 INFO:teuthology.orchestra.run.smithi029.stdout:128849019169 2023-10-09T04:28:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:44 smithi177 ceph-mon[195797]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3119957999' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T04:28:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2354600111' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T04:28:45.098 INFO:teuthology.orchestra.run.smithi029.stdout:163208757533 2023-10-09T04:28:45.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3026541377' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:28:45.792 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1556320291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:28:45.792 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3026541377' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:28:45.792 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1556320291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:28:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3026541377' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T04:28:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1556320291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T04:28:46.566 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024040 got 390842024042 for osd.0 2023-10-09T04:28:46.566 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:46.692 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019166 got 128849019169 for osd.4 2023-10-09T04:28:46.692 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:46 smithi029 ceph-mon[223858]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:46 smithi029 ceph-mon[216903]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:46 smithi177 ceph-mon[195797]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:47.827 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757530 got 163208757533 for osd.5 2023-10-09T04:28:47.827 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:47.924 INFO:tasks.cephadm.ceph_manager.ceph:need seq 609885356041 got 609885356043 for osd.3 2023-10-09T04:28:47.924 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:48 smithi029 ceph-mon[216903]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:48 smithi029 ceph-mon[223858]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:48 smithi177 ceph-mon[195797]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:49.682 INFO:teuthology.orchestra.run.smithi029.stdout:231928234262 2023-10-09T04:28:49.955 INFO:teuthology.orchestra.run.smithi029.stdout:579820584978 2023-10-09T04:28:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:50 smithi029 ceph-mon[216903]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3961371189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:28:50.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2852654215' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:28:50.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:50 smithi029 ceph-mon[223858]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:50.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3961371189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:28:50.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2852654215' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:28:50.930 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234260 got 231928234262 for osd.7 2023-10-09T04:28:50.930 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:50 smithi177 ceph-mon[195797]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3961371189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T04:28:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2852654215' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T04:28:51.141 INFO:tasks.cephadm.ceph_manager.ceph:need seq 579820584974 got 579820584978 for osd.2 2023-10-09T04:28:51.141 DEBUG:teuthology.parallel:result is None 2023-10-09T04:28:51.141 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:28:51.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:28:51.507 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:52.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:52 smithi029 ceph-mon[216903]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:52.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:52 smithi029 ceph-mon[223858]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:52 smithi177 ceph-mon[195797]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:28:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:28:52] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.43.0" 2023-10-09T04:28:53.352 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:28:53.353 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:28:54.191 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":511,"stamp":"2023-10-09T04:28:52.790219+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2612516,"kb_used_data":12900,"kb_used_omap":20,"kb_used_meta":2599531,"kb_avail":747348700,"statfs":{"total":767960285184,"available":765285068800,"internally_reserved":0,"allocated":13209600,"data_stored":9010590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2661920398},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003145"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974877+0000","last_change":"2023-10-09T04:27:30.004963+0000","last_active":"2023-10-09T04:27:56.974877+0000","last_peered":"2023-10-09T04:27:56.974877+0000","last_clean":"2023-10-09T04:27:56.974877+0000","last_became_active":"2023-10-09T04:27:30.004581+0000","last_became_peered":"2023-10-09T04:27:30.004581+0000","last_unstale":"2023-10-09T04:27:56.974877+0000","last_undegraded":"2023-10-09T04:27:56.974877+0000","last_fullsized":"2023-10-09T04:27:56.974877+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:40.077805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988720+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:27:57.988720+0000","last_peered":"2023-10-09T04:27:57.988720+0000","last_clean":"2023-10-09T04:27:57.988720+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:27:57.988720+0000","last_undegraded":"2023-10-09T04:27:57.988720+0000","last_fullsized":"2023-10-09T04:27:57.988720+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.982645+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:27:57.982645+0000","last_peered":"2023-10-09T04:27:57.982645+0000","last_clean":"2023-10-09T04:27:57.982645+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:27:57.982645+0000","last_undegraded":"2023-10-09T04:27:57.982645+0000","last_fullsized":"2023-10-09T04:27:57.982645+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990988+0000","last_change":"2023-10-09T04:27:57.985924+0000","last_active":"2023-10-09T04:27:57.990988+0000","last_peered":"2023-10-09T04:27:57.990988+0000","last_clean":"2023-10-09T04:27:57.990988+0000","last_became_active":"2023-10-09T04:27:57.985559+0000","last_became_peered":"2023-10-09T04:27:57.985559+0000","last_unstale":"2023-10-09T04:27:57.990988+0000","last_undegraded":"2023-10-09T04:27:57.990988+0000","last_fullsized":"2023-10-09T04:27:57.990988+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:01.749257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986393+0000","last_change":"2023-10-09T04:27:57.986393+0000","last_active":"2023-10-09T04:27:57.986393+0000","last_peered":"2023-10-09T04:27:57.986393+0000","last_clean":"2023-10-09T04:27:57.986393+0000","last_became_active":"2023-10-09T04:27:57.981584+0000","last_became_peered":"2023-10-09T04:27:57.981584+0000","last_unstale":"2023-10-09T04:27:57.986393+0000","last_undegraded":"2023-10-09T04:27:57.986393+0000","last_fullsized":"2023-10-09T04:27:57.986393+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:34.073778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981290+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:27:56.981290+0000","last_peered":"2023-10-09T04:27:56.981290+0000","last_clean":"2023-10-09T04:27:56.981290+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:27:56.981290+0000","last_undegraded":"2023-10-09T04:27:56.981290+0000","last_fullsized":"2023-10-09T04:27:56.981290+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989523+0000","last_change":"2023-10-09T04:27:57.989523+0000","last_active":"2023-10-09T04:27:57.989523+0000","last_peered":"2023-10-09T04:27:57.989523+0000","last_clean":"2023-10-09T04:27:57.989523+0000","last_became_active":"2023-10-09T04:27:57.989226+0000","last_became_peered":"2023-10-09T04:27:57.989226+0000","last_unstale":"2023-10-09T04:27:57.989523+0000","last_undegraded":"2023-10-09T04:27:57.989523+0000","last_fullsized":"2023-10-09T04:27:57.989523+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:47.573373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987735+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:27:57.987735+0000","last_peered":"2023-10-09T04:27:57.987735+0000","last_clean":"2023-10-09T04:27:57.987735+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:27:57.987735+0000","last_undegraded":"2023-10-09T04:27:57.987735+0000","last_fullsized":"2023-10-09T04:27:57.987735+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988398+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:27:56.988398+0000","last_peered":"2023-10-09T04:27:56.988398+0000","last_clean":"2023-10-09T04:27:56.988398+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:27:56.988398+0000","last_undegraded":"2023-10-09T04:27:56.988398+0000","last_fullsized":"2023-10-09T04:27:56.988398+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990152+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:27:57.990152+0000","last_peered":"2023-10-09T04:27:57.990152+0000","last_clean":"2023-10-09T04:27:57.990152+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:27:57.990152+0000","last_undegraded":"2023-10-09T04:27:57.990152+0000","last_fullsized":"2023-10-09T04:27:57.990152+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":106,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986830+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:27:57.986830+0000","last_peered":"2023-10-09T04:27:57.986830+0000","last_clean":"2023-10-09T04:27:57.986830+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:27:57.986830+0000","last_undegraded":"2023-10-09T04:27:57.986830+0000","last_fullsized":"2023-10-09T04:27:57.986830+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972995+0000","last_change":"2023-10-09T04:27:29.983053+0000","last_active":"2023-10-09T04:27:57.972995+0000","last_peered":"2023-10-09T04:27:57.972995+0000","last_clean":"2023-10-09T04:27:57.972995+0000","last_became_active":"2023-10-09T04:27:29.982725+0000","last_became_peered":"2023-10-09T04:27:29.982725+0000","last_unstale":"2023-10-09T04:27:57.972995+0000","last_undegraded":"2023-10-09T04:27:57.972995+0000","last_fullsized":"2023-10-09T04:27:57.972995+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:33.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976512+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:27:56.976512+0000","last_peered":"2023-10-09T04:27:56.976512+0000","last_clean":"2023-10-09T04:27:56.976512+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:27:56.976512+0000","last_undegraded":"2023-10-09T04:27:56.976512+0000","last_fullsized":"2023-10-09T04:27:56.976512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975162+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:27:56.975162+0000","last_peered":"2023-10-09T04:27:56.975162+0000","last_clean":"2023-10-09T04:27:56.975162+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:27:56.975162+0000","last_undegraded":"2023-10-09T04:27:56.975162+0000","last_fullsized":"2023-10-09T04:27:56.975162+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981381+0000","last_change":"2023-10-09T04:27:29.989226+0000","last_active":"2023-10-09T04:27:56.981381+0000","last_peered":"2023-10-09T04:27:56.981381+0000","last_clean":"2023-10-09T04:27:56.981381+0000","last_became_active":"2023-10-09T04:27:29.988867+0000","last_became_peered":"2023-10-09T04:27:29.988867+0000","last_unstale":"2023-10-09T04:27:56.981381+0000","last_undegraded":"2023-10-09T04:27:56.981381+0000","last_fullsized":"2023-10-09T04:27:56.981381+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:15.559571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987345+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:27:57.987345+0000","last_peered":"2023-10-09T04:27:57.987345+0000","last_clean":"2023-10-09T04:27:57.987345+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:27:57.987345+0000","last_undegraded":"2023-10-09T04:27:57.987345+0000","last_fullsized":"2023-10-09T04:27:57.987345+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.985231+0000","last_change":"2023-10-09T04:27:57.985231+0000","last_active":"2023-10-09T04:27:57.985231+0000","last_peered":"2023-10-09T04:27:57.985231+0000","last_clean":"2023-10-09T04:27:57.985231+0000","last_became_active":"2023-10-09T04:27:57.984869+0000","last_became_peered":"2023-10-09T04:27:57.984869+0000","last_unstale":"2023-10-09T04:27:57.985231+0000","last_undegraded":"2023-10-09T04:27:57.985231+0000","last_fullsized":"2023-10-09T04:27:57.985231+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:39.521778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992866+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:27:57.992866+0000","last_peered":"2023-10-09T04:27:57.992866+0000","last_clean":"2023-10-09T04:27:57.992866+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:27:57.992866+0000","last_undegraded":"2023-10-09T04:27:57.992866+0000","last_fullsized":"2023-10-09T04:27:57.992866+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980748+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:27:56.980748+0000","last_peered":"2023-10-09T04:27:56.980748+0000","last_clean":"2023-10-09T04:27:56.980748+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:27:56.980748+0000","last_undegraded":"2023-10-09T04:27:56.980748+0000","last_fullsized":"2023-10-09T04:27:56.980748+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.994143+0000","last_change":"2023-10-09T04:27:57.994143+0000","last_active":"2023-10-09T04:27:57.994143+0000","last_peered":"2023-10-09T04:27:57.994143+0000","last_clean":"2023-10-09T04:27:57.994143+0000","last_became_active":"2023-10-09T04:27:57.993862+0000","last_became_peered":"2023-10-09T04:27:57.993862+0000","last_unstale":"2023-10-09T04:27:57.994143+0000","last_undegraded":"2023-10-09T04:27:57.994143+0000","last_fullsized":"2023-10-09T04:27:57.994143+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:24.434239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980940+0000","last_change":"2023-10-09T04:20:03.030960+0000","last_active":"2023-10-09T04:27:56.980940+0000","last_peered":"2023-10-09T04:27:56.980940+0000","last_clean":"2023-10-09T04:27:56.980940+0000","last_became_active":"2023-10-09T04:20:03.030148+0000","last_became_peered":"2023-10-09T04:20:03.030148+0000","last_unstale":"2023-10-09T04:27:56.980940+0000","last_undegraded":"2023-10-09T04:27:56.980940+0000","last_fullsized":"2023-10-09T04:27:56.980940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:09.000584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986461+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:27:57.986461+0000","last_peered":"2023-10-09T04:27:57.986461+0000","last_clean":"2023-10-09T04:27:57.986461+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:27:57.986461+0000","last_undegraded":"2023-10-09T04:27:57.986461+0000","last_fullsized":"2023-10-09T04:27:57.986461+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983931+0000","last_change":"2023-10-09T04:20:03.033517+0000","last_active":"2023-10-09T04:27:56.983931+0000","last_peered":"2023-10-09T04:27:56.983931+0000","last_clean":"2023-10-09T04:27:56.983931+0000","last_became_active":"2023-10-09T04:20:03.033132+0000","last_became_peered":"2023-10-09T04:20:03.033132+0000","last_unstale":"2023-10-09T04:27:56.983931+0000","last_undegraded":"2023-10-09T04:27:56.983931+0000","last_fullsized":"2023-10-09T04:27:56.983931+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:14.247924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988550+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:27:56.988550+0000","last_peered":"2023-10-09T04:27:56.988550+0000","last_clean":"2023-10-09T04:27:56.988550+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:27:56.988550+0000","last_undegraded":"2023-10-09T04:27:56.988550+0000","last_fullsized":"2023-10-09T04:27:56.988550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983774+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:27:56.983774+0000","last_peered":"2023-10-09T04:27:56.983774+0000","last_clean":"2023-10-09T04:27:56.983774+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:27:56.983774+0000","last_undegraded":"2023-10-09T04:27:56.983774+0000","last_fullsized":"2023-10-09T04:27:56.983774+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988386+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:27:57.988386+0000","last_peered":"2023-10-09T04:27:57.988386+0000","last_clean":"2023-10-09T04:27:57.988386+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:27:57.988386+0000","last_undegraded":"2023-10-09T04:27:57.988386+0000","last_fullsized":"2023-10-09T04:27:57.988386+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":87,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982249+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:27:56.982249+0000","last_peered":"2023-10-09T04:27:56.982249+0000","last_clean":"2023-10-09T04:27:56.982249+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:27:56.982249+0000","last_undegraded":"2023-10-09T04:27:56.982249+0000","last_fullsized":"2023-10-09T04:27:56.982249+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982155+0000","last_change":"2023-10-09T04:27:30.005025+0000","last_active":"2023-10-09T04:27:56.982155+0000","last_peered":"2023-10-09T04:27:56.982155+0000","last_clean":"2023-10-09T04:27:56.982155+0000","last_became_active":"2023-10-09T04:27:30.004692+0000","last_became_peered":"2023-10-09T04:27:30.004692+0000","last_unstale":"2023-10-09T04:27:56.982155+0000","last_undegraded":"2023-10-09T04:27:56.982155+0000","last_fullsized":"2023-10-09T04:27:56.982155+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:10.737398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992844+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:27:57.992844+0000","last_peered":"2023-10-09T04:27:57.992844+0000","last_clean":"2023-10-09T04:27:57.992844+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:27:57.992844+0000","last_undegraded":"2023-10-09T04:27:57.992844+0000","last_fullsized":"2023-10-09T04:27:57.992844+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":90,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982864+0000","last_change":"2023-10-09T04:20:03.025205+0000","last_active":"2023-10-09T04:27:56.982864+0000","last_peered":"2023-10-09T04:27:56.982864+0000","last_clean":"2023-10-09T04:27:56.982864+0000","last_became_active":"2023-10-09T04:20:03.024858+0000","last_became_peered":"2023-10-09T04:20:03.024858+0000","last_unstale":"2023-10-09T04:27:56.982864+0000","last_undegraded":"2023-10-09T04:27:56.982864+0000","last_fullsized":"2023-10-09T04:27:56.982864+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:59.239890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992940+0000","last_change":"2023-10-09T04:27:57.992940+0000","last_active":"2023-10-09T04:27:57.992940+0000","last_peered":"2023-10-09T04:27:57.992940+0000","last_clean":"2023-10-09T04:27:57.992940+0000","last_became_active":"2023-10-09T04:27:57.991337+0000","last_became_peered":"2023-10-09T04:27:57.991337+0000","last_unstale":"2023-10-09T04:27:57.992940+0000","last_undegraded":"2023-10-09T04:27:57.992940+0000","last_fullsized":"2023-10-09T04:27:57.992940+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:10.795994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988243+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:27:57.988243+0000","last_peered":"2023-10-09T04:27:57.988243+0000","last_clean":"2023-10-09T04:27:57.988243+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:27:57.988243+0000","last_undegraded":"2023-10-09T04:27:57.988243+0000","last_fullsized":"2023-10-09T04:27:57.988243+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982044+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:27:56.982044+0000","last_peered":"2023-10-09T04:27:56.982044+0000","last_clean":"2023-10-09T04:27:56.982044+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:27:56.982044+0000","last_undegraded":"2023-10-09T04:27:56.982044+0000","last_fullsized":"2023-10-09T04:27:56.982044+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976221+0000","last_change":"2023-10-09T04:27:29.982774+0000","last_active":"2023-10-09T04:27:56.976221+0000","last_peered":"2023-10-09T04:27:56.976221+0000","last_clean":"2023-10-09T04:27:56.976221+0000","last_became_active":"2023-10-09T04:27:29.982507+0000","last_became_peered":"2023-10-09T04:27:29.982507+0000","last_unstale":"2023-10-09T04:27:56.976221+0000","last_undegraded":"2023-10-09T04:27:56.976221+0000","last_fullsized":"2023-10-09T04:27:56.976221+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:14.613302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976415+0000","last_change":"2023-10-09T04:27:30.010756+0000","last_active":"2023-10-09T04:27:56.976415+0000","last_peered":"2023-10-09T04:27:56.976415+0000","last_clean":"2023-10-09T04:27:56.976415+0000","last_became_active":"2023-10-09T04:27:30.010403+0000","last_became_peered":"2023-10-09T04:27:30.010403+0000","last_unstale":"2023-10-09T04:27:56.976415+0000","last_undegraded":"2023-10-09T04:27:56.976415+0000","last_fullsized":"2023-10-09T04:27:56.976415+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:43:10.557818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976602+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:27:56.976602+0000","last_peered":"2023-10-09T04:27:56.976602+0000","last_clean":"2023-10-09T04:27:56.976602+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:27:56.976602+0000","last_undegraded":"2023-10-09T04:27:56.976602+0000","last_fullsized":"2023-10-09T04:27:56.976602+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986421+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:27:57.986421+0000","last_peered":"2023-10-09T04:27:57.986421+0000","last_clean":"2023-10-09T04:27:57.986421+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:27:57.986421+0000","last_undegraded":"2023-10-09T04:27:57.986421+0000","last_fullsized":"2023-10-09T04:27:57.986421+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.983436+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:27:57.983436+0000","last_peered":"2023-10-09T04:27:57.983436+0000","last_clean":"2023-10-09T04:27:57.983436+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:27:57.983436+0000","last_undegraded":"2023-10-09T04:27:57.983436+0000","last_fullsized":"2023-10-09T04:27:57.983436+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973721+0000","last_change":"2023-10-09T04:24:07.916771+0000","last_active":"2023-10-09T04:27:57.973721+0000","last_peered":"2023-10-09T04:27:57.973721+0000","last_clean":"2023-10-09T04:27:57.973721+0000","last_became_active":"2023-10-09T04:24:07.916030+0000","last_became_peered":"2023-10-09T04:24:07.916030+0000","last_unstale":"2023-10-09T04:27:57.973721+0000","last_undegraded":"2023-10-09T04:27:57.973721+0000","last_fullsized":"2023-10-09T04:27:57.973721+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.033444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993021+0000","last_change":"2023-10-09T04:27:57.993021+0000","last_active":"2023-10-09T04:27:57.993021+0000","last_peered":"2023-10-09T04:27:57.993021+0000","last_clean":"2023-10-09T04:27:57.993021+0000","last_became_active":"2023-10-09T04:27:57.992486+0000","last_became_peered":"2023-10-09T04:27:57.992486+0000","last_unstale":"2023-10-09T04:27:57.993021+0000","last_undegraded":"2023-10-09T04:27:57.993021+0000","last_fullsized":"2023-10-09T04:27:57.993021+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:56.130811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972915+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:27:57.972915+0000","last_peered":"2023-10-09T04:27:57.972915+0000","last_clean":"2023-10-09T04:27:57.972915+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:27:57.972915+0000","last_undegraded":"2023-10-09T04:27:57.972915+0000","last_fullsized":"2023-10-09T04:27:57.972915+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988701+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:27:56.988701+0000","last_peered":"2023-10-09T04:27:56.988701+0000","last_clean":"2023-10-09T04:27:56.988701+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:27:56.988701+0000","last_undegraded":"2023-10-09T04:27:56.988701+0000","last_fullsized":"2023-10-09T04:27:56.988701+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975059+0000","last_change":"2023-10-09T04:20:03.030040+0000","last_active":"2023-10-09T04:27:56.975059+0000","last_peered":"2023-10-09T04:27:56.975059+0000","last_clean":"2023-10-09T04:27:56.975059+0000","last_became_active":"2023-10-09T04:20:03.029675+0000","last_became_peered":"2023-10-09T04:20:03.029675+0000","last_unstale":"2023-10-09T04:27:56.975059+0000","last_undegraded":"2023-10-09T04:27:56.975059+0000","last_fullsized":"2023-10-09T04:27:56.975059+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:25.875934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993874+0000","last_change":"2023-10-09T04:27:57.991197+0000","last_active":"2023-10-09T04:27:57.993874+0000","last_peered":"2023-10-09T04:27:57.993874+0000","last_clean":"2023-10-09T04:27:57.993874+0000","last_became_active":"2023-10-09T04:27:57.990548+0000","last_became_peered":"2023-10-09T04:27:57.990548+0000","last_unstale":"2023-10-09T04:27:57.993874+0000","last_undegraded":"2023-10-09T04:27:57.993874+0000","last_fullsized":"2023-10-09T04:27:57.993874+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:05.799943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989081+0000","last_change":"2023-10-09T04:27:57.989081+0000","last_active":"2023-10-09T04:27:57.989081+0000","last_peered":"2023-10-09T04:27:57.989081+0000","last_clean":"2023-10-09T04:27:57.989081+0000","last_became_active":"2023-10-09T04:27:57.988802+0000","last_became_peered":"2023-10-09T04:27:57.988802+0000","last_unstale":"2023-10-09T04:27:57.989081+0000","last_undegraded":"2023-10-09T04:27:57.989081+0000","last_fullsized":"2023-10-09T04:27:57.989081+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:45.574939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972611+0000","last_change":"2023-10-09T04:27:29.982732+0000","last_active":"2023-10-09T04:27:57.972611+0000","last_peered":"2023-10-09T04:27:57.972611+0000","last_clean":"2023-10-09T04:27:57.972611+0000","last_became_active":"2023-10-09T04:27:29.982275+0000","last_became_peered":"2023-10-09T04:27:29.982275+0000","last_unstale":"2023-10-09T04:27:57.972611+0000","last_undegraded":"2023-10-09T04:27:57.972611+0000","last_fullsized":"2023-10-09T04:27:57.972611+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:43.712715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973308+0000","last_change":"2023-10-09T04:24:07.922314+0000","last_active":"2023-10-09T04:27:57.973308+0000","last_peered":"2023-10-09T04:27:57.973308+0000","last_clean":"2023-10-09T04:27:57.973308+0000","last_became_active":"2023-10-09T04:24:07.916895+0000","last_became_peered":"2023-10-09T04:24:07.916895+0000","last_unstale":"2023-10-09T04:27:57.973308+0000","last_undegraded":"2023-10-09T04:27:57.973308+0000","last_fullsized":"2023-10-09T04:27:57.973308+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:13.277124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988253+0000","last_change":"2023-10-09T04:27:29.988702+0000","last_active":"2023-10-09T04:27:56.988253+0000","last_peered":"2023-10-09T04:27:56.988253+0000","last_clean":"2023-10-09T04:27:56.988253+0000","last_became_active":"2023-10-09T04:27:29.988390+0000","last_became_peered":"2023-10-09T04:27:29.988390+0000","last_unstale":"2023-10-09T04:27:56.988253+0000","last_undegraded":"2023-10-09T04:27:56.988253+0000","last_fullsized":"2023-10-09T04:27:56.988253+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:33.897326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":484,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981404+0000","last_change":"2023-10-09T04:20:03.031289+0000","last_active":"2023-10-09T04:27:56.981404+0000","last_peered":"2023-10-09T04:27:56.981404+0000","last_clean":"2023-10-09T04:27:56.981404+0000","last_became_active":"2023-10-09T04:20:03.030125+0000","last_became_peered":"2023-10-09T04:20:03.030125+0000","last_unstale":"2023-10-09T04:27:56.981404+0000","last_undegraded":"2023-10-09T04:27:56.981404+0000","last_fullsized":"2023-10-09T04:27:56.981404+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:33.277598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988547+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:27:57.988547+0000","last_peered":"2023-10-09T04:27:57.988547+0000","last_clean":"2023-10-09T04:27:57.988547+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:27:57.988547+0000","last_undegraded":"2023-10-09T04:27:57.988547+0000","last_fullsized":"2023-10-09T04:27:57.988547+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981058+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:27:56.981058+0000","last_peered":"2023-10-09T04:27:56.981058+0000","last_clean":"2023-10-09T04:27:56.981058+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:27:56.981058+0000","last_undegraded":"2023-10-09T04:27:56.981058+0000","last_fullsized":"2023-10-09T04:27:56.981058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":124,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981273+0000","last_change":"2023-10-09T04:24:07.962764+0000","last_active":"2023-10-09T04:27:56.981273+0000","last_peered":"2023-10-09T04:27:56.981273+0000","last_clean":"2023-10-09T04:27:56.981273+0000","last_became_active":"2023-10-09T04:24:07.962508+0000","last_became_peered":"2023-10-09T04:24:07.962508+0000","last_unstale":"2023-10-09T04:27:56.981273+0000","last_undegraded":"2023-10-09T04:27:56.981273+0000","last_fullsized":"2023-10-09T04:27:56.981273+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:14.078794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989841+0000","last_change":"2023-10-09T04:27:57.988062+0000","last_active":"2023-10-09T04:27:57.989841+0000","last_peered":"2023-10-09T04:27:57.989841+0000","last_clean":"2023-10-09T04:27:57.989841+0000","last_became_active":"2023-10-09T04:27:57.987732+0000","last_became_peered":"2023-10-09T04:27:57.987732+0000","last_unstale":"2023-10-09T04:27:57.989841+0000","last_undegraded":"2023-10-09T04:27:57.989841+0000","last_fullsized":"2023-10-09T04:27:57.989841+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:01.399437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978739+0000","last_change":"2023-10-09T04:27:29.973734+0000","last_active":"2023-10-09T04:27:57.978739+0000","last_peered":"2023-10-09T04:27:57.978739+0000","last_clean":"2023-10-09T04:27:57.978739+0000","last_became_active":"2023-10-09T04:27:29.973431+0000","last_became_peered":"2023-10-09T04:27:29.973431+0000","last_unstale":"2023-10-09T04:27:57.978739+0000","last_undegraded":"2023-10-09T04:27:57.978739+0000","last_fullsized":"2023-10-09T04:27:57.978739+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:14.788158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.985955+0000","last_change":"2023-10-09T04:20:03.030432+0000","last_active":"2023-10-09T04:27:56.985955+0000","last_peered":"2023-10-09T04:27:56.985955+0000","last_clean":"2023-10-09T04:27:56.985955+0000","last_became_active":"2023-10-09T04:20:03.030103+0000","last_became_peered":"2023-10-09T04:20:03.030103+0000","last_unstale":"2023-10-09T04:27:56.985955+0000","last_undegraded":"2023-10-09T04:27:56.985955+0000","last_fullsized":"2023-10-09T04:27:56.985955+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.759695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.974229+0000","last_change":"2023-10-09T04:27:29.981956+0000","last_active":"2023-10-09T04:27:57.974229+0000","last_peered":"2023-10-09T04:27:57.974229+0000","last_clean":"2023-10-09T04:27:57.974229+0000","last_became_active":"2023-10-09T04:27:29.981663+0000","last_became_peered":"2023-10-09T04:27:29.981663+0000","last_unstale":"2023-10-09T04:27:57.974229+0000","last_undegraded":"2023-10-09T04:27:57.974229+0000","last_fullsized":"2023-10-09T04:27:57.974229+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:47.639333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.977902+0000","last_change":"2023-10-09T04:20:03.032817+0000","last_active":"2023-10-09T04:27:57.977902+0000","last_peered":"2023-10-09T04:27:57.977902+0000","last_clean":"2023-10-09T04:27:57.977902+0000","last_became_active":"2023-10-09T04:20:03.029228+0000","last_became_peered":"2023-10-09T04:20:03.029228+0000","last_unstale":"2023-10-09T04:27:57.977902+0000","last_undegraded":"2023-10-09T04:27:57.977902+0000","last_fullsized":"2023-10-09T04:27:57.977902+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.429820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.995379+0000","last_change":"2023-10-09T04:27:57.995379+0000","last_active":"2023-10-09T04:27:57.995379+0000","last_peered":"2023-10-09T04:27:57.995379+0000","last_clean":"2023-10-09T04:27:57.995379+0000","last_became_active":"2023-10-09T04:27:57.995057+0000","last_became_peered":"2023-10-09T04:27:57.995057+0000","last_unstale":"2023-10-09T04:27:57.995379+0000","last_undegraded":"2023-10-09T04:27:57.995379+0000","last_fullsized":"2023-10-09T04:27:57.995379+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:46.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973460+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:27:57.973460+0000","last_peered":"2023-10-09T04:27:57.973460+0000","last_clean":"2023-10-09T04:27:57.973460+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:27:57.973460+0000","last_undegraded":"2023-10-09T04:27:57.973460+0000","last_fullsized":"2023-10-09T04:27:57.973460+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.991611+0000","last_change":"2023-10-09T04:27:57.991611+0000","last_active":"2023-10-09T04:27:57.991611+0000","last_peered":"2023-10-09T04:27:57.991611+0000","last_clean":"2023-10-09T04:27:57.991611+0000","last_became_active":"2023-10-09T04:27:57.991279+0000","last_became_peered":"2023-10-09T04:27:57.991279+0000","last_unstale":"2023-10-09T04:27:57.991611+0000","last_undegraded":"2023-10-09T04:27:57.991611+0000","last_fullsized":"2023-10-09T04:27:57.991611+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:54.806159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981184+0000","last_change":"2023-10-09T04:27:30.000853+0000","last_active":"2023-10-09T04:27:56.981184+0000","last_peered":"2023-10-09T04:27:56.981184+0000","last_clean":"2023-10-09T04:27:56.981184+0000","last_became_active":"2023-10-09T04:27:30.000545+0000","last_became_peered":"2023-10-09T04:27:30.000545+0000","last_unstale":"2023-10-09T04:27:56.981184+0000","last_undegraded":"2023-10-09T04:27:56.981184+0000","last_fullsized":"2023-10-09T04:27:56.981184+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:47.506034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978683+0000","last_change":"2023-10-09T04:24:07.919208+0000","last_active":"2023-10-09T04:27:57.978683+0000","last_peered":"2023-10-09T04:27:57.978683+0000","last_clean":"2023-10-09T04:27:57.978683+0000","last_became_active":"2023-10-09T04:24:07.918797+0000","last_became_peered":"2023-10-09T04:24:07.918797+0000","last_unstale":"2023-10-09T04:27:57.978683+0000","last_undegraded":"2023-10-09T04:27:57.978683+0000","last_fullsized":"2023-10-09T04:27:57.978683+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:11.399352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976517+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:27:56.976517+0000","last_peered":"2023-10-09T04:27:56.976517+0000","last_clean":"2023-10-09T04:27:56.976517+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:27:56.976517+0000","last_undegraded":"2023-10-09T04:27:56.976517+0000","last_fullsized":"2023-10-09T04:27:56.976517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":80,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982962+0000","last_change":"2023-10-09T04:20:03.037673+0000","last_active":"2023-10-09T04:27:56.982962+0000","last_peered":"2023-10-09T04:27:56.982962+0000","last_clean":"2023-10-09T04:27:56.982962+0000","last_became_active":"2023-10-09T04:20:03.032398+0000","last_became_peered":"2023-10-09T04:20:03.032398+0000","last_unstale":"2023-10-09T04:27:56.982962+0000","last_undegraded":"2023-10-09T04:27:56.982962+0000","last_fullsized":"2023-10-09T04:27:56.982962+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:20.306433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989106+0000","last_change":"2023-10-09T04:27:57.989106+0000","last_active":"2023-10-09T04:27:57.989106+0000","last_peered":"2023-10-09T04:27:57.989106+0000","last_clean":"2023-10-09T04:27:57.989106+0000","last_became_active":"2023-10-09T04:27:57.988778+0000","last_became_peered":"2023-10-09T04:27:57.988778+0000","last_unstale":"2023-10-09T04:27:57.989106+0000","last_undegraded":"2023-10-09T04:27:57.989106+0000","last_fullsized":"2023-10-09T04:27:57.989106+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:33.473019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.989866+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:27:56.989866+0000","last_peered":"2023-10-09T04:27:56.989866+0000","last_clean":"2023-10-09T04:27:56.989866+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:27:56.989866+0000","last_undegraded":"2023-10-09T04:27:56.989866+0000","last_fullsized":"2023-10-09T04:27:56.989866+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988397+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:27:57.988397+0000","last_peered":"2023-10-09T04:27:57.988397+0000","last_clean":"2023-10-09T04:27:57.988397+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:27:57.988397+0000","last_undegraded":"2023-10-09T04:27:57.988397+0000","last_fullsized":"2023-10-09T04:27:57.988397+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":88,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.991512+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:27:56.991512+0000","last_peered":"2023-10-09T04:27:56.991512+0000","last_clean":"2023-10-09T04:27:56.991512+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:27:56.991512+0000","last_undegraded":"2023-10-09T04:27:56.991512+0000","last_fullsized":"2023-10-09T04:27:56.991512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988337+0000","last_change":"2023-10-09T04:27:57.988337+0000","last_active":"2023-10-09T04:27:57.988337+0000","last_peered":"2023-10-09T04:27:57.988337+0000","last_clean":"2023-10-09T04:27:57.988337+0000","last_became_active":"2023-10-09T04:27:57.987918+0000","last_became_peered":"2023-10-09T04:27:57.987918+0000","last_unstale":"2023-10-09T04:27:57.988337+0000","last_undegraded":"2023-10-09T04:27:57.988337+0000","last_fullsized":"2023-10-09T04:27:57.988337+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:54.931130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989240+0000","last_change":"2023-10-09T04:27:57.989240+0000","last_active":"2023-10-09T04:27:57.989240+0000","last_peered":"2023-10-09T04:27:57.989240+0000","last_clean":"2023-10-09T04:27:57.989240+0000","last_became_active":"2023-10-09T04:27:57.988856+0000","last_became_peered":"2023-10-09T04:27:57.988856+0000","last_unstale":"2023-10-09T04:27:57.989240+0000","last_undegraded":"2023-10-09T04:27:57.989240+0000","last_fullsized":"2023-10-09T04:27:57.989240+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:52.380796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990600+0000","last_change":"2023-10-09T04:27:57.990600+0000","last_active":"2023-10-09T04:27:57.990600+0000","last_peered":"2023-10-09T04:27:57.990600+0000","last_clean":"2023-10-09T04:27:57.990600+0000","last_became_active":"2023-10-09T04:27:57.989478+0000","last_became_peered":"2023-10-09T04:27:57.989478+0000","last_unstale":"2023-10-09T04:27:57.990600+0000","last_undegraded":"2023-10-09T04:27:57.990600+0000","last_fullsized":"2023-10-09T04:27:57.990600+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:42.167307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978163+0000","last_change":"2023-10-09T04:27:29.977785+0000","last_active":"2023-10-09T04:27:57.978163+0000","last_peered":"2023-10-09T04:27:57.978163+0000","last_clean":"2023-10-09T04:27:57.978163+0000","last_became_active":"2023-10-09T04:27:29.977397+0000","last_became_peered":"2023-10-09T04:27:29.977397+0000","last_unstale":"2023-10-09T04:27:57.978163+0000","last_undegraded":"2023-10-09T04:27:57.978163+0000","last_fullsized":"2023-10-09T04:27:57.978163+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:16.267931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974728+0000","last_change":"2023-10-09T04:27:29.981763+0000","last_active":"2023-10-09T04:27:56.974728+0000","last_peered":"2023-10-09T04:27:56.974728+0000","last_clean":"2023-10-09T04:27:56.974728+0000","last_became_active":"2023-10-09T04:27:29.981401+0000","last_became_peered":"2023-10-09T04:27:29.981401+0000","last_unstale":"2023-10-09T04:27:56.974728+0000","last_undegraded":"2023-10-09T04:27:56.974728+0000","last_fullsized":"2023-10-09T04:27:56.974728+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:35.242521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234263,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303340,"kb_used_data":1956,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441812,"statfs":{"total":95995035648,"available":95684415488,"internally_reserved":0,"allocated":2002944,"data_stored":1492631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":7.8869999999999996,"15min":3.9350000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22900000000000001},"max":{"1min":8.5909999999999993,"5min":629.22500000000002,"15min":629.22500000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":7.5739999999999998,"15min":3.5899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.1849999999999996,"5min":629.43100000000004,"15min":629.43100000000004},"last":0.54400000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":2.5960000000000001,"15min":2.0579999999999998},"min":{"1min":0.41099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.3330000000000002,"5min":80.215000000000003,"15min":80.215000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7309999999999999,"15min":2.0880000000000001},"min":{"1min":0.35199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.2450000000000001,"5min":80.274000000000001,"15min":80.274000000000001},"last":0.69299999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.64400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":9.3070000000000004,"15min":4.5659999999999998},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.254},"max":{"1min":7.968,"5min":686.07500000000005,"15min":686.07500000000005},"last":0.73299999999999998},{"interface":"front","average":{"1min":2.3740000000000001,"5min":9.7650000000000006,"15min":4.7990000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":8.7050000000000001,"5min":686.04399999999998,"15min":686.04399999999998},"last":0.628}]},{"osd":4,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.546,"15min":1.0700000000000001},"min":{"1min":0.38400000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":0.65500000000000003,"5min":8.5500000000000007,"15min":8.5500000000000007},"last":0.755},{"interface":"front","average":{"1min":0.495,"5min":1.1299999999999999,"15min":1.1850000000000001},"min":{"1min":0.36499999999999999,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":0.59699999999999998,"5min":5.9939999999999998,"15min":94.522999999999996},"last":0.59099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.2749999999999999,"15min":0.98499999999999999},"min":{"1min":0.39300000000000002,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":0.753,"5min":8.8610000000000007,"15min":8.8610000000000007},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.3300000000000001,"15min":1.1759999999999999},"min":{"1min":0.371,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.76700000000000002,"5min":6.3319999999999999,"15min":18.125},"last":0.71499999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.4399999999999999,"15min":1.6819999999999999},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.29999999999999999},"max":{"1min":0.78100000000000003,"5min":12.007,"15min":203.19399999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.4319999999999999,"15min":1.786},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.312},"max":{"1min":0.71899999999999997,"5min":7.4800000000000004,"15min":204.303},"last":0.79200000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495897,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299228,"kb_used_data":1940,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445924,"statfs":{"total":95995035648,"available":95688626176,"internally_reserved":0,"allocated":1986560,"data_stored":1492162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.149,"15min":1.956},"min":{"1min":0.32400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.5410000000000004,"5min":10.77,"15min":13.125999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1309999999999998,"15min":2.5960000000000001},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.274,"5min":10.631,"15min":13.551},"last":0.5}]},{"osd":1,"last update":"Mon Oct 9 04:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.0739999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.5600000000000005,"5min":12.728,"15min":12.728},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.4340000000000002,"15min":2.4540000000000002},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":20.181000000000001,"5min":20.181000000000001,"15min":20.181000000000001},"last":0.66200000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.46600000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":2.5840000000000001,"15min":3.222},"min":{"1min":0.439,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.1230000000000002,"5min":16.373999999999999,"15min":119.26900000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":3.903,"5min":2.7999999999999998,"15min":3.323},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.663,"5min":15.663,"15min":119.372},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.351,"15min":1.784},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":5.2439999999999998,"5min":12.743,"15min":92.650000000000006},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.494,"5min":1.6970000000000001,"15min":1.796},"min":{"1min":0.437,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2379999999999995,"5min":12.028,"15min":91.906999999999996},"last":0.69499999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.367,"15min":1.389},"min":{"1min":0.44500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.8920000000000003,"5min":12.710000000000001,"15min":19.759},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.603,"5min":1.7390000000000001,"15min":1.5029999999999999},"min":{"1min":0.503,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":8.4090000000000007,"5min":12.249000000000001,"15min":19.904},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 04:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.6200000000000001,"15min":1.454},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.29899999999999999},"max":{"1min":0.84999999999999998,"5min":14.718999999999999,"15min":55.838999999999999},"last":0.626},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.452,"15min":1.486},"min":{"1min":0.497,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78200000000000003,"5min":20.155000000000001,"15min":55.718000000000004},"last":0.67600000000000005}]}]},{"osd":1,"up_from":118,"seq":506806140987,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":261036,"kb_used_data":1444,"kb_used_omap":4,"kb_used_meta":259579,"kb_avail":93484116,"statfs":{"total":95995035648,"available":95727734784,"internally_reserved":0,"allocated":1478656,"data_stored":912057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":265809806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.7240000000000002,"15min":2.98},"min":{"1min":0.21099999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.016,"5min":11.81,"15min":11.81},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.157,"5min":2.9249999999999998,"15min":3.1280000000000001},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.573,"5min":12.573,"15min":12.573},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"last":0.39400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:15 2023","interfaces":[{"interface":"back","average":{"1min":4.0289999999999999,"5min":1.9219999999999999,"15min":1.3160000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.415,"15min":3.3700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.170999999999999,"5min":13.170999999999999,"15min":13.170999999999999},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":3.8809999999999998,"15min":3.109},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":11.744,"5min":107.795,"15min":107.795},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":2.609,"15min":1.617},"min":{"1min":0.373,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.1120000000000001,"5min":109.167,"15min":109.167},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":2.5270000000000001,"15min":2.6709999999999998},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.9930000000000003,"5min":12.414999999999999,"15min":12.414999999999999},"last":0.629},{"interface":"front","average":{"1min":1.484,"5min":2.8839999999999999,"15min":3.0270000000000001},"min":{"1min":0.40100000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.686,"5min":13.933999999999999,"15min":13.933999999999999},"last":0.55700000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3650000000000002,"15min":2.5},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.7490000000000001,"5min":13.388,"15min":13.388},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.508,"5min":3.048,"15min":3.133},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.776999999999999,"5min":13.882,"15min":13.882},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.2349999999999999,"15min":3.1840000000000002},"min":{"1min":0.443,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":11.805,"5min":13.907,"15min":13.907},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.802,"15min":1.6339999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.1419999999999999,"5min":7.8239999999999998,"15min":7.8239999999999998},"last":0.64400000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024044,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":280852,"kb_used_data":2060,"kb_used_omap":5,"kb_used_meta":278778,"kb_avail":93464300,"statfs":{"total":95995035648,"available":95707443200,"internally_reserved":0,"allocated":2109440,"data_stored":1505553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":285469306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":2.911,"15min":3.419},"min":{"1min":0.24199999999999999,"5min":0.192,"15min":0.192},"max":{"1min":9.1280000000000001,"5min":17.210000000000001,"15min":17.210000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.6779999999999999,"15min":2.8639999999999999},"min":{"1min":0.26600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.5190000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.27200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:19 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":5.351,"15min":3.7799999999999998},"min":{"1min":0.307,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.753,"5min":131.08000000000001,"15min":131.08000000000001},"last":0.311},{"interface":"front","average":{"1min":3.7970000000000002,"5min":4.4939999999999998,"15min":3.472},"min":{"1min":0.27800000000000002,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.85,"5min":128.12200000000001,"15min":128.12200000000001},"last":0.40300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.742,"15min":1.5269999999999999},"min":{"1min":0.38900000000000001,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":1.3720000000000001,"5min":10.73,"15min":10.73},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.78,"15min":1.7270000000000001},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5009999999999999,"5min":8.1300000000000008,"15min":9.5190000000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.032,"15min":1.819},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.4610000000000001,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.718,"15min":1.911},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":2.0779999999999998,"5min":14.144,"15min":14.144},"last":0.65300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":2.145,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.8019999999999996,"5min":9.7469999999999999,"15min":10.403},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.66100000000000003,"5min":1.9890000000000001,"15min":2.2440000000000002},"min":{"1min":0.46700000000000003,"5min":0.378,"15min":0.378},"max":{"1min":2.1949999999999998,"5min":14.090999999999999,"15min":14.090999999999999},"last":0.68500000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.9670000000000001,"15min":2.0470000000000002},"min":{"1min":0.505,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":5.7919999999999998,"5min":11.890000000000001,"15min":11.890000000000001},"last":0.623},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.1139999999999999,"15min":1.984},"min":{"1min":0.45300000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.8250000000000002,"5min":10.765000000000001,"15min":11.176},"last":0.5}]}]},{"osd":2,"up_from":135,"seq":579820584978,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156100,"kb_used_data":1404,"kb_used_omap":7,"kb_used_meta":154680,"kb_avail":93589052,"statfs":{"total":95995035648,"available":95835189248,"internally_reserved":0,"allocated":1437696,"data_stored":907571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158392401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.41599999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.87}]},{"osd":5,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.79400000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.85799999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.81200000000000006}]}]},{"osd":3,"up_from":142,"seq":609885356045,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":121504,"kb_used_data":1368,"kb_used_omap":2,"kb_used_meta":120125,"kb_avail":93623648,"statfs":{"total":95995035648,"available":95870615552,"internally_reserved":0,"allocated":1400832,"data_stored":900486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":123008053},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019171,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599332,"kb_used_data":1372,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":93145820,"statfs":{"total":95995035648,"available":95381319680,"internally_reserved":0,"allocated":1404928,"data_stored":902193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.7110000000000001,"15min":1.879},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.89200000000000002,"5min":11.442,"15min":16.553999999999998},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.7849999999999999,"15min":2.274},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.1699999999999999,"5min":8.4909999999999997,"15min":16.529},"last":0.623}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":2.0510000000000002,"15min":1.9850000000000001},"min":{"1min":0.35099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.7879999999999998,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.073,"5min":2.1779999999999999,"15min":2.306},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.8970000000000002,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.79900000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.66200000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:27:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":2.2679999999999998,"15min":4.3899999999999997},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.30199999999999999},"max":{"1min":7.3040000000000003,"5min":8.4079999999999995,"15min":337.08100000000002},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.2269999999999999,"15min":4.5490000000000004},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":6.2839999999999998,"5min":8.5820000000000007,"15min":342.04399999999998},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:27:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.2310000000000001,"15min":1.0900000000000001},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.22900000000000001},"max":{"1min":6.5540000000000003,"5min":10.255000000000001,"15min":16.602},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.53,"5min":1.1950000000000001,"15min":1.2030000000000001},"min":{"1min":0.27300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.7009999999999996,"5min":6.7009999999999996,"15min":18.739000000000001},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.097,"15min":1.131},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.97599999999999998,"5min":6.5810000000000004,"15min":9.9640000000000004},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.119,"15min":1.1140000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":4.5199999999999996,"15min":9.3859999999999992},"last":0.60699999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.3400000000000001,"15min":1.0900000000000001},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":7.8250000000000002,"15min":10.34},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.355,"15min":1.0800000000000001},"min":{"1min":0.46800000000000003,"5min":0.34399999999999997,"15min":0.25900000000000001},"max":{"1min":0.82699999999999996,"5min":9.7200000000000006,"15min":9.7200000000000006},"last":0.64900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757535,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591124,"kb_used_data":1356,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93154028,"statfs":{"total":95995035648,"available":95389724672,"internally_reserved":0,"allocated":1388544,"data_stored":897937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.1400000000000001,"15min":1.8759999999999999},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":9.4589999999999996,"5min":13.111000000000001,"15min":13.111000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.716,"15min":1.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.133,"5min":9.3190000000000008,"15min":9.3190000000000008},"last":0.48399999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.9119999999999999,"15min":1.8009999999999999},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.1680000000000001,"5min":10.788,"15min":10.788},"last":0.625},{"interface":"front","average":{"1min":7.883,"5min":3.423,"15min":2.468},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":116.04000000000001,"5min":116.04000000000001,"15min":116.04000000000001},"last":0.50600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.56999999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:26:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":4.6500000000000004,"15min":4.8879999999999999},"min":{"1min":0.441,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":9.8399999999999999,"5min":179.53200000000001,"15min":227.672},"last":0.38800000000000001},{"interface":"front","average":{"1min":4.5650000000000004,"5min":4.5300000000000002,"15min":4.8369999999999997},"min":{"1min":0.40100000000000002,"5min":0.35699999999999998,"15min":0.29099999999999998},"max":{"1min":11.632999999999999,"5min":179.52000000000001,"15min":227.52799999999999},"last":0.71099999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:27:49 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1539999999999999,"15min":1.1739999999999999},"min":{"1min":0.29299999999999998,"5min":0.28100000000000003,"15min":0.23499999999999999},"max":{"1min":5.4080000000000004,"5min":9.641,"15min":10.561999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.629,"5min":1.2190000000000001,"15min":1.198},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.245},"max":{"1min":8.9280000000000008,"5min":8.9280000000000008,"15min":11.356},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:28:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":1.268,"15min":1.0649999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.26000000000000001},"max":{"1min":1.649,"5min":9.4019999999999992,"15min":10.31},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.488,"15min":1.238},"min":{"1min":0.44500000000000001,"5min":0.32000000000000001,"15min":0.30399999999999999},"max":{"1min":1.4119999999999999,"5min":14.334,"15min":16.123000000000001},"last":0.53700000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.46,"15min":1.2090000000000001},"min":{"1min":0.48299999999999998,"5min":0.34499999999999997,"15min":0.27400000000000002},"max":{"1min":0.80100000000000005,"5min":9.9000000000000004,"15min":9.9589999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.3460000000000001,"15min":1.089},"min":{"1min":0.502,"5min":0.376,"15min":0.33000000000000002},"max":{"1min":0.86899999999999999,"5min":11.315,"15min":11.315},"last":0.74199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:28:54.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:28:54.563 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:54.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:54 smithi029 ceph-mon[223858]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:54.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:54 smithi029 ceph-mon[223858]: from='client.35295 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:54.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:54 smithi029 ceph-mon[216903]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:54.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:54 smithi029 ceph-mon[216903]: from='client.35295 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:54 smithi177 ceph-mon[195797]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:54 smithi177 ceph-mon[195797]: from='client.35295 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:56.414 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:28:56.415 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:28:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:56 smithi029 ceph-mon[216903]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:56 smithi029 ceph-mon[223858]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:56 smithi177 ceph-mon[195797]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:57.241 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":512,"stamp":"2023-10-09T04:28:54.790647+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2612516,"kb_used_data":12900,"kb_used_omap":20,"kb_used_meta":2599531,"kb_avail":747348700,"statfs":{"total":767960285184,"available":765285068800,"internally_reserved":0,"allocated":13209600,"data_stored":9010590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2661920398},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003083"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974877+0000","last_change":"2023-10-09T04:27:30.004963+0000","last_active":"2023-10-09T04:27:56.974877+0000","last_peered":"2023-10-09T04:27:56.974877+0000","last_clean":"2023-10-09T04:27:56.974877+0000","last_became_active":"2023-10-09T04:27:30.004581+0000","last_became_peered":"2023-10-09T04:27:30.004581+0000","last_unstale":"2023-10-09T04:27:56.974877+0000","last_undegraded":"2023-10-09T04:27:56.974877+0000","last_fullsized":"2023-10-09T04:27:56.974877+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:40.077805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988720+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:27:57.988720+0000","last_peered":"2023-10-09T04:27:57.988720+0000","last_clean":"2023-10-09T04:27:57.988720+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:27:57.988720+0000","last_undegraded":"2023-10-09T04:27:57.988720+0000","last_fullsized":"2023-10-09T04:27:57.988720+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.982645+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:27:57.982645+0000","last_peered":"2023-10-09T04:27:57.982645+0000","last_clean":"2023-10-09T04:27:57.982645+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:27:57.982645+0000","last_undegraded":"2023-10-09T04:27:57.982645+0000","last_fullsized":"2023-10-09T04:27:57.982645+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990988+0000","last_change":"2023-10-09T04:27:57.985924+0000","last_active":"2023-10-09T04:27:57.990988+0000","last_peered":"2023-10-09T04:27:57.990988+0000","last_clean":"2023-10-09T04:27:57.990988+0000","last_became_active":"2023-10-09T04:27:57.985559+0000","last_became_peered":"2023-10-09T04:27:57.985559+0000","last_unstale":"2023-10-09T04:27:57.990988+0000","last_undegraded":"2023-10-09T04:27:57.990988+0000","last_fullsized":"2023-10-09T04:27:57.990988+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:01.749257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986393+0000","last_change":"2023-10-09T04:27:57.986393+0000","last_active":"2023-10-09T04:27:57.986393+0000","last_peered":"2023-10-09T04:27:57.986393+0000","last_clean":"2023-10-09T04:27:57.986393+0000","last_became_active":"2023-10-09T04:27:57.981584+0000","last_became_peered":"2023-10-09T04:27:57.981584+0000","last_unstale":"2023-10-09T04:27:57.986393+0000","last_undegraded":"2023-10-09T04:27:57.986393+0000","last_fullsized":"2023-10-09T04:27:57.986393+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:34.073778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981290+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:27:56.981290+0000","last_peered":"2023-10-09T04:27:56.981290+0000","last_clean":"2023-10-09T04:27:56.981290+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:27:56.981290+0000","last_undegraded":"2023-10-09T04:27:56.981290+0000","last_fullsized":"2023-10-09T04:27:56.981290+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989523+0000","last_change":"2023-10-09T04:27:57.989523+0000","last_active":"2023-10-09T04:27:57.989523+0000","last_peered":"2023-10-09T04:27:57.989523+0000","last_clean":"2023-10-09T04:27:57.989523+0000","last_became_active":"2023-10-09T04:27:57.989226+0000","last_became_peered":"2023-10-09T04:27:57.989226+0000","last_unstale":"2023-10-09T04:27:57.989523+0000","last_undegraded":"2023-10-09T04:27:57.989523+0000","last_fullsized":"2023-10-09T04:27:57.989523+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:47.573373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987735+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:27:57.987735+0000","last_peered":"2023-10-09T04:27:57.987735+0000","last_clean":"2023-10-09T04:27:57.987735+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:27:57.987735+0000","last_undegraded":"2023-10-09T04:27:57.987735+0000","last_fullsized":"2023-10-09T04:27:57.987735+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988398+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:27:56.988398+0000","last_peered":"2023-10-09T04:27:56.988398+0000","last_clean":"2023-10-09T04:27:56.988398+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:27:56.988398+0000","last_undegraded":"2023-10-09T04:27:56.988398+0000","last_fullsized":"2023-10-09T04:27:56.988398+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990152+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:27:57.990152+0000","last_peered":"2023-10-09T04:27:57.990152+0000","last_clean":"2023-10-09T04:27:57.990152+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:27:57.990152+0000","last_undegraded":"2023-10-09T04:27:57.990152+0000","last_fullsized":"2023-10-09T04:27:57.990152+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":106,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986830+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:27:57.986830+0000","last_peered":"2023-10-09T04:27:57.986830+0000","last_clean":"2023-10-09T04:27:57.986830+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:27:57.986830+0000","last_undegraded":"2023-10-09T04:27:57.986830+0000","last_fullsized":"2023-10-09T04:27:57.986830+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972995+0000","last_change":"2023-10-09T04:27:29.983053+0000","last_active":"2023-10-09T04:27:57.972995+0000","last_peered":"2023-10-09T04:27:57.972995+0000","last_clean":"2023-10-09T04:27:57.972995+0000","last_became_active":"2023-10-09T04:27:29.982725+0000","last_became_peered":"2023-10-09T04:27:29.982725+0000","last_unstale":"2023-10-09T04:27:57.972995+0000","last_undegraded":"2023-10-09T04:27:57.972995+0000","last_fullsized":"2023-10-09T04:27:57.972995+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:33.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976512+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:27:56.976512+0000","last_peered":"2023-10-09T04:27:56.976512+0000","last_clean":"2023-10-09T04:27:56.976512+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:27:56.976512+0000","last_undegraded":"2023-10-09T04:27:56.976512+0000","last_fullsized":"2023-10-09T04:27:56.976512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975162+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:27:56.975162+0000","last_peered":"2023-10-09T04:27:56.975162+0000","last_clean":"2023-10-09T04:27:56.975162+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:27:56.975162+0000","last_undegraded":"2023-10-09T04:27:56.975162+0000","last_fullsized":"2023-10-09T04:27:56.975162+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981381+0000","last_change":"2023-10-09T04:27:29.989226+0000","last_active":"2023-10-09T04:27:56.981381+0000","last_peered":"2023-10-09T04:27:56.981381+0000","last_clean":"2023-10-09T04:27:56.981381+0000","last_became_active":"2023-10-09T04:27:29.988867+0000","last_became_peered":"2023-10-09T04:27:29.988867+0000","last_unstale":"2023-10-09T04:27:56.981381+0000","last_undegraded":"2023-10-09T04:27:56.981381+0000","last_fullsized":"2023-10-09T04:27:56.981381+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:15.559571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987345+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:27:57.987345+0000","last_peered":"2023-10-09T04:27:57.987345+0000","last_clean":"2023-10-09T04:27:57.987345+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:27:57.987345+0000","last_undegraded":"2023-10-09T04:27:57.987345+0000","last_fullsized":"2023-10-09T04:27:57.987345+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.985231+0000","last_change":"2023-10-09T04:27:57.985231+0000","last_active":"2023-10-09T04:27:57.985231+0000","last_peered":"2023-10-09T04:27:57.985231+0000","last_clean":"2023-10-09T04:27:57.985231+0000","last_became_active":"2023-10-09T04:27:57.984869+0000","last_became_peered":"2023-10-09T04:27:57.984869+0000","last_unstale":"2023-10-09T04:27:57.985231+0000","last_undegraded":"2023-10-09T04:27:57.985231+0000","last_fullsized":"2023-10-09T04:27:57.985231+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:39.521778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992866+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:27:57.992866+0000","last_peered":"2023-10-09T04:27:57.992866+0000","last_clean":"2023-10-09T04:27:57.992866+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:27:57.992866+0000","last_undegraded":"2023-10-09T04:27:57.992866+0000","last_fullsized":"2023-10-09T04:27:57.992866+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980748+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:27:56.980748+0000","last_peered":"2023-10-09T04:27:56.980748+0000","last_clean":"2023-10-09T04:27:56.980748+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:27:56.980748+0000","last_undegraded":"2023-10-09T04:27:56.980748+0000","last_fullsized":"2023-10-09T04:27:56.980748+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.994143+0000","last_change":"2023-10-09T04:27:57.994143+0000","last_active":"2023-10-09T04:27:57.994143+0000","last_peered":"2023-10-09T04:27:57.994143+0000","last_clean":"2023-10-09T04:27:57.994143+0000","last_became_active":"2023-10-09T04:27:57.993862+0000","last_became_peered":"2023-10-09T04:27:57.993862+0000","last_unstale":"2023-10-09T04:27:57.994143+0000","last_undegraded":"2023-10-09T04:27:57.994143+0000","last_fullsized":"2023-10-09T04:27:57.994143+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:24.434239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980940+0000","last_change":"2023-10-09T04:20:03.030960+0000","last_active":"2023-10-09T04:27:56.980940+0000","last_peered":"2023-10-09T04:27:56.980940+0000","last_clean":"2023-10-09T04:27:56.980940+0000","last_became_active":"2023-10-09T04:20:03.030148+0000","last_became_peered":"2023-10-09T04:20:03.030148+0000","last_unstale":"2023-10-09T04:27:56.980940+0000","last_undegraded":"2023-10-09T04:27:56.980940+0000","last_fullsized":"2023-10-09T04:27:56.980940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:09.000584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986461+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:27:57.986461+0000","last_peered":"2023-10-09T04:27:57.986461+0000","last_clean":"2023-10-09T04:27:57.986461+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:27:57.986461+0000","last_undegraded":"2023-10-09T04:27:57.986461+0000","last_fullsized":"2023-10-09T04:27:57.986461+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983931+0000","last_change":"2023-10-09T04:20:03.033517+0000","last_active":"2023-10-09T04:27:56.983931+0000","last_peered":"2023-10-09T04:27:56.983931+0000","last_clean":"2023-10-09T04:27:56.983931+0000","last_became_active":"2023-10-09T04:20:03.033132+0000","last_became_peered":"2023-10-09T04:20:03.033132+0000","last_unstale":"2023-10-09T04:27:56.983931+0000","last_undegraded":"2023-10-09T04:27:56.983931+0000","last_fullsized":"2023-10-09T04:27:56.983931+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:14.247924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988550+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:27:56.988550+0000","last_peered":"2023-10-09T04:27:56.988550+0000","last_clean":"2023-10-09T04:27:56.988550+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:27:56.988550+0000","last_undegraded":"2023-10-09T04:27:56.988550+0000","last_fullsized":"2023-10-09T04:27:56.988550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983774+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:27:56.983774+0000","last_peered":"2023-10-09T04:27:56.983774+0000","last_clean":"2023-10-09T04:27:56.983774+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:27:56.983774+0000","last_undegraded":"2023-10-09T04:27:56.983774+0000","last_fullsized":"2023-10-09T04:27:56.983774+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988386+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:27:57.988386+0000","last_peered":"2023-10-09T04:27:57.988386+0000","last_clean":"2023-10-09T04:27:57.988386+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:27:57.988386+0000","last_undegraded":"2023-10-09T04:27:57.988386+0000","last_fullsized":"2023-10-09T04:27:57.988386+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":87,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982249+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:27:56.982249+0000","last_peered":"2023-10-09T04:27:56.982249+0000","last_clean":"2023-10-09T04:27:56.982249+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:27:56.982249+0000","last_undegraded":"2023-10-09T04:27:56.982249+0000","last_fullsized":"2023-10-09T04:27:56.982249+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982155+0000","last_change":"2023-10-09T04:27:30.005025+0000","last_active":"2023-10-09T04:27:56.982155+0000","last_peered":"2023-10-09T04:27:56.982155+0000","last_clean":"2023-10-09T04:27:56.982155+0000","last_became_active":"2023-10-09T04:27:30.004692+0000","last_became_peered":"2023-10-09T04:27:30.004692+0000","last_unstale":"2023-10-09T04:27:56.982155+0000","last_undegraded":"2023-10-09T04:27:56.982155+0000","last_fullsized":"2023-10-09T04:27:56.982155+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:10.737398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992844+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:27:57.992844+0000","last_peered":"2023-10-09T04:27:57.992844+0000","last_clean":"2023-10-09T04:27:57.992844+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:27:57.992844+0000","last_undegraded":"2023-10-09T04:27:57.992844+0000","last_fullsized":"2023-10-09T04:27:57.992844+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":90,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982864+0000","last_change":"2023-10-09T04:20:03.025205+0000","last_active":"2023-10-09T04:27:56.982864+0000","last_peered":"2023-10-09T04:27:56.982864+0000","last_clean":"2023-10-09T04:27:56.982864+0000","last_became_active":"2023-10-09T04:20:03.024858+0000","last_became_peered":"2023-10-09T04:20:03.024858+0000","last_unstale":"2023-10-09T04:27:56.982864+0000","last_undegraded":"2023-10-09T04:27:56.982864+0000","last_fullsized":"2023-10-09T04:27:56.982864+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:59.239890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992940+0000","last_change":"2023-10-09T04:27:57.992940+0000","last_active":"2023-10-09T04:27:57.992940+0000","last_peered":"2023-10-09T04:27:57.992940+0000","last_clean":"2023-10-09T04:27:57.992940+0000","last_became_active":"2023-10-09T04:27:57.991337+0000","last_became_peered":"2023-10-09T04:27:57.991337+0000","last_unstale":"2023-10-09T04:27:57.992940+0000","last_undegraded":"2023-10-09T04:27:57.992940+0000","last_fullsized":"2023-10-09T04:27:57.992940+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:10.795994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988243+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:27:57.988243+0000","last_peered":"2023-10-09T04:27:57.988243+0000","last_clean":"2023-10-09T04:27:57.988243+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:27:57.988243+0000","last_undegraded":"2023-10-09T04:27:57.988243+0000","last_fullsized":"2023-10-09T04:27:57.988243+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982044+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:27:56.982044+0000","last_peered":"2023-10-09T04:27:56.982044+0000","last_clean":"2023-10-09T04:27:56.982044+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:27:56.982044+0000","last_undegraded":"2023-10-09T04:27:56.982044+0000","last_fullsized":"2023-10-09T04:27:56.982044+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976221+0000","last_change":"2023-10-09T04:27:29.982774+0000","last_active":"2023-10-09T04:27:56.976221+0000","last_peered":"2023-10-09T04:27:56.976221+0000","last_clean":"2023-10-09T04:27:56.976221+0000","last_became_active":"2023-10-09T04:27:29.982507+0000","last_became_peered":"2023-10-09T04:27:29.982507+0000","last_unstale":"2023-10-09T04:27:56.976221+0000","last_undegraded":"2023-10-09T04:27:56.976221+0000","last_fullsized":"2023-10-09T04:27:56.976221+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:14.613302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976415+0000","last_change":"2023-10-09T04:27:30.010756+0000","last_active":"2023-10-09T04:27:56.976415+0000","last_peered":"2023-10-09T04:27:56.976415+0000","last_clean":"2023-10-09T04:27:56.976415+0000","last_became_active":"2023-10-09T04:27:30.010403+0000","last_became_peered":"2023-10-09T04:27:30.010403+0000","last_unstale":"2023-10-09T04:27:56.976415+0000","last_undegraded":"2023-10-09T04:27:56.976415+0000","last_fullsized":"2023-10-09T04:27:56.976415+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:43:10.557818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976602+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:27:56.976602+0000","last_peered":"2023-10-09T04:27:56.976602+0000","last_clean":"2023-10-09T04:27:56.976602+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:27:56.976602+0000","last_undegraded":"2023-10-09T04:27:56.976602+0000","last_fullsized":"2023-10-09T04:27:56.976602+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986421+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:27:57.986421+0000","last_peered":"2023-10-09T04:27:57.986421+0000","last_clean":"2023-10-09T04:27:57.986421+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:27:57.986421+0000","last_undegraded":"2023-10-09T04:27:57.986421+0000","last_fullsized":"2023-10-09T04:27:57.986421+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.983436+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:27:57.983436+0000","last_peered":"2023-10-09T04:27:57.983436+0000","last_clean":"2023-10-09T04:27:57.983436+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:27:57.983436+0000","last_undegraded":"2023-10-09T04:27:57.983436+0000","last_fullsized":"2023-10-09T04:27:57.983436+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973721+0000","last_change":"2023-10-09T04:24:07.916771+0000","last_active":"2023-10-09T04:27:57.973721+0000","last_peered":"2023-10-09T04:27:57.973721+0000","last_clean":"2023-10-09T04:27:57.973721+0000","last_became_active":"2023-10-09T04:24:07.916030+0000","last_became_peered":"2023-10-09T04:24:07.916030+0000","last_unstale":"2023-10-09T04:27:57.973721+0000","last_undegraded":"2023-10-09T04:27:57.973721+0000","last_fullsized":"2023-10-09T04:27:57.973721+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.033444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993021+0000","last_change":"2023-10-09T04:27:57.993021+0000","last_active":"2023-10-09T04:27:57.993021+0000","last_peered":"2023-10-09T04:27:57.993021+0000","last_clean":"2023-10-09T04:27:57.993021+0000","last_became_active":"2023-10-09T04:27:57.992486+0000","last_became_peered":"2023-10-09T04:27:57.992486+0000","last_unstale":"2023-10-09T04:27:57.993021+0000","last_undegraded":"2023-10-09T04:27:57.993021+0000","last_fullsized":"2023-10-09T04:27:57.993021+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:56.130811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972915+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:27:57.972915+0000","last_peered":"2023-10-09T04:27:57.972915+0000","last_clean":"2023-10-09T04:27:57.972915+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:27:57.972915+0000","last_undegraded":"2023-10-09T04:27:57.972915+0000","last_fullsized":"2023-10-09T04:27:57.972915+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988701+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:27:56.988701+0000","last_peered":"2023-10-09T04:27:56.988701+0000","last_clean":"2023-10-09T04:27:56.988701+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:27:56.988701+0000","last_undegraded":"2023-10-09T04:27:56.988701+0000","last_fullsized":"2023-10-09T04:27:56.988701+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975059+0000","last_change":"2023-10-09T04:20:03.030040+0000","last_active":"2023-10-09T04:27:56.975059+0000","last_peered":"2023-10-09T04:27:56.975059+0000","last_clean":"2023-10-09T04:27:56.975059+0000","last_became_active":"2023-10-09T04:20:03.029675+0000","last_became_peered":"2023-10-09T04:20:03.029675+0000","last_unstale":"2023-10-09T04:27:56.975059+0000","last_undegraded":"2023-10-09T04:27:56.975059+0000","last_fullsized":"2023-10-09T04:27:56.975059+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:25.875934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993874+0000","last_change":"2023-10-09T04:27:57.991197+0000","last_active":"2023-10-09T04:27:57.993874+0000","last_peered":"2023-10-09T04:27:57.993874+0000","last_clean":"2023-10-09T04:27:57.993874+0000","last_became_active":"2023-10-09T04:27:57.990548+0000","last_became_peered":"2023-10-09T04:27:57.990548+0000","last_unstale":"2023-10-09T04:27:57.993874+0000","last_undegraded":"2023-10-09T04:27:57.993874+0000","last_fullsized":"2023-10-09T04:27:57.993874+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:05.799943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989081+0000","last_change":"2023-10-09T04:27:57.989081+0000","last_active":"2023-10-09T04:27:57.989081+0000","last_peered":"2023-10-09T04:27:57.989081+0000","last_clean":"2023-10-09T04:27:57.989081+0000","last_became_active":"2023-10-09T04:27:57.988802+0000","last_became_peered":"2023-10-09T04:27:57.988802+0000","last_unstale":"2023-10-09T04:27:57.989081+0000","last_undegraded":"2023-10-09T04:27:57.989081+0000","last_fullsized":"2023-10-09T04:27:57.989081+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:45.574939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972611+0000","last_change":"2023-10-09T04:27:29.982732+0000","last_active":"2023-10-09T04:27:57.972611+0000","last_peered":"2023-10-09T04:27:57.972611+0000","last_clean":"2023-10-09T04:27:57.972611+0000","last_became_active":"2023-10-09T04:27:29.982275+0000","last_became_peered":"2023-10-09T04:27:29.982275+0000","last_unstale":"2023-10-09T04:27:57.972611+0000","last_undegraded":"2023-10-09T04:27:57.972611+0000","last_fullsized":"2023-10-09T04:27:57.972611+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:43.712715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973308+0000","last_change":"2023-10-09T04:24:07.922314+0000","last_active":"2023-10-09T04:27:57.973308+0000","last_peered":"2023-10-09T04:27:57.973308+0000","last_clean":"2023-10-09T04:27:57.973308+0000","last_became_active":"2023-10-09T04:24:07.916895+0000","last_became_peered":"2023-10-09T04:24:07.916895+0000","last_unstale":"2023-10-09T04:27:57.973308+0000","last_undegraded":"2023-10-09T04:27:57.973308+0000","last_fullsized":"2023-10-09T04:27:57.973308+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:13.277124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988253+0000","last_change":"2023-10-09T04:27:29.988702+0000","last_active":"2023-10-09T04:27:56.988253+0000","last_peered":"2023-10-09T04:27:56.988253+0000","last_clean":"2023-10-09T04:27:56.988253+0000","last_became_active":"2023-10-09T04:27:29.988390+0000","last_became_peered":"2023-10-09T04:27:29.988390+0000","last_unstale":"2023-10-09T04:27:56.988253+0000","last_undegraded":"2023-10-09T04:27:56.988253+0000","last_fullsized":"2023-10-09T04:27:56.988253+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:33.897326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":484,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981404+0000","last_change":"2023-10-09T04:20:03.031289+0000","last_active":"2023-10-09T04:27:56.981404+0000","last_peered":"2023-10-09T04:27:56.981404+0000","last_clean":"2023-10-09T04:27:56.981404+0000","last_became_active":"2023-10-09T04:20:03.030125+0000","last_became_peered":"2023-10-09T04:20:03.030125+0000","last_unstale":"2023-10-09T04:27:56.981404+0000","last_undegraded":"2023-10-09T04:27:56.981404+0000","last_fullsized":"2023-10-09T04:27:56.981404+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:33.277598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988547+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:27:57.988547+0000","last_peered":"2023-10-09T04:27:57.988547+0000","last_clean":"2023-10-09T04:27:57.988547+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:27:57.988547+0000","last_undegraded":"2023-10-09T04:27:57.988547+0000","last_fullsized":"2023-10-09T04:27:57.988547+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981058+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:27:56.981058+0000","last_peered":"2023-10-09T04:27:56.981058+0000","last_clean":"2023-10-09T04:27:56.981058+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:27:56.981058+0000","last_undegraded":"2023-10-09T04:27:56.981058+0000","last_fullsized":"2023-10-09T04:27:56.981058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":124,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981273+0000","last_change":"2023-10-09T04:24:07.962764+0000","last_active":"2023-10-09T04:27:56.981273+0000","last_peered":"2023-10-09T04:27:56.981273+0000","last_clean":"2023-10-09T04:27:56.981273+0000","last_became_active":"2023-10-09T04:24:07.962508+0000","last_became_peered":"2023-10-09T04:24:07.962508+0000","last_unstale":"2023-10-09T04:27:56.981273+0000","last_undegraded":"2023-10-09T04:27:56.981273+0000","last_fullsized":"2023-10-09T04:27:56.981273+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:14.078794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989841+0000","last_change":"2023-10-09T04:27:57.988062+0000","last_active":"2023-10-09T04:27:57.989841+0000","last_peered":"2023-10-09T04:27:57.989841+0000","last_clean":"2023-10-09T04:27:57.989841+0000","last_became_active":"2023-10-09T04:27:57.987732+0000","last_became_peered":"2023-10-09T04:27:57.987732+0000","last_unstale":"2023-10-09T04:27:57.989841+0000","last_undegraded":"2023-10-09T04:27:57.989841+0000","last_fullsized":"2023-10-09T04:27:57.989841+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:01.399437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978739+0000","last_change":"2023-10-09T04:27:29.973734+0000","last_active":"2023-10-09T04:27:57.978739+0000","last_peered":"2023-10-09T04:27:57.978739+0000","last_clean":"2023-10-09T04:27:57.978739+0000","last_became_active":"2023-10-09T04:27:29.973431+0000","last_became_peered":"2023-10-09T04:27:29.973431+0000","last_unstale":"2023-10-09T04:27:57.978739+0000","last_undegraded":"2023-10-09T04:27:57.978739+0000","last_fullsized":"2023-10-09T04:27:57.978739+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:14.788158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.985955+0000","last_change":"2023-10-09T04:20:03.030432+0000","last_active":"2023-10-09T04:27:56.985955+0000","last_peered":"2023-10-09T04:27:56.985955+0000","last_clean":"2023-10-09T04:27:56.985955+0000","last_became_active":"2023-10-09T04:20:03.030103+0000","last_became_peered":"2023-10-09T04:20:03.030103+0000","last_unstale":"2023-10-09T04:27:56.985955+0000","last_undegraded":"2023-10-09T04:27:56.985955+0000","last_fullsized":"2023-10-09T04:27:56.985955+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.759695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.974229+0000","last_change":"2023-10-09T04:27:29.981956+0000","last_active":"2023-10-09T04:27:57.974229+0000","last_peered":"2023-10-09T04:27:57.974229+0000","last_clean":"2023-10-09T04:27:57.974229+0000","last_became_active":"2023-10-09T04:27:29.981663+0000","last_became_peered":"2023-10-09T04:27:29.981663+0000","last_unstale":"2023-10-09T04:27:57.974229+0000","last_undegraded":"2023-10-09T04:27:57.974229+0000","last_fullsized":"2023-10-09T04:27:57.974229+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:47.639333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.977902+0000","last_change":"2023-10-09T04:20:03.032817+0000","last_active":"2023-10-09T04:27:57.977902+0000","last_peered":"2023-10-09T04:27:57.977902+0000","last_clean":"2023-10-09T04:27:57.977902+0000","last_became_active":"2023-10-09T04:20:03.029228+0000","last_became_peered":"2023-10-09T04:20:03.029228+0000","last_unstale":"2023-10-09T04:27:57.977902+0000","last_undegraded":"2023-10-09T04:27:57.977902+0000","last_fullsized":"2023-10-09T04:27:57.977902+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.429820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.995379+0000","last_change":"2023-10-09T04:27:57.995379+0000","last_active":"2023-10-09T04:27:57.995379+0000","last_peered":"2023-10-09T04:27:57.995379+0000","last_clean":"2023-10-09T04:27:57.995379+0000","last_became_active":"2023-10-09T04:27:57.995057+0000","last_became_peered":"2023-10-09T04:27:57.995057+0000","last_unstale":"2023-10-09T04:27:57.995379+0000","last_undegraded":"2023-10-09T04:27:57.995379+0000","last_fullsized":"2023-10-09T04:27:57.995379+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:46.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973460+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:27:57.973460+0000","last_peered":"2023-10-09T04:27:57.973460+0000","last_clean":"2023-10-09T04:27:57.973460+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:27:57.973460+0000","last_undegraded":"2023-10-09T04:27:57.973460+0000","last_fullsized":"2023-10-09T04:27:57.973460+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.991611+0000","last_change":"2023-10-09T04:27:57.991611+0000","last_active":"2023-10-09T04:27:57.991611+0000","last_peered":"2023-10-09T04:27:57.991611+0000","last_clean":"2023-10-09T04:27:57.991611+0000","last_became_active":"2023-10-09T04:27:57.991279+0000","last_became_peered":"2023-10-09T04:27:57.991279+0000","last_unstale":"2023-10-09T04:27:57.991611+0000","last_undegraded":"2023-10-09T04:27:57.991611+0000","last_fullsized":"2023-10-09T04:27:57.991611+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:54.806159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981184+0000","last_change":"2023-10-09T04:27:30.000853+0000","last_active":"2023-10-09T04:27:56.981184+0000","last_peered":"2023-10-09T04:27:56.981184+0000","last_clean":"2023-10-09T04:27:56.981184+0000","last_became_active":"2023-10-09T04:27:30.000545+0000","last_became_peered":"2023-10-09T04:27:30.000545+0000","last_unstale":"2023-10-09T04:27:56.981184+0000","last_undegraded":"2023-10-09T04:27:56.981184+0000","last_fullsized":"2023-10-09T04:27:56.981184+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:47.506034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978683+0000","last_change":"2023-10-09T04:24:07.919208+0000","last_active":"2023-10-09T04:27:57.978683+0000","last_peered":"2023-10-09T04:27:57.978683+0000","last_clean":"2023-10-09T04:27:57.978683+0000","last_became_active":"2023-10-09T04:24:07.918797+0000","last_became_peered":"2023-10-09T04:24:07.918797+0000","last_unstale":"2023-10-09T04:27:57.978683+0000","last_undegraded":"2023-10-09T04:27:57.978683+0000","last_fullsized":"2023-10-09T04:27:57.978683+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:11.399352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976517+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:27:56.976517+0000","last_peered":"2023-10-09T04:27:56.976517+0000","last_clean":"2023-10-09T04:27:56.976517+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:27:56.976517+0000","last_undegraded":"2023-10-09T04:27:56.976517+0000","last_fullsized":"2023-10-09T04:27:56.976517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":80,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982962+0000","last_change":"2023-10-09T04:20:03.037673+0000","last_active":"2023-10-09T04:27:56.982962+0000","last_peered":"2023-10-09T04:27:56.982962+0000","last_clean":"2023-10-09T04:27:56.982962+0000","last_became_active":"2023-10-09T04:20:03.032398+0000","last_became_peered":"2023-10-09T04:20:03.032398+0000","last_unstale":"2023-10-09T04:27:56.982962+0000","last_undegraded":"2023-10-09T04:27:56.982962+0000","last_fullsized":"2023-10-09T04:27:56.982962+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:20.306433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989106+0000","last_change":"2023-10-09T04:27:57.989106+0000","last_active":"2023-10-09T04:27:57.989106+0000","last_peered":"2023-10-09T04:27:57.989106+0000","last_clean":"2023-10-09T04:27:57.989106+0000","last_became_active":"2023-10-09T04:27:57.988778+0000","last_became_peered":"2023-10-09T04:27:57.988778+0000","last_unstale":"2023-10-09T04:27:57.989106+0000","last_undegraded":"2023-10-09T04:27:57.989106+0000","last_fullsized":"2023-10-09T04:27:57.989106+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:33.473019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.989866+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:27:56.989866+0000","last_peered":"2023-10-09T04:27:56.989866+0000","last_clean":"2023-10-09T04:27:56.989866+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:27:56.989866+0000","last_undegraded":"2023-10-09T04:27:56.989866+0000","last_fullsized":"2023-10-09T04:27:56.989866+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988397+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:27:57.988397+0000","last_peered":"2023-10-09T04:27:57.988397+0000","last_clean":"2023-10-09T04:27:57.988397+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:27:57.988397+0000","last_undegraded":"2023-10-09T04:27:57.988397+0000","last_fullsized":"2023-10-09T04:27:57.988397+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":88,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.991512+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:27:56.991512+0000","last_peered":"2023-10-09T04:27:56.991512+0000","last_clean":"2023-10-09T04:27:56.991512+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:27:56.991512+0000","last_undegraded":"2023-10-09T04:27:56.991512+0000","last_fullsized":"2023-10-09T04:27:56.991512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988337+0000","last_change":"2023-10-09T04:27:57.988337+0000","last_active":"2023-10-09T04:27:57.988337+0000","last_peered":"2023-10-09T04:27:57.988337+0000","last_clean":"2023-10-09T04:27:57.988337+0000","last_became_active":"2023-10-09T04:27:57.987918+0000","last_became_peered":"2023-10-09T04:27:57.987918+0000","last_unstale":"2023-10-09T04:27:57.988337+0000","last_undegraded":"2023-10-09T04:27:57.988337+0000","last_fullsized":"2023-10-09T04:27:57.988337+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:54.931130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989240+0000","last_change":"2023-10-09T04:27:57.989240+0000","last_active":"2023-10-09T04:27:57.989240+0000","last_peered":"2023-10-09T04:27:57.989240+0000","last_clean":"2023-10-09T04:27:57.989240+0000","last_became_active":"2023-10-09T04:27:57.988856+0000","last_became_peered":"2023-10-09T04:27:57.988856+0000","last_unstale":"2023-10-09T04:27:57.989240+0000","last_undegraded":"2023-10-09T04:27:57.989240+0000","last_fullsized":"2023-10-09T04:27:57.989240+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:52.380796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990600+0000","last_change":"2023-10-09T04:27:57.990600+0000","last_active":"2023-10-09T04:27:57.990600+0000","last_peered":"2023-10-09T04:27:57.990600+0000","last_clean":"2023-10-09T04:27:57.990600+0000","last_became_active":"2023-10-09T04:27:57.989478+0000","last_became_peered":"2023-10-09T04:27:57.989478+0000","last_unstale":"2023-10-09T04:27:57.990600+0000","last_undegraded":"2023-10-09T04:27:57.990600+0000","last_fullsized":"2023-10-09T04:27:57.990600+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:42.167307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978163+0000","last_change":"2023-10-09T04:27:29.977785+0000","last_active":"2023-10-09T04:27:57.978163+0000","last_peered":"2023-10-09T04:27:57.978163+0000","last_clean":"2023-10-09T04:27:57.978163+0000","last_became_active":"2023-10-09T04:27:29.977397+0000","last_became_peered":"2023-10-09T04:27:29.977397+0000","last_unstale":"2023-10-09T04:27:57.978163+0000","last_undegraded":"2023-10-09T04:27:57.978163+0000","last_fullsized":"2023-10-09T04:27:57.978163+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:16.267931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974728+0000","last_change":"2023-10-09T04:27:29.981763+0000","last_active":"2023-10-09T04:27:56.974728+0000","last_peered":"2023-10-09T04:27:56.974728+0000","last_clean":"2023-10-09T04:27:56.974728+0000","last_became_active":"2023-10-09T04:27:29.981401+0000","last_became_peered":"2023-10-09T04:27:29.981401+0000","last_unstale":"2023-10-09T04:27:56.974728+0000","last_undegraded":"2023-10-09T04:27:56.974728+0000","last_fullsized":"2023-10-09T04:27:56.974728+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:35.242521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234264,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303340,"kb_used_data":1956,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441812,"statfs":{"total":95995035648,"available":95684415488,"internally_reserved":0,"allocated":2002944,"data_stored":1492631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":7.8869999999999996,"15min":3.9350000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22900000000000001},"max":{"1min":8.5909999999999993,"5min":629.22500000000002,"15min":629.22500000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":7.5739999999999998,"15min":3.5899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.1849999999999996,"5min":629.43100000000004,"15min":629.43100000000004},"last":0.54400000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":2.5960000000000001,"15min":2.0579999999999998},"min":{"1min":0.41099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.3330000000000002,"5min":80.215000000000003,"15min":80.215000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7309999999999999,"15min":2.0880000000000001},"min":{"1min":0.35199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.2450000000000001,"5min":80.274000000000001,"15min":80.274000000000001},"last":0.69299999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.64400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":9.3070000000000004,"15min":4.5659999999999998},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.254},"max":{"1min":7.968,"5min":686.07500000000005,"15min":686.07500000000005},"last":0.73299999999999998},{"interface":"front","average":{"1min":2.3740000000000001,"5min":9.7650000000000006,"15min":4.7990000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":8.7050000000000001,"5min":686.04399999999998,"15min":686.04399999999998},"last":0.628}]},{"osd":4,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.546,"15min":1.0700000000000001},"min":{"1min":0.38400000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":0.65500000000000003,"5min":8.5500000000000007,"15min":8.5500000000000007},"last":0.755},{"interface":"front","average":{"1min":0.495,"5min":1.1299999999999999,"15min":1.1850000000000001},"min":{"1min":0.36499999999999999,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":0.59699999999999998,"5min":5.9939999999999998,"15min":94.522999999999996},"last":0.59099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.2749999999999999,"15min":0.98499999999999999},"min":{"1min":0.39300000000000002,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":0.753,"5min":8.8610000000000007,"15min":8.8610000000000007},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.3300000000000001,"15min":1.1759999999999999},"min":{"1min":0.371,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.76700000000000002,"5min":6.3319999999999999,"15min":18.125},"last":0.71499999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.4399999999999999,"15min":1.6819999999999999},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.29999999999999999},"max":{"1min":0.78100000000000003,"5min":12.007,"15min":203.19399999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.4319999999999999,"15min":1.786},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.312},"max":{"1min":0.71899999999999997,"5min":7.4800000000000004,"15min":204.303},"last":0.79200000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495898,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299228,"kb_used_data":1940,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445924,"statfs":{"total":95995035648,"available":95688626176,"internally_reserved":0,"allocated":1986560,"data_stored":1492162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.149,"15min":1.956},"min":{"1min":0.32400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.5410000000000004,"5min":10.77,"15min":13.125999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1309999999999998,"15min":2.5960000000000001},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.274,"5min":10.631,"15min":13.551},"last":0.51500000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.0739999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.5600000000000005,"5min":12.728,"15min":12.728},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.4340000000000002,"15min":2.4540000000000002},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":20.181000000000001,"5min":20.181000000000001,"15min":20.181000000000001},"last":0.40500000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.65600000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:27:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":2.5840000000000001,"15min":3.222},"min":{"1min":0.439,"5min":0.30199999999999999,"15min":0.28899999999999998},"max":{"1min":6.1230000000000002,"5min":16.373999999999999,"15min":119.26900000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":3.903,"5min":2.7999999999999998,"15min":3.323},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.663,"5min":15.663,"15min":119.372},"last":0.442}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.351,"15min":1.784},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":5.2439999999999998,"5min":12.743,"15min":92.650000000000006},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.494,"5min":1.6970000000000001,"15min":1.796},"min":{"1min":0.437,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2379999999999995,"5min":12.028,"15min":91.906999999999996},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.367,"15min":1.389},"min":{"1min":0.44500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.8920000000000003,"5min":12.710000000000001,"15min":19.759},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.603,"5min":1.7390000000000001,"15min":1.5029999999999999},"min":{"1min":0.503,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":8.4090000000000007,"5min":12.249000000000001,"15min":19.904},"last":0.69099999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.6200000000000001,"15min":1.454},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.29899999999999999},"max":{"1min":0.84999999999999998,"5min":14.718999999999999,"15min":55.838999999999999},"last":0.623},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.452,"15min":1.486},"min":{"1min":0.497,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78200000000000003,"5min":20.155000000000001,"15min":55.718000000000004},"last":0.64300000000000002}]}]},{"osd":1,"up_from":118,"seq":506806140987,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":261036,"kb_used_data":1444,"kb_used_omap":4,"kb_used_meta":259579,"kb_avail":93484116,"statfs":{"total":95995035648,"available":95727734784,"internally_reserved":0,"allocated":1478656,"data_stored":912057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":265809806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.7240000000000002,"15min":2.98},"min":{"1min":0.21099999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.016,"5min":11.81,"15min":11.81},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.157,"5min":2.9249999999999998,"15min":3.1280000000000001},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.573,"5min":12.573,"15min":12.573},"last":0.51400000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"last":0.39400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:27:15 2023","interfaces":[{"interface":"back","average":{"1min":4.0289999999999999,"5min":1.9219999999999999,"15min":1.3160000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.415,"15min":3.3700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.170999999999999,"5min":13.170999999999999,"15min":13.170999999999999},"last":0.371}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":3.8809999999999998,"15min":3.109},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":11.744,"5min":107.795,"15min":107.795},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":2.609,"15min":1.617},"min":{"1min":0.373,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.1120000000000001,"5min":109.167,"15min":109.167},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":2.5270000000000001,"15min":2.6709999999999998},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.9930000000000003,"5min":12.414999999999999,"15min":12.414999999999999},"last":0.629},{"interface":"front","average":{"1min":1.484,"5min":2.8839999999999999,"15min":3.0270000000000001},"min":{"1min":0.40100000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.686,"5min":13.933999999999999,"15min":13.933999999999999},"last":0.55700000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3650000000000002,"15min":2.5},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.7490000000000001,"5min":13.388,"15min":13.388},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.508,"5min":3.048,"15min":3.133},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.776999999999999,"5min":13.882,"15min":13.882},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.2349999999999999,"15min":3.1840000000000002},"min":{"1min":0.443,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":11.805,"5min":13.907,"15min":13.907},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.802,"15min":1.6339999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.1419999999999999,"5min":7.8239999999999998,"15min":7.8239999999999998},"last":0.64400000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024044,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":280852,"kb_used_data":2060,"kb_used_omap":5,"kb_used_meta":278778,"kb_avail":93464300,"statfs":{"total":95995035648,"available":95707443200,"internally_reserved":0,"allocated":2109440,"data_stored":1505553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":285469306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":2.911,"15min":3.419},"min":{"1min":0.24199999999999999,"5min":0.192,"15min":0.192},"max":{"1min":9.1280000000000001,"5min":17.210000000000001,"15min":17.210000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.6779999999999999,"15min":2.8639999999999999},"min":{"1min":0.26600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.5190000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.27200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.42299999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:27:19 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":5.351,"15min":3.7799999999999998},"min":{"1min":0.307,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.753,"5min":131.08000000000001,"15min":131.08000000000001},"last":0.311},{"interface":"front","average":{"1min":3.7970000000000002,"5min":4.4939999999999998,"15min":3.472},"min":{"1min":0.27800000000000002,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.85,"5min":128.12200000000001,"15min":128.12200000000001},"last":0.40300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.742,"15min":1.5269999999999999},"min":{"1min":0.38900000000000001,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":1.3720000000000001,"5min":10.73,"15min":10.73},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.78,"15min":1.7270000000000001},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5009999999999999,"5min":8.1300000000000008,"15min":9.5190000000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.032,"15min":1.819},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.4610000000000001,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.718,"15min":1.911},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":2.0779999999999998,"5min":14.144,"15min":14.144},"last":0.65300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":2.145,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.8019999999999996,"5min":9.7469999999999999,"15min":10.403},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.66100000000000003,"5min":1.9890000000000001,"15min":2.2440000000000002},"min":{"1min":0.46700000000000003,"5min":0.378,"15min":0.378},"max":{"1min":2.1949999999999998,"5min":14.090999999999999,"15min":14.090999999999999},"last":0.68500000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.9670000000000001,"15min":2.0470000000000002},"min":{"1min":0.505,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":5.7919999999999998,"5min":11.890000000000001,"15min":11.890000000000001},"last":0.623},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.1139999999999999,"15min":1.984},"min":{"1min":0.45300000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.8250000000000002,"5min":10.765000000000001,"15min":11.176},"last":0.5}]}]},{"osd":2,"up_from":135,"seq":579820584979,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156100,"kb_used_data":1404,"kb_used_omap":7,"kb_used_meta":154680,"kb_avail":93589052,"statfs":{"total":95995035648,"available":95835189248,"internally_reserved":0,"allocated":1437696,"data_stored":907571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158392401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.308},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.32800000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40600000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.502},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.65800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57199999999999995}]}]},{"osd":3,"up_from":142,"seq":609885356045,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":121504,"kb_used_data":1368,"kb_used_omap":2,"kb_used_meta":120125,"kb_avail":93623648,"statfs":{"total":95995035648,"available":95870615552,"internally_reserved":0,"allocated":1400832,"data_stored":900486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":123008053},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019172,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599332,"kb_used_data":1372,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":93145820,"statfs":{"total":95995035648,"available":95381319680,"internally_reserved":0,"allocated":1404928,"data_stored":902193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.7110000000000001,"15min":1.879},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.89200000000000002,"5min":11.442,"15min":16.553999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.7849999999999999,"15min":2.274},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.1699999999999999,"5min":8.4909999999999997,"15min":16.529},"last":0.65200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":2.0510000000000002,"15min":1.9850000000000001},"min":{"1min":0.35099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.7879999999999998,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.497},{"interface":"front","average":{"1min":1.073,"5min":2.1779999999999999,"15min":2.306},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.8970000000000002,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.69299999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:27:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":2.2679999999999998,"15min":4.3899999999999997},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.30199999999999999},"max":{"1min":7.3040000000000003,"5min":8.4079999999999995,"15min":337.08100000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.2269999999999999,"15min":4.5490000000000004},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":6.2839999999999998,"5min":8.5820000000000007,"15min":342.04399999999998},"last":0.57799999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.1619999999999999,"15min":0.95399999999999996},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.22900000000000001},"max":{"1min":0.73399999999999999,"5min":10.255000000000001,"15min":10.255000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.137,"15min":0.92500000000000004},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.82999999999999996,"5min":6.7009999999999996,"15min":7.194},"last":0.59299999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.097,"15min":1.131},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.97599999999999998,"5min":6.5810000000000004,"15min":9.9640000000000004},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.119,"15min":1.1140000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":4.5199999999999996,"15min":9.3859999999999992},"last":0.56299999999999994}]},{"osd":7,"last update":"Mon Oct 9 04:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.3400000000000001,"15min":1.0900000000000001},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":7.8250000000000002,"15min":10.34},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.355,"15min":1.0800000000000001},"min":{"1min":0.46800000000000003,"5min":0.34399999999999997,"15min":0.25900000000000001},"max":{"1min":0.82699999999999996,"5min":9.7200000000000006,"15min":9.7200000000000006},"last":0.61599999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757536,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591124,"kb_used_data":1356,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93154028,"statfs":{"total":95995035648,"available":95389724672,"internally_reserved":0,"allocated":1388544,"data_stored":897937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.1400000000000001,"15min":1.8759999999999999},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":9.4589999999999996,"5min":13.111000000000001,"15min":13.111000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.716,"15min":1.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.133,"5min":9.3190000000000008,"15min":9.3190000000000008},"last":0.58099999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.9119999999999999,"15min":1.8009999999999999},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.1680000000000001,"5min":10.788,"15min":10.788},"last":0.46300000000000002},{"interface":"front","average":{"1min":7.883,"5min":3.423,"15min":2.468},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":116.04000000000001,"5min":116.04000000000001,"15min":116.04000000000001},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.63700000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:26:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9169999999999998,"5min":4.6500000000000004,"15min":4.8879999999999999},"min":{"1min":0.441,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":9.8399999999999999,"5min":179.53200000000001,"15min":227.672},"last":0.65300000000000002},{"interface":"front","average":{"1min":4.5650000000000004,"5min":4.5300000000000002,"15min":4.8369999999999997},"min":{"1min":0.40100000000000002,"5min":0.35699999999999998,"15min":0.29099999999999998},"max":{"1min":11.632999999999999,"5min":179.52000000000001,"15min":227.52799999999999},"last":0.54800000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.139,"15min":0.94999999999999996},"min":{"1min":0.35499999999999998,"5min":0.28100000000000003,"15min":0.23499999999999999},"max":{"1min":0.71699999999999997,"5min":9.641,"15min":9.641},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.151,"15min":1.0189999999999999},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.245},"max":{"1min":0.73099999999999998,"5min":8.9280000000000008,"15min":11.356},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:28:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":1.268,"15min":1.0649999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.26000000000000001},"max":{"1min":1.649,"5min":9.4019999999999992,"15min":10.31},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.488,"15min":1.238},"min":{"1min":0.44500000000000001,"5min":0.32000000000000001,"15min":0.30399999999999999},"max":{"1min":1.4119999999999999,"5min":14.334,"15min":16.123000000000001},"last":0.71899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.46,"15min":1.2090000000000001},"min":{"1min":0.48299999999999998,"5min":0.34499999999999997,"15min":0.27400000000000002},"max":{"1min":0.80100000000000005,"5min":9.9000000000000004,"15min":9.9589999999999996},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.3460000000000001,"15min":1.089},"min":{"1min":0.502,"5min":0.376,"15min":0.33000000000000002},"max":{"1min":0.86899999999999999,"5min":11.315,"15min":11.315},"last":0.67600000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:28:57.249 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:28:57.249 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-09T04:28:57.249 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-09T04:28:57.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-09T04:28:57.614 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:28:57.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:57 smithi029 ceph-mon[216903]: from='client.45236 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:57.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:57 smithi029 ceph-mon[223858]: from='client.45236 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:57 smithi177 ceph-mon[195797]: from='client.45236 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:28:58.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:58 smithi029 ceph-mon[216903]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:58.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:28:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:58.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:58 smithi029 ceph-mon[223858]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:58.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:28:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:58 smithi177 ceph-mon[195797]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:28:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:28:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:28:59.594 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:28:59.595 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-09T04:29:00.425 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-09T04:29:00.426 INFO:teuthology.run_tasks:Running task print... 2023-10-09T04:29:00.434 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-09T04:29:00.434 INFO:teuthology.run_tasks:Running task parallel... 2023-10-09T04:29:00.442 INFO:teuthology.task.parallel:starting parallel... 2023-10-09T04:29:00.442 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-09T04:29:00.442 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-09T04:29:00.444 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T04:29:00.444 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b4fa663fa77ba2efb5b9f24e2aeb12bf66bce1f0'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'a340fb5463aa3715e93e7472d5f60dfdcf184ab3'}} 2023-10-09T04:29:00.444 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-09T04:29:00.444 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-09T04:29:00.444 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-09T04:29:00.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:29:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:00 smithi029 ceph-mon[223858]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1491778196' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:29:00.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:00 smithi029 ceph-mon[216903]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1491778196' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:29:00.811 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:00 smithi177 ceph-mon[195797]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1491778196' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-09T04:29:02.612 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:29:02.614 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:29:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:02 smithi029 ceph-mon[216903]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:02.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:29:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:29:02] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.43.0" 2023-10-09T04:29:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:02 smithi029 ceph-mon[223858]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:02 smithi177 ceph-mon[195797]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:03.446 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":515,"stamp":"2023-10-09T04:29:00.792063+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2612516,"kb_used_data":12900,"kb_used_omap":20,"kb_used_meta":2599531,"kb_avail":747348700,"statfs":{"total":767960285184,"available":765285068800,"internally_reserved":0,"allocated":13209600,"data_stored":9010590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2661920398},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002872"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974877+0000","last_change":"2023-10-09T04:27:30.004963+0000","last_active":"2023-10-09T04:27:56.974877+0000","last_peered":"2023-10-09T04:27:56.974877+0000","last_clean":"2023-10-09T04:27:56.974877+0000","last_became_active":"2023-10-09T04:27:30.004581+0000","last_became_peered":"2023-10-09T04:27:30.004581+0000","last_unstale":"2023-10-09T04:27:56.974877+0000","last_undegraded":"2023-10-09T04:27:56.974877+0000","last_fullsized":"2023-10-09T04:27:56.974877+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:40.077805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988720+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:27:57.988720+0000","last_peered":"2023-10-09T04:27:57.988720+0000","last_clean":"2023-10-09T04:27:57.988720+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:27:57.988720+0000","last_undegraded":"2023-10-09T04:27:57.988720+0000","last_fullsized":"2023-10-09T04:27:57.988720+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.984643+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:27:57.984643+0000","last_peered":"2023-10-09T04:27:57.984643+0000","last_clean":"2023-10-09T04:27:57.984643+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:27:57.984643+0000","last_undegraded":"2023-10-09T04:27:57.984643+0000","last_fullsized":"2023-10-09T04:27:57.984643+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990988+0000","last_change":"2023-10-09T04:27:57.985924+0000","last_active":"2023-10-09T04:27:57.990988+0000","last_peered":"2023-10-09T04:27:57.990988+0000","last_clean":"2023-10-09T04:27:57.990988+0000","last_became_active":"2023-10-09T04:27:57.985559+0000","last_became_peered":"2023-10-09T04:27:57.985559+0000","last_unstale":"2023-10-09T04:27:57.990988+0000","last_undegraded":"2023-10-09T04:27:57.990988+0000","last_fullsized":"2023-10-09T04:27:57.990988+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:01.749257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986393+0000","last_change":"2023-10-09T04:27:57.986393+0000","last_active":"2023-10-09T04:27:57.986393+0000","last_peered":"2023-10-09T04:27:57.986393+0000","last_clean":"2023-10-09T04:27:57.986393+0000","last_became_active":"2023-10-09T04:27:57.981584+0000","last_became_peered":"2023-10-09T04:27:57.981584+0000","last_unstale":"2023-10-09T04:27:57.986393+0000","last_undegraded":"2023-10-09T04:27:57.986393+0000","last_fullsized":"2023-10-09T04:27:57.986393+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:34.073778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981290+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:27:56.981290+0000","last_peered":"2023-10-09T04:27:56.981290+0000","last_clean":"2023-10-09T04:27:56.981290+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:27:56.981290+0000","last_undegraded":"2023-10-09T04:27:56.981290+0000","last_fullsized":"2023-10-09T04:27:56.981290+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989523+0000","last_change":"2023-10-09T04:27:57.989523+0000","last_active":"2023-10-09T04:27:57.989523+0000","last_peered":"2023-10-09T04:27:57.989523+0000","last_clean":"2023-10-09T04:27:57.989523+0000","last_became_active":"2023-10-09T04:27:57.989226+0000","last_became_peered":"2023-10-09T04:27:57.989226+0000","last_unstale":"2023-10-09T04:27:57.989523+0000","last_undegraded":"2023-10-09T04:27:57.989523+0000","last_fullsized":"2023-10-09T04:27:57.989523+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:47.573373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987735+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:27:57.987735+0000","last_peered":"2023-10-09T04:27:57.987735+0000","last_clean":"2023-10-09T04:27:57.987735+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:27:57.987735+0000","last_undegraded":"2023-10-09T04:27:57.987735+0000","last_fullsized":"2023-10-09T04:27:57.987735+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988398+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:27:56.988398+0000","last_peered":"2023-10-09T04:27:56.988398+0000","last_clean":"2023-10-09T04:27:56.988398+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:27:56.988398+0000","last_undegraded":"2023-10-09T04:27:56.988398+0000","last_fullsized":"2023-10-09T04:27:56.988398+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990152+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:27:57.990152+0000","last_peered":"2023-10-09T04:27:57.990152+0000","last_clean":"2023-10-09T04:27:57.990152+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:27:57.990152+0000","last_undegraded":"2023-10-09T04:27:57.990152+0000","last_fullsized":"2023-10-09T04:27:57.990152+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":106,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986830+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:27:57.986830+0000","last_peered":"2023-10-09T04:27:57.986830+0000","last_clean":"2023-10-09T04:27:57.986830+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:27:57.986830+0000","last_undegraded":"2023-10-09T04:27:57.986830+0000","last_fullsized":"2023-10-09T04:27:57.986830+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972995+0000","last_change":"2023-10-09T04:27:29.983053+0000","last_active":"2023-10-09T04:27:57.972995+0000","last_peered":"2023-10-09T04:27:57.972995+0000","last_clean":"2023-10-09T04:27:57.972995+0000","last_became_active":"2023-10-09T04:27:29.982725+0000","last_became_peered":"2023-10-09T04:27:29.982725+0000","last_unstale":"2023-10-09T04:27:57.972995+0000","last_undegraded":"2023-10-09T04:27:57.972995+0000","last_fullsized":"2023-10-09T04:27:57.972995+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:33.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976512+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:27:56.976512+0000","last_peered":"2023-10-09T04:27:56.976512+0000","last_clean":"2023-10-09T04:27:56.976512+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:27:56.976512+0000","last_undegraded":"2023-10-09T04:27:56.976512+0000","last_fullsized":"2023-10-09T04:27:56.976512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975162+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:27:56.975162+0000","last_peered":"2023-10-09T04:27:56.975162+0000","last_clean":"2023-10-09T04:27:56.975162+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:27:56.975162+0000","last_undegraded":"2023-10-09T04:27:56.975162+0000","last_fullsized":"2023-10-09T04:27:56.975162+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981381+0000","last_change":"2023-10-09T04:27:29.989226+0000","last_active":"2023-10-09T04:27:56.981381+0000","last_peered":"2023-10-09T04:27:56.981381+0000","last_clean":"2023-10-09T04:27:56.981381+0000","last_became_active":"2023-10-09T04:27:29.988867+0000","last_became_peered":"2023-10-09T04:27:29.988867+0000","last_unstale":"2023-10-09T04:27:56.981381+0000","last_undegraded":"2023-10-09T04:27:56.981381+0000","last_fullsized":"2023-10-09T04:27:56.981381+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:15.559571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987345+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:27:57.987345+0000","last_peered":"2023-10-09T04:27:57.987345+0000","last_clean":"2023-10-09T04:27:57.987345+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:27:57.987345+0000","last_undegraded":"2023-10-09T04:27:57.987345+0000","last_fullsized":"2023-10-09T04:27:57.987345+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.985231+0000","last_change":"2023-10-09T04:27:57.985231+0000","last_active":"2023-10-09T04:27:57.985231+0000","last_peered":"2023-10-09T04:27:57.985231+0000","last_clean":"2023-10-09T04:27:57.985231+0000","last_became_active":"2023-10-09T04:27:57.984869+0000","last_became_peered":"2023-10-09T04:27:57.984869+0000","last_unstale":"2023-10-09T04:27:57.985231+0000","last_undegraded":"2023-10-09T04:27:57.985231+0000","last_fullsized":"2023-10-09T04:27:57.985231+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:39.521778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992866+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:27:57.992866+0000","last_peered":"2023-10-09T04:27:57.992866+0000","last_clean":"2023-10-09T04:27:57.992866+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:27:57.992866+0000","last_undegraded":"2023-10-09T04:27:57.992866+0000","last_fullsized":"2023-10-09T04:27:57.992866+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980748+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:27:56.980748+0000","last_peered":"2023-10-09T04:27:56.980748+0000","last_clean":"2023-10-09T04:27:56.980748+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:27:56.980748+0000","last_undegraded":"2023-10-09T04:27:56.980748+0000","last_fullsized":"2023-10-09T04:27:56.980748+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.994143+0000","last_change":"2023-10-09T04:27:57.994143+0000","last_active":"2023-10-09T04:27:57.994143+0000","last_peered":"2023-10-09T04:27:57.994143+0000","last_clean":"2023-10-09T04:27:57.994143+0000","last_became_active":"2023-10-09T04:27:57.993862+0000","last_became_peered":"2023-10-09T04:27:57.993862+0000","last_unstale":"2023-10-09T04:27:57.994143+0000","last_undegraded":"2023-10-09T04:27:57.994143+0000","last_fullsized":"2023-10-09T04:27:57.994143+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:24.434239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980940+0000","last_change":"2023-10-09T04:20:03.030960+0000","last_active":"2023-10-09T04:27:56.980940+0000","last_peered":"2023-10-09T04:27:56.980940+0000","last_clean":"2023-10-09T04:27:56.980940+0000","last_became_active":"2023-10-09T04:20:03.030148+0000","last_became_peered":"2023-10-09T04:20:03.030148+0000","last_unstale":"2023-10-09T04:27:56.980940+0000","last_undegraded":"2023-10-09T04:27:56.980940+0000","last_fullsized":"2023-10-09T04:27:56.980940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:09.000584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986461+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:27:57.986461+0000","last_peered":"2023-10-09T04:27:57.986461+0000","last_clean":"2023-10-09T04:27:57.986461+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:27:57.986461+0000","last_undegraded":"2023-10-09T04:27:57.986461+0000","last_fullsized":"2023-10-09T04:27:57.986461+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983931+0000","last_change":"2023-10-09T04:20:03.033517+0000","last_active":"2023-10-09T04:27:56.983931+0000","last_peered":"2023-10-09T04:27:56.983931+0000","last_clean":"2023-10-09T04:27:56.983931+0000","last_became_active":"2023-10-09T04:20:03.033132+0000","last_became_peered":"2023-10-09T04:20:03.033132+0000","last_unstale":"2023-10-09T04:27:56.983931+0000","last_undegraded":"2023-10-09T04:27:56.983931+0000","last_fullsized":"2023-10-09T04:27:56.983931+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:14.247924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988550+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:27:56.988550+0000","last_peered":"2023-10-09T04:27:56.988550+0000","last_clean":"2023-10-09T04:27:56.988550+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:27:56.988550+0000","last_undegraded":"2023-10-09T04:27:56.988550+0000","last_fullsized":"2023-10-09T04:27:56.988550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983774+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:27:56.983774+0000","last_peered":"2023-10-09T04:27:56.983774+0000","last_clean":"2023-10-09T04:27:56.983774+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:27:56.983774+0000","last_undegraded":"2023-10-09T04:27:56.983774+0000","last_fullsized":"2023-10-09T04:27:56.983774+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988386+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:27:57.988386+0000","last_peered":"2023-10-09T04:27:57.988386+0000","last_clean":"2023-10-09T04:27:57.988386+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:27:57.988386+0000","last_undegraded":"2023-10-09T04:27:57.988386+0000","last_fullsized":"2023-10-09T04:27:57.988386+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":87,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982249+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:27:56.982249+0000","last_peered":"2023-10-09T04:27:56.982249+0000","last_clean":"2023-10-09T04:27:56.982249+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:27:56.982249+0000","last_undegraded":"2023-10-09T04:27:56.982249+0000","last_fullsized":"2023-10-09T04:27:56.982249+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982155+0000","last_change":"2023-10-09T04:27:30.005025+0000","last_active":"2023-10-09T04:27:56.982155+0000","last_peered":"2023-10-09T04:27:56.982155+0000","last_clean":"2023-10-09T04:27:56.982155+0000","last_became_active":"2023-10-09T04:27:30.004692+0000","last_became_peered":"2023-10-09T04:27:30.004692+0000","last_unstale":"2023-10-09T04:27:56.982155+0000","last_undegraded":"2023-10-09T04:27:56.982155+0000","last_fullsized":"2023-10-09T04:27:56.982155+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:10.737398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992844+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:27:57.992844+0000","last_peered":"2023-10-09T04:27:57.992844+0000","last_clean":"2023-10-09T04:27:57.992844+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:27:57.992844+0000","last_undegraded":"2023-10-09T04:27:57.992844+0000","last_fullsized":"2023-10-09T04:27:57.992844+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":90,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982864+0000","last_change":"2023-10-09T04:20:03.025205+0000","last_active":"2023-10-09T04:27:56.982864+0000","last_peered":"2023-10-09T04:27:56.982864+0000","last_clean":"2023-10-09T04:27:56.982864+0000","last_became_active":"2023-10-09T04:20:03.024858+0000","last_became_peered":"2023-10-09T04:20:03.024858+0000","last_unstale":"2023-10-09T04:27:56.982864+0000","last_undegraded":"2023-10-09T04:27:56.982864+0000","last_fullsized":"2023-10-09T04:27:56.982864+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:59.239890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992940+0000","last_change":"2023-10-09T04:27:57.992940+0000","last_active":"2023-10-09T04:27:57.992940+0000","last_peered":"2023-10-09T04:27:57.992940+0000","last_clean":"2023-10-09T04:27:57.992940+0000","last_became_active":"2023-10-09T04:27:57.991337+0000","last_became_peered":"2023-10-09T04:27:57.991337+0000","last_unstale":"2023-10-09T04:27:57.992940+0000","last_undegraded":"2023-10-09T04:27:57.992940+0000","last_fullsized":"2023-10-09T04:27:57.992940+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:10.795994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988243+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:27:57.988243+0000","last_peered":"2023-10-09T04:27:57.988243+0000","last_clean":"2023-10-09T04:27:57.988243+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:27:57.988243+0000","last_undegraded":"2023-10-09T04:27:57.988243+0000","last_fullsized":"2023-10-09T04:27:57.988243+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982044+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:27:56.982044+0000","last_peered":"2023-10-09T04:27:56.982044+0000","last_clean":"2023-10-09T04:27:56.982044+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:27:56.982044+0000","last_undegraded":"2023-10-09T04:27:56.982044+0000","last_fullsized":"2023-10-09T04:27:56.982044+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976221+0000","last_change":"2023-10-09T04:27:29.982774+0000","last_active":"2023-10-09T04:27:56.976221+0000","last_peered":"2023-10-09T04:27:56.976221+0000","last_clean":"2023-10-09T04:27:56.976221+0000","last_became_active":"2023-10-09T04:27:29.982507+0000","last_became_peered":"2023-10-09T04:27:29.982507+0000","last_unstale":"2023-10-09T04:27:56.976221+0000","last_undegraded":"2023-10-09T04:27:56.976221+0000","last_fullsized":"2023-10-09T04:27:56.976221+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:14.613302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976415+0000","last_change":"2023-10-09T04:27:30.010756+0000","last_active":"2023-10-09T04:27:56.976415+0000","last_peered":"2023-10-09T04:27:56.976415+0000","last_clean":"2023-10-09T04:27:56.976415+0000","last_became_active":"2023-10-09T04:27:30.010403+0000","last_became_peered":"2023-10-09T04:27:30.010403+0000","last_unstale":"2023-10-09T04:27:56.976415+0000","last_undegraded":"2023-10-09T04:27:56.976415+0000","last_fullsized":"2023-10-09T04:27:56.976415+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:43:10.557818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976602+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:27:56.976602+0000","last_peered":"2023-10-09T04:27:56.976602+0000","last_clean":"2023-10-09T04:27:56.976602+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:27:56.976602+0000","last_undegraded":"2023-10-09T04:27:56.976602+0000","last_fullsized":"2023-10-09T04:27:56.976602+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986421+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:27:57.986421+0000","last_peered":"2023-10-09T04:27:57.986421+0000","last_clean":"2023-10-09T04:27:57.986421+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:27:57.986421+0000","last_undegraded":"2023-10-09T04:27:57.986421+0000","last_fullsized":"2023-10-09T04:27:57.986421+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.983436+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:27:57.983436+0000","last_peered":"2023-10-09T04:27:57.983436+0000","last_clean":"2023-10-09T04:27:57.983436+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:27:57.983436+0000","last_undegraded":"2023-10-09T04:27:57.983436+0000","last_fullsized":"2023-10-09T04:27:57.983436+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973721+0000","last_change":"2023-10-09T04:24:07.916771+0000","last_active":"2023-10-09T04:27:57.973721+0000","last_peered":"2023-10-09T04:27:57.973721+0000","last_clean":"2023-10-09T04:27:57.973721+0000","last_became_active":"2023-10-09T04:24:07.916030+0000","last_became_peered":"2023-10-09T04:24:07.916030+0000","last_unstale":"2023-10-09T04:27:57.973721+0000","last_undegraded":"2023-10-09T04:27:57.973721+0000","last_fullsized":"2023-10-09T04:27:57.973721+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.033444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993021+0000","last_change":"2023-10-09T04:27:57.993021+0000","last_active":"2023-10-09T04:27:57.993021+0000","last_peered":"2023-10-09T04:27:57.993021+0000","last_clean":"2023-10-09T04:27:57.993021+0000","last_became_active":"2023-10-09T04:27:57.992486+0000","last_became_peered":"2023-10-09T04:27:57.992486+0000","last_unstale":"2023-10-09T04:27:57.993021+0000","last_undegraded":"2023-10-09T04:27:57.993021+0000","last_fullsized":"2023-10-09T04:27:57.993021+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:56.130811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972915+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:27:57.972915+0000","last_peered":"2023-10-09T04:27:57.972915+0000","last_clean":"2023-10-09T04:27:57.972915+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:27:57.972915+0000","last_undegraded":"2023-10-09T04:27:57.972915+0000","last_fullsized":"2023-10-09T04:27:57.972915+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988701+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:27:56.988701+0000","last_peered":"2023-10-09T04:27:56.988701+0000","last_clean":"2023-10-09T04:27:56.988701+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:27:56.988701+0000","last_undegraded":"2023-10-09T04:27:56.988701+0000","last_fullsized":"2023-10-09T04:27:56.988701+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975059+0000","last_change":"2023-10-09T04:20:03.030040+0000","last_active":"2023-10-09T04:27:56.975059+0000","last_peered":"2023-10-09T04:27:56.975059+0000","last_clean":"2023-10-09T04:27:56.975059+0000","last_became_active":"2023-10-09T04:20:03.029675+0000","last_became_peered":"2023-10-09T04:20:03.029675+0000","last_unstale":"2023-10-09T04:27:56.975059+0000","last_undegraded":"2023-10-09T04:27:56.975059+0000","last_fullsized":"2023-10-09T04:27:56.975059+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:25.875934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993874+0000","last_change":"2023-10-09T04:27:57.991197+0000","last_active":"2023-10-09T04:27:57.993874+0000","last_peered":"2023-10-09T04:27:57.993874+0000","last_clean":"2023-10-09T04:27:57.993874+0000","last_became_active":"2023-10-09T04:27:57.990548+0000","last_became_peered":"2023-10-09T04:27:57.990548+0000","last_unstale":"2023-10-09T04:27:57.993874+0000","last_undegraded":"2023-10-09T04:27:57.993874+0000","last_fullsized":"2023-10-09T04:27:57.993874+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:05.799943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989081+0000","last_change":"2023-10-09T04:27:57.989081+0000","last_active":"2023-10-09T04:27:57.989081+0000","last_peered":"2023-10-09T04:27:57.989081+0000","last_clean":"2023-10-09T04:27:57.989081+0000","last_became_active":"2023-10-09T04:27:57.988802+0000","last_became_peered":"2023-10-09T04:27:57.988802+0000","last_unstale":"2023-10-09T04:27:57.989081+0000","last_undegraded":"2023-10-09T04:27:57.989081+0000","last_fullsized":"2023-10-09T04:27:57.989081+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:45.574939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972611+0000","last_change":"2023-10-09T04:27:29.982732+0000","last_active":"2023-10-09T04:27:57.972611+0000","last_peered":"2023-10-09T04:27:57.972611+0000","last_clean":"2023-10-09T04:27:57.972611+0000","last_became_active":"2023-10-09T04:27:29.982275+0000","last_became_peered":"2023-10-09T04:27:29.982275+0000","last_unstale":"2023-10-09T04:27:57.972611+0000","last_undegraded":"2023-10-09T04:27:57.972611+0000","last_fullsized":"2023-10-09T04:27:57.972611+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:43.712715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973308+0000","last_change":"2023-10-09T04:24:07.922314+0000","last_active":"2023-10-09T04:27:57.973308+0000","last_peered":"2023-10-09T04:27:57.973308+0000","last_clean":"2023-10-09T04:27:57.973308+0000","last_became_active":"2023-10-09T04:24:07.916895+0000","last_became_peered":"2023-10-09T04:24:07.916895+0000","last_unstale":"2023-10-09T04:27:57.973308+0000","last_undegraded":"2023-10-09T04:27:57.973308+0000","last_fullsized":"2023-10-09T04:27:57.973308+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:13.277124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988253+0000","last_change":"2023-10-09T04:27:29.988702+0000","last_active":"2023-10-09T04:27:56.988253+0000","last_peered":"2023-10-09T04:27:56.988253+0000","last_clean":"2023-10-09T04:27:56.988253+0000","last_became_active":"2023-10-09T04:27:29.988390+0000","last_became_peered":"2023-10-09T04:27:29.988390+0000","last_unstale":"2023-10-09T04:27:56.988253+0000","last_undegraded":"2023-10-09T04:27:56.988253+0000","last_fullsized":"2023-10-09T04:27:56.988253+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:33.897326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":484,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981404+0000","last_change":"2023-10-09T04:20:03.031289+0000","last_active":"2023-10-09T04:27:56.981404+0000","last_peered":"2023-10-09T04:27:56.981404+0000","last_clean":"2023-10-09T04:27:56.981404+0000","last_became_active":"2023-10-09T04:20:03.030125+0000","last_became_peered":"2023-10-09T04:20:03.030125+0000","last_unstale":"2023-10-09T04:27:56.981404+0000","last_undegraded":"2023-10-09T04:27:56.981404+0000","last_fullsized":"2023-10-09T04:27:56.981404+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:33.277598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988547+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:27:57.988547+0000","last_peered":"2023-10-09T04:27:57.988547+0000","last_clean":"2023-10-09T04:27:57.988547+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:27:57.988547+0000","last_undegraded":"2023-10-09T04:27:57.988547+0000","last_fullsized":"2023-10-09T04:27:57.988547+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981058+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:27:56.981058+0000","last_peered":"2023-10-09T04:27:56.981058+0000","last_clean":"2023-10-09T04:27:56.981058+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:27:56.981058+0000","last_undegraded":"2023-10-09T04:27:56.981058+0000","last_fullsized":"2023-10-09T04:27:56.981058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":124,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981273+0000","last_change":"2023-10-09T04:24:07.962764+0000","last_active":"2023-10-09T04:27:56.981273+0000","last_peered":"2023-10-09T04:27:56.981273+0000","last_clean":"2023-10-09T04:27:56.981273+0000","last_became_active":"2023-10-09T04:24:07.962508+0000","last_became_peered":"2023-10-09T04:24:07.962508+0000","last_unstale":"2023-10-09T04:27:56.981273+0000","last_undegraded":"2023-10-09T04:27:56.981273+0000","last_fullsized":"2023-10-09T04:27:56.981273+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:14.078794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989841+0000","last_change":"2023-10-09T04:27:57.988062+0000","last_active":"2023-10-09T04:27:57.989841+0000","last_peered":"2023-10-09T04:27:57.989841+0000","last_clean":"2023-10-09T04:27:57.989841+0000","last_became_active":"2023-10-09T04:27:57.987732+0000","last_became_peered":"2023-10-09T04:27:57.987732+0000","last_unstale":"2023-10-09T04:27:57.989841+0000","last_undegraded":"2023-10-09T04:27:57.989841+0000","last_fullsized":"2023-10-09T04:27:57.989841+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:01.399437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978739+0000","last_change":"2023-10-09T04:27:29.973734+0000","last_active":"2023-10-09T04:27:57.978739+0000","last_peered":"2023-10-09T04:27:57.978739+0000","last_clean":"2023-10-09T04:27:57.978739+0000","last_became_active":"2023-10-09T04:27:29.973431+0000","last_became_peered":"2023-10-09T04:27:29.973431+0000","last_unstale":"2023-10-09T04:27:57.978739+0000","last_undegraded":"2023-10-09T04:27:57.978739+0000","last_fullsized":"2023-10-09T04:27:57.978739+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:14.788158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.985955+0000","last_change":"2023-10-09T04:20:03.030432+0000","last_active":"2023-10-09T04:27:56.985955+0000","last_peered":"2023-10-09T04:27:56.985955+0000","last_clean":"2023-10-09T04:27:56.985955+0000","last_became_active":"2023-10-09T04:20:03.030103+0000","last_became_peered":"2023-10-09T04:20:03.030103+0000","last_unstale":"2023-10-09T04:27:56.985955+0000","last_undegraded":"2023-10-09T04:27:56.985955+0000","last_fullsized":"2023-10-09T04:27:56.985955+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.759695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.974229+0000","last_change":"2023-10-09T04:27:29.981956+0000","last_active":"2023-10-09T04:27:57.974229+0000","last_peered":"2023-10-09T04:27:57.974229+0000","last_clean":"2023-10-09T04:27:57.974229+0000","last_became_active":"2023-10-09T04:27:29.981663+0000","last_became_peered":"2023-10-09T04:27:29.981663+0000","last_unstale":"2023-10-09T04:27:57.974229+0000","last_undegraded":"2023-10-09T04:27:57.974229+0000","last_fullsized":"2023-10-09T04:27:57.974229+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:47.639333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.977902+0000","last_change":"2023-10-09T04:20:03.032817+0000","last_active":"2023-10-09T04:27:57.977902+0000","last_peered":"2023-10-09T04:27:57.977902+0000","last_clean":"2023-10-09T04:27:57.977902+0000","last_became_active":"2023-10-09T04:20:03.029228+0000","last_became_peered":"2023-10-09T04:20:03.029228+0000","last_unstale":"2023-10-09T04:27:57.977902+0000","last_undegraded":"2023-10-09T04:27:57.977902+0000","last_fullsized":"2023-10-09T04:27:57.977902+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.429820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.995379+0000","last_change":"2023-10-09T04:27:57.995379+0000","last_active":"2023-10-09T04:27:57.995379+0000","last_peered":"2023-10-09T04:27:57.995379+0000","last_clean":"2023-10-09T04:27:57.995379+0000","last_became_active":"2023-10-09T04:27:57.995057+0000","last_became_peered":"2023-10-09T04:27:57.995057+0000","last_unstale":"2023-10-09T04:27:57.995379+0000","last_undegraded":"2023-10-09T04:27:57.995379+0000","last_fullsized":"2023-10-09T04:27:57.995379+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:46.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973460+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:27:57.973460+0000","last_peered":"2023-10-09T04:27:57.973460+0000","last_clean":"2023-10-09T04:27:57.973460+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:27:57.973460+0000","last_undegraded":"2023-10-09T04:27:57.973460+0000","last_fullsized":"2023-10-09T04:27:57.973460+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.991611+0000","last_change":"2023-10-09T04:27:57.991611+0000","last_active":"2023-10-09T04:27:57.991611+0000","last_peered":"2023-10-09T04:27:57.991611+0000","last_clean":"2023-10-09T04:27:57.991611+0000","last_became_active":"2023-10-09T04:27:57.991279+0000","last_became_peered":"2023-10-09T04:27:57.991279+0000","last_unstale":"2023-10-09T04:27:57.991611+0000","last_undegraded":"2023-10-09T04:27:57.991611+0000","last_fullsized":"2023-10-09T04:27:57.991611+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:54.806159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981184+0000","last_change":"2023-10-09T04:27:30.000853+0000","last_active":"2023-10-09T04:27:56.981184+0000","last_peered":"2023-10-09T04:27:56.981184+0000","last_clean":"2023-10-09T04:27:56.981184+0000","last_became_active":"2023-10-09T04:27:30.000545+0000","last_became_peered":"2023-10-09T04:27:30.000545+0000","last_unstale":"2023-10-09T04:27:56.981184+0000","last_undegraded":"2023-10-09T04:27:56.981184+0000","last_fullsized":"2023-10-09T04:27:56.981184+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:47.506034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978683+0000","last_change":"2023-10-09T04:24:07.919208+0000","last_active":"2023-10-09T04:27:57.978683+0000","last_peered":"2023-10-09T04:27:57.978683+0000","last_clean":"2023-10-09T04:27:57.978683+0000","last_became_active":"2023-10-09T04:24:07.918797+0000","last_became_peered":"2023-10-09T04:24:07.918797+0000","last_unstale":"2023-10-09T04:27:57.978683+0000","last_undegraded":"2023-10-09T04:27:57.978683+0000","last_fullsized":"2023-10-09T04:27:57.978683+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:11.399352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976517+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:27:56.976517+0000","last_peered":"2023-10-09T04:27:56.976517+0000","last_clean":"2023-10-09T04:27:56.976517+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:27:56.976517+0000","last_undegraded":"2023-10-09T04:27:56.976517+0000","last_fullsized":"2023-10-09T04:27:56.976517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":80,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982962+0000","last_change":"2023-10-09T04:20:03.037673+0000","last_active":"2023-10-09T04:27:56.982962+0000","last_peered":"2023-10-09T04:27:56.982962+0000","last_clean":"2023-10-09T04:27:56.982962+0000","last_became_active":"2023-10-09T04:20:03.032398+0000","last_became_peered":"2023-10-09T04:20:03.032398+0000","last_unstale":"2023-10-09T04:27:56.982962+0000","last_undegraded":"2023-10-09T04:27:56.982962+0000","last_fullsized":"2023-10-09T04:27:56.982962+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:20.306433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989106+0000","last_change":"2023-10-09T04:27:57.989106+0000","last_active":"2023-10-09T04:27:57.989106+0000","last_peered":"2023-10-09T04:27:57.989106+0000","last_clean":"2023-10-09T04:27:57.989106+0000","last_became_active":"2023-10-09T04:27:57.988778+0000","last_became_peered":"2023-10-09T04:27:57.988778+0000","last_unstale":"2023-10-09T04:27:57.989106+0000","last_undegraded":"2023-10-09T04:27:57.989106+0000","last_fullsized":"2023-10-09T04:27:57.989106+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:33.473019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.989866+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:27:56.989866+0000","last_peered":"2023-10-09T04:27:56.989866+0000","last_clean":"2023-10-09T04:27:56.989866+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:27:56.989866+0000","last_undegraded":"2023-10-09T04:27:56.989866+0000","last_fullsized":"2023-10-09T04:27:56.989866+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988397+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:27:57.988397+0000","last_peered":"2023-10-09T04:27:57.988397+0000","last_clean":"2023-10-09T04:27:57.988397+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:27:57.988397+0000","last_undegraded":"2023-10-09T04:27:57.988397+0000","last_fullsized":"2023-10-09T04:27:57.988397+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":88,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.991512+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:27:56.991512+0000","last_peered":"2023-10-09T04:27:56.991512+0000","last_clean":"2023-10-09T04:27:56.991512+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:27:56.991512+0000","last_undegraded":"2023-10-09T04:27:56.991512+0000","last_fullsized":"2023-10-09T04:27:56.991512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988337+0000","last_change":"2023-10-09T04:27:57.988337+0000","last_active":"2023-10-09T04:27:57.988337+0000","last_peered":"2023-10-09T04:27:57.988337+0000","last_clean":"2023-10-09T04:27:57.988337+0000","last_became_active":"2023-10-09T04:27:57.987918+0000","last_became_peered":"2023-10-09T04:27:57.987918+0000","last_unstale":"2023-10-09T04:27:57.988337+0000","last_undegraded":"2023-10-09T04:27:57.988337+0000","last_fullsized":"2023-10-09T04:27:57.988337+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:54.931130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989240+0000","last_change":"2023-10-09T04:27:57.989240+0000","last_active":"2023-10-09T04:27:57.989240+0000","last_peered":"2023-10-09T04:27:57.989240+0000","last_clean":"2023-10-09T04:27:57.989240+0000","last_became_active":"2023-10-09T04:27:57.988856+0000","last_became_peered":"2023-10-09T04:27:57.988856+0000","last_unstale":"2023-10-09T04:27:57.989240+0000","last_undegraded":"2023-10-09T04:27:57.989240+0000","last_fullsized":"2023-10-09T04:27:57.989240+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:52.380796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990600+0000","last_change":"2023-10-09T04:27:57.990600+0000","last_active":"2023-10-09T04:27:57.990600+0000","last_peered":"2023-10-09T04:27:57.990600+0000","last_clean":"2023-10-09T04:27:57.990600+0000","last_became_active":"2023-10-09T04:27:57.989478+0000","last_became_peered":"2023-10-09T04:27:57.989478+0000","last_unstale":"2023-10-09T04:27:57.990600+0000","last_undegraded":"2023-10-09T04:27:57.990600+0000","last_fullsized":"2023-10-09T04:27:57.990600+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:42.167307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978163+0000","last_change":"2023-10-09T04:27:29.977785+0000","last_active":"2023-10-09T04:27:57.978163+0000","last_peered":"2023-10-09T04:27:57.978163+0000","last_clean":"2023-10-09T04:27:57.978163+0000","last_became_active":"2023-10-09T04:27:29.977397+0000","last_became_peered":"2023-10-09T04:27:29.977397+0000","last_unstale":"2023-10-09T04:27:57.978163+0000","last_undegraded":"2023-10-09T04:27:57.978163+0000","last_fullsized":"2023-10-09T04:27:57.978163+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:16.267931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974728+0000","last_change":"2023-10-09T04:27:29.981763+0000","last_active":"2023-10-09T04:27:56.974728+0000","last_peered":"2023-10-09T04:27:56.974728+0000","last_clean":"2023-10-09T04:27:56.974728+0000","last_became_active":"2023-10-09T04:27:29.981401+0000","last_became_peered":"2023-10-09T04:27:29.981401+0000","last_unstale":"2023-10-09T04:27:56.974728+0000","last_undegraded":"2023-10-09T04:27:56.974728+0000","last_fullsized":"2023-10-09T04:27:56.974728+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:35.242521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234265,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303340,"kb_used_data":1956,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441812,"statfs":{"total":95995035648,"available":95684415488,"internally_reserved":0,"allocated":2002944,"data_stored":1492631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":7.8869999999999996,"15min":3.9350000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22900000000000001},"max":{"1min":8.5909999999999993,"5min":629.22500000000002,"15min":629.22500000000002},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.93400000000000005,"5min":7.5739999999999998,"15min":3.5899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.1849999999999996,"5min":629.43100000000004,"15min":629.43100000000004},"last":0.52400000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":2.5960000000000001,"15min":2.0579999999999998},"min":{"1min":0.41099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.3330000000000002,"5min":80.215000000000003,"15min":80.215000000000003},"last":0.752},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7309999999999999,"15min":2.0880000000000001},"min":{"1min":0.35199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.2450000000000001,"5min":80.274000000000001,"15min":80.274000000000001},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.67000000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":9.3070000000000004,"15min":4.5659999999999998},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.254},"max":{"1min":7.968,"5min":686.07500000000005,"15min":686.07500000000005},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.3740000000000001,"5min":9.7650000000000006,"15min":4.7990000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":8.7050000000000001,"5min":686.04399999999998,"15min":686.04399999999998},"last":0.70199999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.546,"15min":1.0700000000000001},"min":{"1min":0.38400000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":0.65500000000000003,"5min":8.5500000000000007,"15min":8.5500000000000007},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.495,"5min":1.1299999999999999,"15min":1.1850000000000001},"min":{"1min":0.36499999999999999,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":0.59699999999999998,"5min":5.9939999999999998,"15min":94.522999999999996},"last":0.59999999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.2749999999999999,"15min":0.98499999999999999},"min":{"1min":0.39300000000000002,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":0.753,"5min":8.8610000000000007,"15min":8.8610000000000007},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.3300000000000001,"15min":1.1759999999999999},"min":{"1min":0.371,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.76700000000000002,"5min":6.3319999999999999,"15min":18.125},"last":0.73399999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.4399999999999999,"15min":1.6819999999999999},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.29999999999999999},"max":{"1min":0.78100000000000003,"5min":12.007,"15min":203.19399999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.4319999999999999,"15min":1.786},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.312},"max":{"1min":0.71899999999999997,"5min":7.4800000000000004,"15min":204.303},"last":0.76900000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495899,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299228,"kb_used_data":1940,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445924,"statfs":{"total":95995035648,"available":95688626176,"internally_reserved":0,"allocated":1986560,"data_stored":1492162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.149,"15min":1.956},"min":{"1min":0.32400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.5410000000000004,"5min":10.77,"15min":13.125999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1309999999999998,"15min":2.5960000000000001},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.274,"5min":10.631,"15min":13.551},"last":0.58899999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.0739999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.5600000000000005,"5min":12.728,"15min":12.728},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.4340000000000002,"15min":2.4540000000000002},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":20.181000000000001,"5min":20.181000000000001,"15min":20.181000000000001},"last":0.41799999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.46600000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.68700000000000006}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.351,"15min":1.784},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":5.2439999999999998,"5min":12.743,"15min":92.650000000000006},"last":0.622},{"interface":"front","average":{"1min":1.494,"5min":1.6970000000000001,"15min":1.796},"min":{"1min":0.437,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2379999999999995,"5min":12.028,"15min":91.906999999999996},"last":0.72799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.367,"15min":1.389},"min":{"1min":0.44500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.8920000000000003,"5min":12.710000000000001,"15min":19.759},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.603,"5min":1.7390000000000001,"15min":1.5029999999999999},"min":{"1min":0.503,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":8.4090000000000007,"5min":12.249000000000001,"15min":19.904},"last":0.67500000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.6200000000000001,"15min":1.454},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.29899999999999999},"max":{"1min":0.84999999999999998,"5min":14.718999999999999,"15min":55.838999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.452,"15min":1.486},"min":{"1min":0.497,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78200000000000003,"5min":20.155000000000001,"15min":55.718000000000004},"last":0.65900000000000003}]}]},{"osd":1,"up_from":118,"seq":506806140988,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":261036,"kb_used_data":1444,"kb_used_omap":4,"kb_used_meta":259579,"kb_avail":93484116,"statfs":{"total":95995035648,"available":95727734784,"internally_reserved":0,"allocated":1478656,"data_stored":912057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":265809806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.7240000000000002,"15min":2.98},"min":{"1min":0.21099999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.016,"5min":11.81,"15min":11.81},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.157,"5min":2.9249999999999998,"15min":3.1280000000000001},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.573,"5min":12.573,"15min":12.573},"last":0.26400000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:27:15 2023","interfaces":[{"interface":"back","average":{"1min":4.0289999999999999,"5min":1.9219999999999999,"15min":1.3160000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.4249999999999998,"5min":3.415,"15min":3.3700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.170999999999999,"5min":13.170999999999999,"15min":13.170999999999999},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":3.8809999999999998,"15min":3.109},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":11.744,"5min":107.795,"15min":107.795},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":2.609,"15min":1.617},"min":{"1min":0.373,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.1120000000000001,"5min":109.167,"15min":109.167},"last":0.55300000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":2.5270000000000001,"15min":2.6709999999999998},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.9930000000000003,"5min":12.414999999999999,"15min":12.414999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.484,"5min":2.8839999999999999,"15min":3.0270000000000001},"min":{"1min":0.40100000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.686,"5min":13.933999999999999,"15min":13.933999999999999},"last":0.79400000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3650000000000002,"15min":2.5},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.7490000000000001,"5min":13.388,"15min":13.388},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.508,"5min":3.048,"15min":3.133},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.776999999999999,"5min":13.882,"15min":13.882},"last":0.84999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.2349999999999999,"15min":3.1840000000000002},"min":{"1min":0.443,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":11.805,"5min":13.907,"15min":13.907},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.802,"15min":1.6339999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.1419999999999999,"5min":7.8239999999999998,"15min":7.8239999999999998},"last":0.62}]}]},{"osd":0,"up_from":91,"seq":390842024045,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":280852,"kb_used_data":2060,"kb_used_omap":5,"kb_used_meta":278778,"kb_avail":93464300,"statfs":{"total":95995035648,"available":95707443200,"internally_reserved":0,"allocated":2109440,"data_stored":1505553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":285469306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":2.911,"15min":3.419},"min":{"1min":0.24199999999999999,"5min":0.192,"15min":0.192},"max":{"1min":9.1280000000000001,"5min":17.210000000000001,"15min":17.210000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.6779999999999999,"15min":2.8639999999999999},"min":{"1min":0.26600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.5190000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.44}]},{"osd":3,"last update":"Mon Oct 9 04:27:19 2023","interfaces":[{"interface":"back","average":{"1min":4.1500000000000004,"5min":5.351,"15min":3.7799999999999998},"min":{"1min":0.307,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.753,"5min":131.08000000000001,"15min":131.08000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.7970000000000002,"5min":4.4939999999999998,"15min":3.472},"min":{"1min":0.27800000000000002,"5min":0.253,"15min":0.24199999999999999},"max":{"1min":11.85,"5min":128.12200000000001,"15min":128.12200000000001},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.742,"15min":1.5269999999999999},"min":{"1min":0.38900000000000001,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":1.3720000000000001,"5min":10.73,"15min":10.73},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.78,"15min":1.7270000000000001},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5009999999999999,"5min":8.1300000000000008,"15min":9.5190000000000001},"last":0.76100000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.032,"15min":1.819},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.4610000000000001,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.718,"15min":1.911},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":2.0779999999999998,"5min":14.144,"15min":14.144},"last":0.76900000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":2.145,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.8019999999999996,"5min":9.7469999999999999,"15min":10.403},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.66100000000000003,"5min":1.9890000000000001,"15min":2.2440000000000002},"min":{"1min":0.46700000000000003,"5min":0.378,"15min":0.378},"max":{"1min":2.1949999999999998,"5min":14.090999999999999,"15min":14.090999999999999},"last":0.71899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.9670000000000001,"15min":2.0470000000000002},"min":{"1min":0.505,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":5.7919999999999998,"5min":11.890000000000001,"15min":11.890000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.1139999999999999,"15min":1.984},"min":{"1min":0.45300000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.8250000000000002,"5min":10.765000000000001,"15min":11.176},"last":0.79500000000000004}]}]},{"osd":2,"up_from":135,"seq":579820584980,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156100,"kb_used_data":1404,"kb_used_omap":7,"kb_used_meta":154680,"kb_avail":93589052,"statfs":{"total":95995035648,"available":95835189248,"internally_reserved":0,"allocated":1437696,"data_stored":907571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158392401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.20200000000000001},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.30599999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.245}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.71999999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.65700000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.67700000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.53400000000000003}]}]},{"osd":3,"up_from":142,"seq":609885356046,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":121504,"kb_used_data":1368,"kb_used_omap":2,"kb_used_meta":120125,"kb_avail":93623648,"statfs":{"total":95995035648,"available":95870615552,"internally_reserved":0,"allocated":1400832,"data_stored":900486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":123008053},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34000000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29999999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40500000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69999999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48599999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019173,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599332,"kb_used_data":1372,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":93145820,"statfs":{"total":95995035648,"available":95381319680,"internally_reserved":0,"allocated":1404928,"data_stored":902193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.7110000000000001,"15min":1.879},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.89200000000000002,"5min":11.442,"15min":16.553999999999998},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.7849999999999999,"15min":2.274},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.1699999999999999,"5min":8.4909999999999997,"15min":16.529},"last":0.71799999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":2.0510000000000002,"15min":1.9850000000000001},"min":{"1min":0.35099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.7879999999999998,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.073,"5min":2.1779999999999999,"15min":2.306},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.8970000000000002,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.67000000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.68200000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.752}]},{"osd":5,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.1619999999999999,"15min":0.95399999999999996},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.22900000000000001},"max":{"1min":0.73399999999999999,"5min":10.255000000000001,"15min":10.255000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.137,"15min":0.92500000000000004},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.82999999999999996,"5min":6.7009999999999996,"15min":7.194},"last":0.58499999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.097,"15min":1.131},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.97599999999999998,"5min":6.5810000000000004,"15min":9.9640000000000004},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.119,"15min":1.1140000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":4.5199999999999996,"15min":9.3859999999999992},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.3400000000000001,"15min":1.0900000000000001},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":7.8250000000000002,"15min":10.34},"last":0.746},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.355,"15min":1.0800000000000001},"min":{"1min":0.46800000000000003,"5min":0.34399999999999997,"15min":0.25900000000000001},"max":{"1min":0.82699999999999996,"5min":9.7200000000000006,"15min":9.7200000000000006},"last":0.70799999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757537,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591124,"kb_used_data":1356,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93154028,"statfs":{"total":95995035648,"available":95389724672,"internally_reserved":0,"allocated":1388544,"data_stored":897937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.1400000000000001,"15min":1.8759999999999999},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":9.4589999999999996,"5min":13.111000000000001,"15min":13.111000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.716,"15min":1.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.133,"5min":9.3190000000000008,"15min":9.3190000000000008},"last":0.495}]},{"osd":1,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.9119999999999999,"15min":1.8009999999999999},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.1680000000000001,"5min":10.788,"15min":10.788},"last":0.51900000000000002},{"interface":"front","average":{"1min":7.883,"5min":3.423,"15min":2.468},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":116.04000000000001,"5min":116.04000000000001,"15min":116.04000000000001},"last":0.39700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.47599999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.57099999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.139,"15min":0.94999999999999996},"min":{"1min":0.35499999999999998,"5min":0.28100000000000003,"15min":0.23499999999999999},"max":{"1min":0.71699999999999997,"5min":9.641,"15min":9.641},"last":0.623},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.151,"15min":1.0189999999999999},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.245},"max":{"1min":0.73099999999999998,"5min":8.9280000000000008,"15min":11.356},"last":0.58499999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:28:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":1.268,"15min":1.0649999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.26000000000000001},"max":{"1min":1.649,"5min":9.4019999999999992,"15min":10.31},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.488,"15min":1.238},"min":{"1min":0.44500000000000001,"5min":0.32000000000000001,"15min":0.30399999999999999},"max":{"1min":1.4119999999999999,"5min":14.334,"15min":16.123000000000001},"last":0.64000000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.46,"15min":1.2090000000000001},"min":{"1min":0.48299999999999998,"5min":0.34499999999999997,"15min":0.27400000000000002},"max":{"1min":0.80100000000000005,"5min":9.9000000000000004,"15min":9.9589999999999996},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.3460000000000001,"15min":1.089},"min":{"1min":0.502,"5min":0.376,"15min":0.33000000000000002},"max":{"1min":0.86899999999999999,"5min":11.315,"15min":11.315},"last":0.65900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:29:03.452 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:29:03.826 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:03 smithi029 ceph-mon[216903]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:03 smithi029 ceph-mon[223858]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:03 smithi177 ceph-mon[195797]: from='client.35313 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:04 smithi029 ceph-mon[216903]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:04 smithi029 ceph-mon[223858]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:04 smithi177 ceph-mon[195797]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:05.693 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:29:05.695 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:29:06.603 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":517,"stamp":"2023-10-09T04:29:04.793205+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2612516,"kb_used_data":12900,"kb_used_omap":20,"kb_used_meta":2599531,"kb_avail":747348700,"statfs":{"total":767960285184,"available":765285068800,"internally_reserved":0,"allocated":13209600,"data_stored":9010590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2661920398},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002985"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974877+0000","last_change":"2023-10-09T04:27:30.004963+0000","last_active":"2023-10-09T04:27:56.974877+0000","last_peered":"2023-10-09T04:27:56.974877+0000","last_clean":"2023-10-09T04:27:56.974877+0000","last_became_active":"2023-10-09T04:27:30.004581+0000","last_became_peered":"2023-10-09T04:27:30.004581+0000","last_unstale":"2023-10-09T04:27:56.974877+0000","last_undegraded":"2023-10-09T04:27:56.974877+0000","last_fullsized":"2023-10-09T04:27:56.974877+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:40.077805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988720+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:27:57.988720+0000","last_peered":"2023-10-09T04:27:57.988720+0000","last_clean":"2023-10-09T04:27:57.988720+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:27:57.988720+0000","last_undegraded":"2023-10-09T04:27:57.988720+0000","last_fullsized":"2023-10-09T04:27:57.988720+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.984643+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:27:57.984643+0000","last_peered":"2023-10-09T04:27:57.984643+0000","last_clean":"2023-10-09T04:27:57.984643+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:27:57.984643+0000","last_undegraded":"2023-10-09T04:27:57.984643+0000","last_fullsized":"2023-10-09T04:27:57.984643+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990988+0000","last_change":"2023-10-09T04:27:57.985924+0000","last_active":"2023-10-09T04:27:57.990988+0000","last_peered":"2023-10-09T04:27:57.990988+0000","last_clean":"2023-10-09T04:27:57.990988+0000","last_became_active":"2023-10-09T04:27:57.985559+0000","last_became_peered":"2023-10-09T04:27:57.985559+0000","last_unstale":"2023-10-09T04:27:57.990988+0000","last_undegraded":"2023-10-09T04:27:57.990988+0000","last_fullsized":"2023-10-09T04:27:57.990988+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:01.749257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986393+0000","last_change":"2023-10-09T04:27:57.986393+0000","last_active":"2023-10-09T04:27:57.986393+0000","last_peered":"2023-10-09T04:27:57.986393+0000","last_clean":"2023-10-09T04:27:57.986393+0000","last_became_active":"2023-10-09T04:27:57.981584+0000","last_became_peered":"2023-10-09T04:27:57.981584+0000","last_unstale":"2023-10-09T04:27:57.986393+0000","last_undegraded":"2023-10-09T04:27:57.986393+0000","last_fullsized":"2023-10-09T04:27:57.986393+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:34.073778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981290+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:27:56.981290+0000","last_peered":"2023-10-09T04:27:56.981290+0000","last_clean":"2023-10-09T04:27:56.981290+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:27:56.981290+0000","last_undegraded":"2023-10-09T04:27:56.981290+0000","last_fullsized":"2023-10-09T04:27:56.981290+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989523+0000","last_change":"2023-10-09T04:27:57.989523+0000","last_active":"2023-10-09T04:27:57.989523+0000","last_peered":"2023-10-09T04:27:57.989523+0000","last_clean":"2023-10-09T04:27:57.989523+0000","last_became_active":"2023-10-09T04:27:57.989226+0000","last_became_peered":"2023-10-09T04:27:57.989226+0000","last_unstale":"2023-10-09T04:27:57.989523+0000","last_undegraded":"2023-10-09T04:27:57.989523+0000","last_fullsized":"2023-10-09T04:27:57.989523+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:47.573373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987735+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:27:57.987735+0000","last_peered":"2023-10-09T04:27:57.987735+0000","last_clean":"2023-10-09T04:27:57.987735+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:27:57.987735+0000","last_undegraded":"2023-10-09T04:27:57.987735+0000","last_fullsized":"2023-10-09T04:27:57.987735+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988398+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:27:56.988398+0000","last_peered":"2023-10-09T04:27:56.988398+0000","last_clean":"2023-10-09T04:27:56.988398+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:27:56.988398+0000","last_undegraded":"2023-10-09T04:27:56.988398+0000","last_fullsized":"2023-10-09T04:27:56.988398+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990152+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:27:57.990152+0000","last_peered":"2023-10-09T04:27:57.990152+0000","last_clean":"2023-10-09T04:27:57.990152+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:27:57.990152+0000","last_undegraded":"2023-10-09T04:27:57.990152+0000","last_fullsized":"2023-10-09T04:27:57.990152+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":106,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986830+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:27:57.986830+0000","last_peered":"2023-10-09T04:27:57.986830+0000","last_clean":"2023-10-09T04:27:57.986830+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:27:57.986830+0000","last_undegraded":"2023-10-09T04:27:57.986830+0000","last_fullsized":"2023-10-09T04:27:57.986830+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972995+0000","last_change":"2023-10-09T04:27:29.983053+0000","last_active":"2023-10-09T04:27:57.972995+0000","last_peered":"2023-10-09T04:27:57.972995+0000","last_clean":"2023-10-09T04:27:57.972995+0000","last_became_active":"2023-10-09T04:27:29.982725+0000","last_became_peered":"2023-10-09T04:27:29.982725+0000","last_unstale":"2023-10-09T04:27:57.972995+0000","last_undegraded":"2023-10-09T04:27:57.972995+0000","last_fullsized":"2023-10-09T04:27:57.972995+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:33.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976512+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:27:56.976512+0000","last_peered":"2023-10-09T04:27:56.976512+0000","last_clean":"2023-10-09T04:27:56.976512+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:27:56.976512+0000","last_undegraded":"2023-10-09T04:27:56.976512+0000","last_fullsized":"2023-10-09T04:27:56.976512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975162+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:27:56.975162+0000","last_peered":"2023-10-09T04:27:56.975162+0000","last_clean":"2023-10-09T04:27:56.975162+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:27:56.975162+0000","last_undegraded":"2023-10-09T04:27:56.975162+0000","last_fullsized":"2023-10-09T04:27:56.975162+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981381+0000","last_change":"2023-10-09T04:27:29.989226+0000","last_active":"2023-10-09T04:27:56.981381+0000","last_peered":"2023-10-09T04:27:56.981381+0000","last_clean":"2023-10-09T04:27:56.981381+0000","last_became_active":"2023-10-09T04:27:29.988867+0000","last_became_peered":"2023-10-09T04:27:29.988867+0000","last_unstale":"2023-10-09T04:27:56.981381+0000","last_undegraded":"2023-10-09T04:27:56.981381+0000","last_fullsized":"2023-10-09T04:27:56.981381+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:15.559571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.987345+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:27:57.987345+0000","last_peered":"2023-10-09T04:27:57.987345+0000","last_clean":"2023-10-09T04:27:57.987345+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:27:57.987345+0000","last_undegraded":"2023-10-09T04:27:57.987345+0000","last_fullsized":"2023-10-09T04:27:57.987345+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.985231+0000","last_change":"2023-10-09T04:27:57.985231+0000","last_active":"2023-10-09T04:27:57.985231+0000","last_peered":"2023-10-09T04:27:57.985231+0000","last_clean":"2023-10-09T04:27:57.985231+0000","last_became_active":"2023-10-09T04:27:57.984869+0000","last_became_peered":"2023-10-09T04:27:57.984869+0000","last_unstale":"2023-10-09T04:27:57.985231+0000","last_undegraded":"2023-10-09T04:27:57.985231+0000","last_fullsized":"2023-10-09T04:27:57.985231+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:39.521778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992866+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:27:57.992866+0000","last_peered":"2023-10-09T04:27:57.992866+0000","last_clean":"2023-10-09T04:27:57.992866+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:27:57.992866+0000","last_undegraded":"2023-10-09T04:27:57.992866+0000","last_fullsized":"2023-10-09T04:27:57.992866+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980748+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:27:56.980748+0000","last_peered":"2023-10-09T04:27:56.980748+0000","last_clean":"2023-10-09T04:27:56.980748+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:27:56.980748+0000","last_undegraded":"2023-10-09T04:27:56.980748+0000","last_fullsized":"2023-10-09T04:27:56.980748+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.994143+0000","last_change":"2023-10-09T04:27:57.994143+0000","last_active":"2023-10-09T04:27:57.994143+0000","last_peered":"2023-10-09T04:27:57.994143+0000","last_clean":"2023-10-09T04:27:57.994143+0000","last_became_active":"2023-10-09T04:27:57.993862+0000","last_became_peered":"2023-10-09T04:27:57.993862+0000","last_unstale":"2023-10-09T04:27:57.994143+0000","last_undegraded":"2023-10-09T04:27:57.994143+0000","last_fullsized":"2023-10-09T04:27:57.994143+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:24.434239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.980940+0000","last_change":"2023-10-09T04:20:03.030960+0000","last_active":"2023-10-09T04:27:56.980940+0000","last_peered":"2023-10-09T04:27:56.980940+0000","last_clean":"2023-10-09T04:27:56.980940+0000","last_became_active":"2023-10-09T04:20:03.030148+0000","last_became_peered":"2023-10-09T04:20:03.030148+0000","last_unstale":"2023-10-09T04:27:56.980940+0000","last_undegraded":"2023-10-09T04:27:56.980940+0000","last_fullsized":"2023-10-09T04:27:56.980940+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:09.000584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986461+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:27:57.986461+0000","last_peered":"2023-10-09T04:27:57.986461+0000","last_clean":"2023-10-09T04:27:57.986461+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:27:57.986461+0000","last_undegraded":"2023-10-09T04:27:57.986461+0000","last_fullsized":"2023-10-09T04:27:57.986461+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983931+0000","last_change":"2023-10-09T04:20:03.033517+0000","last_active":"2023-10-09T04:27:56.983931+0000","last_peered":"2023-10-09T04:27:56.983931+0000","last_clean":"2023-10-09T04:27:56.983931+0000","last_became_active":"2023-10-09T04:20:03.033132+0000","last_became_peered":"2023-10-09T04:20:03.033132+0000","last_unstale":"2023-10-09T04:27:56.983931+0000","last_undegraded":"2023-10-09T04:27:56.983931+0000","last_fullsized":"2023-10-09T04:27:56.983931+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:54:14.247924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988550+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:27:56.988550+0000","last_peered":"2023-10-09T04:27:56.988550+0000","last_clean":"2023-10-09T04:27:56.988550+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:27:56.988550+0000","last_undegraded":"2023-10-09T04:27:56.988550+0000","last_fullsized":"2023-10-09T04:27:56.988550+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.983774+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:27:56.983774+0000","last_peered":"2023-10-09T04:27:56.983774+0000","last_clean":"2023-10-09T04:27:56.983774+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:27:56.983774+0000","last_undegraded":"2023-10-09T04:27:56.983774+0000","last_fullsized":"2023-10-09T04:27:56.983774+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988386+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:27:57.988386+0000","last_peered":"2023-10-09T04:27:57.988386+0000","last_clean":"2023-10-09T04:27:57.988386+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:27:57.988386+0000","last_undegraded":"2023-10-09T04:27:57.988386+0000","last_fullsized":"2023-10-09T04:27:57.988386+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":87,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982249+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:27:56.982249+0000","last_peered":"2023-10-09T04:27:56.982249+0000","last_clean":"2023-10-09T04:27:56.982249+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:27:56.982249+0000","last_undegraded":"2023-10-09T04:27:56.982249+0000","last_fullsized":"2023-10-09T04:27:56.982249+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982155+0000","last_change":"2023-10-09T04:27:30.005025+0000","last_active":"2023-10-09T04:27:56.982155+0000","last_peered":"2023-10-09T04:27:56.982155+0000","last_clean":"2023-10-09T04:27:56.982155+0000","last_became_active":"2023-10-09T04:27:30.004692+0000","last_became_peered":"2023-10-09T04:27:30.004692+0000","last_unstale":"2023-10-09T04:27:56.982155+0000","last_undegraded":"2023-10-09T04:27:56.982155+0000","last_fullsized":"2023-10-09T04:27:56.982155+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:10.737398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992844+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:27:57.992844+0000","last_peered":"2023-10-09T04:27:57.992844+0000","last_clean":"2023-10-09T04:27:57.992844+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:27:57.992844+0000","last_undegraded":"2023-10-09T04:27:57.992844+0000","last_fullsized":"2023-10-09T04:27:57.992844+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978109+0000","last_change":"2023-10-09T04:20:03.025205+0000","last_active":"2023-10-09T04:27:57.978109+0000","last_peered":"2023-10-09T04:27:57.978109+0000","last_clean":"2023-10-09T04:27:57.978109+0000","last_became_active":"2023-10-09T04:20:03.024858+0000","last_became_peered":"2023-10-09T04:20:03.024858+0000","last_unstale":"2023-10-09T04:27:57.978109+0000","last_undegraded":"2023-10-09T04:27:57.978109+0000","last_fullsized":"2023-10-09T04:27:57.978109+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:59.239890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.992940+0000","last_change":"2023-10-09T04:27:57.992940+0000","last_active":"2023-10-09T04:27:57.992940+0000","last_peered":"2023-10-09T04:27:57.992940+0000","last_clean":"2023-10-09T04:27:57.992940+0000","last_became_active":"2023-10-09T04:27:57.991337+0000","last_became_peered":"2023-10-09T04:27:57.991337+0000","last_unstale":"2023-10-09T04:27:57.992940+0000","last_undegraded":"2023-10-09T04:27:57.992940+0000","last_fullsized":"2023-10-09T04:27:57.992940+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:46:10.795994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988243+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:27:57.988243+0000","last_peered":"2023-10-09T04:27:57.988243+0000","last_clean":"2023-10-09T04:27:57.988243+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:27:57.988243+0000","last_undegraded":"2023-10-09T04:27:57.988243+0000","last_fullsized":"2023-10-09T04:27:57.988243+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982044+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:27:56.982044+0000","last_peered":"2023-10-09T04:27:56.982044+0000","last_clean":"2023-10-09T04:27:56.982044+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:27:56.982044+0000","last_undegraded":"2023-10-09T04:27:56.982044+0000","last_fullsized":"2023-10-09T04:27:56.982044+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976221+0000","last_change":"2023-10-09T04:27:29.982774+0000","last_active":"2023-10-09T04:27:56.976221+0000","last_peered":"2023-10-09T04:27:56.976221+0000","last_clean":"2023-10-09T04:27:56.976221+0000","last_became_active":"2023-10-09T04:27:29.982507+0000","last_became_peered":"2023-10-09T04:27:29.982507+0000","last_unstale":"2023-10-09T04:27:56.976221+0000","last_undegraded":"2023-10-09T04:27:56.976221+0000","last_fullsized":"2023-10-09T04:27:56.976221+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:14.613302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976415+0000","last_change":"2023-10-09T04:27:30.010756+0000","last_active":"2023-10-09T04:27:56.976415+0000","last_peered":"2023-10-09T04:27:56.976415+0000","last_clean":"2023-10-09T04:27:56.976415+0000","last_became_active":"2023-10-09T04:27:30.010403+0000","last_became_peered":"2023-10-09T04:27:30.010403+0000","last_unstale":"2023-10-09T04:27:56.976415+0000","last_undegraded":"2023-10-09T04:27:56.976415+0000","last_fullsized":"2023-10-09T04:27:56.976415+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:43:10.557818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976602+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:27:56.976602+0000","last_peered":"2023-10-09T04:27:56.976602+0000","last_clean":"2023-10-09T04:27:56.976602+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:27:56.976602+0000","last_undegraded":"2023-10-09T04:27:56.976602+0000","last_fullsized":"2023-10-09T04:27:56.976602+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.986421+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:27:57.986421+0000","last_peered":"2023-10-09T04:27:57.986421+0000","last_clean":"2023-10-09T04:27:57.986421+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:27:57.986421+0000","last_undegraded":"2023-10-09T04:27:57.986421+0000","last_fullsized":"2023-10-09T04:27:57.986421+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.983436+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:27:57.983436+0000","last_peered":"2023-10-09T04:27:57.983436+0000","last_clean":"2023-10-09T04:27:57.983436+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:27:57.983436+0000","last_undegraded":"2023-10-09T04:27:57.983436+0000","last_fullsized":"2023-10-09T04:27:57.983436+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973721+0000","last_change":"2023-10-09T04:24:07.916771+0000","last_active":"2023-10-09T04:27:57.973721+0000","last_peered":"2023-10-09T04:27:57.973721+0000","last_clean":"2023-10-09T04:27:57.973721+0000","last_became_active":"2023-10-09T04:24:07.916030+0000","last_became_peered":"2023-10-09T04:24:07.916030+0000","last_unstale":"2023-10-09T04:27:57.973721+0000","last_undegraded":"2023-10-09T04:27:57.973721+0000","last_fullsized":"2023-10-09T04:27:57.973721+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:07.033444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993021+0000","last_change":"2023-10-09T04:27:57.993021+0000","last_active":"2023-10-09T04:27:57.993021+0000","last_peered":"2023-10-09T04:27:57.993021+0000","last_clean":"2023-10-09T04:27:57.993021+0000","last_became_active":"2023-10-09T04:27:57.992486+0000","last_became_peered":"2023-10-09T04:27:57.992486+0000","last_unstale":"2023-10-09T04:27:57.993021+0000","last_undegraded":"2023-10-09T04:27:57.993021+0000","last_fullsized":"2023-10-09T04:27:57.993021+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:56.130811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972915+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:27:57.972915+0000","last_peered":"2023-10-09T04:27:57.972915+0000","last_clean":"2023-10-09T04:27:57.972915+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:27:57.972915+0000","last_undegraded":"2023-10-09T04:27:57.972915+0000","last_fullsized":"2023-10-09T04:27:57.972915+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988701+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:27:56.988701+0000","last_peered":"2023-10-09T04:27:56.988701+0000","last_clean":"2023-10-09T04:27:56.988701+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:27:56.988701+0000","last_undegraded":"2023-10-09T04:27:56.988701+0000","last_fullsized":"2023-10-09T04:27:56.988701+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.975059+0000","last_change":"2023-10-09T04:20:03.030040+0000","last_active":"2023-10-09T04:27:56.975059+0000","last_peered":"2023-10-09T04:27:56.975059+0000","last_clean":"2023-10-09T04:27:56.975059+0000","last_became_active":"2023-10-09T04:20:03.029675+0000","last_became_peered":"2023-10-09T04:20:03.029675+0000","last_unstale":"2023-10-09T04:27:56.975059+0000","last_undegraded":"2023-10-09T04:27:56.975059+0000","last_fullsized":"2023-10-09T04:27:56.975059+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:25.875934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.993874+0000","last_change":"2023-10-09T04:27:57.991197+0000","last_active":"2023-10-09T04:27:57.993874+0000","last_peered":"2023-10-09T04:27:57.993874+0000","last_clean":"2023-10-09T04:27:57.993874+0000","last_became_active":"2023-10-09T04:27:57.990548+0000","last_became_peered":"2023-10-09T04:27:57.990548+0000","last_unstale":"2023-10-09T04:27:57.993874+0000","last_undegraded":"2023-10-09T04:27:57.993874+0000","last_fullsized":"2023-10-09T04:27:57.993874+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:05.799943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989081+0000","last_change":"2023-10-09T04:27:57.989081+0000","last_active":"2023-10-09T04:27:57.989081+0000","last_peered":"2023-10-09T04:27:57.989081+0000","last_clean":"2023-10-09T04:27:57.989081+0000","last_became_active":"2023-10-09T04:27:57.988802+0000","last_became_peered":"2023-10-09T04:27:57.988802+0000","last_unstale":"2023-10-09T04:27:57.989081+0000","last_undegraded":"2023-10-09T04:27:57.989081+0000","last_fullsized":"2023-10-09T04:27:57.989081+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:45.574939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.972611+0000","last_change":"2023-10-09T04:27:29.982732+0000","last_active":"2023-10-09T04:27:57.972611+0000","last_peered":"2023-10-09T04:27:57.972611+0000","last_clean":"2023-10-09T04:27:57.972611+0000","last_became_active":"2023-10-09T04:27:29.982275+0000","last_became_peered":"2023-10-09T04:27:29.982275+0000","last_unstale":"2023-10-09T04:27:57.972611+0000","last_undegraded":"2023-10-09T04:27:57.972611+0000","last_fullsized":"2023-10-09T04:27:57.972611+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:43.712715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973308+0000","last_change":"2023-10-09T04:24:07.922314+0000","last_active":"2023-10-09T04:27:57.973308+0000","last_peered":"2023-10-09T04:27:57.973308+0000","last_clean":"2023-10-09T04:27:57.973308+0000","last_became_active":"2023-10-09T04:24:07.916895+0000","last_became_peered":"2023-10-09T04:24:07.916895+0000","last_unstale":"2023-10-09T04:27:57.973308+0000","last_undegraded":"2023-10-09T04:27:57.973308+0000","last_fullsized":"2023-10-09T04:27:57.973308+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:13.277124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.988253+0000","last_change":"2023-10-09T04:27:29.988702+0000","last_active":"2023-10-09T04:27:56.988253+0000","last_peered":"2023-10-09T04:27:56.988253+0000","last_clean":"2023-10-09T04:27:56.988253+0000","last_became_active":"2023-10-09T04:27:29.988390+0000","last_became_peered":"2023-10-09T04:27:29.988390+0000","last_unstale":"2023-10-09T04:27:56.988253+0000","last_undegraded":"2023-10-09T04:27:56.988253+0000","last_fullsized":"2023-10-09T04:27:56.988253+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:33.897326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":484,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981404+0000","last_change":"2023-10-09T04:20:03.031289+0000","last_active":"2023-10-09T04:27:56.981404+0000","last_peered":"2023-10-09T04:27:56.981404+0000","last_clean":"2023-10-09T04:27:56.981404+0000","last_became_active":"2023-10-09T04:20:03.030125+0000","last_became_peered":"2023-10-09T04:20:03.030125+0000","last_unstale":"2023-10-09T04:27:56.981404+0000","last_undegraded":"2023-10-09T04:27:56.981404+0000","last_fullsized":"2023-10-09T04:27:56.981404+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:44:33.277598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988547+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:27:57.988547+0000","last_peered":"2023-10-09T04:27:57.988547+0000","last_clean":"2023-10-09T04:27:57.988547+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:27:57.988547+0000","last_undegraded":"2023-10-09T04:27:57.988547+0000","last_fullsized":"2023-10-09T04:27:57.988547+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981058+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:27:56.981058+0000","last_peered":"2023-10-09T04:27:56.981058+0000","last_clean":"2023-10-09T04:27:56.981058+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:27:56.981058+0000","last_undegraded":"2023-10-09T04:27:56.981058+0000","last_fullsized":"2023-10-09T04:27:56.981058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":124,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981273+0000","last_change":"2023-10-09T04:24:07.962764+0000","last_active":"2023-10-09T04:27:56.981273+0000","last_peered":"2023-10-09T04:27:56.981273+0000","last_clean":"2023-10-09T04:27:56.981273+0000","last_became_active":"2023-10-09T04:24:07.962508+0000","last_became_peered":"2023-10-09T04:24:07.962508+0000","last_unstale":"2023-10-09T04:27:56.981273+0000","last_undegraded":"2023-10-09T04:27:56.981273+0000","last_fullsized":"2023-10-09T04:27:56.981273+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:14.078794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989841+0000","last_change":"2023-10-09T04:27:57.988062+0000","last_active":"2023-10-09T04:27:57.989841+0000","last_peered":"2023-10-09T04:27:57.989841+0000","last_clean":"2023-10-09T04:27:57.989841+0000","last_became_active":"2023-10-09T04:27:57.987732+0000","last_became_peered":"2023-10-09T04:27:57.987732+0000","last_unstale":"2023-10-09T04:27:57.989841+0000","last_undegraded":"2023-10-09T04:27:57.989841+0000","last_fullsized":"2023-10-09T04:27:57.989841+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:01.399437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978739+0000","last_change":"2023-10-09T04:27:29.973734+0000","last_active":"2023-10-09T04:27:57.978739+0000","last_peered":"2023-10-09T04:27:57.978739+0000","last_clean":"2023-10-09T04:27:57.978739+0000","last_became_active":"2023-10-09T04:27:29.973431+0000","last_became_peered":"2023-10-09T04:27:29.973431+0000","last_unstale":"2023-10-09T04:27:57.978739+0000","last_undegraded":"2023-10-09T04:27:57.978739+0000","last_fullsized":"2023-10-09T04:27:57.978739+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:14.788158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.985955+0000","last_change":"2023-10-09T04:20:03.030432+0000","last_active":"2023-10-09T04:27:56.985955+0000","last_peered":"2023-10-09T04:27:56.985955+0000","last_clean":"2023-10-09T04:27:56.985955+0000","last_became_active":"2023-10-09T04:20:03.030103+0000","last_became_peered":"2023-10-09T04:20:03.030103+0000","last_unstale":"2023-10-09T04:27:56.985955+0000","last_undegraded":"2023-10-09T04:27:56.985955+0000","last_fullsized":"2023-10-09T04:27:56.985955+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.759695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.974229+0000","last_change":"2023-10-09T04:27:29.981956+0000","last_active":"2023-10-09T04:27:57.974229+0000","last_peered":"2023-10-09T04:27:57.974229+0000","last_clean":"2023-10-09T04:27:57.974229+0000","last_became_active":"2023-10-09T04:27:29.981663+0000","last_became_peered":"2023-10-09T04:27:29.981663+0000","last_unstale":"2023-10-09T04:27:57.974229+0000","last_undegraded":"2023-10-09T04:27:57.974229+0000","last_fullsized":"2023-10-09T04:27:57.974229+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:47.639333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.977902+0000","last_change":"2023-10-09T04:20:03.032817+0000","last_active":"2023-10-09T04:27:57.977902+0000","last_peered":"2023-10-09T04:27:57.977902+0000","last_clean":"2023-10-09T04:27:57.977902+0000","last_became_active":"2023-10-09T04:20:03.029228+0000","last_became_peered":"2023-10-09T04:20:03.029228+0000","last_unstale":"2023-10-09T04:27:57.977902+0000","last_undegraded":"2023-10-09T04:27:57.977902+0000","last_fullsized":"2023-10-09T04:27:57.977902+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:14.429820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.995379+0000","last_change":"2023-10-09T04:27:57.995379+0000","last_active":"2023-10-09T04:27:57.995379+0000","last_peered":"2023-10-09T04:27:57.995379+0000","last_clean":"2023-10-09T04:27:57.995379+0000","last_became_active":"2023-10-09T04:27:57.995057+0000","last_became_peered":"2023-10-09T04:27:57.995057+0000","last_unstale":"2023-10-09T04:27:57.995379+0000","last_undegraded":"2023-10-09T04:27:57.995379+0000","last_fullsized":"2023-10-09T04:27:57.995379+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:46.440110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.973460+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:27:57.973460+0000","last_peered":"2023-10-09T04:27:57.973460+0000","last_clean":"2023-10-09T04:27:57.973460+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:27:57.973460+0000","last_undegraded":"2023-10-09T04:27:57.973460+0000","last_fullsized":"2023-10-09T04:27:57.973460+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.991611+0000","last_change":"2023-10-09T04:27:57.991611+0000","last_active":"2023-10-09T04:27:57.991611+0000","last_peered":"2023-10-09T04:27:57.991611+0000","last_clean":"2023-10-09T04:27:57.991611+0000","last_became_active":"2023-10-09T04:27:57.991279+0000","last_became_peered":"2023-10-09T04:27:57.991279+0000","last_unstale":"2023-10-09T04:27:57.991611+0000","last_undegraded":"2023-10-09T04:27:57.991611+0000","last_fullsized":"2023-10-09T04:27:57.991611+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:54.806159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.981184+0000","last_change":"2023-10-09T04:27:30.000853+0000","last_active":"2023-10-09T04:27:56.981184+0000","last_peered":"2023-10-09T04:27:56.981184+0000","last_clean":"2023-10-09T04:27:56.981184+0000","last_became_active":"2023-10-09T04:27:30.000545+0000","last_became_peered":"2023-10-09T04:27:30.000545+0000","last_unstale":"2023-10-09T04:27:56.981184+0000","last_undegraded":"2023-10-09T04:27:56.981184+0000","last_fullsized":"2023-10-09T04:27:56.981184+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:47.506034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978683+0000","last_change":"2023-10-09T04:24:07.919208+0000","last_active":"2023-10-09T04:27:57.978683+0000","last_peered":"2023-10-09T04:27:57.978683+0000","last_clean":"2023-10-09T04:27:57.978683+0000","last_became_active":"2023-10-09T04:24:07.918797+0000","last_became_peered":"2023-10-09T04:24:07.918797+0000","last_unstale":"2023-10-09T04:27:57.978683+0000","last_undegraded":"2023-10-09T04:27:57.978683+0000","last_fullsized":"2023-10-09T04:27:57.978683+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:11.399352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.976517+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:27:56.976517+0000","last_peered":"2023-10-09T04:27:56.976517+0000","last_clean":"2023-10-09T04:27:56.976517+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:27:56.976517+0000","last_undegraded":"2023-10-09T04:27:56.976517+0000","last_fullsized":"2023-10-09T04:27:56.976517+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":80,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.982962+0000","last_change":"2023-10-09T04:20:03.037673+0000","last_active":"2023-10-09T04:27:56.982962+0000","last_peered":"2023-10-09T04:27:56.982962+0000","last_clean":"2023-10-09T04:27:56.982962+0000","last_became_active":"2023-10-09T04:20:03.032398+0000","last_became_peered":"2023-10-09T04:20:03.032398+0000","last_unstale":"2023-10-09T04:27:56.982962+0000","last_undegraded":"2023-10-09T04:27:56.982962+0000","last_fullsized":"2023-10-09T04:27:56.982962+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:20.306433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989106+0000","last_change":"2023-10-09T04:27:57.989106+0000","last_active":"2023-10-09T04:27:57.989106+0000","last_peered":"2023-10-09T04:27:57.989106+0000","last_clean":"2023-10-09T04:27:57.989106+0000","last_became_active":"2023-10-09T04:27:57.988778+0000","last_became_peered":"2023-10-09T04:27:57.988778+0000","last_unstale":"2023-10-09T04:27:57.989106+0000","last_undegraded":"2023-10-09T04:27:57.989106+0000","last_fullsized":"2023-10-09T04:27:57.989106+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:33.473019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.989866+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:27:56.989866+0000","last_peered":"2023-10-09T04:27:56.989866+0000","last_clean":"2023-10-09T04:27:56.989866+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:27:56.989866+0000","last_undegraded":"2023-10-09T04:27:56.989866+0000","last_fullsized":"2023-10-09T04:27:56.989866+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988397+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:27:57.988397+0000","last_peered":"2023-10-09T04:27:57.988397+0000","last_clean":"2023-10-09T04:27:57.988397+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:27:57.988397+0000","last_undegraded":"2023-10-09T04:27:57.988397+0000","last_fullsized":"2023-10-09T04:27:57.988397+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":88,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.991512+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:27:56.991512+0000","last_peered":"2023-10-09T04:27:56.991512+0000","last_clean":"2023-10-09T04:27:56.991512+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:27:56.991512+0000","last_undegraded":"2023-10-09T04:27:56.991512+0000","last_fullsized":"2023-10-09T04:27:56.991512+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.988337+0000","last_change":"2023-10-09T04:27:57.988337+0000","last_active":"2023-10-09T04:27:57.988337+0000","last_peered":"2023-10-09T04:27:57.988337+0000","last_clean":"2023-10-09T04:27:57.988337+0000","last_became_active":"2023-10-09T04:27:57.987918+0000","last_became_peered":"2023-10-09T04:27:57.987918+0000","last_unstale":"2023-10-09T04:27:57.988337+0000","last_undegraded":"2023-10-09T04:27:57.988337+0000","last_fullsized":"2023-10-09T04:27:57.988337+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:44:54.931130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.989240+0000","last_change":"2023-10-09T04:27:57.989240+0000","last_active":"2023-10-09T04:27:57.989240+0000","last_peered":"2023-10-09T04:27:57.989240+0000","last_clean":"2023-10-09T04:27:57.989240+0000","last_became_active":"2023-10-09T04:27:57.988856+0000","last_became_peered":"2023-10-09T04:27:57.988856+0000","last_unstale":"2023-10-09T04:27:57.989240+0000","last_undegraded":"2023-10-09T04:27:57.989240+0000","last_fullsized":"2023-10-09T04:27:57.989240+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:52.380796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.990600+0000","last_change":"2023-10-09T04:27:57.990600+0000","last_active":"2023-10-09T04:27:57.990600+0000","last_peered":"2023-10-09T04:27:57.990600+0000","last_clean":"2023-10-09T04:27:57.990600+0000","last_became_active":"2023-10-09T04:27:57.989478+0000","last_became_peered":"2023-10-09T04:27:57.989478+0000","last_unstale":"2023-10-09T04:27:57.990600+0000","last_undegraded":"2023-10-09T04:27:57.990600+0000","last_fullsized":"2023-10-09T04:27:57.990600+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:42.167307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-09T04:27:57.978163+0000","last_change":"2023-10-09T04:27:29.977785+0000","last_active":"2023-10-09T04:27:57.978163+0000","last_peered":"2023-10-09T04:27:57.978163+0000","last_clean":"2023-10-09T04:27:57.978163+0000","last_became_active":"2023-10-09T04:27:29.977397+0000","last_became_peered":"2023-10-09T04:27:29.977397+0000","last_unstale":"2023-10-09T04:27:57.978163+0000","last_undegraded":"2023-10-09T04:27:57.978163+0000","last_fullsized":"2023-10-09T04:27:57.978163+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:16.267931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-10-09T04:27:56.974728+0000","last_change":"2023-10-09T04:27:29.981763+0000","last_active":"2023-10-09T04:27:56.974728+0000","last_peered":"2023-10-09T04:27:56.974728+0000","last_clean":"2023-10-09T04:27:56.974728+0000","last_became_active":"2023-10-09T04:27:29.981401+0000","last_became_peered":"2023-10-09T04:27:29.981401+0000","last_unstale":"2023-10-09T04:27:56.974728+0000","last_undegraded":"2023-10-09T04:27:56.974728+0000","last_fullsized":"2023-10-09T04:27:56.974728+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:35.242521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234266,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303340,"kb_used_data":1956,"kb_used_omap":0,"kb_used_meta":301376,"kb_avail":93441812,"statfs":{"total":95995035648,"available":95684415488,"internally_reserved":0,"allocated":2002944,"data_stored":1492631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308609024},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":7.8869999999999996,"15min":3.9350000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22900000000000001},"max":{"1min":8.5909999999999993,"5min":629.22500000000002,"15min":629.22500000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.93400000000000005,"5min":7.5739999999999998,"15min":3.5899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.1849999999999996,"5min":629.43100000000004,"15min":629.43100000000004},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":2.5960000000000001,"15min":2.0579999999999998},"min":{"1min":0.41099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.3330000000000002,"5min":80.215000000000003,"15min":80.215000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.7309999999999999,"15min":2.0880000000000001},"min":{"1min":0.35199999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.2450000000000001,"5min":80.274000000000001,"15min":80.274000000000001},"last":0.65300000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:30 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.501}]},{"osd":3,"last update":"Mon Oct 9 04:29:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.52000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.546,"15min":1.0700000000000001},"min":{"1min":0.38400000000000001,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":0.65500000000000003,"5min":8.5500000000000007,"15min":8.5500000000000007},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.495,"5min":1.1299999999999999,"15min":1.1850000000000001},"min":{"1min":0.36499999999999999,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":0.59699999999999998,"5min":5.9939999999999998,"15min":94.522999999999996},"last":0.436}]},{"osd":5,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.2749999999999999,"15min":0.98499999999999999},"min":{"1min":0.39300000000000002,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":0.753,"5min":8.8610000000000007,"15min":8.8610000000000007},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.3300000000000001,"15min":1.1759999999999999},"min":{"1min":0.371,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.76700000000000002,"5min":6.3319999999999999,"15min":18.125},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:28:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.4399999999999999,"15min":1.6819999999999999},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.29999999999999999},"max":{"1min":0.78100000000000003,"5min":12.007,"15min":203.19399999999999},"last":0.627},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.4319999999999999,"15min":1.786},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.312},"max":{"1min":0.71899999999999997,"5min":7.4800000000000004,"15min":204.303},"last":0.59199999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495900,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299228,"kb_used_data":1940,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93445924,"statfs":{"total":95995035648,"available":95688626176,"internally_reserved":0,"allocated":1986560,"data_stored":1492162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":2.149,"15min":1.956},"min":{"1min":0.32400000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.5410000000000004,"5min":10.77,"15min":13.125999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.1309999999999998,"15min":2.5960000000000001},"min":{"1min":0.28399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.274,"5min":10.631,"15min":13.551},"last":0.47699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:28:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.0739999999999998,"15min":2.2559999999999998},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.5600000000000005,"5min":12.728,"15min":12.728},"last":0.432},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.4340000000000002,"15min":2.4540000000000002},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":20.181000000000001,"5min":20.181000000000001,"15min":20.181000000000001},"last":0.65500000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.68200000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.41399999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.351,"15min":1.784},"min":{"1min":0.372,"5min":0.246,"15min":0.246},"max":{"1min":5.2439999999999998,"5min":12.743,"15min":92.650000000000006},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.494,"5min":1.6970000000000001,"15min":1.796},"min":{"1min":0.437,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2379999999999995,"5min":12.028,"15min":91.906999999999996},"last":0.52800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.367,"15min":1.389},"min":{"1min":0.44500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.8920000000000003,"5min":12.710000000000001,"15min":19.759},"last":0.498},{"interface":"front","average":{"1min":1.603,"5min":1.7390000000000001,"15min":1.5029999999999999},"min":{"1min":0.503,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":8.4090000000000007,"5min":12.249000000000001,"15min":19.904},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:28:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.6200000000000001,"15min":1.454},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.29899999999999999},"max":{"1min":0.84999999999999998,"5min":14.718999999999999,"15min":55.838999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.452,"15min":1.486},"min":{"1min":0.497,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78200000000000003,"5min":20.155000000000001,"15min":55.718000000000004},"last":0.59899999999999998}]}]},{"osd":1,"up_from":118,"seq":506806140989,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":261036,"kb_used_data":1444,"kb_used_omap":4,"kb_used_meta":259579,"kb_avail":93484116,"statfs":{"total":95995035648,"available":95727734784,"internally_reserved":0,"allocated":1478656,"data_stored":912057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":265809806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.7240000000000002,"15min":2.98},"min":{"1min":0.21099999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":9.016,"5min":11.81,"15min":11.81},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.157,"5min":2.9249999999999998,"15min":3.1280000000000001},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.573,"5min":12.573,"15min":12.573},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"last":0.39000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:28:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"last":0.433},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.377}]},{"osd":4,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":3.8809999999999998,"15min":3.109},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":11.744,"5min":107.795,"15min":107.795},"last":0.62},{"interface":"front","average":{"1min":0.64800000000000002,"5min":2.609,"15min":1.617},"min":{"1min":0.373,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.1120000000000001,"5min":109.167,"15min":109.167},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":2.5270000000000001,"15min":2.6709999999999998},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.9930000000000003,"5min":12.414999999999999,"15min":12.414999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.484,"5min":2.8839999999999999,"15min":3.0270000000000001},"min":{"1min":0.40100000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.686,"5min":13.933999999999999,"15min":13.933999999999999},"last":0.753}]},{"osd":6,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3650000000000002,"15min":2.5},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.7490000000000001,"5min":13.388,"15min":13.388},"last":0.70399999999999996},{"interface":"front","average":{"1min":1.508,"5min":3.048,"15min":3.133},"min":{"1min":0.433,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.776999999999999,"5min":13.882,"15min":13.882},"last":0.67700000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":3.2349999999999999,"15min":3.1840000000000002},"min":{"1min":0.443,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":11.805,"5min":13.907,"15min":13.907},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.802,"15min":1.6339999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.1419999999999999,"5min":7.8239999999999998,"15min":7.8239999999999998},"last":0.69199999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024046,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":280852,"kb_used_data":2060,"kb_used_omap":5,"kb_used_meta":278778,"kb_avail":93464300,"statfs":{"total":95995035648,"available":95707443200,"internally_reserved":0,"allocated":2109440,"data_stored":1505553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":285469306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":2.911,"15min":3.419},"min":{"1min":0.24199999999999999,"5min":0.192,"15min":0.192},"max":{"1min":9.1280000000000001,"5min":17.210000000000001,"15min":17.210000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.6779999999999999,"15min":2.8639999999999999},"min":{"1min":0.26600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":2.5190000000000001,"5min":18.004000000000001,"15min":18.004000000000001},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:31 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.34100000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:28:58 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.317},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.504,"5min":0.504,"15min":0.504},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.742,"15min":1.5269999999999999},"min":{"1min":0.38900000000000001,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":1.3720000000000001,"5min":10.73,"15min":10.73},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.78,"15min":1.7270000000000001},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5009999999999999,"5min":8.1300000000000008,"15min":9.5190000000000001},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":2.032,"15min":1.819},"min":{"1min":0.40000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.4610000000000001,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.718,"15min":1.911},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":2.0779999999999998,"5min":14.144,"15min":14.144},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":2.145,"15min":1.988},"min":{"1min":0.46300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.8019999999999996,"5min":9.7469999999999999,"15min":10.403},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":1.9890000000000001,"15min":2.2440000000000002},"min":{"1min":0.46700000000000003,"5min":0.378,"15min":0.378},"max":{"1min":2.1949999999999998,"5min":14.090999999999999,"15min":14.090999999999999},"last":0.57999999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:28:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.9670000000000001,"15min":2.0470000000000002},"min":{"1min":0.505,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":5.7919999999999998,"5min":11.890000000000001,"15min":11.890000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":2.1139999999999999,"15min":1.984},"min":{"1min":0.45300000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.8250000000000002,"5min":10.765000000000001,"15min":11.176},"last":0.59599999999999997}]}]},{"osd":2,"up_from":135,"seq":579820584981,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156100,"kb_used_data":1404,"kb_used_omap":7,"kb_used_meta":154680,"kb_avail":93589052,"statfs":{"total":95995035648,"available":95835189248,"internally_reserved":0,"allocated":1437696,"data_stored":907571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158392401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"last":0.39200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.31}]},{"osd":3,"last update":"Mon Oct 9 04:29:01 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.36899999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.59399999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68200000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.67300000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:28:32 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.63400000000000001}]}]},{"osd":3,"up_from":142,"seq":609885356047,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":121504,"kb_used_data":1368,"kb_used_omap":2,"kb_used_meta":120125,"kb_avail":93623648,"statfs":{"total":95995035648,"available":95870615552,"internally_reserved":0,"allocated":1400832,"data_stored":900486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":123008053},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.0629999999999997,"5min":6.0629999999999997,"15min":6.0629999999999997},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.39100000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":23.699999999999999,"5min":23.699999999999999,"15min":23.699999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.0060000000000002,"5min":8.0060000000000002,"15min":8.0060000000000002},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.514,"5min":1.514,"15min":1.514},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":23.661000000000001,"5min":23.661000000000001,"15min":23.661000000000001},"last":0.34999999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.802,"15min":1.802},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":23.526,"5min":23.526,"15min":23.526},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":6.2519999999999998,"5min":6.2519999999999998,"15min":6.2519999999999998},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":6.2149999999999999,"5min":6.2149999999999999,"15min":6.2149999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.59999999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":23.59,"5min":23.59,"15min":23.59},"last":0.63800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:28:57 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":1.758,"15min":1.758},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":23.626999999999999,"5min":23.626999999999999,"15min":23.626999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":6.282,"5min":6.282,"15min":6.282},"last":0.53600000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019174,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599332,"kb_used_data":1372,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":93145820,"statfs":{"total":95995035648,"available":95381319680,"internally_reserved":0,"allocated":1404928,"data_stored":902193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.7110000000000001,"15min":1.879},"min":{"1min":0.23799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.89200000000000002,"5min":11.442,"15min":16.553999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.7849999999999999,"15min":2.274},"min":{"1min":0.33700000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.1699999999999999,"5min":8.4909999999999997,"15min":16.529},"last":0.499}]},{"osd":1,"last update":"Mon Oct 9 04:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":2.0510000000000002,"15min":1.9850000000000001},"min":{"1min":0.35099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.7879999999999998,"5min":13.164999999999999,"15min":13.164999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.073,"5min":2.1779999999999999,"15min":2.306},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.8970000000000002,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.73199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.624}]},{"osd":3,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.69299999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.1619999999999999,"15min":0.95399999999999996},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.22900000000000001},"max":{"1min":0.73399999999999999,"5min":10.255000000000001,"15min":10.255000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.137,"15min":0.92500000000000004},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.82999999999999996,"5min":6.7009999999999996,"15min":7.194},"last":0.71099999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.097,"15min":1.131},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.97599999999999998,"5min":6.5810000000000004,"15min":9.9640000000000004},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.119,"15min":1.1140000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":4.5199999999999996,"15min":9.3859999999999992},"last":0.78900000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:28:42 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.3400000000000001,"15min":1.0900000000000001},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":7.8250000000000002,"15min":10.34},"last":0.753},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.355,"15min":1.0800000000000001},"min":{"1min":0.46800000000000003,"5min":0.34399999999999997,"15min":0.25900000000000001},"max":{"1min":0.82699999999999996,"5min":9.7200000000000006,"15min":9.7200000000000006},"last":0.80300000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757538,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591124,"kb_used_data":1356,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":93154028,"statfs":{"total":95995035648,"available":95389724672,"internally_reserved":0,"allocated":1388544,"data_stored":897937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.1400000000000001,"15min":1.8759999999999999},"min":{"1min":0.28100000000000003,"5min":0.23899999999999999,"15min":0.23499999999999999},"max":{"1min":9.4589999999999996,"5min":13.111000000000001,"15min":13.111000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.716,"15min":1.4279999999999999},"min":{"1min":0.39200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.133,"5min":9.3190000000000008,"15min":9.3190000000000008},"last":0.57199999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:28:15 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.9119999999999999,"15min":1.8009999999999999},"min":{"1min":0.40999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.1680000000000001,"5min":10.788,"15min":10.788},"last":0.68899999999999995},{"interface":"front","average":{"1min":7.883,"5min":3.423,"15min":2.468},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":116.04000000000001,"5min":116.04000000000001,"15min":116.04000000000001},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.55200000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:28:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.627},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.64700000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.139,"15min":0.94999999999999996},"min":{"1min":0.35499999999999998,"5min":0.28100000000000003,"15min":0.23499999999999999},"max":{"1min":0.71699999999999997,"5min":9.641,"15min":9.641},"last":0.502},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.151,"15min":1.0189999999999999},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.245},"max":{"1min":0.73099999999999998,"5min":8.9280000000000008,"15min":11.356},"last":0.59499999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:28:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":1.268,"15min":1.0649999999999999},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.26000000000000001},"max":{"1min":1.649,"5min":9.4019999999999992,"15min":10.31},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.488,"15min":1.238},"min":{"1min":0.44500000000000001,"5min":0.32000000000000001,"15min":0.30399999999999999},"max":{"1min":1.4119999999999999,"5min":14.334,"15min":16.123000000000001},"last":0.66900000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.46,"15min":1.2090000000000001},"min":{"1min":0.48299999999999998,"5min":0.34499999999999997,"15min":0.27400000000000002},"max":{"1min":0.80100000000000005,"5min":9.9000000000000004,"15min":9.9589999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.3460000000000001,"15min":1.089},"min":{"1min":0.502,"5min":0.376,"15min":0.33000000000000002},"max":{"1min":0.86899999999999999,"5min":11.315,"15min":11.315},"last":0.76500000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:29:06.609 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-09T04:29:06.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-09T04:29:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:06 smithi029 ceph-mon[216903]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:06 smithi029 ceph-mon[223858]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:06.976 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:06 smithi177 ceph-mon[195797]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:07.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:07 smithi029 ceph-mon[216903]: from='client.45254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:07.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:07 smithi029 ceph-mon[223858]: from='client.45254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:07 smithi177 ceph-mon[195797]: from='client.45254 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:29:08.913 INFO:teuthology.orchestra.run.smithi029.stdout:epoch 143 2023-10-09T04:29:08.913 INFO:teuthology.orchestra.run.smithi029.stdout:fsid 486585fa-6658-11ee-8db5-212e2dc638e7 2023-10-09T04:29:08.913 INFO:teuthology.orchestra.run.smithi029.stdout:created 2023-10-09T04:00:46.050683+0000 2023-10-09T04:29:08.913 INFO:teuthology.orchestra.run.smithi029.stdout:modified 2023-10-09T04:27:57.969171+0000 2023-10-09T04:29:08.913 INFO:teuthology.orchestra.run.smithi029.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:crush_version 22 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:full_ratio 0.95 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:backfillfull_ratio 0.9 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:nearfull_ratio 0.85 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:require_min_compat_client luminous 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:min_compat_client luminous 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:require_osd_release quincy 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:stretch_mode_enabled false 2023-10-09T04:29:08.914 INFO:teuthology.orchestra.run.smithi029.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 63 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:max_osd 8 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 up in weight 1 up_from 91 up_thru 142 down_at 88 last_clean_interval [8,87) [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403] [v2:172.21.15.29:6804/709667403,v1:172.21.15.29:6805/709667403] exists,up a3e888dc-7381-4dc9-9e9e-ea492336810d 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 up in weight 1 up_from 118 up_thru 142 down_at 115 last_clean_interval [13,114) [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802] [v2:172.21.15.29:6812/1230632802,v1:172.21.15.29:6813/1230632802] exists,up 12e83e56-ffd0-4eb6-bb51-5612a7768792 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 up in weight 1 up_from 135 up_thru 142 down_at 131 last_clean_interval [18,130) [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261] [v2:172.21.15.29:6820/193382261,v1:172.21.15.29:6821/193382261] exists,up 6620b1f9-fce6-4509-8c69-741afdb0332d 2023-10-09T04:29:08.915 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 up in weight 1 up_from 142 up_thru 142 down_at 139 last_clean_interval [25,138) [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922] [v2:172.21.15.29:6828/2977593922,v1:172.21.15.29:6829/2977593922] exists,up 266c755e-25de-4dd5-a2a8-1b67b5fc8e3a 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 up in weight 1 up_from 30 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139] [v2:172.21.15.177:6802/3399759139,v1:172.21.15.177:6803/3399759139] exists,up 4eed67c8-385c-41ab-9afc-a65ad85104d0 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 up in weight 1 up_from 38 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946] [v2:172.21.15.177:6810/3350634946,v1:172.21.15.177:6811/3350634946] exists,up d1c822a8-9ffa-4586-9773-007ad79e1b5f 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 up in weight 1 up_from 46 up_thru 139 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886] [v2:172.21.15.177:6818/4114806886,v1:172.21.15.177:6819/4114806886] exists,up 41a89acf-f32f-41bf-9eb6-5a6edeea9043 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 up in weight 1 up_from 54 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577] [v2:172.21.15.177:6826/3009136577,v1:172.21.15.177:6827/3009136577] exists,up cf54f5e9-1efd-44de-968c-59accfa44021 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:pg_upmap_items 4.a [1,2] 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:pg_upmap_items 4.b [1,5] 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:pg_upmap_items 4.18 [1,2] 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3682933143 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/1003732865 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.916 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/1003732865 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1432692814 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1371585021 expires 2023-10-10T04:11:25.861678+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.177:0/3560371706 expires 2023-10-10T04:12:57.728727+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/2869772012 expires 2023-10-10T04:01:15.592153+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3925658916 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1647324901 expires 2023-10-10T04:01:43.065145+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1481182 expires 2023-10-10T04:06:26.739366+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2363485910 expires 2023-10-10T04:01:43.065145+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.177:0/2501999039 expires 2023-10-10T04:12:57.728727+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1007503695 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1305222045 expires 2023-10-10T04:06:26.739366+0000 2023-10-09T04:29:08.917 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/817958524 expires 2023-10-10T04:01:15.592153+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3506139723 expires 2023-10-10T04:11:25.861678+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/2869772012 expires 2023-10-10T04:01:15.592153+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/613592146 expires 2023-10-10T04:01:15.592153+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.177:6834/1319489652 expires 2023-10-10T04:12:57.728727+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/673424689 expires 2023-10-10T04:01:43.065145+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/4232895263 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/3407458368 expires 2023-10-10T04:11:25.861678+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/4212129823 expires 2023-10-10T04:06:26.739366+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6800/3451944530 expires 2023-10-10T04:01:43.065145+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2321967128 expires 2023-10-10T04:11:25.861678+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/2548128137 expires 2023-10-10T04:06:26.739366+0000 2023-10-09T04:29:08.918 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/4212129823 expires 2023-10-10T04:06:26.739366+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1787157051 expires 2023-10-10T04:11:25.861678+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.177:0/1665676424 expires 2023-10-10T04:12:57.728727+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/3335564331 expires 2023-10-10T04:06:26.739366+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/3407458368 expires 2023-10-10T04:11:25.861678+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/40150084 expires 2023-10-10T04:01:15.592153+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:6801/3451944530 expires 2023-10-10T04:01:43.065145+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.29:0/1566968162 expires 2023-10-10T04:13:58.298189+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.177:0/63096837 expires 2023-10-10T04:12:57.728727+0000 2023-10-09T04:29:08.919 INFO:teuthology.orchestra.run.smithi029.stdout:blocklist 172.21.15.177:6835/1319489652 expires 2023-10-10T04:12:57.728727+0000 2023-10-09T04:29:08.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:08 smithi029 ceph-mon[216903]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:08 smithi029 ceph-mon[223858]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:08 smithi177 ceph-mon[195797]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:09.810 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 91 up_thru 142 down_at 88 last_clean_interval [8,87) [v2:172.21.15.29:6802/709667403,v1:172.21.15.29:6803/709667403] [v2:172.21.15.29:6804/709667403,v1:172.21.15.29:6805/709667403] exists,up a3e888dc-7381-4dc9-9e9e-ea492336810d', 'osd.1 up in weight 1 up_from 118 up_thru 142 down_at 115 last_clean_interval [13,114) [v2:172.21.15.29:6810/1230632802,v1:172.21.15.29:6811/1230632802] [v2:172.21.15.29:6812/1230632802,v1:172.21.15.29:6813/1230632802] exists,up 12e83e56-ffd0-4eb6-bb51-5612a7768792', 'osd.2 up in weight 1 up_from 135 up_thru 142 down_at 131 last_clean_interval [18,130) [v2:172.21.15.29:6818/193382261,v1:172.21.15.29:6819/193382261] [v2:172.21.15.29:6820/193382261,v1:172.21.15.29:6821/193382261] exists,up 6620b1f9-fce6-4509-8c69-741afdb0332d', 'osd.3 up in weight 1 up_from 142 up_thru 142 down_at 139 last_clean_interval [25,138) [v2:172.21.15.29:6826/2977593922,v1:172.21.15.29:6827/2977593922] [v2:172.21.15.29:6828/2977593922,v1:172.21.15.29:6829/2977593922] exists,up 266c755e-25de-4dd5-a2a8-1b67b5fc8e3a', 'osd.4 up in weight 1 up_from 30 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6800/3399759139,v1:172.21.15.177:6801/3399759139] [v2:172.21.15.177:6802/3399759139,v1:172.21.15.177:6803/3399759139] exists,up 4eed67c8-385c-41ab-9afc-a65ad85104d0', 'osd.5 up in weight 1 up_from 38 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6808/3350634946,v1:172.21.15.177:6809/3350634946] [v2:172.21.15.177:6810/3350634946,v1:172.21.15.177:6811/3350634946] exists,up d1c822a8-9ffa-4586-9773-007ad79e1b5f', 'osd.6 up in weight 1 up_from 46 up_thru 139 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6816/4114806886,v1:172.21.15.177:6817/4114806886] [v2:172.21.15.177:6818/4114806886,v1:172.21.15.177:6819/4114806886] exists,up 41a89acf-f32f-41bf-9eb6-5a6edeea9043', 'osd.7 up in weight 1 up_from 54 up_thru 142 down_at 0 last_clean_interval [0,0) [v2:172.21.15.177:6824/3009136577,v1:172.21.15.177:6825/3009136577] [v2:172.21.15.177:6826/3009136577,v1:172.21.15.177:6827/3009136577] exists,up cf54f5e9-1efd-44de-968c-59accfa44021'] 2023-10-09T04:29:09.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-09T04:29:09.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/815856085' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:29:09.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/815856085' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:29:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/815856085' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T04:29:10.190 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:10.797 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:10 smithi029 ceph-mon[216903]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:10.797 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:10 smithi029 ceph-mon[223858]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:10 smithi177 ceph-mon[195797]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:11.814 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "name": "mon.a", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster": "ceph", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "admin_socket_mode": "", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_client_required": "cephx, none", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_cluster_required": "cephx", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_debug": "true", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_service_required": "cephx", 2023-10-09T04:29:11.815 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "auth_supported": "", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio": "true", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio_poll_ms": "250", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_aio_reap_max": "16", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_async_discard": "false", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_block_size": "4096", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_aio": "false", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_debug_inflight_ios": "false", 2023-10-09T04:29:11.816 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_enable_discard": "false", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_flock_retry": "3", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_inject_crash": "0", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_ioring": "false", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_ioring_hipri": "false", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bdev_type": "", 2023-10-09T04:29:11.817 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_alloc_size": "1048576", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_allocator": "hybrid", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_buffered_io": "true", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_check_for_zeros": "false", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_compact_log_sync": "false", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_max_log_runway": "4194304", 2023-10-09T04:29:11.818 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_max_prefetch": "1048576", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_min_flush_size": "524288", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_min_log_runway": "1048576", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_replay_recovery": "false", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluefs_sync_write": "false", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_allocation_from_file": "true", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_allocator": "hybrid", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-09T04:29:11.819 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_create": "true", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_db_create": "false", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_db_path": "", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_db_size": "0", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_path": "", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_preallocate_file": "false", 2023-10-09T04:29:11.820 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_size": "107374182400", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_wal_create": "false", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_wal_path": "", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_block_wal_size": "100663296", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs": "true", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-09T04:29:11.821 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_autotune": "true", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_size": "0", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cache_type": "2q", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-09T04:29:11.822 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_clone_cow": "true", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_mode": "none", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_csum_type": "crc32c", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-09T04:29:11.823 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_freelist": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-09T04:29:11.824 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_small_allocations": "0", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_default_buffered_read": "true", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_default_buffered_write": "false", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-09T04:29:11.825 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fail_eio": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mount": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_umount": "false", 2023-10-09T04:29:11.826 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_ignore_data_csum": "false", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-09T04:29:11.827 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_log_op_age": "5.000000", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_alloc_size": "0", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_blob_size": "0", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_max_deferred_txc": "32", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_min_alloc_size": "0", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_nid_prealloc": "1024", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-09T04:29:11.828 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_retry_disk_reads": "3", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_cf": "true", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_spdk_mem": "512", 2023-10-09T04:29:11.829 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_tracing": "false", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-09T04:29:11.830 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "bluestore_zero_block_detection": "false", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-09T04:29:11.831 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_cluster_require_signatures": "false", 2023-10-09T04:29:11.832 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_cluster_require_version": "2", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_require_signatures": "false", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_require_version": "2", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_service_require_signatures": "false", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_service_require_version": "2", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "cephx_sign_messages": "true", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "chdir": "", 2023-10-09T04:29:11.833 INFO:teuthology.orchestra.run.smithi029.stdout: "client_acl_type": "", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_asio_thread_count": "2", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_cache_mid": "0.750000", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_cache_size": "16384", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_caps_release_delay": "5", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_check_pool_perm": "true", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_debug_force_sync_read": "false", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_debug_getattr_caps": "false", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_debug_inject_tick_delay": "0", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_die_on_failed_remount": "false", 2023-10-09T04:29:11.834 INFO:teuthology.orchestra.run.smithi029.stdout: "client_dirsize_rbytes": "true", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_force_lazyio": "false", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_fs": "", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_inject_release_failure": "false", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_max_inline_size": "4096", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mds_namespace": "", 2023-10-09T04:29:11.835 INFO:teuthology.orchestra.run.smithi029.stdout: "client_metadata": "", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mount_gid": "-1", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mount_timeout": "300", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mount_uid": "-1", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_mountpoint": "/", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_notify_timeout": "10", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc": "true", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_max_dirty": "104857600", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_max_objects": "1000", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_size": "209715200", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_oc_target_dirty": "8388608", 2023-10-09T04:29:11.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_permissions": "true", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_quota": "true", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_quota_df": "true", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_readahead_max_bytes": "0", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_readahead_max_periods": "4", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_readahead_min": "131072", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_reconnect_stale": "false", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_shutdown_timeout": "30", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_snapdir": ".snap", 2023-10-09T04:29:11.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_tick_interval": "1", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client_trace": "", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client_try_dentry_invalidate": "false", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client_use_faked_inos": "false", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "client_use_random_mds": "false", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_graylog": "false", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_graylog_port": "12201", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_monitors": "default=true", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_syslog": "false", 2023-10-09T04:29:11.838 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "clog_to_syslog_level": "info", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster_addr": "-", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster_network": "", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "cluster_network_interface": "", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zlib_isal": "false", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zlib_level": "5", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zlib_winsize": "-15", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "compressor_zstd_level": "1", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:f1b604d5f2ca70c029abed5d4751ce47aa1c979454bd17275a62f3ffa2ac6e8c", 2023-10-09T04:29:11.839 INFO:teuthology.orchestra.run.smithi029.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_alien_op_num_threads": "6", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crimson_seastar_smp": "1", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crush_location": "", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crush_location_hook": "", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "crush_location_hook_timeout": "10", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "daemonize": "false", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "daos_pool": "tank", 2023-10-09T04:29:11.840 INFO:teuthology.orchestra.run.smithi029.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_alienstore": "0/5", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_allow_any_pool_priority": "false", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_asok": "1/5", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_asok_assert_abort": "false", 2023-10-09T04:29:11.841 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_asserts_on_shutdown": "true", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_auth": "1/5", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_bdev": "1/3", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_bluefs": "1/5", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_bluestore": "1/5", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_buffer": "0/1", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_ceph_exporter": "1/5", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_cephfs_mirror": "0/5", 2023-10-09T04:29:11.842 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_cephsqlite": "0/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_client": "0/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_compressor": "1/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_context": "0/1", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_crush": "1/1", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_crypto": "1/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_cyanstore": "0/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_deliberately_leak_memory": "false", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_disable_randomized_ping": "false", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_dpdk": "1/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_eventtrace": "1/5", 2023-10-09T04:29:11.843 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_filer": "0/1", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_filestore": "1/3", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_finisher": "1/1", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_fuse": "1/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_heartbeat_testing_span": "0", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_heartbeatmap": "1/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_javaclient": "1/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_journal": "1/3", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_journaler": "0/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_kstore": "1/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_leveldb": "4/5", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_lockdep": "0/1", 2023-10-09T04:29:11.844 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mclock": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_balancer": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_locker": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_log": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_log_expire": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mds_migrator": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_memstore": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mgr": "2/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mgrc": "1/5", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_mon": "20/20", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_monc": "0/10", 2023-10-09T04:29:11.845 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_ms": "1/1", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_none": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_objclass": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_objectcacher": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_objecter": "0/1", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_optracker": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_osd": "1/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_paxos": "20/20", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_perfcounter": "1/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_prioritycache": "1/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rados": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd_mirror": "0/5", 2023-10-09T04:29:11.846 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd_pwl": "0/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rbd_replay": "0/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_refs": "0/0", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_reserver": "1/1", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw": "1/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_access": "1/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_datacache": "1/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_dbstore": "1/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_flight": "1/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rgw_sync": "1/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_rocksdb": "4/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore": "0/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_backref": "0/5", 2023-10-09T04:29:11.847 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_cache": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_cleaner": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_device": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_epm": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_journal": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_lba": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_odata": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_omap": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_onode": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_t": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_seastore_tm": "0/5", 2023-10-09T04:29:11.848 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_striper": "0/1", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_test": "0/5", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_throttle": "1/1", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_timer": "0/1", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "debug_tp": "0/5", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "device_failure_prediction_mode": "none", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_graylog": "false", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_journald": "false", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_stderr": "true", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "err_to_syslog": "false", 2023-10-09T04:29:11.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event_tracing": "false", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_addr": "0.0.0.0", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_http_port": "9926", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_prio_limit": "5", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_sort_metrics": "true", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "exporter_stats_period": "5", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "fake_statfs_for_testing": "0", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "fatal_signal_handlers": "true", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "filer_max_purge_ops": "10", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "filer_max_truncate_ops": "128", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_apply_finisher_threads": "1", 2023-10-09T04:29:11.850 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_blackhole": "false", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_btrfs_clone_range": "true", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_btrfs_snap": "true", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_caller_concurrency": "10", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_collect_device_partition_information": "true", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_commit_timeout": "600.000000", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_inject_read_err": "false", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_omap_check": "false", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_debug_verify_split": "false", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_dump_file": "", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-09T04:29:11.851 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fadvise": "true", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fail_eio": "true", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fd_cache_shards": "16", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fd_cache_size": "128", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fiemap": "false", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fiemap_threshold": "4096", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_inject_stall": "0", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_journal_parallel": "false", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_journal_trailing": "false", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_journal_writeahead": "false", 2023-10-09T04:29:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_kill_at": "0", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs": "0", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size": "0", 2023-10-09T04:29:11.853 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_merge_threshold": "-10", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_odsync_write": "false", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_omap_backend": "rocksdb", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_omap_backend_path": "", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_op_thread_timeout": "60", 2023-10-09T04:29:11.854 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_op_threads": "2", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_punch_hole": "false", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_queue_max_ops": "50", 2023-10-09T04:29:11.855 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_seek_data_hole": "false", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_sloppy_crc": "false", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_splice": "false", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_split_multiple": "2", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_split_rand_factor": "20", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_update_to": "1000", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-09T04:29:11.856 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_enable": "true", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_xfs_extsize": "false", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "filestore_zfs_snap": "false", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "fio_dir": "/tmp/fio", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "fsid": "486585fa-6658-11ee-8db5-212e2dc638e7", 2023-10-09T04:29:11.857 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_allow_other": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_atomic_o_trunc": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_big_writes": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_debug": "false", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_default_permissions": "false", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_disable_pagecache": "false", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_max_write": "0", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_multithreaded": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_require_active_mds": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_set_user_groups": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_splice_move": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_splice_read": "true", 2023-10-09T04:29:11.858 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_splice_write": "true", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "fuse_use_invalidate_cb": "true", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "gss_target_name": "ceph", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "heartbeat_file": "", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "heartbeat_inject_failure": "0", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "heartbeat_interval": "5", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "host": "smithi029", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-09T04:29:11.859 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "inject_early_sigterm": "false", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "jaeger_agent_port": "6799", 2023-10-09T04:29:11.860 INFO:teuthology.orchestra.run.smithi029.stdout: "jaeger_tracing_enable": "false", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_aio": "true", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_align_min_size": "65536", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_block_align": "true", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_block_size": "4096", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_dio": "true", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_discard": "false", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_force_aio": "false", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_ignore_corruption": "false", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_max_write_bytes": "10485760", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_max_write_entries": "100", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_replay_from": "0", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-09T04:29:11.861 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_write_header_frequency": "0", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journal_zero_on_create": "false", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journaler_prefetch_periods": "10", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journaler_prezero_periods": "5", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "journaler_write_head_interval": "15", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "key": "", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "keyfile": "", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-09T04:29:11.862 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_backend": "rocksdb", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_default_stripe_size": "65536", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_fsck_on_mount": "false", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_max_bytes": "67108864", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_max_ops": "512", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_nid_prealloc": "1024", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_onode_map_size": "1024", 2023-10-09T04:29:11.863 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_sync_submit_transaction": "false", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "kstore_sync_transaction": "false", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_block_size": "65536", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_bloom_size": "0", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_cache_size": "536870912", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_compact_on_mount": "false", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_compression": "false", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_log": "", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_max_open_files": "0", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_paranoid": "false", 2023-10-09T04:29:11.864 INFO:teuthology.orchestra.run.smithi029.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "librados_thread_count": "2", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "lockdep": "false", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "lockdep_force_backtrace": "false", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "log_coarse_timestamps": "true", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "log_flush_on_exit": "false", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "log_graylog_host": "127.0.0.1", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "log_graylog_port": "12201", 2023-10-09T04:29:11.865 INFO:teuthology.orchestra.run.smithi029.stdout: "log_max_new": "1000", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_max_recent": "10000", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_stderr_prefix": "", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_stop_at_utilization": "0.970000", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_file": "true", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_graylog": "false", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_journald": "false", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_stderr": "false", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "log_to_syslog": "false", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "max_rotating_auth_attempts": "10", 2023-10-09T04:29:11.866 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_action_on_write_error": "1", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_alternate_name_max": "8192", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_asio_thread_count": "2", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_export_pin": "true", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_dirs": "true", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_interval": "5", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_interval": "10", 2023-10-09T04:29:11.867 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_max": "-1", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_max_until": "-1", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_merge_size": "50", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_midchunk": "0.300000", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_min_start": "0.200000", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_minchunk": "0.001000", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_mode": "0", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_need_max": "1.200000", 2023-10-09T04:29:11.868 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_need_min": "0.800000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_bits": "3", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_size": "10000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_target_decay": "10.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_beacon_grace": "15.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_beacon_interval": "4.000000", 2023-10-09T04:29:11.869 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_mid": "0.700000", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_release_free_interval": "10", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_reservation": "0.050000", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_trim_interval": "1", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cache_trim_threshold": "262144", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-09T04:29:11.870 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_client_prealloc_inos": "1000", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_connect_bootstrapping": "false", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_damage_table_max_entries": "10000", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_auth_pins": "false", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_frag": "false", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_scatterstat": "false", 2023-10-09T04:29:11.871 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_debug_subtrees": "false", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_decay_halflife": "5.000000", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_default_dir_hash": "2", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_defer_session_stale": "true", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_deny_all_reconnect": "false", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_keys_per_op": "16384", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_max_commit_size": "10", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_max_entries": "0", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dir_prefetch": "true", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-09T04:29:11.872 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_on_map": "false", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_early_reply": "true", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_enable_op_tracker": "true", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_enforce_unique_name": "true", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_random": "true", 2023-10-09T04:29:11.873 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_health_summarize_threshold": "10", 2023-10-09T04:29:11.874 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_health_dummy": "false", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_migrator_session_race": "false", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_join_fs": "", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_journal_format": "1", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_create_at": "0", 2023-10-09T04:29:11.875 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_export_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_import_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_journal_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_journal_expire_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_journal_replay_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_link_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_mdstable_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_openc_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_rename_at": "0", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_events_per_segment": "1024", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_max_events": "-1", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_max_segments": "128", 2023-10-09T04:29:11.876 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_pause": "false", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_segment_size": "0", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_log_warn_factor": "2.000000", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_caps_per_client": "1048576", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_completed_flushes": "100000", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_completed_requests": "100000", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_export_size": "20971520", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_file_recover": "32", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_purge_files": "64", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_purge_ops": "8192", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-09T04:29:11.877 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_snaps_per_dir": "100", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_metrics_update_interval": "2", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_min_caps_per_client": "100", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_min_caps_working_set": "10000", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_numa_node": "-1", 2023-10-09T04:29:11.878 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_complaint_time": "30.000000", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_history_duration": "600", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_history_size": "20", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_op_log_threshold": "5", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_ping_grace": "15", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_ping_interval": "5", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_max_caps": "30000", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-09T04:29:11.879 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_recall_warning_threshold": "262144", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_replay_interval": "1.000000", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_root_ino_gid": "0", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_root_ino_uid": "0", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-09T04:29:11.880 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_shutdown_check": "0", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_skip_ino": "0", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_snap_max_uid": "4294967294", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_snap_min_uid": "0", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_snap_rstat": "false", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_standby_replay_damaged": "false", 2023-10-09T04:29:11.881 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_symlink_recovery": "true", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_thrash_exports": "0", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_thrash_fragments": "0", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_tick_interval": "5.000000", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_valgrind_exit": "false", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_verify_backtrace": "1", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_verify_scatter": "false", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mds_wipe_sessions": "false", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "mempool_debug": "false", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-09T04:29:11.882 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_device_bytes": "1073741824", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_page_set": "false", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "memstore_page_size": "65536", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_client_bytes": "134217728", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_client_messages": "512", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_disabled_modules": "", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_max_pg_num_change": "128", 2023-10-09T04:29:11.883 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mds_bytes": "134217728", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mds_messages": "128", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mon_bytes": "134217728", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_mon_messages": "128", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_osd_bytes": "536870912", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_osd_messages": "8192", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_pool": "true", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_standby_modules": "true", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_stats_period": "5", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_stats_threshold": "5", 2023-10-09T04:29:11.884 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_tick_period": "2", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_allow_pool_delete": "true", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_allow_pool_size_one": "false", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_auth_validate_all_caps": "true", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_bytes": "104857600", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_directed_command_retry": "2", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-09T04:29:11.885 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_hunt_parallel": "3", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_log_interval": "1.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_ping_interval": "10.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_file_level": "debug", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_file": "false", 2023-10-09T04:29:11.886 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_journald": "true", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_compact_on_bootstrap": "false", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_compact_on_start": "false", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_compact_on_trim": "true", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-09T04:29:11.887 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_cpu_threads": "4", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_crush_min_required_version": "hammer", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_daemon_bytes": "419430400", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data_avail_crit": "5", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data_avail_warn": "5", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_data_size_warn": "16106127360", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_dump_json": "false", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-09T04:29:11.888 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_dump_transactions": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_extra_checks": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_require_quincy": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_no_require_reef": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_down_mkfs_grace": "60", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_election_default_strategy": "1", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_election_timeout": "5.000000", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-09T04:29:11.889 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_elector_ping_divisor": "2", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_enable_op_tracker": "true", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_fake_pool_delete": "false", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_force_quorum_join": "false", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_globalid_prealloc": "10000", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_detail_to_clog": "true", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_log_update_period": "5", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_max_detail": "50", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_to_clog": "true", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_to_clog_interval": "600", 2023-10-09T04:29:11.890 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_host": "[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_host_override": "", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_initial_members": "", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_lease": "5.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-09T04:29:11.891 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_log_full_interval": "50", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_log_max": "10000", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_log_max_summary": "50", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_log_epochs": "500", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_osd": "10000", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_pg_per_osd": "10000", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_pool_pg_num": "65536", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-09T04:29:11.892 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mds_force_trim_to": "0", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mds_skip_sanity": "true", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_memory_autotune": "true", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_memory_target": "2147483648", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_beacon_grace": "30", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_digest_period": "5", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_inactive_grace": "60", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_min_osdmap_epochs": "500", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_complaint_time": "30", 2023-10-09T04:29:11.893 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_duration": "600", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_size": "20", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_slow_op_size": "20", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_op_log_threshold": "5", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_auto_mark_in": "false", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-09T04:29:11.894 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_cache_size": "500", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_down_out_interval": "600", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_force_trim_to": "0", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-09T04:29:11.895 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_min_down_reporters": "2", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_report_timeout": "900", 2023-10-09T04:29:11.896 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_stuck_threshold": "60", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-09T04:29:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_probe_timeout": "2.000000", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_max_change": "0.050000", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_max_osds": "4", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_interval": "86400", 2023-10-09T04:29:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_max_keys": "100", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_scrub_timeout": "300", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_session_timeout": "300", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_smart_report_timeout": "5", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stat_smooth_intervals": "6", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_pool_min_size": "2", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_pool_size": "4", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-09T04:29:11.899 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_debug": "false", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_provider_kill_at": "0", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_requester_kill_at": "0", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_sync_timeout": "60.000000", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_target_pg_per_osd": "100", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_tick_interval": "5", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_timecheck_interval": "300.000000", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_use_min_delay_socket": "false", 2023-10-09T04:29:11.900 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_older_version_delay": "604800", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-09T04:29:11.901 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_misplaced": "false", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_older_version": "true", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-09T04:29:11.902 INFO:teuthology.orchestra.run.smithi029.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "monmap": "", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "motr_tracing_enabled": "false", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_op_threads": "3", 2023-10-09T04:29:11.903 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_cm": "false", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_device_name": "", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_dscp": "96", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_local_gid": "", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_port_num": "1", 2023-10-09T04:29:11.904 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_sl": "3", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_support_srq": "true", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_rdma_type": "ib", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_async_reap_threshold": "5", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_before_connect": "false", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_ipv4": "true", 2023-10-09T04:29:11.905 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_ipv6": "false", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_msgr1": "true", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_msgr2": "true", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_port_max": "7568", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_port_min": "6800", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_retry_count": "3", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_bind_retry_delay": "5", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_client": "false", 2023-10-09T04:29:11.906 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_mds": "false", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_mgr": "false", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_mon": "false", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_blackhole_osd": "false", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_client_mode": "crc secure", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_cluster_mode": "crc secure", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_cluster_type": "", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_compress_secure": "false", 2023-10-09T04:29:11.907 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_connection_idle_timeout": "900", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_connection_ready_timeout": "10", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_crc_data": "true", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_crc_header": "true", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_bad_msg": "false", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_bug": "true", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_old_message": "true", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_skipped_message": "false", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_coremask": "0xF", 2023-10-09T04:29:11.908 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_enable_tso": "true", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_hugepages": "", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_lro": "true", 2023-10-09T04:29:11.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_memory_channel": "4", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_pmd": "", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_port_id": "0", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_dump_on_send": "false", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_initial_backoff": "0.200000", 2023-10-09T04:29:11.910 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_delay_max": "1.000000", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_delay_type": "", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_network_congestion": "0", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_inject_socket_failures": "0", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_learn_addr_from_peer": "true", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_max_accept_failures": "4", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_max_backoff": "15.000000", 2023-10-09T04:29:11.911 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_mon_client_mode": "secure crc", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_mon_service_mode": "secure crc", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_osd_compress_min_size": "1024", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_osd_compress_mode": "none", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_pq_min_cost": "65536", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_public_type": "", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_service_mode": "crc secure", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_listen_backlog": "512", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_nodelay": "true", 2023-10-09T04:29:11.912 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_tcp_rcvbuf": "0", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "ms_type": "async+posix", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "no_config_file": "false", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_completion_locks_per_session": "32", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_inflight_ops": "1024", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_tick_interval": "5.000000", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objecter_timeout": "10.000000", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "objectstore_blackhole": "false", 2023-10-09T04:29:11.913 INFO:teuthology.orchestra.run.smithi029.stdout: "openssl_engine_opts": "", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_delay_time": "5.000000", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_hist_halflife": "1000", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_max_low_ops": "2", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_max_ops": "4", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_agent_slop": "0.020000", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_asio_thread_count": "2", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_async_recovery_min_cost": "100", 2023-10-09T04:29:11.914 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backfill_scan_max": "512", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backfill_scan_min": "64", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backoff_on_degraded": "false", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backoff_on_peering": "false", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_backoff_on_unfound": "true", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_beacon_report_interval": "300", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_duration": "30", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_max_block_size": "67108864", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-09T04:29:11.915 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_blkin_trace_all": "false", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_check_for_log_corruption": "false", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_class_update_on_start": "true", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_message_cap": "256", 2023-10-09T04:29:11.916 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_message_size_cap": "524288000", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_op_priority": "63", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_client_watch_timeout": "30", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_command_max_records": "256", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_compact_on_start": "false", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_update_on_start": "true", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_crush_update_weight_set": "true", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-09T04:29:11.917 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_feed_pullee": "-1", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_misdirected_ops": "false", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_no_acting_change": "false", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_no_purge_strays": "false", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_op_order": "false", 2023-10-09T04:29:11.918 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_shutdown": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_snaps": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-09T04:29:11.919 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_keys": "1024", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_stride": "524288", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_default_notify_timeout": "30", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep": "0.000000", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-09T04:29:11.920 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_discard_disconnected_ops": "true", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_enable_op_tracker": "true", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_info": "true", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_shutdown": "true", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-09T04:29:11.921 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_function_tracing": "false", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_grace": "20", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_interval": "6", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_min_peers": "10", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_min_size": "2000", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_stale": "600", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_hit_set_max_size": "100000", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_hit_set_min_size": "1000", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-09T04:29:11.922 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_journal_size": "5120", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_kill_backfill_at": "0", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_cache_size": "50", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_dedup": "true", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_message_max": "40", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_map_share_max_epochs": "40", 2023-10-09T04:29:11.923 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_attr_name_len": "100", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_attr_size": "0", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_backfills": "1", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_markdown_count": "5", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_markdown_period": "600", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_object_name_len": "2048", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_object_namespace_len": "256", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_object_size": "134217728", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pg_blocked_by": "16", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pg_log_entries": "10000", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-09T04:29:11.924 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_pgls": "1024", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_push_cost": "8388608", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_push_objects": "10", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_scrubs": "1", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_trimming_pgs": "2", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_write_op_reply_len": "64", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_max_write_size": "90", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-09T04:29:11.925 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_profile": "balanced", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-09T04:29:11.926 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_base": "805306368", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_cache_min": "134217728", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_target": "4294967296", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_target_autotune": "false", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-09T04:29:11.927 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_min_pg_log_entries": "250", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_min_recovery_priority": "0", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_report_interval": "5", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_num_cache_shards": "32", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_num_op_tracker_shard": "32", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_numa_auto_affinity": "true", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_numa_node": "-1", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_numa_prefer_iface": "true", 2023-10-09T04:29:11.928 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objecter_finishers": "1", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objectstore": "bluestore", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objectstore_fuse": "false", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_objectstore_tracing": "false", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_complaint_time": "30.000000", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_duration": "600", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_size": "20", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_slow_op_size": "20", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_log_threshold": "5", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_shards": "0", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_shards_hdd": "5", 2023-10-09T04:29:11.929 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_shards_ssd": "8", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_pq_min_cost": "65536", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_queue_cut_off": "high", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_op_thread_timeout": "15", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_open_classes_on_start": "true", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_os_flags": "0", 2023-10-09T04:29:11.930 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_peering_op_priority": "255", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_delete_cost": "1048576", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_delete_priority": "5", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_log_trim_max": "10000", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_log_trim_min": "100", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-09T04:29:11.931 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_crimson": "false", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-09T04:29:11.932 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_flags": "0", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_min_size": "0", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_pg_num": "32", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_pgp_num": "0", 2023-10-09T04:29:11.933 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_size": "3", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_default_type": "replicated", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_push_per_object_cost": "1000", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recover_clone_overlap": "true", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-09T04:29:11.934 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_cost": "20971520", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_active": "0", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_max_single_start": "1", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_op_priority": "3", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_priority": "5", 2023-10-09T04:29:11.935 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep": "0.000000", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_repair_during_recovery": "false", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_requested_scrub_priority": "120", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-09T04:29:11.936 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_auto_repair": "false", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_begin_hour": "0", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_begin_week_day": "0", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_chunk_max": "25", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_chunk_min": "5", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_cost": "52428800", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_during_recovery": "false", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_end_hour": "0", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_end_week_day": "0", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_event_cost": "4096", 2023-10-09T04:29:11.937 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_invalid_stats": "true", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_max_preemptions": "5", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_priority": "5", 2023-10-09T04:29:11.938 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_sleep": "0.000000", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_skip_data_digest": "false", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_smart_report_timeout": "5", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_cost": "1048576", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_priority": "5", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-09T04:29:11.939 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_target_transaction_size": "30", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-09T04:29:11.940 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_tracing": "false", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_use_stale_snap": "false", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-09T04:29:11.941 INFO:teuthology.orchestra.run.smithi029.stdout: "osdc_blkin_trace_all": "false", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_kill_at": "0", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_max_join_drift": "10", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_min": "500", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_min_wait": "0.050000", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_propose_interval": "1.000000", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_service_trim_max": "500", 2023-10-09T04:29:11.942 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_service_trim_min": "250", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_stash_full_interval": "25", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_trim_max": "500", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "paxos_trim_min": "250", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "perf": "true", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "pid_file": "", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-09T04:29:11.943 INFO:teuthology.orchestra.run.smithi029.stdout: "public_addr": "-", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "public_addrv": "", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "public_bind_addr": "-", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "public_network": "172.21.0.0/20", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "public_network_interface": "", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "qat_compressor_enabled": "false", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "qat_compressor_session_max_number": "256", 2023-10-09T04:29:11.944 INFO:teuthology.orchestra.run.smithi029.stdout: "rados_mon_op_timeout": "0", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rados_osd_op_timeout": "0", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rados_tracing": "false", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_atime_update_interval": "60", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_balance_parent_reads": "false", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_balance_snap_reads": "false", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_blkin_trace_all": "false", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache": "true", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-09T04:29:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_policy": "writearound", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_size": "33554432", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_clone_copy_on_read": "false", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_compression_hint": "none", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_concurrent_management_ops": "10", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_clone_format": "auto", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_data_pool": "", 2023-10-09T04:29:11.946 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_features": "61", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_format": "2", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_map_options": "", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_order": "22", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_pool": "rbd", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_stripe_count": "0", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_default_stripe_unit": "0", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_enable_alloc_hint": "true", 2023-10-09T04:29:11.947 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_io_scheduler": "simple", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_order": "24", 2023-10-09T04:29:11.948 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_pool": "", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_journal_splay_width": "4", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_localize_parent_reads": "false", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_localize_snap_reads": "false", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-09T04:29:11.949 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-09T04:29:11.950 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_mtime_update_interval": "60", 2023-10-09T04:29:11.951 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_non_blocking_aio": "true", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_op_thread_timeout": "60", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_op_threads": "1", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_parent_cache_enabled": "false", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_plugins": "", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_bps_burst": "0", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_bps_limit": "0", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_exclude_ops": "0", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_iops_burst": "0", 2023-10-09T04:29:11.952 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_iops_limit": "0", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-09T04:29:11.953 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_read_from_replica_policy": "default", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_skip_partial_discard": "true", 2023-10-09T04:29:11.954 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_tracing": "false", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_validate_names": "true", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rbd_validate_pool": "true", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "restapi_base_url": "", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "restapi_log_level": "", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_acl_grants_max_num": "100", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_admin_entry": "admin", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_backend_store": "rados", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_barbican_url": "", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_beast_enable_async": "true", 2023-10-09T04:29:11.955 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cache_enabled": "true", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cache_expiry_interval": "900", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cache_lru_size": "10000", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_config_store": "rados", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_content_length_compat": "false", 2023-10-09T04:29:11.956 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_copy_obj_progress": "true", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cors_rules_max_num": "100", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_cross_domain_policy": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_addr": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_password": "", 2023-10-09T04:29:11.957 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_kmip_username": "", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_require_ssl": "true", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-09T04:29:11.958 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_addr": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_auth": "token", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_namespace": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_prefix": "", 2023-10-09T04:29:11.959 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_token_file": "", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_buffersize": "524288", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_low_speed_time": "300", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-09T04:29:11.960 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_changes_size": "1000", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_num_shards": "128", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_log_window": "30", 2023-10-09T04:29:11.961 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-09T04:29:11.962 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dns_name": "", 2023-10-09T04:29:11.963 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dns_s3website_name": "", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_dynamic_resharding": "true", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_gc_threads": "true", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_lc_threads": "true", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_ops_log": "false", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_quota_threads": "true", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_static_website": "false", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enable_usage_log": "false", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_enforce_swift_acls": "true", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_exit_timeout_secs": "120", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_expose_bucket": "false", 2023-10-09T04:29:11.964 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_extended_http_attrs": "", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_filter": "none", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_frontends": "beast port=7480", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_deferred": "50", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_objs": "32", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_gc_processor_period": "3600", 2023-10-09T04:29:11.965 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_init_timeout": "300", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_domain": "", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_password": "", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_password_path": "", 2023-10-09T04:29:11.966 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_project": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_tenant": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_token": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_token_path": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_admin_user": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_api_version": "2", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_domain": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_password": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_project": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_barbican_user": "", 2023-10-09T04:29:11.967 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_url": "", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_debug_interval": "-1", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_lock_max_time": "90", 2023-10-09T04:29:11.968 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_objs": "32", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_rules": "1000", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_worker": "3", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lc_thread_delay": "0", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_dnattr": "uid", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_searchfilter": "", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-09T04:29:11.969 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_http_headers": "", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_log_object_name_utc": "false", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-09T04:29:11.970 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_attr_name_len": "0", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_attr_size": "0", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_chunk_size": "4194304", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_listing_results": "1000", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_notify_retries": "10", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_put_param_size": "1048576", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_put_size": "5368709120", 2023-10-09T04:29:11.971 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_max_slo_entries": "1000", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_md_log_max_shards": "64", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_mp_lock_max_time": "600", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-09T04:29:11.972 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_max_gc": "300", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-09T04:29:11.973 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_num_async_rados_threads": "32", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_num_control_oids": "8", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_numa_node": "-1", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-09T04:29:11.974 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_objexp_chunk_size": "100", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_objexp_gc_interval": "600", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_op_thread_timeout": "600", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_op_tracing": "false", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_opa_token": "", 2023-10-09T04:29:11.977 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_opa_url": "", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_opa_verify_ssl": "true", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_rados": "false", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_ops_log_socket_path": "", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_push_interval": "2.000000", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-09T04:29:11.978 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_print_continue": "true", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rados_tracing": "false", 2023-10-09T04:29:11.979 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_realm": "", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_realm_id": "", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_region": "", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-09T04:29:11.980 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_request_uri": "", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_batch_size": "64", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_max_aio": "128", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_num_logs": "16", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_reshard_thread_interval": "600", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_resolve_cname": "false", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_run_sync_thread": "true", 2023-10-09T04:29:11.981 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_scheduler_type": "throttler", 2023-10-09T04:29:11.982 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_script_uri": "", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_service_provider_name": "", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_client_id": "", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_client_secret": "", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_entry": "sts", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_key": "sts", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_min_session_duration": "900", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sts_token_introspection_url": "", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_account_in_url": "false", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_auth_entry": "auth", 2023-10-09T04:29:11.983 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_auth_url": "", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_custom_header": "", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_need_stats": "true", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_tenant_name": "", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_token_expiration": "86400", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_url": "", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_url_prefix": "swift", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_lease_period": "120", 2023-10-09T04:29:11.984 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_thread_pool_size": "512", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_comment": "", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_createby": "", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_encoding": "", 2023-10-09T04:29:11.985 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_flag": "false", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_origin": "", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_torrent_tracker": "", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_trust_forwarded_https": "false", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_max_shards": "32", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_usage_max_user_shards": "1", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_use_opa_authz": "false", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-09T04:29:11.986 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_max_buckets": "1000", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_policies_max_num": "100", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_user_unique_email": "true", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_verify_ssl": "true", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zone": "", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zone_id": "", 2023-10-09T04:29:11.987 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zonegroup": "", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zonegroup_id": "", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rgwlc_auto_session_clear": "true", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_block_size": "4096", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-09T04:29:11.988 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_size": "536870912", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_index_type": "binary_search", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-09T04:29:11.989 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_partition_filters": "false", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_perf": "false", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "run_dir": "/var/run/ceph", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_block_create": "true", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_cache_lru_size": "67108864", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_default_max_object_size": "16777216", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_device_size": "53687091200", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_batch_capacity": "16", 2023-10-09T04:29:11.990 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_main_device_type": "SSD", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "seastore_segment_size": "67108864", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "set_keepcaps": "false", 2023-10-09T04:29:11.991 INFO:teuthology.orchestra.run.smithi029.stdout: "setgroup": "ceph", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "setuser": "ceph", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "setuser_match_path": "", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "thp": "false", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "threadpool_default_timeout": "60", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout: "throttler_perf_counter": "true" 2023-10-09T04:29:11.992 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:29:12.644 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-09T04:29:12.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:29:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:29:12] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.43.0" 2023-10-09T04:29:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:12 smithi029 ceph-mon[216903]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:29:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:12 smithi029 ceph-mon[223858]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:29:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:12 smithi177 ceph-mon[195797]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:29:13.346 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:14 smithi029 ceph-mon[216903]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:14 smithi029 ceph-mon[223858]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:14 smithi177 ceph-mon[195797]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:15.234 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a: {} 2023-10-09T04:29:15.245 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b: {} 2023-10-09T04:29:15.255 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c: {} 2023-10-09T04:29:15.256 INFO:teuthology.orchestra.run.smithi029.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:29:15.256 INFO:teuthology.orchestra.run.smithi029.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:29:15.257 INFO:teuthology.orchestra.run.smithi029.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-09T04:29:16.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[223858]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:16.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1029351554' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:29:16.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:16.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:16.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[216903]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:16.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1029351554' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:29:16.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:16.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:16.239 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-09T04:29:16.239 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-09T04:29:16.239 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:29:16.240 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:29:16.240 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-09T04:29:16.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-09T04:29:16.240 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-09T04:29:16.241 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:29:16.241 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:29:16.241 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T04:29:16.241 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:29:16.241 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:29:16.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T04:29:16.242 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-09T04:29:16.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:16.243 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-09T04:29:16.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:29:16.243 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-09T04:29:16.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:29:16.244 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-09T04:29:16.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:29:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:15 smithi177 ceph-mon[195797]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1029351554' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T04:29:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:16.324 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:16.448 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:16.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:16.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:16.595 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:16.603 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:29:16.813 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:16.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:16.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:16.946 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.106 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:17.157 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:17.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:17.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:17.288 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:17.288 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:17.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:17.334 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.474 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:17.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:17.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:17.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:17.607 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:17.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[216903]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:17.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:17.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:17.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[223858]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:17.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:17.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:17.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:17.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:17 smithi177 ceph-mon[195797]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:17.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:29:17.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:29:17.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:18.152 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:29:18.152 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:29:18.152 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:29:19.711 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:19.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:19.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:19.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:19.847 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:19 smithi029 ceph-mon[216903]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:20.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:19 smithi029 ceph-mon[223858]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:19 smithi177 ceph-mon[195797]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:20.350 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:20.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:20.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:20.490 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:20.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:20.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:20.644 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:20.689 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:21.196 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:29:21.196 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:29:21.196 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:29:21.196 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:29:21.994 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:21.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:21.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:22.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.134 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:22.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:22.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:21 smithi177 ceph-mon[195797]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:22.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:22.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:21 smithi029 ceph-mon[216903]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:22.322 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:21 smithi029 ceph-mon[223858]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:22.329 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:29:22.386 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:22.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:22.455 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:29:22.478 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:29:22.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:22.499 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:29:22.519 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:29:22.525 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.539 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:29:22.559 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:29:22.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:22.579 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:29:22.599 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:29:22.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:22.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:22.707 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:22.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:22.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:22.884 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:22.888 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:29:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:29:22] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.43.0" 2023-10-09T04:29:22.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:22.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:23.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:23.061 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:23.159 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_4' created 2023-10-09T04:29:23.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1147969969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1147969969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1147969969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:23.359 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:23.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:23.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:23.494 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:23.696 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:23.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:23.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:23.828 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:23.833 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:29:23.833 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:29:23.833 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:29:23.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T04:29:23.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:23.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:23.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:23.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:24.004 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:24.156 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/780998145' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T04:29:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/326156725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/780998145' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:29:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T04:29:24.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/326156725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:24.405 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:24.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:24.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:24.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/780998145' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: osdmap e144: 8 total, 8 up, 8 in 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: failed to encode map e144 with expected crc 2023-10-09T04:29:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/326156725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:29:24.536 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:24.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-09T04:29:25.051 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:25.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:25.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:25.071 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:25.154 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:25.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:25.188 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:25.198 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:25.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:25.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:25.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:25.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:25.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:25.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/326156725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:25.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T04:29:25.738 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.739 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.739 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:25 smithi177 ceph-mon[195797]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.739 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/326156725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[216903]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/326156725' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:29:25.740 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: osdmap e145: 8 total, 8 up, 8 in 2023-10-09T04:29:25.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:25 smithi029 ceph-mon[223858]: failed to encode map e145 with expected crc 2023-10-09T04:29:25.770 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:25.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:25.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:25.934 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:25.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:29:25.961 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:26.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:26.141 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[216903]: pgmap v529: 89 pgs: 9 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:26.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[216903]: failed to encode map e146 with expected crc 2023-10-09T04:29:26.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[223858]: pgmap v529: 89 pgs: 9 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:26.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[223858]: failed to encode map e146 with expected crc 2023-10-09T04:29:26.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[223858]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T04:29:26.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[223858]: failed to encode map e146 with expected crc 2023-10-09T04:29:26.194 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:26.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:26.326 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:26.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:26.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[216903]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T04:29:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:26 smithi029 ceph-mon[216903]: failed to encode map e146 with expected crc 2023-10-09T04:29:26.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:26.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:26 smithi177 ceph-mon[195797]: pgmap v529: 89 pgs: 9 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:26.506 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:26 smithi177 ceph-mon[195797]: failed to encode map e146 with expected crc 2023-10-09T04:29:26.506 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:26 smithi177 ceph-mon[195797]: osdmap e146: 8 total, 8 up, 8 in 2023-10-09T04:29:26.506 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:26 smithi177 ceph-mon[195797]: failed to encode map e146 with expected crc 2023-10-09T04:29:26.507 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:26.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:26.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:26.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:26.682 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:27.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:29:27.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:27.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:27.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:28.018 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:28.390 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:28.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:28 smithi029 ceph-mon[216903]: pgmap v531: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:28 smithi029 ceph-mon[223858]: pgmap v531: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:28.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:28.525 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:28 smithi177 ceph-mon[195797]: pgmap v531: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:28.528 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:28.535 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:28.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:28.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:28.685 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:28.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:28.714 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:28.756 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:28.756 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:28.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:28.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:28.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:28.891 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:28.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:28.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:29.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:29.069 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:29.503 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-09T04:29:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3192221671' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:29:29.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:29 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:29:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3192221671' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:29:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:29:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:29.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:29.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3192221671' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:29:29.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:29 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:29:29.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:29 smithi029 conmon[216878]: 2023-10-09T04:29:29.717+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e147 definitely_dead 0 2023-10-09T04:29:30.069 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:30.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:30.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:30.205 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:30.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:29:30.513 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.0. 2023-10-09T04:29:30.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: pgmap v532: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:30.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:29:30.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T04:29:30.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/487388105' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: pgmap v532: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/487388105' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:29:30.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:30 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: pgmap v532: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-09T04:29:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:29:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: osdmap e147: 8 total, 8 up, 8 in 2023-10-09T04:29:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/487388105' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:29:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:29:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:29:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:30 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:29:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: osdmap e148: 8 total, 8 up, 7 in 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:31 smithi177 ceph-mon[195797]: osdmap e149: 8 total, 8 up, 7 in 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: osdmap e148: 8 total, 8 up, 7 in 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.802 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[216903]: osdmap e149: 8 total, 8 up, 7 in 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: osdmap e148: 8 total, 8 up, 7 in 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e148 with expected crc 2023-10-09T04:29:31.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: failed to encode map e147 with expected crc 2023-10-09T04:29:31.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:31 smithi029 ceph-mon[223858]: osdmap e149: 8 total, 8 up, 7 in 2023-10-09T04:29:31.813 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:31.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:31.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:31.950 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:31.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:32.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:32.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:32.134 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: pgmap v535: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:32 smithi177 ceph-mon[195797]: osdmap e150: 8 total, 8 up, 7 in 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: pgmap v535: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[216903]: osdmap e150: 8 total, 8 up, 7 in 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: pgmap v535: 89 pgs: 89 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: failed to encode map e149 with expected crc 2023-10-09T04:29:32.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:32 smithi029 ceph-mon[223858]: osdmap e150: 8 total, 8 up, 7 in 2023-10-09T04:29:32.821 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:29:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:29:32] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2023-10-09T04:29:32.821 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-09T04:29:32.823 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:29:32.823 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:29:33.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:33.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:33.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:29:33.037 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:29:33.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_299184 2023-10-09T04:29:33.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:29:33.038 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:29:33.105 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:33.153 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:33.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:33.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:33.304 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:33.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:33.378 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:33.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:33.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:33.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:33.645 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:33 smithi177 ceph-mon[195797]: failed to encode map e150 with expected crc 2023-10-09T04:29:33.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:33 smithi029 ceph-mon[216903]: failed to encode map e150 with expected crc 2023-10-09T04:29:33.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:33 smithi029 ceph-mon[223858]: failed to encode map e150 with expected crc 2023-10-09T04:29:33.911 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:33.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:33.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:34.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:34.045 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3379 3363 209.019 210.188 0.00315664 0.00477171 2023-10-09T04:29:34.081 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:34.189 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:34.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:34.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:34.333 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:29:34.341 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:34.368 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:29:34.405 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:29:34.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:34.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:34.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:34.446 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:29:34.485 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:29:34.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:34.529 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:29:34.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: pgmap v538: 89 pgs: 11 peering, 78 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:29:34.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:34.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: osdmap e151: 8 total, 8 up, 7 in 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1804797188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:34.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1365208662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:34.571 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:29:34.571 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:29:34.594 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:34.609 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:29:34.621 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.621 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.621 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.621 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.622 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.622 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.622 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.622 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:29:34.728 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:34.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: pgmap v538: 89 pgs: 11 peering, 78 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: osdmap e151: 8 total, 8 up, 7 in 2023-10-09T04:29:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1804797188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1365208662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:34.833 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: pgmap v538: 89 pgs: 11 peering, 78 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: osdmap e151: 8 total, 8 up, 7 in 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: failed to encode map e151 with expected crc 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1804797188' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:29:34.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1365208662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:34.884 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:35.043 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6448 6432 200.431 191.812 0.00656455 0.00498337 2023-10-09T04:29:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[223858]: osdmap e152: 8 total, 8 up, 7 in 2023-10-09T04:29:35.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[223858]: failed to encode map e152 with expected crc 2023-10-09T04:29:35.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[223858]: failed to encode map e152 with expected crc 2023-10-09T04:29:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[216903]: osdmap e152: 8 total, 8 up, 7 in 2023-10-09T04:29:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[216903]: failed to encode map e152 with expected crc 2023-10-09T04:29:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:35 smithi029 ceph-mon[216903]: failed to encode map e152 with expected crc 2023-10-09T04:29:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:35 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:29:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:35 smithi177 ceph-mon[195797]: osdmap e152: 8 total, 8 up, 7 in 2023-10-09T04:29:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:35 smithi177 ceph-mon[195797]: failed to encode map e152 with expected crc 2023-10-09T04:29:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:35 smithi177 ceph-mon[195797]: failed to encode map e152 with expected crc 2023-10-09T04:29:36.043 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10062 10046 208.89 225.875 0.0064107 0.00478046 2023-10-09T04:29:36.759 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:29:36.760 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:29:36.760 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T04:29:36.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T04:29:36.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:36 smithi029 ceph-mon[216903]: pgmap v541: 89 pgs: 11 peering, 78 active+clean; 218 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 144 KiB/s, 2 objects/s recovering 2023-10-09T04:29:36.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:36 smithi029 ceph-mon[223858]: pgmap v541: 89 pgs: 11 peering, 78 active+clean; 218 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 144 KiB/s, 2 objects/s recovering 2023-10-09T04:29:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:36 smithi177 ceph-mon[195797]: pgmap v541: 89 pgs: 11 peering, 78 active+clean; 218 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 144 KiB/s, 2 objects/s recovering 2023-10-09T04:29:37.043 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 12744 12728 198.584 167.625 0.00281305 0.00489025 2023-10-09T04:29:37.159 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:37.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:37.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:37.264 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:37.319 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:37 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T04:29:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:37 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T04:29:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:37 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-09T04:29:38.044 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 15 15544 15529 193.859 175.062 0.00423092 0.00515301 2023-10-09T04:29:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:38 smithi029 ceph-mon[216903]: pgmap v542: 89 pgs: 89 active+clean; 453 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 109 KiB/s, 2 objects/s recovering 2023-10-09T04:29:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:38 smithi029 ceph-mon[223858]: pgmap v542: 89 pgs: 89 active+clean; 453 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 109 KiB/s, 2 objects/s recovering 2023-10-09T04:29:39.004 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:39.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:38 smithi177 ceph-mon[195797]: pgmap v542: 89 pgs: 89 active+clean; 453 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 109 KiB/s, 2 objects/s recovering 2023-10-09T04:29:39.044 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 18317 18301 190.424 173.25 0.00691176 0.00524542 2023-10-09T04:29:39.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:39.170 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:39.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:39.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:39.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:39.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:39.579 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:40.045 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 21009 20993 187.256 168.25 0.00321776 0.00532725 2023-10-09T04:29:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:40 smithi029 ceph-mon[216903]: pgmap v543: 89 pgs: 89 active+clean; 453 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 92 KiB/s, 1 objects/s recovering 2023-10-09T04:29:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:40 smithi029 ceph-mon[223858]: pgmap v543: 89 pgs: 89 active+clean; 453 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 92 KiB/s, 1 objects/s recovering 2023-10-09T04:29:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:40 smithi177 ceph-mon[195797]: pgmap v543: 89 pgs: 89 active+clean; 453 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 92 KiB/s, 1 objects/s recovering 2023-10-09T04:29:41.045 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 23342 23326 182.061 145.812 0.00799912 0.00548796 2023-10-09T04:29:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:41 smithi177 ceph-mon[195797]: pgmap v544: 89 pgs: 89 active+clean; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 72 KiB/s, 1 objects/s recovering 2023-10-09T04:29:42.045 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 25259 25243 175.149 119.812 0.00392323 0.00570432 2023-10-09T04:29:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:41 smithi029 ceph-mon[216903]: pgmap v544: 89 pgs: 89 active+clean; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 72 KiB/s, 1 objects/s recovering 2023-10-09T04:29:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:41 smithi029 ceph-mon[223858]: pgmap v544: 89 pgs: 89 active+clean; 1.0 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 72 KiB/s, 1 objects/s recovering 2023-10-09T04:29:43.046 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 26848 26832 167.57 99.3125 0.00183828 0.00593812 2023-10-09T04:29:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:29:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:29:42] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2023-10-09T04:29:43.563 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:43.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:43.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:43.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:43.874 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:43.940 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:44.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:44.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:44.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:44.046 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 28568 28552 162.109 107.5 0.00920986 0.00616202 2023-10-09T04:29:44.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:43 smithi029 ceph-mon[216903]: pgmap v545: 89 pgs: 89 active+clean; 1.3 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s; 62 KiB/s, 1 objects/s recovering 2023-10-09T04:29:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:43 smithi029 ceph-mon[223858]: pgmap v545: 89 pgs: 89 active+clean; 1.3 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s; 62 KiB/s, 1 objects/s recovering 2023-10-09T04:29:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:44.211 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:43 smithi177 ceph-mon[195797]: pgmap v545: 89 pgs: 89 active+clean; 1.3 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s; 62 KiB/s, 1 objects/s recovering 2023-10-09T04:29:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:44.368 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:44.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:44.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:44.546 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:124501 7.6 7.f187b2f6 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e91)", 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:20:02.056427+0000", 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.51393370100004, 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96062819099999996, 2023-10-09T04:29:44.575 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124501 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056427+0000", 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056427+0000", 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056425+0000", 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056432+0000", 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5399999999999998e-06 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056444+0000", 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1849e-05 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056457+0000", 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2566e-05 2023-10-09T04:29:44.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056935+0000", 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00047851700000000003 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056946+0000", 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0151e-05 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.017041+0000", 2023-10-09T04:29:44.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96009593400000004 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.017055+0000", 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3586e-05 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:124505 7.4 7.6a7cd8f4 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e91)", 2023-10-09T04:29:44.580 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:20:02.056464+0000", 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.51389683100001, 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95814426699999999, 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124505 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056464+0000", 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056464+0000", 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056463+0000", 2023-10-09T04:29:44.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056467+0000", 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.817e-06 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056474+0000", 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3159999999999999e-06 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056480+0000", 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.117e-06 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056550+0000", 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9851000000000006e-05 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056563+0000", 2023-10-09T04:29:44.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2625e-05 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.014596+0000", 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95803304700000003 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.014608+0000", 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1995e-05 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.585 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:124506 7.9 7.2a283429 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e91)", 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:20:02.056494+0000", 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.51386640600003, 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95763316799999998, 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124506 2023-10-09T04:29:44.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056494+0000", 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056494+0000", 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056494+0000", 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056497+0000", 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8200000000000001e-06 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056503+0000", 2023-10-09T04:29:44.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9009999999999997e-06 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056509+0000", 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8549999999999998e-06 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.057034+0000", 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052449099999999998 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.057043+0000", 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5820000000000008e-06 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.014109+0000", 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95706524299999995 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.014127+0000", 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8740999999999999e-05 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:124511 7.9 7.f04792e9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e91)", 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:20:02.056788+0000", 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.513572178, 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95738236700000001, 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.591 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124511 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056788+0000", 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056788+0000", 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056788+0000", 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056793+0000", 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9540000000000003e-06 2023-10-09T04:29:44.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056804+0000", 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1085e-05 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.056812+0000", 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5140000000000001e-06 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.057067+0000", 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025491700000000001 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:02.057075+0000", 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.014160+0000", 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95708543099999999 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:20:03.014171+0000", 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0417999999999999e-05 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:189609 7.9 7:90b3e6a1:::benchmark_data_smithi029_237125_object72261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:21:44.130471+0000", 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 480.43988979800002, 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.011063998, 2023-10-09T04:29:44.596 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189609 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130471+0000", 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130471+0000", 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130469+0000", 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130479+0000", 2023-10-09T04:29:44.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0298e-05 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130493+0000", 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.327e-05 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130501+0000", 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6799999999999999e-06 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.131409+0000", 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00090790200000000004 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.131420+0000", 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0669000000000001e-05 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.141445+0000", 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0100251840000001 2023-10-09T04:29:44.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.141535+0000", 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9577e-05 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:189619 7.9 7:9da4be6b:::benchmark_data_smithi029_237125_object72271:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:21:44.130508+0000", 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 480.43985286200001, 2023-10-09T04:29:44.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0114232620000001, 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189619 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130508+0000", 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130508+0000", 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130507+0000", 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130510+0000", 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1519999999999999e-06 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130515+0000", 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5199999999999997e-06 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130520+0000", 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.131443+0000", 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00092270999999999998 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.131449+0000", 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6030000000000001e-06 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.141863+0000", 2023-10-09T04:29:44.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.010414224 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.141931+0000", 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7556000000000002e-05 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:189626 7.9 7:9fae71e5:::benchmark_data_smithi029_237125_object72278:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:21:44.130524+0000", 2023-10-09T04:29:44.606 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 480.43983696399999, 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0217415190000001, 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189626 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130524+0000", 2023-10-09T04:29:44.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130524+0000", 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130523+0000", 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130525+0000", 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9099999999999999e-06 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130531+0000", 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.575e-06 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.130536+0000", 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8180000000000003e-06 2023-10-09T04:29:44.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.131689+0000", 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001153243 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.131698+0000", 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5310000000000007e-06 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.141949+0000", 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0102501420000001 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.142072+0000", 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012301900000000001 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.142176+0000", 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104329 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.152149+0000", 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099729250000000005 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.152159+0000", 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0685e-05 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.152190+0000", 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0763999999999999e-05 2023-10-09T04:29:44.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.152222+0000", 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1946000000000002e-05 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.152265+0000", 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2930999999999999e-05 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:189638 7.9 7:9d98455b:::benchmark_data_smithi029_237125_object72290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:21:44.142436+0000", 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 480.427924061, 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0240938070000001, 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189638 2023-10-09T04:29:44.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142436+0000", 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142436+0000", 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142434+0000", 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142444+0000", 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5990000000000008e-06 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142463+0000", 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9276e-05 2023-10-09T04:29:44.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142474+0000", 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1025e-05 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142540+0000", 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5257000000000002e-05 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.142552+0000", 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2027e-05 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.145618+0000", 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0030666210000001 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.145708+0000", 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9592999999999994e-05 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.145833+0000", 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012463200000000001 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.152164+0000", 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0063319550000000002 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.159794+0000", 2023-10-09T04:29:44.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076293819999999997 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.166433+0000", 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066390930000000004 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.166484+0000", 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1085999999999998e-05 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.166530+0000", 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6260000000000001e-05 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:189667 7.9 7:95bcee2a:::benchmark_data_smithi029_237125_object72319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e100)", 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:21:44.176948+0000", 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 480.39341280899998, 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99742425300000004, 2023-10-09T04:29:44.621 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 189667 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.176948+0000", 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.176948+0000", 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.176946+0000", 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.176954+0000", 2023-10-09T04:29:44.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6369999999999999e-06 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.176968+0000", 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4623999999999999e-05 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.176979+0000", 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0958000000000001e-05 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.177060+0000", 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0370999999999993e-05 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:44.177075+0000", 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5373e-05 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.149294+0000", 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97221882800000003 2023-10-09T04:29:44.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.149397+0000", 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102985 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.149516+0000", 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011911100000000001 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.156043+0000", 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065273470000000002 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.159838+0000", 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037943809999999999 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.174270+0000", 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014432281 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.174323+0000", 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3375999999999999e-05 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:21:45.174372+0000", 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8695e-05 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:210863 7.f 7:fc4d7303:::benchmark_data_smithi029_237125_object93515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:22:15.821467+0000", 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 448.74889306699998, 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0150586100000001, 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 210863 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821467+0000", 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821467+0000", 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821465+0000", 2023-10-09T04:29:44.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821476+0000", 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0606e-05 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821497+0000", 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0846e-05 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821510+0000", 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2764e-05 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821570+0000", 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0084000000000003e-05 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.821583+0000", 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3638999999999999e-05 2023-10-09T04:29:44.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.828070+0000", 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.006486735 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.828179+0000", 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108725 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:29:44.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.828287+0000", 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108399 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.833714+0000", 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054263769999999996 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836410+0000", 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00269615 2023-10-09T04:29:44.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836448+0000", 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8597999999999999e-05 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836483+0000", 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4431000000000003e-05 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836526+0000", 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3282000000000003e-05 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:210874 7.f 7:fa056383:::benchmark_data_smithi029_237125_object93526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:22:15.833254+0000", 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 448.73710668699999, 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0116534420000001, 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.636 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 210874 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833254+0000", 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833254+0000", 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833252+0000", 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833261+0000", 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.217e-06 2023-10-09T04:29:44.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833278+0000", 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6460000000000002e-05 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833288+0000", 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0013000000000001e-05 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833353+0000", 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5388000000000005e-05 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.833365+0000", 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1705000000000001e-05 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.831321+0000", 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.997956489 2023-10-09T04:29:44.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.831418+0000", 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6613999999999994e-05 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.831495+0000", 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7334000000000002e-05 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836578+0000", 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050824829999999996 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836624+0000", 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6180000000000002e-05 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.844838+0000", 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082138109999999997 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.844870+0000", 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1996000000000003e-05 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.844907+0000", 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7747000000000003e-05 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.54127.0:210897 7.f 7:f58c3dba:::benchmark_data_smithi029_237125_object93549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e102)", 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:22:15.876316+0000", 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 448.69404425099998, 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97533070899999996, 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.54127", 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3047949529", 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 210897 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.644 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876316+0000", 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876316+0000", 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876315+0000", 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876323+0000", 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6279999999999996e-06 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876335+0000", 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2796000000000001e-05 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876345+0000", 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6930000000000002e-06 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876415+0000", 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0080999999999995e-05 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:15.876431+0000", 2023-10-09T04:29:44.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5914999999999999e-05 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.833753+0000", 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95732169600000006 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.833882+0000", 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129216 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.834005+0000", 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012278899999999999 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836615+0000", 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002610313 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.836629+0000", 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3552e-05 2023-10-09T04:29:44.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.851586+0000", 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014957094000000001 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.851618+0000", 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2333999999999999e-05 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:22:16.851647+0000", 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8938999999999999e-05 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:60692 8.1 8:873c9826:::benchmark_data_smithi029_252925_object60691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:23:44.509116+0000", 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 360.06124464800001, 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99391049099999995, 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.651 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 60692 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509116+0000", 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509116+0000", 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509114+0000", 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509141+0000", 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6594e-05 2023-10-09T04:29:44.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509160+0000", 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8842e-05 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509171+0000", 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1097999999999999e-05 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509224+0000", 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2828000000000003e-05 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:44.509249+0000", 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5482000000000001e-05 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.497355+0000", 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98810600199999998 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.498869+0000", 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001513468 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2", 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.499051+0000", 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000182795 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.501325+0000", 2023-10-09T04:29:44.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022738979999999999 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.502883+0000", 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015582 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.502941+0000", 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7503999999999997e-05 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:23:45.503026+0000", 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5443000000000002e-05 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:88376 8.1 8:88006b61:::benchmark_data_smithi029_252925_object88375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:07.903122+0000", 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 336.66723860899998, 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1049434119999999, 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.658 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88376 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903122+0000", 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903122+0000", 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903120+0000", 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903165+0000", 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5729000000000002e-05 2023-10-09T04:29:44.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903192+0000", 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6563999999999998e-05 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903204+0000", 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1545e-05 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903284+0000", 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0809999999999994e-05 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903303+0000", 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8732999999999998e-05 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.906977+0000", 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036738930000000001 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.906996+0000", 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8550999999999999e-05 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.007957+0000", 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1009610590000001 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.008065+0000", 2023-10-09T04:29:44.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108768 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:88377 8.1 8:833f4176:::benchmark_data_smithi029_252925_object88376:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:07.903224+0000", 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 336.66713646199997, 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1062361759999999, 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88377 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903224+0000", 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903224+0000", 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903223+0000", 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903260+0000", 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7178999999999998e-05 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903269+0000", 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0429999999999996e-06 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903276+0000", 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8460000000000002e-06 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903444+0000", 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016738199999999999 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903452+0000", 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5599999999999994e-06 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907372+0000", 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039196220000000002 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907382+0000", 2023-10-09T04:29:44.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0662000000000001e-05 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009358+0000", 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.101975511 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009460+0000", 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102259 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.669 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:88383 8.1 8:8ae67f58:::benchmark_data_smithi029_252925_object88382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:07.903283+0000", 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 336.66707772900003, 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1166282439999999, 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88383 2023-10-09T04:29:44.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903283+0000", 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903283+0000", 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903282+0000", 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903321+0000", 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9394999999999998e-05 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903367+0000", 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5163000000000001e-05 2023-10-09T04:29:44.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903376+0000", 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6520000000000001e-06 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903555+0000", 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017835799999999999 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903566+0000", 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1619e-05 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907415+0000", 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038489359999999999 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907424+0000", 2023-10-09T04:29:44.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.884e-06 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009625+0000", 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1022007709999999 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009778+0000", 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015287800000000001 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.010541+0000", 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000762848 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.014175+0000", 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036340040000000001 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.016651+0000", 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024762690000000001 2023-10-09T04:29:44.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.018480+0000", 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00182884 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.018557+0000", 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6822000000000004e-05 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.019911+0000", 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013545639999999999 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:99095 8.c 8:335eeb1f:::benchmark_data_smithi029_252925_object99094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:21.580231+0000", 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 322.99012932900001, 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99713536999999997, 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.678 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 99095 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:21.580231+0000", 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:21.580231+0000", 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:21.580230+0000", 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:21.580251+0000", 2023-10-09T04:29:44.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0692000000000001e-05 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.501380+0000", 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92112977100000004 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.501394+0000", 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3570999999999999e-05 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.501760+0000", 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036583500000000001 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.501921+0000", 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016125100000000001 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:29:44.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.502091+0000", 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000170419 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.509045+0000", 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069530989999999999 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.511719+0000", 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026744799999999999 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.576788+0000", 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.065069302999999995 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.576850+0000", 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.19e-05 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:22.577367+0000", 2023-10-09T04:29:44.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00051640699999999998 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216843 8.e 8:7402676a:::benchmark_data_smithi029_252925_object100219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.014427+0000", 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.55593394300001, 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.084552424, 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.685 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216843 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014427+0000", 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014427+0000", 2023-10-09T04:29:44.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014425+0000", 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014433+0000", 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.276e-06 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014452+0000", 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9042000000000002e-05 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014461+0000", 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5099999999999998e-06 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014499+0000", 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8621999999999997e-05 2023-10-09T04:29:44.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014611+0000", 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011174400000000001 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014719+0000", 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108366 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019596+0000", 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048769490000000002 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.028391+0000", 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087943320000000002 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.043835+0000", 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015444599999999999 2023-10-09T04:29:44.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.043852+0000", 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6378000000000001e-05 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.098894+0000", 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0550420890000001 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.098979+0000", 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5234000000000004e-05 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216846 8.e 8:7d36edfd:::benchmark_data_smithi029_252925_object100222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.014473+0000", 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.555887246, 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0848377520000001, 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:29:44.692 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216846 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014473+0000", 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014473+0000", 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014473+0000", 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014476+0000", 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3459999999999998e-06 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014483+0000", 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0439999999999998e-06 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014489+0000", 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9349999999999999e-06 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.020049+0000", 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055601820000000003 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.020181+0000", 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131445 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.020297+0000", 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011656599999999999 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.027555+0000", 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072572000000000001 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.028431+0000", 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00087675700000000004 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.044106+0000", 2023-10-09T04:29:44.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015674246999999999 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.044111+0000", 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4929999999999998e-06 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.099246+0000", 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.055135369 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.099311+0000", 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4633000000000003e-05 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216864 8.e 8:72237c5d:::benchmark_data_smithi029_252925_object100240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.047328+0000", 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.52303268099999, 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0669346900000001, 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:29:44.699 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216864 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047328+0000", 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047328+0000", 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047326+0000", 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047332+0000", 2023-10-09T04:29:44.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7309999999999998e-06 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047347+0000", 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4698999999999999e-05 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047357+0000", 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0254999999999999e-05 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047400+0000", 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3334000000000002e-05 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.047407+0000", 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0530000000000001e-06 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.099353+0000", 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0519452709999999 2023-10-09T04:29:44.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.099468+0000", 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115332 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4", 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.099595+0000", 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012707300000000001 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:29:44.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.113095+0000", 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013500349 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.114137+0000", 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010418929999999999 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.114173+0000", 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5460000000000003e-05 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:29:44.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.114263+0000", 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9765000000000005e-05 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:29:44.706 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:29:44.749 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:44.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:44.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:44.910 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:45.046 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 30560 30544 158.976 124.5 0.00612864 0.00628469 2023-10-09T04:29:45.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:29:45.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:29:46.047 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 32098 32082 154.131 96.125 0.00590256 0.00648275 2023-10-09T04:29:46.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:46 smithi029 ceph-mon[216903]: pgmap v546: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 168 MiB/s wr, 2.68k op/s; 56 KiB/s, 1 objects/s recovering 2023-10-09T04:29:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:46 smithi029 ceph-mon[223858]: pgmap v546: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 168 MiB/s wr, 2.68k op/s; 56 KiB/s, 1 objects/s recovering 2023-10-09T04:29:46.297 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:46 smithi177 ceph-mon[195797]: pgmap v546: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 168 MiB/s wr, 2.68k op/s; 56 KiB/s, 1 objects/s recovering 2023-10-09T04:29:46.821 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:46.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:46.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:29:47.047 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 33947 33931 151.377 115.562 0.0109187 0.00660175 2023-10-09T04:29:47.049 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:47.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:29:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:47.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:29:47.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:47.346 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:47.867 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:47.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:47.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:47.997 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:48.047 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 35477 35461 147.661 95.625 0.00256956 0.00665169 2023-10-09T04:29:48.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:48.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:48.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:48.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:48 smithi029 ceph-mon[216903]: pgmap v547: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:29:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:48 smithi029 ceph-mon[223858]: pgmap v547: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:29:48.190 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:48.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:48 smithi177 ceph-mon[195797]: pgmap v547: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:29:48.599 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:48.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:48.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:29:48.705 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:48.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:48.751 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:48.808 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:48.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:48.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:48.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:48.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:48.968 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:49.047 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 36921 36905 144.074 90.25 0.00696579 0.00693547 2023-10-09T04:29:49.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:49.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:49.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:29:49.240 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:49.489 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:29:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:49.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:29:49.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:49.681 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:49.979 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:49.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:49.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:50.031 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.0. 2023-10-09T04:29:50.047 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 38526 38510 141.501 100.312 0.0177521 0.007056 2023-10-09T04:29:50.080 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:29:50.087 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:29:50.112 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:50.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:29:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:50.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:29:50.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:50 smithi029 ceph-mon[223858]: pgmap v548: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-10-09T04:29:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3888947622' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:29:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:29:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:49 smithi029 conmon[216878]: 2023-10-09T04:29:49.963+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e152 definitely_dead 0 2023-10-09T04:29:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:50 smithi029 ceph-mon[216903]: pgmap v548: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-10-09T04:29:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3888947622' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:29:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:29:50.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:50 smithi177 ceph-mon[195797]: pgmap v548: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s 2023-10-09T04:29:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3888947622' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:29:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:29:50.296 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:50.643 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:50.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:50.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:50.776 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:50.820 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:50.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:50.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:50.958 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:51.049 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 38552 38536 133.733 1.625 0.00227397 0.0070546 2023-10-09T04:29:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:29:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:29:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[223858]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:51 smithi029 ceph-mon[216903]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:29:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: osdmap e153: 8 total, 8 up, 8 in 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: failed to encode map e153 with expected crc 2023-10-09T04:29:51.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:51 smithi177 ceph-mon[195797]: osdmap e154: 8 total, 8 up, 8 in 2023-10-09T04:29:52.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: pgmap v550: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T04:29:52.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.047 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 38552 38536 126.698 0 - 0.0070546 2023-10-09T04:29:52.227 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:52.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:52.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:52.328 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: pgmap v550: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[216903]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.329 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:52 smithi029 ceph-mon[223858]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:29:52.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: pgmap v550: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: failed to encode map e154 with expected crc 2023-10-09T04:29:52.378 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:52 smithi177 ceph-mon[195797]: osdmap e155: 8 total, 8 up, 8 in 2023-10-09T04:29:52.380 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:52.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:52.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:52.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:52.590 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:52.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:52.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:52.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:52.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:52.809 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:52.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:52.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:52.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:52.993 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:53.050 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:29:53.051038+0000 min lat: 0.00115182 max lat: 2.19982 avg lat: 0.00815447 2023-10-09T04:29:53.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:29:53.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 39257 39241 122.551 22.0312 0.0173035 0.00815447 2023-10-09T04:29:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[223858]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[223858]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[223858]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[223858]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[223858]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[216903]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[216903]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[216903]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[216903]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:53 smithi029 ceph-mon[216903]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:29:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:29:52] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-10-09T04:29:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:53 smithi177 ceph-mon[195797]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:53 smithi177 ceph-mon[195797]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:53 smithi177 ceph-mon[195797]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:53 smithi177 ceph-mon[195797]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:53 smithi177 ceph-mon[195797]: failed to encode map e155 with expected crc 2023-10-09T04:29:53.558 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T04:29:53.849 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:53.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:53.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:53.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:54.010 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:54.050 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 40579 40563 120.65 82.625 0.0113837 0.00828161 2023-10-09T04:29:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[223858]: pgmap v553: 89 pgs: 2 remapped+peering, 5 activating, 82 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 2/114942 objects degraded (0.002%); 0 B/s, 0 objects/s recovering 2023-10-09T04:29:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[223858]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:29:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[223858]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[223858]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[223858]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[223858]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[216903]: pgmap v553: 89 pgs: 2 remapped+peering, 5 activating, 82 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 2/114942 objects degraded (0.002%); 0 B/s, 0 objects/s recovering 2023-10-09T04:29:54.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[216903]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:29:54.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[216903]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[216903]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[216903]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:54 smithi029 ceph-mon[216903]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.285 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:54.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:29:54.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:54.323 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:29:54.365 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:29:54.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:29:54.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:29:54.455 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:29:54.476 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:54.502 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:29:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:54 smithi177 ceph-mon[195797]: pgmap v553: 89 pgs: 2 remapped+peering, 5 activating, 82 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 2/114942 objects degraded (0.002%); 0 B/s, 0 objects/s recovering 2023-10-09T04:29:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:54 smithi177 ceph-mon[195797]: osdmap e156: 8 total, 8 up, 8 in 2023-10-09T04:29:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:54 smithi177 ceph-mon[195797]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:54 smithi177 ceph-mon[195797]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:54 smithi177 ceph-mon[195797]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:54 smithi177 ceph-mon[195797]: failed to encode map e156 with expected crc 2023-10-09T04:29:54.544 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:29:54.584 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:29:54.587 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:29:54.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:29:54.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.600 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:29:54.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:54.734 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:54.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:54.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:54.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:55.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:55.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 41739 41723 118.462 72.5 0.0101298 0.0084353 2023-10-09T04:29:55.072 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:55.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:55.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:55.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:55.310 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:55.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[216903]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:29:55.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[216903]: failed to encode map e157 with expected crc 2023-10-09T04:29:55.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[216903]: failed to encode map e157 with expected crc 2023-10-09T04:29:55.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1029930947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:55.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[223858]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:29:55.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[223858]: failed to encode map e157 with expected crc 2023-10-09T04:29:55.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[223858]: failed to encode map e157 with expected crc 2023-10-09T04:29:55.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1029930947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:55 smithi177 ceph-mon[195797]: osdmap e157: 8 total, 8 up, 8 in 2023-10-09T04:29:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:55 smithi177 ceph-mon[195797]: failed to encode map e157 with expected crc 2023-10-09T04:29:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:55 smithi177 ceph-mon[195797]: failed to encode map e157 with expected crc 2023-10-09T04:29:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1029930947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:29:56.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 15 42548 42533 115.513 50.625 0.0164412 0.00864936 2023-10-09T04:29:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:56 smithi029 ceph-mon[216903]: pgmap v556: 89 pgs: 1 active+recovery_wait+remapped, 2 remapped+peering, 5 activating, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 4/120174 objects degraded (0.003%); 4613/120174 objects misplaced (3.839%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:29:56.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:56 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 4/120174 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:29:56.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:56 smithi029 ceph-mon[223858]: pgmap v556: 89 pgs: 1 active+recovery_wait+remapped, 2 remapped+peering, 5 activating, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 4/120174 objects degraded (0.003%); 4613/120174 objects misplaced (3.839%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:29:56.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:56 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 4/120174 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:29:56.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:56 smithi177 ceph-mon[195797]: pgmap v556: 89 pgs: 1 active+recovery_wait+remapped, 2 remapped+peering, 5 activating, 1 active+recovering+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 4/120174 objects degraded (0.003%); 4613/120174 objects misplaced (3.839%); 3.3 MiB/s, 52 objects/s recovering 2023-10-09T04:29:56.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:56 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 4/120174 objects degraded (0.003%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:29:56.766 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:56.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:56.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:29:56.931 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:57.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 43594 43578 113.422 65.3125 0.00792798 0.00881193 2023-10-09T04:29:57.345 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:29:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:57.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:29:57.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:29:57.505 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:57.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:29:57.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:57.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:29:57.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:57.756 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:57.916 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:29:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:57.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:29:57.979 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:29:57.980 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:29:58.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:29:58.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 44749 44733 111.773 72.1875 0.0102463 0.00894014 2023-10-09T04:29:58.072 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:58.120 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:29:58.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:29:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:29:58.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:29:58.234 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:58 smithi029 ceph-mon[216903]: pgmap v557: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2359/125436 objects degraded (1.881%); 7041/125436 objects misplaced (5.613%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:29:58.234 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1336535624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:29:58.234 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:58 smithi029 ceph-mon[223858]: pgmap v557: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2359/125436 objects degraded (1.881%); 7041/125436 objects misplaced (5.613%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:29:58.234 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1336535624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:29:58.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:29:58.326 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:29:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:58 smithi177 ceph-mon[195797]: pgmap v557: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 2359/125436 objects degraded (1.881%); 7041/125436 objects misplaced (5.613%); 2.3 MiB/s, 36 objects/s recovering 2023-10-09T04:29:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1336535624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:29:58.559 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:29:58.559 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:29:58.559 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:29:58.559 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:29:58.559 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:29:58.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:29:59.051 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 46065 46049 110.637 82.25 0.0137395 0.00902966 2023-10-09T04:29:59.330 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:29:59.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:29:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1336535624' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: failed to encode map e158 with expected crc 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: failed to encode map e158 with expected crc 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1336535624' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:29:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: failed to encode map e158 with expected crc 2023-10-09T04:29:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: failed to encode map e158 with expected crc 2023-10-09T04:29:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:29:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:29:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:29:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1336535624' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:29:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: osdmap e158: 8 total, 8 up, 8 in 2023-10-09T04:29:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: failed to encode map e158 with expected crc 2023-10-09T04:29:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: failed to encode map e158 with expected crc 2023-10-09T04:29:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:29:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:29:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:00.052 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 46986 46970 108.67 57.5625 0.0577574 0.00918037 2023-10-09T04:30:00.520 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:00.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:00.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:00 smithi177 ceph-mon[195797]: pgmap v559: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 40 MiB/s wr, 642 op/s; 2359/126504 objects degraded (1.865%); 7041/126504 objects misplaced (5.566%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T04:30:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:00 smithi177 ceph-mon[195797]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 2359/126504 objects degraded (1.865%), 2 pgs degraded 2023-10-09T04:30:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:00 smithi177 ceph-mon[195797]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T04:30:00.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:00 smithi177 ceph-mon[195797]: [WRN] PG_DEGRADED: Degraded data redundancy: 2359/126504 objects degraded (1.865%), 2 pgs degraded 2023-10-09T04:30:00.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:00 smithi177 ceph-mon[195797]: pg 9.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-10-09T04:30:00.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:00 smithi177 ceph-mon[195797]: pg 9.9 is active+recovering+undersized+degraded+remapped, acting [5,7] 2023-10-09T04:30:00.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[216903]: pgmap v559: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 40 MiB/s wr, 642 op/s; 2359/126504 objects degraded (1.865%); 7041/126504 objects misplaced (5.566%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[216903]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 2359/126504 objects degraded (1.865%), 2 pgs degraded 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[216903]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[216903]: [WRN] PG_DEGRADED: Degraded data redundancy: 2359/126504 objects degraded (1.865%), 2 pgs degraded 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[216903]: pg 9.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[216903]: pg 9.9 is active+recovering+undersized+degraded+remapped, acting [5,7] 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[223858]: pgmap v559: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 40 MiB/s wr, 642 op/s; 2359/126504 objects degraded (1.865%); 7041/126504 objects misplaced (5.566%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T04:30:00.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[223858]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 2359/126504 objects degraded (1.865%), 2 pgs degraded 2023-10-09T04:30:00.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[223858]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-09T04:30:00.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[223858]: [WRN] PG_DEGRADED: Degraded data redundancy: 2359/126504 objects degraded (1.865%), 2 pgs degraded 2023-10-09T04:30:00.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[223858]: pg 9.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-10-09T04:30:00.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:00 smithi029 ceph-mon[223858]: pg 9.9 is active+recovering+undersized+degraded+remapped, acting [5,7] 2023-10-09T04:30:00.825 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:01.052 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 47852 47836 106.723 54.125 0.0134592 0.00936304 2023-10-09T04:30:01.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:30:01.122 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:01.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:01.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:01.314 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:01.315 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:01.470 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:01.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:01.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:01.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:01.681 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:01.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2359/134589 objects degraded (1.753%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:01.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2359/134589 objects degraded (1.753%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:01.795 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:01.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:01.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2359/134589 objects degraded (1.753%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:02.052 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 48988 48972 105.491 71 0.0160478 0.00947343 2023-10-09T04:30:02.202 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:02.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:02.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:02.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:02.453 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:02.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:02.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:02.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:02.681 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:02.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:02 smithi177 ceph-mon[195797]: pgmap v560: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 2359/134589 objects degraded (1.753%); 6594/134589 objects misplaced (4.899%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:30:02.801 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:02.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:02.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:02 smithi029 ceph-mon[216903]: pgmap v560: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 2359/134589 objects degraded (1.753%); 6594/134589 objects misplaced (4.899%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:30:02.821 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:30:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:30:02] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-10-09T04:30:02.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:02 smithi029 ceph-mon[223858]: pgmap v560: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 2359/134589 objects degraded (1.753%); 6594/134589 objects misplaced (4.899%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:30:02.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:02.964 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.046 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:03.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:03.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:03.052 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 50263 50247 104.631 79.6875 0.0146685 0.00955212 2023-10-09T04:30:03.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:30:03.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:03.179 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.225 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:03.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:03.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:03.329 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:03.355 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:03.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:03.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:03.569 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:03.996 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:04.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:04.053 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 50951 50935 102.641 43 0.0312622 0.00973887 2023-10-09T04:30:04.124 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:04.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:04.204 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:04.386 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:04.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:04.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:04.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:04.570 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:04.692 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:04.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:04.727 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:04 smithi029 ceph-mon[216903]: pgmap v561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 2359/141492 objects degraded (1.667%); 6594/141492 objects misplaced (4.660%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:30:04.728 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:04 smithi029 ceph-mon[223858]: pgmap v561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 2359/141492 objects degraded (1.667%); 6594/141492 objects misplaced (4.660%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:30:04.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:04.864 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:04 smithi177 ceph-mon[195797]: pgmap v561: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 2359/141492 objects degraded (1.667%); 6594/141492 objects misplaced (4.660%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:30:05.053 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 52016 52000 101.514 66.5625 0.00606318 0.00984608 2023-10-09T04:30:05.169 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:30:05.210 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:30:05.254 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:30:05.298 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:30:05.337 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:30:05.383 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:30:05.423 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:30:05.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:30:05.470 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.470 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.470 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:06.055 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 52870 52854 100.049 53.375 0.00920691 0.00998921 2023-10-09T04:30:06.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:06 smithi177 ceph-mon[195797]: pgmap v562: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 2359/150219 objects degraded (1.570%); 6148/150219 objects misplaced (4.093%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:30:06.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4014550293' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:06 smithi029 ceph-mon[216903]: pgmap v562: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 2359/150219 objects degraded (1.570%); 6148/150219 objects misplaced (4.093%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:30:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4014550293' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:06 smithi029 ceph-mon[223858]: pgmap v562: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 2359/150219 objects degraded (1.570%); 6148/150219 objects misplaced (4.093%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:30:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4014550293' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:06.554 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:30:06.554 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:30:06.554 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:30:07.055 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 53507 53491 98.2783 39.8125 0.00571684 0.010165 2023-10-09T04:30:07.402 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:07.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:07 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2359/156729 objects degraded (1.505%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:07.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:07 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2359/156729 objects degraded (1.505%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:07.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:07.514 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:07 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2359/156729 objects degraded (1.505%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:07.633 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:07.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:07.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:07.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:07.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:07.869 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:08.055 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 15 54321 54306 96.9262 50.9375 0.00767664 0.0103131 2023-10-09T04:30:08.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:08 smithi029 ceph-mon[216903]: pgmap v563: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 2359/156729 objects degraded (1.505%); 6148/156729 objects misplaced (3.923%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:30:08.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:08 smithi029 ceph-mon[223858]: pgmap v563: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 2359/156729 objects degraded (1.505%); 6148/156729 objects misplaced (3.923%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:30:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:08 smithi177 ceph-mon[195797]: pgmap v563: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 2359/156729 objects degraded (1.505%); 6148/156729 objects misplaced (3.923%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:30:08.937 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:08.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:08.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:09.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:09.055 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 55133 55117 95.642 50.6875 0.011297 0.01045 2023-10-09T04:30:09.099 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:09.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:09.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:09.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:09.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:09.313 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:09.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:09.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:09.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:09.496 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:09.536 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:09.599 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:09.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:09.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:09.787 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:09.864 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:30:09.864 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:30:09.864 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:30:09.864 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:30:09.864 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:30:09.905 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:09.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:10.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:10.055 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:10.057 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 56135 56119 94.7455 62.625 0.0109347 0.0105504 2023-10-09T04:30:10.076 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:10 smithi029 ceph-mon[223858]: pgmap v564: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 2359/157515 objects degraded (1.498%); 6148/157515 objects misplaced (3.903%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:30:10.076 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3727533247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:10.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:10 smithi029 ceph-mon[216903]: pgmap v564: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 2359/157515 objects degraded (1.498%); 6148/157515 objects misplaced (3.903%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:30:10.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3727533247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:10.115 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:30:10.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:10.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:10.229 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:10.264 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:10.311 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:10.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:10.415 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:10.456 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:10 smithi177 ceph-mon[195797]: pgmap v564: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 2359/157515 objects degraded (1.498%); 6148/157515 objects misplaced (3.903%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:30:10.457 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3727533247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:10.464 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:10.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:10.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:10.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:10.706 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:11.057 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 56925 56909 93.5519 49.375 0.00290254 0.0106644 2023-10-09T04:30:11.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3727533247' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[216903]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3727533247' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:11 smithi029 ceph-mon[223858]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:30:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3727533247' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: osdmap e159: 8 total, 8 up, 8 in 2023-10-09T04:30:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: failed to encode map e159 with expected crc 2023-10-09T04:30:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:11 smithi177 ceph-mon[195797]: failed to encode map e159 with expected crc 2023-10-09T04:30:12.058 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 57379 57363 91.8813 28.375 0.00285014 0.0108586 2023-10-09T04:30:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:12 smithi029 ceph-mon[223858]: pgmap v566: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 2359/163629 objects degraded (1.442%); 5616/163629 objects misplaced (3.432%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:30:12.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:12 smithi029 ceph-mon[216903]: pgmap v566: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 2359/163629 objects degraded (1.442%); 5616/163629 objects misplaced (3.432%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:30:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:12 smithi177 ceph-mon[195797]: pgmap v566: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s; 2359/163629 objects degraded (1.442%); 5616/163629 objects misplaced (3.432%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:30:12.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:30:12.608 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:12.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:12.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:12.725 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:12.759 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:12.806 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:30:12.806 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:30:12.806 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T04:30:12.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T04:30:12.819 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:30:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:30:12] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-10-09T04:30:12.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:12.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:12.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:12.994 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:13.061 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:30:13.061081+0000 min lat: 0.00115182 max lat: 2.19982 avg lat: 0.0109592 2023-10-09T04:30:13.061 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:30:13.061 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 58407 58391 91.1843 64.25 0.0262043 0.0109592 2023-10-09T04:30:13.065 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:13.148 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:13 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2359/168837 objects degraded (1.397%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:13.201 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:13.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:13.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:13.341 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:13.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:13.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:13 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2359/168837 objects degraded (1.397%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:13.424 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:13 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2359/168837 objects degraded (1.397%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:30:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:13.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:13.603 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:13.813 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:13.837 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:13.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:13.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:14.040 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:14.066 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 59389 59373 90.4551 61.375 0.0210887 0.0110503 2023-10-09T04:30:14.172 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:14.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:14.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:14.363 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:14.486 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:14.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:14.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:14.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:30:14.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:14 smithi029 ceph-mon[216903]: pgmap v567: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 2359/168837 objects degraded (1.397%); 5616/168837 objects misplaced (3.326%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:30:14.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:14.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:14.655 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:14 smithi029 ceph-mon[223858]: pgmap v567: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 2359/168837 objects degraded (1.397%); 5616/168837 objects misplaced (3.326%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:30:14.655 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:14.655 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:14.658 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:14.773 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:14.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:14.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:14 smithi177 ceph-mon[195797]: pgmap v567: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 2359/168837 objects degraded (1.397%); 5616/168837 objects misplaced (3.326%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:30:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:14.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:14.919 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:14.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:14.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:15.067 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 60523 60507 89.9768 70.875 0.0148796 0.0111071 2023-10-09T04:30:15.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:15.192 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:15.843 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:15.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:15.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:16.019 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:16.067 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 61538 61522 89.3599 63.4375 0.0138191 0.0111845 2023-10-09T04:30:16.112 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:16.157 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:16.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:16.295 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:16.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:16.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:16.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:16.553 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:16.704 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:16 smithi177 ceph-mon[195797]: pgmap v568: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 2359/176040 objects degraded (1.340%); 4990/176040 objects misplaced (2.835%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:30:16.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:16.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:16 smithi029 ceph-mon[216903]: pgmap v568: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 2359/176040 objects degraded (1.340%); 4990/176040 objects misplaced (2.835%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:30:16.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:16 smithi029 ceph-mon[223858]: pgmap v568: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 2359/176040 objects degraded (1.340%); 4990/176040 objects misplaced (2.835%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:30:16.847 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:16.902 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:16.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:17.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:17.033 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.067 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 62218 62202 88.2954 42.5 0.00278665 0.0112209 2023-10-09T04:30:17.075 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:17.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:17.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:17.210 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.337 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:17.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:17.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:17.466 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:17.515 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[216903]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[216903]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[216903]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[216903]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[223858]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[223858]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[223858]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[223858]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:17.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:17.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:17.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:17.644 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.696 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:17.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:17 smithi177 ceph-mon[195797]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:17 smithi177 ceph-mon[195797]: osdmap e160: 8 total, 8 up, 8 in 2023-10-09T04:30:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:17 smithi177 ceph-mon[195797]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:17 smithi177 ceph-mon[195797]: failed to encode map e160 with expected crc 2023-10-09T04:30:17.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:30:17.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:17.830 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:17.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:17.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:18.004 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:18.063 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:18.067 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 62492 62476 86.7147 17.125 0.0220246 0.0115257 2023-10-09T04:30:18.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:18.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:18.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:18.302 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:18.391 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:18.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:18.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:18.499 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:18.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:18 smithi177 ceph-mon[195797]: pgmap v570: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 2359/182265 objects degraded (1.294%); 4990/182265 objects misplaced (2.738%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T04:30:18.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:18 smithi177 ceph-mon[195797]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:30:18.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:18 smithi177 ceph-mon[195797]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.518 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:18 smithi177 ceph-mon[195797]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.518 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:18 smithi177 ceph-mon[195797]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.518 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:18 smithi177 ceph-mon[195797]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.660 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:18.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[216903]: pgmap v570: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 2359/182265 objects degraded (1.294%); 4990/182265 objects misplaced (2.738%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T04:30:18.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[216903]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[216903]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[216903]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[216903]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[216903]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[223858]: pgmap v570: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 2359/182265 objects degraded (1.294%); 4990/182265 objects misplaced (2.738%); 9.0 MiB/s, 144 objects/s recovering 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[223858]: osdmap e161: 8 total, 8 up, 8 in 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[223858]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[223858]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[223858]: failed to encode map e161 with expected crc 2023-10-09T04:30:18.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:18 smithi029 ceph-mon[223858]: failed to encode map e161 with expected crc 2023-10-09T04:30:19.019 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:19.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:19.068 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 63537 63521 86.2486 65.3125 0.0311192 0.0115871 2023-10-09T04:30:19.126 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:19.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:19.174 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:30:19.241 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:30:19.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:19.293 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:30:19.336 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:30:19.388 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:30:19.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:30:19.468 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:30:19.510 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:30:19.532 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.533 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:30:19.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/755796585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:19.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/755796585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/755796585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:20.068 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 64357 64341 85.5043 51.25 0.038036 0.0116901 2023-10-09T04:30:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:20 smithi029 ceph-mon[216903]: pgmap v572: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 2357/183327 objects degraded (1.286%); 4600/183327 objects misplaced (2.509%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:30:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2357/183327 objects degraded (1.286%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:20 smithi029 ceph-mon[223858]: pgmap v572: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 2357/183327 objects degraded (1.286%); 4600/183327 objects misplaced (2.509%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:30:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2357/183327 objects degraded (1.286%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:20 smithi177 ceph-mon[195797]: pgmap v572: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 2357/183327 objects degraded (1.286%); 4600/183327 objects misplaced (2.509%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:30:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2357/183327 objects degraded (1.286%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:21.069 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 65382 65366 85.0576 64.0625 0.00952337 0.0117521 2023-10-09T04:30:22.069 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 66339 66323 84.5426 59.8125 0.00307056 0.0117862 2023-10-09T04:30:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:22 smithi029 ceph-mon[216903]: pgmap v573: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 2357/188502 objects degraded (1.250%); 4600/188502 objects misplaced (2.440%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:30:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:22.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:22 smithi029 ceph-mon[223858]: pgmap v573: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 2357/188502 objects degraded (1.250%); 4600/188502 objects misplaced (2.440%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:30:22.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:22.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:22 smithi177 ceph-mon[195797]: pgmap v573: 89 pgs: 1 active+recovering+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 2357/188502 objects degraded (1.250%); 4600/188502 objects misplaced (2.440%); 8.0 MiB/s, 127 objects/s recovering 2023-10-09T04:30:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:23.069 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 67165 67149 83.8845 51.625 0.0135333 0.0119157 2023-10-09T04:30:23.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:30:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:30:22] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-10-09T04:30:23.269 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:23.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:23.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:23.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:23.401 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:23.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:23.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:23.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:23.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:23.575 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:23.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:23.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:23.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:23.824 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:24.072 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 67903 67887 83.1387 46.125 0.0165226 0.0120226 2023-10-09T04:30:24.342 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:24.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:24.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:24.479 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:24.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:24.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:24.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:24 smithi029 ceph-mon[216903]: pgmap v574: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 2357/194343 objects degraded (1.213%); 4600/194343 objects misplaced (2.367%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:30:24.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:24 smithi029 ceph-mon[223858]: pgmap v574: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 2357/194343 objects degraded (1.213%); 4600/194343 objects misplaced (2.367%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:30:24.667 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:24.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:24.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:24.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:24 smithi177 ceph-mon[195797]: pgmap v574: 89 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 2357/194343 objects degraded (1.213%); 4600/194343 objects misplaced (2.367%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:30:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:24.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:24.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:24.895 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:24.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:24.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:24.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:25.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 68869 68853 82.701 60.375 0.0114144 0.012087 2023-10-09T04:30:25.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:25.159 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:25.561 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:30:25.562 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:30:25.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2357/194343 objects degraded (1.213%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:25.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2357/194343 objects degraded (1.213%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2357/194343 objects degraded (1.213%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:26.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 69851 69835 82.2972 61.375 0.0179141 0.0121456 2023-10-09T04:30:26.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:30:26.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:26 smithi029 ceph-mon[223858]: pgmap v575: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 2357/201051 objects degraded (1.172%); 4266/201051 objects misplaced (2.122%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:30:26.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:26 smithi029 ceph-mon[216903]: pgmap v575: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 2357/201051 objects degraded (1.172%); 4266/201051 objects misplaced (2.122%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:30:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:26 smithi177 ceph-mon[195797]: pgmap v575: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 85 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 2357/201051 objects degraded (1.172%); 4266/201051 objects misplaced (2.122%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:30:27.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 70280 70264 81.2702 26.8125 0.00906072 0.0122706 2023-10-09T04:30:28.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 71039 71023 80.6553 47.4375 0.0144913 0.0123735 2023-10-09T04:30:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:28 smithi029 ceph-mon[216903]: pgmap v576: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 2357/206853 objects degraded (1.139%); 4266/206853 objects misplaced (2.062%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:30:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:28 smithi029 ceph-mon[223858]: pgmap v576: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 2357/206853 objects degraded (1.139%); 4266/206853 objects misplaced (2.062%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:30:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:28 smithi177 ceph-mon[195797]: pgmap v576: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 2357/206853 objects degraded (1.139%); 4266/206853 objects misplaced (2.062%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:30:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:29.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 71771 71755 80.0322 45.75 0.00277527 0.0124662 2023-10-09T04:30:29.750 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:29.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:29.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:29.929 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:29.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:30:30.069 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:30.075 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 72510 72494 79.4386 46.1875 0.0177486 0.0125824 2023-10-09T04:30:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:30.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:30.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:30.362 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:30.571 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:30.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:30.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:30.596 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:30 smithi029 ceph-mon[216903]: pgmap v577: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 2357/208362 objects degraded (1.131%); 3876/208362 objects misplaced (1.860%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:30:30.596 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2357/208362 objects degraded (1.131%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:30.696 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:30.813 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:30.821 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 267 (0%) 2023-10-09T04:30:30.821 INFO:teuthology.orchestra.run.smithi029.stdout:avg 33.375 2023-10-09T04:30:30.821 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 6.04023 -> 6.04023 (expected baseline 5.40399) 2023-10-09T04:30:30.821 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-10-09T04:30:30.821 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T04:30:30.821 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:30:30.822 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T04:30:30.822 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 1 2023-10-09T04:30:30.822 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:30:30.822 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0232 2023-10-09T04:30:30.822 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.2319 2023-10-09T04:30:30.825 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:30:30.842 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:30.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:30 smithi029 ceph-mon[223858]: pgmap v577: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 2357/208362 objects degraded (1.131%); 3876/208362 objects misplaced (1.860%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:30:30.866 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2357/208362 objects degraded (1.131%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:30 smithi177 ceph-mon[195797]: pgmap v577: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 2357/208362 objects degraded (1.131%); 3876/208362 objects misplaced (1.860%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:30:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2357/208362 objects degraded (1.131%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:31.076 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 73184 73168 78.7924 42.125 0.0166395 0.012685 2023-10-09T04:30:31.268 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:31.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:31.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:31.433 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:31.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:31.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:31.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1788336655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:31.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:31.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1788336655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:31.595 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:31.637 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:30:31.640 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:31.729 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:31.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1788336655' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:31.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:30:32.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 74136 74120 78.4655 59.5 0.015263 0.0127389 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: pgmap v578: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 2357/213336 objects degraded (1.105%); 3876/213336 objects misplaced (1.817%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: from='client.45344 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[223858]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: pgmap v578: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 2357/213336 objects degraded (1.105%); 3876/213336 objects misplaced (1.817%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: from='client.45344 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:32 smithi029 ceph-mon[216903]: failed to encode map e162 with expected crc 2023-10-09T04:30:32.924 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:30:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:30:32] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T04:30:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: pgmap v578: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 2357/213336 objects degraded (1.105%); 3876/213336 objects misplaced (1.817%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:30:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: from='client.45344 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:30:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: failed to encode map e162 with expected crc 2023-10-09T04:30:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:30:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: osdmap e162: 8 total, 8 up, 8 in 2023-10-09T04:30:33.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: failed to encode map e162 with expected crc 2023-10-09T04:30:33.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: failed to encode map e162 with expected crc 2023-10-09T04:30:33.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:32 smithi177 ceph-mon[195797]: failed to encode map e162 with expected crc 2023-10-09T04:30:33.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:30:33.077235+0000 min lat: 0.00115182 max lat: 2.19982 avg lat: 0.0127727 2023-10-09T04:30:33.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:30:33.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 15 75199 75184 78.2661 66.5 0.0151274 0.0127727 2023-10-09T04:30:33.959 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:33.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:34.069 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:34.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 76094 76078 77.8991 55.875 0.010406 0.0128319 2023-10-09T04:30:34.111 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:34.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:34.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:34.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:34.389 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:34 smithi029 ceph-mon[216903]: pgmap v580: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 2357/218415 objects degraded (1.079%); 3876/218415 objects misplaced (1.775%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:30:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:34 smithi029 ceph-mon[223858]: pgmap v580: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 2357/218415 objects degraded (1.079%); 3876/218415 objects misplaced (1.775%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:30:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:34 smithi177 ceph-mon[195797]: pgmap v580: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 2357/218415 objects degraded (1.079%); 3876/218415 objects misplaced (1.775%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:30:35.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 77094 77078 77.6478 62.5 0.0149353 0.0128736 2023-10-09T04:30:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2357/218415 objects degraded (1.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:35.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2357/218415 objects degraded (1.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:35.961 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:36.012 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:30:36.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T04:30:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2357/218415 objects degraded (1.079%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:36.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 77707 77691 77.0238 38.3125 0.0216876 0.0129783 2023-10-09T04:30:36.339 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:36.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:36.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:36.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:36.494 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:36.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:36.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:36.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:36.725 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:36 smithi177 ceph-mon[195797]: pgmap v581: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 2357/225396 objects degraded (1.046%); 3494/225396 objects misplaced (1.550%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:30:36.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:36.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:36.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:36 smithi029 ceph-mon[216903]: pgmap v581: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 2357/225396 objects degraded (1.046%); 3494/225396 objects misplaced (1.550%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:30:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:36 smithi029 ceph-mon[223858]: pgmap v581: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 2357/225396 objects degraded (1.046%); 3494/225396 objects misplaced (1.550%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:30:36.945 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:37.080 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 78506 78490 76.6004 49.9375 0.0102836 0.0130499 2023-10-09T04:30:37.464 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:37.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:37.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:37.622 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:37.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:37.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:37.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:37.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:37.812 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:38.080 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 79066 79050 75.9607 35 0.126897 0.0131215 2023-10-09T04:30:38.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:30:38.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:38 smithi029 ceph-mon[223858]: pgmap v582: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 2357/231114 objects degraded (1.020%); 3494/231114 objects misplaced (1.512%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:30:38.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:38 smithi029 ceph-mon[216903]: pgmap v582: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 2357/231114 objects degraded (1.020%); 3494/231114 objects misplaced (1.512%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:30:38.884 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:38.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:38.994 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:38 smithi177 ceph-mon[195797]: pgmap v582: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 85 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 2357/231114 objects degraded (1.020%); 3494/231114 objects misplaced (1.512%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:30:39.030 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.080 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 79693 79677 75.4038 39.1875 0.0128359 0.0132573 2023-10-09T04:30:39.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:39.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:39.242 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:39.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:39.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:39.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:39.554 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.081 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 80214 80198 74.7637 32.5625 0.0189089 0.0133709 2023-10-09T04:30:40.298 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:40.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:40.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:40.453 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:40.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:40.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:40.654 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:40.723 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:40 smithi177 ceph-mon[195797]: pgmap v583: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 2357/232569 objects degraded (1.013%); 2961/232569 objects misplaced (1.273%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:30:40.723 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2357/232569 objects degraded (1.013%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:40.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:40.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:40.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:40.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:40 smithi029 ceph-mon[223858]: pgmap v583: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 2357/232569 objects degraded (1.013%); 2961/232569 objects misplaced (1.273%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:30:40.879 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2357/232569 objects degraded (1.013%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:40.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:40 smithi029 ceph-mon[216903]: pgmap v583: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 2357/232569 objects degraded (1.013%); 2961/232569 objects misplaced (1.273%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:30:40.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2357/232569 objects degraded (1.013%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:40.889 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:40.978 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:30:41.024 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:41.030 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:41.040 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:30:41.085 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 15 81025 81010 74.4067 50.75 0.0353216 0.0134314 2023-10-09T04:30:41.111 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:30:41.138 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:41.169 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.186 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:30:41.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:41.228 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:30:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:41.264 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:30:41.308 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:30:41.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:41.351 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:30:41.367 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.367 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.367 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:30:41.368 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:41.378 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:81236 9.b 9:d62afb4c:::benchmark_data_smithi029_299184_object81235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:41.373881+0000", 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0036092210000000001, 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003653819, 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:30:41.385 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 81236 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.373881+0000", 2023-10-09T04:30:41.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.373881+0000", 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.373880+0000", 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:30:41.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.373906+0000", 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6927000000000002e-05 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.373929+0000", 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2555e-05 2023-10-09T04:30:41.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.373937+0000", 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0860000000000002e-06 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.374101+0000", 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016408699999999999 2023-10-09T04:30:41.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.374204+0000", 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103131 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.374317+0000", 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011226900000000001 2023-10-09T04:30:41.390 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:81226 9.b 9:d7ffa78c:::benchmark_data_smithi029_299184_object81225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:41.352942+0000", 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.024548190000000001, 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024684985, 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:30:41.391 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 81226 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:30:41.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.352942+0000", 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.352942+0000", 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.352942+0000", 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:30:41.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.352967+0000", 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5514999999999999e-05 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.352976+0000", 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2910000000000007e-06 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.352982+0000", 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.618e-06 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.357637+0000", 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046548830000000003 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:30:41.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.357808+0000", 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000171058 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.358004+0000", 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000196193 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.365975+0000", 2023-10-09T04:30:41.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079704960000000005 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.365994+0000", 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8966000000000001e-05 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.372759+0000", 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067648669999999999 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:30:41.397 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:81231 9.b 9:dc143c91:::benchmark_data_smithi029_299184_object81230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:41.361169+0000", 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.016321684999999999, 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016557678999999999, 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:30:41.398 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 81231 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:30:41.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.361169+0000", 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.361169+0000", 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.361167+0000", 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.361195+0000", 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7336000000000001e-05 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.361206+0000", 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0922e-05 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:30:41.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.361213+0000", 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.097e-06 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.366402+0000", 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051895179999999997 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.366572+0000", 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000170225 2023-10-09T04:30:41.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.366764+0000", 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019182100000000001 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.372772+0000", 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060074109999999998 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:30:41.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:41.374085+0000", 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013133610000000001 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 3 2023-10-09T04:30:41.404 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:30:41.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:41.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:41.596 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:41.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3449393897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3449393897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:41.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:41.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3449393897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:30:41.831 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:41.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:41.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:42.026 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:42.086 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 81881 81865 74.1009 53.4375 0.00877695 0.0134894 2023-10-09T04:30:42.090 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:42 smithi029 ceph-mon[223858]: pgmap v584: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 2357/236877 objects degraded (0.995%); 2961/236877 objects misplaced (1.250%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:30:42.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:30:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:30:42] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T04:30:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:42 smithi029 ceph-mon[216903]: pgmap v584: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 2357/236877 objects degraded (0.995%); 2961/236877 objects misplaced (1.250%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:30:42.926 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:42.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:42 smithi177 ceph-mon[195797]: pgmap v584: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 2357/236877 objects degraded (0.995%); 2961/236877 objects misplaced (1.250%); 5.7 MiB/s, 91 objects/s recovering 2023-10-09T04:30:43.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:43.070 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:43.086 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 82808 82792 73.8701 57.9375 0.00371382 0.0135184 2023-10-09T04:30:43.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:43.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:43.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:43.272 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:44.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 83440 83424 73.3862 39.5 0.0125937 0.0136199 2023-10-09T04:30:44.569 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:44.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:44.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:44.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:44.734 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:44 smithi177 ceph-mon[195797]: pgmap v585: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 712 op/s; 2357/241719 objects degraded (0.975%); 2961/241719 objects misplaced (1.225%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:30:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:44.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:44 smithi029 ceph-mon[216903]: pgmap v585: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 712 op/s; 2357/241719 objects degraded (0.975%); 2961/241719 objects misplaced (1.225%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:30:44.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:44.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:44.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:44 smithi029 ceph-mon[223858]: pgmap v585: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 712 op/s; 2357/241719 objects degraded (0.975%); 2961/241719 objects misplaced (1.225%); 5.1 MiB/s, 81 objects/s recovering 2023-10-09T04:30:44.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:30:44.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:45.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 84127 84111 72.9635 42.9375 0.00333085 0.0136608 2023-10-09T04:30:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:45 smithi177 ceph-mon[195797]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:45 smithi177 ceph-mon[195797]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:30:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:45 smithi177 ceph-mon[195797]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:45 smithi177 ceph-mon[195797]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:45 smithi177 ceph-mon[195797]: pgmap v587: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s; 2357/247713 objects degraded (0.952%); 2483/247713 objects misplaced (1.002%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T04:30:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2357/241719 objects degraded (0.975%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:46.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 84328 84312 72.1366 12.5625 0.00826516 0.0138175 2023-10-09T04:30:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[223858]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[223858]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:30:46.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[223858]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[223858]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[223858]: pgmap v587: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s; 2357/247713 objects degraded (0.952%); 2483/247713 objects misplaced (1.002%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T04:30:46.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2357/241719 objects degraded (0.975%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[216903]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[216903]: osdmap e163: 8 total, 8 up, 8 in 2023-10-09T04:30:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[216903]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[216903]: failed to encode map e163 with expected crc 2023-10-09T04:30:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[216903]: pgmap v587: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 743 op/s; 2357/247713 objects degraded (0.952%); 2483/247713 objects misplaced (1.002%); 6.3 MiB/s, 101 objects/s recovering 2023-10-09T04:30:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2357/241719 objects degraded (0.975%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:30:46.676 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:46.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:46.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:46.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:46.839 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:46.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:30:46.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:46.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:46.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:46.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[216903]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:30:46.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[216903]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[216903]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[216903]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[216903]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[216903]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[223858]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:30:46.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[223858]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[223858]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[223858]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[223858]: failed to encode map e164 with expected crc 2023-10-09T04:30:46.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:46 smithi029 ceph-mon[223858]: failed to encode map e164 with expected crc 2023-10-09T04:30:47.013 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:30:47.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:30:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:46 smithi177 ceph-mon[195797]: osdmap e164: 8 total, 8 up, 8 in 2023-10-09T04:30:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:46 smithi177 ceph-mon[195797]: failed to encode map e164 with expected crc 2023-10-09T04:30:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:46 smithi177 ceph-mon[195797]: failed to encode map e164 with expected crc 2023-10-09T04:30:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:46 smithi177 ceph-mon[195797]: failed to encode map e164 with expected crc 2023-10-09T04:30:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:46 smithi177 ceph-mon[195797]: failed to encode map e164 with expected crc 2023-10-09T04:30:47.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:46 smithi177 ceph-mon[195797]: failed to encode map e164 with expected crc 2023-10-09T04:30:47.047 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:47.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 85223 85207 71.9177 55.9375 0.0298222 0.0138985 2023-10-09T04:30:47.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:30:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:47.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:30:47.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:30:47.391 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:47 smithi029 ceph-mon[223858]: pgmap v589: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 4624/251952 objects misplaced (1.835%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:30:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:47 smithi029 ceph-mon[216903]: pgmap v589: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 4624/251952 objects misplaced (1.835%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:30:48.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:47 smithi177 ceph-mon[195797]: pgmap v589: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 85 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 4624/251952 objects misplaced (1.835%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:30:48.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 86084 86068 71.6764 53.8125 0.0257868 0.0139469 2023-10-09T04:30:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:48 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2357/247713 objects degraded (0.952%), 1 pg degraded) 2023-10-09T04:30:49.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 87103 87087 71.5712 63.6875 0.011163 0.0139671 2023-10-09T04:30:49.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:48 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2357/247713 objects degraded (0.952%), 1 pg degraded) 2023-10-09T04:30:49.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:48 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2357/247713 objects degraded (0.952%), 1 pg degraded) 2023-10-09T04:30:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:49 smithi177 ceph-mon[195797]: pgmap v590: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 4569/253023 objects misplaced (1.806%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:30:50.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 88079 88063 71.4337 61 0.0286511 0.013993 2023-10-09T04:30:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:49 smithi029 ceph-mon[216903]: pgmap v590: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 4569/253023 objects misplaced (1.806%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:30:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:49 smithi029 ceph-mon[223858]: pgmap v590: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 4569/253023 objects misplaced (1.806%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:30:50.368 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:30:50.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:50.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:30:50.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:50.544 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:50.908 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:50.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:50.966 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:51.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:30:51.080 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:30:51.101 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 88885 88869 71.163 50.375 0.0280573 0.014048 2023-10-09T04:30:51.101 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:51.243 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 267 (0%) 2023-10-09T04:30:51.243 INFO:teuthology.orchestra.run.smithi029.stdout:avg 33.375 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 6.04023 -> 6.04023 (expected baseline 5.40399) 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0274 2023-10-09T04:30:51.244 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.2736 2023-10-09T04:30:51.282 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:30:51.340 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:30:51.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:51.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:30:51.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:30:51.485 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:51.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:30:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:51.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:30:51.671 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:30:51.748 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:52.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 89670 89654 70.8833 49.0625 0.0196013 0.0141033 2023-10-09T04:30:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:52 smithi029 ceph-mon[216903]: pgmap v591: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 4569/258090 objects misplaced (1.770%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:30:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:52 smithi029 ceph-mon[216903]: from='client.45353 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:30:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:52 smithi029 ceph-mon[223858]: pgmap v591: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 4569/258090 objects misplaced (1.770%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:30:52.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:52 smithi029 ceph-mon[223858]: from='client.45353 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:30:52.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:52 smithi177 ceph-mon[195797]: pgmap v591: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 4569/258090 objects misplaced (1.770%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:30:52.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:52 smithi177 ceph-mon[195797]: from='client.45353 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:30:53.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:30:53.089210+0000 min lat: 0.00115182 max lat: 2.19982 avg lat: 0.0141295 2023-10-09T04:30:53.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:30:53.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 90630 90614 70.7473 60 0.019488 0.0141295 2023-10-09T04:30:53.109 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:30:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1052707036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:30:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:30:52] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-10-09T04:30:53.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1052707036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:53.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1052707036' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:30:54.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 91520 91504 70.5606 55.625 0.0179701 0.014167 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: pgmap v592: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 4334/265707 objects misplaced (1.631%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1052707036' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[223858]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: pgmap v592: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 4334/265707 objects misplaced (1.631%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1052707036' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:54 smithi029 ceph-mon[216903]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: pgmap v592: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 4334/265707 objects misplaced (1.631%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T04:30:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2023-10-09T04:30:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:30:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1052707036' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:30:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: osdmap e165: 8 total, 8 up, 8 in 2023-10-09T04:30:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: failed to encode map e165 with expected crc 2023-10-09T04:30:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:54 smithi177 ceph-mon[195797]: failed to encode map e165 with expected crc 2023-10-09T04:30:55.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 92445 92429 70.4052 57.8125 0.0229616 0.0141987 2023-10-09T04:30:56.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 93254 93238 70.1661 50.5625 0.00330379 0.0142284 2023-10-09T04:30:56.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:56 smithi029 ceph-mon[216903]: pgmap v594: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 4334/271641 objects misplaced (1.595%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:30:56.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:56 smithi029 ceph-mon[223858]: pgmap v594: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 4334/271641 objects misplaced (1.595%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:30:56.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:56 smithi177 ceph-mon[195797]: pgmap v594: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s; 4334/271641 objects misplaced (1.595%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:30:56.682 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:30:56.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:56.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:30:56.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:56.837 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:56.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:56.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:56.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:57.030 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:30:57.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 93950 93934 69.8488 43.5 0.00778033 0.0143111 2023-10-09T04:30:57.109 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:57.491 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:30:57.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-09T04:30:58.089 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 94568 94552 69.4816 38.625 0.0170863 0.0143869 2023-10-09T04:30:58.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:58 smithi029 ceph-mon[216903]: pgmap v595: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 4084/277206 objects misplaced (1.473%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T04:30:58.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:58 smithi029 ceph-mon[223858]: pgmap v595: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 4084/277206 objects misplaced (1.473%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T04:30:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:58 smithi177 ceph-mon[195797]: pgmap v595: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s; 4084/277206 objects misplaced (1.473%); 3.4 MiB/s, 53 objects/s recovering 2023-10-09T04:30:59.090 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 95404 95388 69.2802 52.25 0.0177155 0.0144289 2023-10-09T04:30:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:30:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:30:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:59.519 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:30:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:30:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:30:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:59.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:30:59.629 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:30:59.664 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:59.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:30:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:59.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:30:59.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:30:59.868 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:30:59.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:30:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:30:59.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:30:59.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:31:00.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:00.058 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 96154 96138 69.0227 46.875 0.0366549 0.0144805 2023-10-09T04:31:00.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:00.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:00.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:00.131 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:00.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:00.259 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:00.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:00.345 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:31:00.399 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:31:00.424 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:00.434 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:31:00.455 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.509 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:31:00.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:00.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:00.556 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:31:00.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:31:00.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:00.641 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:31:00.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[216903]: pgmap v596: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 4084/278679 objects misplaced (1.465%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:31:00.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:00.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:00.651 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4113803803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:00.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[223858]: pgmap v596: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 4084/278679 objects misplaced (1.465%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:31:00.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:00.655 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:00.655 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4113803803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:00.658 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:00.682 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.708 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.709 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.709 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:00.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:00.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:00 smithi177 ceph-mon[195797]: pgmap v596: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 854 op/s; 4084/278679 objects misplaced (1.465%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:31:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:00 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:00 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4113803803' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:00.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:00.850 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:00.851 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:00.932 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:01.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 96800 96784 68.6973 40.375 0.0152447 0.0145516 2023-10-09T04:31:01.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:01.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:31:01.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:01.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:01.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:31:01.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:31:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:31:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:02.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 97633 97617 68.5104 52.0625 0.00701688 0.0145913 2023-10-09T04:31:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:02 smithi029 ceph-mon[216903]: pgmap v597: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 4084/283815 objects misplaced (1.439%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:31:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:02 smithi029 ceph-mon[223858]: pgmap v597: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 4084/283815 objects misplaced (1.439%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:31:02.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:31:02] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T04:31:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:02 smithi177 ceph-mon[195797]: pgmap v597: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 4084/283815 objects misplaced (1.439%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:31:03.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0087 2023-10-09T04:31:03.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 98300 2023-10-09T04:31:03.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:31:03.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:31:03.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 68.2573 2023-10-09T04:31:03.091 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 42.9427 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 225.875 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1092 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 687.089 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3614 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0146464 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0389806 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.19982 2023-10-09T04:31:03.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00115182 2023-10-09T04:31:03.095 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:31:03.440 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:03.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:03.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:03.602 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:03.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:03.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:03.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:03.828 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:03.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:03.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:03.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:04.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:04.073 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:04 smithi029 ceph-mon[216903]: pgmap v598: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 3827/289812 objects misplaced (1.321%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:04 smithi029 ceph-mon[223858]: pgmap v598: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 3827/289812 objects misplaced (1.321%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:04 smithi177 ceph-mon[195797]: pgmap v598: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 3827/289812 objects misplaced (1.321%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:05.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:31:05.745 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:05.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:05.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:05.911 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:05.923 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:05.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:05.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:06.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:06.211 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:06.404 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:06.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:06.542 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:06.614 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:06.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:06 smithi029 ceph-mon[216903]: pgmap v599: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 87 B/s rd, 48 MiB/s wr, 810 op/s; 3827/291450 objects misplaced (1.313%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:31:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:06 smithi029 ceph-mon[223858]: pgmap v599: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 87 B/s rd, 48 MiB/s wr, 810 op/s; 3827/291450 objects misplaced (1.313%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:31:06.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:06.825 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:06.898 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:06.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:06.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout:moved 9 / 267 (3.37079%) 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout:avg 33.375 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 6.04023 -> 6.22369 (expected baseline 5.40399) 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 41 -> 39 pgs (1.22846 -> 1.16854 * mean) 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:31:06.970 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0304 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0335 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 1.0000 -> 0.9500 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 weight 1.0000 -> 0.9500 2023-10-09T04:31:06.971 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 weight 1.0000 -> 0.9500 2023-10-09T04:31:06.981 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:31:07.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:06 smithi177 ceph-mon[195797]: pgmap v599: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 87 B/s rd, 48 MiB/s wr, 810 op/s; 3827/291450 objects misplaced (1.313%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:31:07.061 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.073 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:06 smithi029 conmon[202582]: 2023-10-09T04:31:06.959+0000 7f6049533700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-09T04:31:07.140 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:07.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:07.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:07.305 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:07.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:07.513 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:07.552 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:07.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:07.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:07.760 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:07.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:08 smithi029 ceph-mon[216903]: pgmap v600: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 762 op/s; 3529/290442 objects misplaced (1.215%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:08 smithi029 ceph-mon[216903]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:08 smithi029 ceph-mon[223858]: pgmap v600: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 762 op/s; 3529/290442 objects misplaced (1.215%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:08 smithi029 ceph-mon[223858]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:09.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:08 smithi177 ceph-mon[195797]: pgmap v600: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 762 op/s; 3529/290442 objects misplaced (1.215%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:09.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:08 smithi177 ceph-mon[195797]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:09.607 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:09.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:09.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:09.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:09.791 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:09.884 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:09.922 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:31:09.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T04:31:09.950 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:09.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:09.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:10.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:10.123 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.197 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:10.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:10.210 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:10.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:10.333 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.401 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:10.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:10.547 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:10.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:10.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:10 smithi177 ceph-mon[195797]: pgmap v601: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 646 op/s; 3529/289467 objects misplaced (1.219%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:10.775 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:10.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:10.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:10.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:10 smithi029 ceph-mon[216903]: pgmap v601: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 646 op/s; 3529/289467 objects misplaced (1.219%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:10.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:10 smithi029 ceph-mon[223858]: pgmap v601: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 646 op/s; 3529/289467 objects misplaced (1.219%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:10.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:10.986 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:11.042 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:11.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:11.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:11.185 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:11.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2444633075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:11.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:11.730 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:31:11.837 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:11.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:11.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2444633075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:11.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:11.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:11.995 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2444633075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:11.995 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:31:11.998 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:12.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:12.221 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:12.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:12.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:12.427 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.512 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:12.623 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:12.664 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:12.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:12.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:12.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: pgmap v602: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 735 op/s; 3529/284778 objects misplaced (1.239%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:31:12.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[223858]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:31:12] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: pgmap v602: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 735 op/s; 3529/284778 objects misplaced (1.239%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:12 smithi029 ceph-mon[216903]: failed to encode map e166 with expected crc 2023-10-09T04:31:12.932 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:13.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: pgmap v602: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 735 op/s; 3529/284778 objects misplaced (1.239%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:13.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:31:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: osdmap e166: 8 total, 8 up, 8 in 2023-10-09T04:31:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: failed to encode map e166 with expected crc 2023-10-09T04:31:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: failed to encode map e166 with expected crc 2023-10-09T04:31:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: failed to encode map e166 with expected crc 2023-10-09T04:31:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:12 smithi177 ceph-mon[195797]: failed to encode map e166 with expected crc 2023-10-09T04:31:13.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:13.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:14.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:14.185 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:14.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:14.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:14.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:14.323 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:14.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:14.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:14.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:14.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:14.502 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:14 smithi029 ceph-mon[216903]: pgmap v604: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 701 op/s; 3250/279090 objects misplaced (1.164%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:31:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:14 smithi029 ceph-mon[223858]: pgmap v604: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 701 op/s; 3250/279090 objects misplaced (1.164%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:31:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:14 smithi177 ceph-mon[195797]: pgmap v604: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 701 op/s; 3250/279090 objects misplaced (1.164%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:31:15.560 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:15.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:15.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:15.745 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:15.820 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:15.839 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:15.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:15.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:16.002 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:16.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:16.296 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.324 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:31:16.358 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:31:16.398 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:31:16.449 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:31:16.489 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:31:16.547 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:31:16.559 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:16.579 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:31:16.615 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:31:16.630 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.630 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.630 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.630 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.631 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.631 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.631 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.631 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:31:16.651 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:35396 9.2 e152/144 9:4b31cc3b:::benchmark_data_smithi029_299184_object35395:head v 152'2203, mlcod=152'2201)", 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:47.768288+0000", 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 88.876981408999995, 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33991231199999999, 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.652 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768288+0000", 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768288+0000", 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768287+0000", 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768311+0000", 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3723999999999999e-05 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768321+0000", 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0747999999999999e-05 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768329+0000", 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1250000000000004e-06 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768434+0000", 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105723 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.768457+0000", 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3067000000000001e-05 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.108158+0000", 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33970039800000001 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.108200+0000", 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2620999999999997e-05 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:35401 9.b e152/148 9:db72b310:::benchmark_data_smithi029_299184_object35400:head v 152'2225, mlcod=152'2221)", 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:47.769271+0000", 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 88.875998316999997, 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33943419899999999, 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.769271+0000", 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.769271+0000", 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.769270+0000", 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.769336+0000", 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5698999999999996e-05 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105027+0000", 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33569097599999997 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105034+0000", 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2490000000000004e-06 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105097+0000", 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2528999999999997e-05 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105133+0000", 2023-10-09T04:31:16.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6018e-05 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.108665+0000", 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035325209999999998 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.108705+0000", 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9857000000000001e-05 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.662 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:35404 9.2 e152/144 9:432034d1:::benchmark_data_smithi029_299184_object35403:head v 152'2204, mlcod=152'2201)", 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:47.773049+0000", 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 88.872220463999994, 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.335181964, 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.773049+0000", 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.773049+0000", 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.773048+0000", 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:16.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.773076+0000", 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7684000000000001e-05 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.104994+0000", 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33191881499999998 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105034+0000", 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9637e-05 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.106120+0000", 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001086011 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.106139+0000", 2023-10-09T04:31:16.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9040000000000001e-05 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.108206+0000", 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020667089999999999 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.108231+0000", 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5113000000000002e-05 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:35424 9.0 e152/144 9:01d44fc1:::benchmark_data_smithi029_299184_object35423:head v 152'2204, mlcod=152'2202)", 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:47.781049+0000", 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 88.864220614999994, 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.32636962899999999, 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.781049+0000", 2023-10-09T04:31:16.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.781049+0000", 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.781048+0000", 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:16.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.781075+0000", 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6934e-05 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.104989+0000", 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.3239148 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105526+0000", 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00053624 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105571+0000", 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5791000000000003e-05 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.105585+0000", 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3728e-05 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.107368+0000", 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017825709999999999 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.107418+0000", 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0834000000000002e-05 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38514 9.b 9:d453250c:::benchmark_data_smithi029_299184_object38513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020440+0000", 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.624829872000007, 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1984075160000001, 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38514 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.673 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020440+0000", 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020440+0000", 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020438+0000", 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020473+0000", 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4394000000000003e-05 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020480+0000", 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.593e-06 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020487+0000", 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8170000000000001e-06 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021885+0000", 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001398299 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021895+0000", 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0147e-05 2023-10-09T04:31:16.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.200648+0000", 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.178752359 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.201605+0000", 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00095710200000000004 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.201829+0000", 2023-10-09T04:31:16.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022408899999999999 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206278+0000", 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044487629999999997 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206331+0000", 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3090999999999997e-05 2023-10-09T04:31:16.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.209876+0000", 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035455370000000001 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217416+0000", 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075398699999999997 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217479+0000", 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3187000000000003e-05 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.218847+0000", 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013676809999999999 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38525 9.b 9:d3e20ee7:::benchmark_data_smithi029_299184_object38524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:31:16.680 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.021836+0000", 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.623433532999996, 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.198257849, 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38525 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021836+0000", 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021836+0000", 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021835+0000", 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021862+0000", 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6475999999999999e-05 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021873+0000", 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.169e-05 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021882+0000", 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9539999999999993e-06 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021932+0000", 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9437000000000001e-05 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:16.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021942+0000", 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.039e-05 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.204518+0000", 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.182575868 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.204666+0000", 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014826499999999999 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.204849+0000", 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000182497 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.209797+0000", 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049479670000000002 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210470+0000", 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00067308500000000002 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216710+0000", 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062401000000000002 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.218945+0000", 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022349760000000001 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219015+0000", 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9538999999999999e-05 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220094+0000", 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010792709999999999 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38552 9.b 9:d4209d55:::benchmark_data_smithi029_299184_object38551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.021895+0000", 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.623374072000004, 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.19943095, 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38552 2023-10-09T04:31:16.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021895+0000", 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021895+0000", 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021895+0000", 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021919+0000", 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3975999999999999e-05 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021927+0000", 2023-10-09T04:31:16.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8320000000000006e-06 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021933+0000", 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1800000000000001e-06 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021965+0000", 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2361999999999999e-05 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:16.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.021973+0000", 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5989999999999996e-06 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206359+0000", 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.184385883 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206495+0000", 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013616400000000001 2023-10-09T04:31:16.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206660+0000", 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016552599999999999 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212382+0000", 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057219289999999997 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216677+0000", 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042949540000000001 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216766+0000", 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8632000000000001e-05 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220137+0000", 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033710139999999999 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220186+0000", 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8896000000000001e-05 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221326+0000", 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011406420000000001 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:52512 9.b 9:d9494fe2:::benchmark_data_smithi029_299184_object52511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:05.611455+0000", 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.033814421000002, 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20043435000000001, 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 52512 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611455+0000", 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611455+0000", 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611453+0000", 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611493+0000", 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9919999999999997e-05 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611508+0000", 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5288999999999999e-05 2023-10-09T04:31:16.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611518+0000", 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.628e-06 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611565+0000", 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7409e-05 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611721+0000", 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015581699999999999 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611887+0000", 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166429 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614480+0000", 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025920549999999998 2023-10-09T04:31:16.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810554+0000", 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.196074518 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810586+0000", 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2033000000000003e-05 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810690+0000", 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103776 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810748+0000", 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8559e-05 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.811889+0000", 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001140987 2023-10-09T04:31:16.703 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:52514 9.b 9:db551996:::benchmark_data_smithi029_299184_object52513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:05.611533+0000", 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.03373637, 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.210039583, 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 52514 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611533+0000", 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611533+0000", 2023-10-09T04:31:16.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611533+0000", 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611554+0000", 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1905000000000001e-05 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611561+0000", 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8970000000000002e-06 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.611567+0000", 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.648e-06 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.616230+0000", 2023-10-09T04:31:16.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046628909999999997 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.808782+0000", 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19255225300000001 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.808974+0000", 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000192034 2023-10-09T04:31:16.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.814110+0000", 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051358560000000003 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.814147+0000", 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7394000000000001e-05 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.818175+0000", 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040272770000000001 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.820273+0000", 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020986379999999999 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.820332+0000", 2023-10-09T04:31:16.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8884000000000001e-05 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.821573+0000", 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012404180000000001 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:52517 9.0 e158/144 9:01f71211:::benchmark_data_smithi029_299184_object52516:head v 158'3265, mlcod=158'3263)", 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:05.612296+0000", 2023-10-09T04:31:16.711 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.032973612999996, 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19813614900000001, 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612296+0000", 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612296+0000", 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612295+0000", 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612319+0000", 2023-10-09T04:31:16.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3711e-05 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612330+0000", 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1e-05 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612337+0000", 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2289999999999997e-06 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612405+0000", 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7787000000000007e-05 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.612427+0000", 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1755000000000001e-05 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810395+0000", 2023-10-09T04:31:16.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19796820800000001 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810432+0000", 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7282e-05 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:52513 9.2 e158/144 9:49872d1e:::benchmark_data_smithi029_299184_object52512:head v 158'3280, mlcod=158'3278)", 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:05.614429+0000", 2023-10-09T04:31:16.716 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.030840377999994, 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19771755199999999, 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614429+0000", 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614429+0000", 2023-10-09T04:31:16.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614428+0000", 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614450+0000", 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.177e-05 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614462+0000", 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2159e-05 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614469+0000", 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0589999999999997e-06 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614513+0000", 2023-10-09T04:31:16.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3819000000000002e-05 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614533+0000", 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0030999999999998e-05 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.812108+0000", 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19757470899999999 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.812147+0000", 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8761999999999999e-05 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:52520 9.0 e158/144 9:0fde2b3e:::benchmark_data_smithi029_299184_object52519:head v 158'3266, mlcod=158'3263)", 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:05.614650+0000", 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.030619311999999, 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19600045299999999, 2023-10-09T04:31:16.721 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614650+0000", 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614650+0000", 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614650+0000", 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614671+0000", 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1528000000000001e-05 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614681+0000", 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0125e-05 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.614688+0000", 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4300000000000003e-06 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.808778+0000", 2023-10-09T04:31:16.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19409079500000001 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.808802+0000", 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3810000000000001e-05 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810620+0000", 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001817719 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810651+0000", 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0673999999999998e-05 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:52512 9.b e158/154 ondisk, result = 0)", 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:05.618024+0000", 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 71.027245128999994, 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19253448300000001, 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.726 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.618024+0000", 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.618024+0000", 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.618023+0000", 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.618030+0000", 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0800000000000002e-06 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.808744+0000", 2023-10-09T04:31:16.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19071475600000001 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.808757+0000", 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2678e-05 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810537+0000", 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017801570000000001 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810547+0000", 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6199999999999994e-06 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:05.810559+0000", 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2069000000000001e-05 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:53385 9.b 9:d908701a:::benchmark_data_smithi029_299184_object53384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:06.575483+0000", 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 70.069786749000002, 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.31189256900000001, 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 53385 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.731 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575483+0000", 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575483+0000", 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575481+0000", 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575510+0000", 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9079999999999999e-05 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575528+0000", 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7781000000000001e-05 2023-10-09T04:31:16.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575538+0000", 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0349999999999999e-05 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575609+0000", 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1182999999999999e-05 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575761+0000", 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015233 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.575919+0000", 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015779999999999999 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.578932+0000", 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030130859999999999 2023-10-09T04:31:16.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.581528+0000", 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025951440000000002 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.582118+0000", 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000590535 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.886134+0000", 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30401578699999998 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.886204+0000", 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0610000000000003e-05 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.887375+0000", 2023-10-09T04:31:16.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011708409999999999 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:53406 9.b 9:d8d93016:::benchmark_data_smithi029_299184_object53405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:06.589247+0000", 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 70.056022912000003, 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29914559699999999, 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.738 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 53406 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589247+0000", 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589247+0000", 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589245+0000", 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589271+0000", 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6168000000000001e-05 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589284+0000", 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2651e-05 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589294+0000", 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9119999999999993e-06 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589360+0000", 2023-10-09T04:31:16.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6172000000000003e-05 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589525+0000", 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016528300000000001 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.589678+0000", 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015280900000000001 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.745847+0000", 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.156168951 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.747465+0000", 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016182939999999999 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.747504+0000", 2023-10-09T04:31:16.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9264000000000002e-05 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.887465+0000", 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13996059799999999 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.887533+0000", 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8095000000000004e-05 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.888392+0000", 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00085900799999999997 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:53412 9.b 9:de6a45f1:::benchmark_data_smithi029_299184_object53411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:06.594169+0000", 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 70.051100142999999, 2023-10-09T04:31:16.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29495328500000001, 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 53412 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.594169+0000", 2023-10-09T04:31:16.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.594169+0000", 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.594168+0000", 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.594194+0000", 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6366000000000001e-05 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.745790+0000", 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.151595588 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.745826+0000", 2023-10-09T04:31:16.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6157e-05 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.745895+0000", 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8801999999999996e-05 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.746027+0000", 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013207 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.746142+0000", 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115497 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.747658+0000", 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001516117 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.748351+0000", 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000692619 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.748408+0000", 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7080000000000002e-05 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.888423+0000", 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14001533399999999 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.888452+0000", 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8904e-05 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.889123+0000", 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00067038500000000001 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:56746 9.b 9:d53fd6af:::benchmark_data_smithi029_299184_object56745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:10.677897+0000", 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 65.967372259000001, 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.206152004, 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 56746 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677897+0000", 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677897+0000", 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677896+0000", 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677916+0000", 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0373999999999998e-05 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677931+0000", 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4793999999999999e-05 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677938+0000", 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4309999999999999e-06 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.677991+0000", 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3504999999999997e-05 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.678162+0000", 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017126700000000001 2023-10-09T04:31:16.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.678280+0000", 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117484 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.680321+0000", 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020407849999999998 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.681559+0000", 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001238259 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.681951+0000", 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000392352 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.882841+0000", 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20088920299999999 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.882897+0000", 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6786999999999997e-05 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.884049+0000", 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001151918 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.759 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.760 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:56757 9.b 9:d4b48ac5:::benchmark_data_smithi029_299184_object56756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:10.688501+0000", 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 65.956768323000006, 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19656225299999999, 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.761 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 56757 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688501+0000", 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688501+0000", 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688499+0000", 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688532+0000", 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2449000000000001e-05 2023-10-09T04:31:16.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688558+0000", 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6395000000000001e-05 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688569+0000", 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1188e-05 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688635+0000", 2023-10-09T04:31:16.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5325000000000002e-05 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.688821+0000", 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000185973 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.689024+0000", 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020374499999999999 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.691089+0000", 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002065051 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.691715+0000", 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00062576099999999996 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.691777+0000", 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1519999999999994e-05 2023-10-09T04:31:16.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.884110+0000", 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19233318199999999 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.884163+0000", 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3031000000000001e-05 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:10.885063+0000", 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00090049900000000003 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:82903 9.b 9:d6e4affa:::benchmark_data_smithi029_299184_object82902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:43.238168+0000", 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 33.407101064999999, 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22869874100000001, 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.768 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 82903 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238168+0000", 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.769 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238168+0000", 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238167+0000", 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238191+0000", 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4471e-05 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238203+0000", 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1708e-05 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238212+0000", 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5229999999999995e-06 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238410+0000", 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019886700000000001 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238630+0000", 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000219896 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.238815+0000", 2023-10-09T04:31:16.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018461700000000001 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.242380+0000", 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035654649999999999 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.249555+0000", 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071744670000000003 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.773 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.250814+0000", 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012590699999999999 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.466600+0000", 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21578592499999999 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.466671+0000", 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0751000000000001e-05 2023-10-09T04:31:16.774 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.466867+0000", 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000196474 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:82918 9.b 9:dd50c8de:::benchmark_data_smithi029_299184_object82917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:43.254803+0000", 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 33.390466502000002, 2023-10-09T04:31:16.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21342626100000001, 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 82918 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254803+0000", 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254803+0000", 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254801+0000", 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254828+0000", 2023-10-09T04:31:16.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6282999999999999e-05 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254842+0000", 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4814000000000001e-05 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254858+0000", 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5919000000000001e-05 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.254917+0000", 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8683e-05 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.255107+0000", 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190132 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,6", 2023-10-09T04:31:16.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.255297+0000", 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019020199999999999 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.257531+0000", 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022334109999999998 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.258335+0000", 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00080466399999999997 2023-10-09T04:31:16.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.258392+0000", 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.694e-05 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.468019+0000", 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.209626162 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:16.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.468074+0000", 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.507e-05 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:43.468229+0000", 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000155556 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:16.782 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:31:16.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:16.813 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:16.883 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:16.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:16 smithi029 ceph-mon[216903]: pgmap v605: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 4.3 MiB/s wr, 699 op/s; 3250/274626 objects misplaced (1.183%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:31:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1984346428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:16 smithi029 ceph-mon[223858]: pgmap v605: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 4.3 MiB/s wr, 699 op/s; 3250/274626 objects misplaced (1.183%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:31:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1984346428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:16.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:16 smithi177 ceph-mon[195797]: pgmap v605: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 4.3 MiB/s wr, 699 op/s; 3250/274626 objects misplaced (1.183%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:31:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1984346428' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:17.068 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:17.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:31:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:18 smithi177 ceph-mon[195797]: pgmap v606: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 429 KiB/s wr, 696 op/s; 2832/269955 objects misplaced (1.049%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:19.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:18 smithi029 ceph-mon[216903]: pgmap v606: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 429 KiB/s wr, 696 op/s; 2832/269955 objects misplaced (1.049%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:18 smithi029 ceph-mon[223858]: pgmap v606: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 429 KiB/s wr, 696 op/s; 2832/269955 objects misplaced (1.049%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:19.266 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:19.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:19.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:19.402 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:19 smithi177 ceph-mon[195797]: pgmap v607: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2832/268680 objects misplaced (1.054%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:19 smithi029 ceph-mon[216903]: pgmap v607: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2832/268680 objects misplaced (1.054%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:19 smithi029 ceph-mon[223858]: pgmap v607: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 2832/268680 objects misplaced (1.054%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:20.383 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:20.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:20.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:31:20.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:20.642 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.011 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:21.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:21.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.189 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.267 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:21.353 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:21.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:21.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:21.434 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:21.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:21.514 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:21.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:21.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:21.788 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:moved 51 / 267 (19.1011%) 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:avg 33.375 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 6.04023 -> 6.14283 (expected baseline 5.40399) 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 22 -> 31 pgs (0.659176 -> 0.928839 * mean) 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 41 -> 33 pgs (1.22846 -> 0.988764 * mean) 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 1 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0281 2023-10-09T04:31:21.797 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0310 2023-10-09T04:31:21.798 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 1.0000 -> 0.6545 2023-10-09T04:31:21.798 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 1.0000 -> 0.6792 2023-10-09T04:31:21.798 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 weight 1.0000 -> 0.6802 2023-10-09T04:31:21.798 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 weight 1.0000 -> 0.8248 2023-10-09T04:31:21.807 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:31:21.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:21 smithi029 conmon[202582]: 2023-10-09T04:31:21.789+0000 7f6049533700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-09T04:31:22.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:22 smithi029 ceph-mon[216903]: pgmap v608: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 2832/264093 objects misplaced (1.072%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:22.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:22 smithi029 ceph-mon[223858]: pgmap v608: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 2832/264093 objects misplaced (1.072%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:22 smithi177 ceph-mon[195797]: pgmap v608: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 2832/264093 objects misplaced (1.072%); 4.4 MiB/s, 69 objects/s recovering 2023-10-09T04:31:22.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:31:22.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:31:22] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-10-09T04:31:23.245 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:23.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:23.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:23.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:23 smithi177 ceph-mon[195797]: from='client.35436 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:23.409 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:23.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:23 smithi029 ceph-mon[216903]: from='client.35436 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:23.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:23 smithi029 ceph-mon[223858]: from='client.35436 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:23.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:23.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:23.564 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:23.590 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:23.633 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:23.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:23.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:23.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:23.769 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[216903]: pgmap v609: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 2501/258018 objects misplaced (0.969%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[216903]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[216903]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[216903]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[216903]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[216903]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[223858]: pgmap v609: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 2501/258018 objects misplaced (0.969%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[223858]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[223858]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:31:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[223858]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[223858]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:24 smithi029 ceph-mon[223858]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:24 smithi177 ceph-mon[195797]: pgmap v609: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 2501/258018 objects misplaced (0.969%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:24 smithi177 ceph-mon[195797]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:24 smithi177 ceph-mon[195797]: osdmap e167: 8 total, 8 up, 8 in 2023-10-09T04:31:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:24 smithi177 ceph-mon[195797]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:24 smithi177 ceph-mon[195797]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:24 smithi177 ceph-mon[195797]: failed to encode map e167 with expected crc 2023-10-09T04:31:24.760 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:24.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:24.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:24.914 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:25.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[216903]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:31:25.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[216903]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[216903]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[216903]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[223858]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:31:25.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[223858]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[223858]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:25 smithi029 ceph-mon[223858]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:25 smithi177 ceph-mon[195797]: osdmap e168: 8 total, 8 up, 8 in 2023-10-09T04:31:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:25 smithi177 ceph-mon[195797]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:25 smithi177 ceph-mon[195797]: failed to encode map e168 with expected crc 2023-10-09T04:31:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:25 smithi177 ceph-mon[195797]: failed to encode map e168 with expected crc 2023-10-09T04:31:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:26 smithi029 ceph-mon[216903]: pgmap v612: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 2501/253224 objects misplaced (0.988%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:31:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:26 smithi029 ceph-mon[223858]: pgmap v612: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 2501/253224 objects misplaced (0.988%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:31:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:26 smithi177 ceph-mon[195797]: pgmap v612: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 87 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 2501/253224 objects misplaced (0.988%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:31:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:27 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T04:31:27.536 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:27.543 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:27 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T04:31:27.544 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:27 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2023-10-09T04:31:27.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:27.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:27.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:27.684 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:28.117 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T04:31:28.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T04:31:28.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:28 smithi029 ceph-mon[216903]: pgmap v613: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 2138/249000 objects misplaced (0.859%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:31:28.357 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:28 smithi029 ceph-mon[223858]: pgmap v613: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 2138/249000 objects misplaced (0.859%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:31:28.384 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:28.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:28.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:28 smithi177 ceph-mon[195797]: pgmap v613: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 2138/249000 objects misplaced (0.859%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:31:28.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:28.563 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:28.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:28.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:28.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:28.833 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:28.936 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:29.031 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:29.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:115889 9.c e168/144)", 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.076714+0000", 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0031150069999999999, 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031452670000000002, 2023-10-09T04:31:29.084 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076714+0000", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076714+0000", 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076713+0000", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076719+0000", 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5500000000000002e-06 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076731+0000", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1953000000000001e-05 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076739+0000", 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6690000000000006e-06 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:115886 9.1 9:8e66ad9d:::benchmark_data_smithi029_299184_object17583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.058732+0000", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.021096737000000001, 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021186553, 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 115886 2023-10-09T04:31:29.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058732+0000", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058732+0000", 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058731+0000", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058738+0000", 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5619999999999998e-06 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058757+0000", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8508e-05 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058767+0000", 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8290000000000003e-06 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.063369+0000", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046025980000000003 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.063489+0000", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011948 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.063601+0000", 2023-10-09T04:31:29.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000111749 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067605+0000", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040043120000000003 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067820+0000", 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000215317 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.093 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:115889 9.c 9:35a01c25:::benchmark_data_smithi029_299184_object17586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.058773+0000", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.021055991, 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021214452000000002, 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:29.094 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 115889 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058773+0000", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058773+0000", 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058773+0000", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058777+0000", 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9600000000000002e-06 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058782+0000", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5459999999999999e-06 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058788+0000", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2290000000000002e-06 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068332+0000", 2023-10-09T04:31:29.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095445770000000003 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068468+0000", 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013604399999999999 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068593+0000", 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012484799999999999 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.098 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:115891 9.c 9:3dca0dd6:::benchmark_data_smithi029_299184_object17588:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.058802+0000", 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.021027301000000002, 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021276890999999999, 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:29.099 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 115891 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058802+0000", 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058802+0000", 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058802+0000", 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058804+0000", 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.52e-06 2023-10-09T04:31:29.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058810+0000", 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.818e-06 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.058815+0000", 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8459999999999999e-06 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076560+0000", 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017745438999999998 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076712+0000", 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000152227 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.076848+0000", 2023-10-09T04:31:29.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013566000000000001 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:115892 9.c 9.518d83ec (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.067875+0000", 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.011954453, 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012296850999999999, 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:29.104 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 115892 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067875+0000", 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067875+0000", 2023-10-09T04:31:29.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067873+0000", 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067880+0000", 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2990000000000002e-06 2023-10-09T04:31:29.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067890+0000", 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0528e-05 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.067899+0000", 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5280000000000003e-06 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:115897 9.1 9:8e721cd0:::benchmark_data_smithi029_299184_object17594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.068561+0000", 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.011267935, 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01167145, 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 115897 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.108 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068561+0000", 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068561+0000", 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068560+0000", 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068566+0000", 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0719999999999996e-06 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068584+0000", 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7203000000000001e-05 2023-10-09T04:31:29.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068592+0000", 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4679999999999992e-06 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068667+0000", 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4958999999999998e-05 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:29.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068785+0000", 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118057 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.068903+0000", 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011794399999999999 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.073598+0000", 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046948550000000004 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.077728+0000", 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041299730000000003 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:115883 9.4 e168/144 9:23406e04:::benchmark_data_smithi029_299184_object17580:head v 168'7103, mlcod=168'7101)", 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.070098+0000", 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0097311600000000009, 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010223494, 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.113 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.070098+0000", 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.070098+0000", 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.070097+0000", 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.070104+0000", 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8229999999999997e-06 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.070114+0000", 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0195e-05 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.070121+0000", 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8399999999999997e-06 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:29.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.072702+0000", 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025812410000000002 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.072728+0000", 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5904000000000001e-05 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:115894 9.f e168/144 9:fa5d982f:::benchmark_data_smithi029_299184_object17591:head v 168'7367, mlcod=168'7365)", 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:29.075615+0000", 2023-10-09T04:31:29.117 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0042137479999999998, 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047709220000000004, 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075615+0000", 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075615+0000", 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075614+0000", 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075621+0000", 2023-10-09T04:31:29.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3050000000000004e-06 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075641+0000", 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9528999999999999e-05 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075651+0000", 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.029e-05 2023-10-09T04:31:29.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075700+0000", 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9129999999999999e-05 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:29.075728+0000", 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8306e-05 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:29.121 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.122 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:29.122 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:31:29.122 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 8 2023-10-09T04:31:29.122 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:31:29.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:29.193 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:29.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:29.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:29.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:29.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:29.425 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:29.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:29.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:29.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:29.641 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:29.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:29.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:29.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:29.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:29.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:29.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:29.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:29.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:29.896 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:30.589 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:31:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:30 smithi177 ceph-mon[195797]: pgmap v614: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2138/247959 objects misplaced (0.862%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:31:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3669056075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:30 smithi029 ceph-mon[216903]: pgmap v614: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2138/247959 objects misplaced (0.862%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:31:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3669056075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:30 smithi029 ceph-mon[223858]: pgmap v614: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2138/247959 objects misplaced (0.862%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:31:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3669056075' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:31:31.334 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:31.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:31.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:31.516 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3669056075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[216903]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3669056075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: failed to encode map e169 with expected crc 2023-10-09T04:31:31.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:31 smithi029 ceph-mon[223858]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:31:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3669056075' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: osdmap e169: 8 total, 8 up, 8 in 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:31 smithi177 ceph-mon[195797]: failed to encode map e169 with expected crc 2023-10-09T04:31:32.653 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:32.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:32 smithi029 ceph-mon[216903]: pgmap v616: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2138/243822 objects misplaced (0.877%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:32.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:32 smithi029 ceph-mon[223858]: pgmap v616: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2138/243822 objects misplaced (0.877%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:32.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:32.822 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:32 smithi177 ceph-mon[195797]: pgmap v616: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2138/243822 objects misplaced (0.877%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:31:32.825 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:32.854 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:32.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:31:32.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:32.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:33.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:33.036 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:33.073 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:31:32] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T04:31:33.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:33.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:33.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:33.214 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:33.302 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.070 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:34.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:34.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:34.251 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.315 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:34.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:34.462 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:34.567 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:34 smithi029 ceph-mon[223858]: pgmap v617: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 1915/238269 objects misplaced (0.804%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:34.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:34.679 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:34.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:31:34.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:34.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:34.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:34 smithi029 ceph-mon[216903]: pgmap v617: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 1915/238269 objects misplaced (0.804%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:34.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:35.008 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:34 smithi177 ceph-mon[195797]: pgmap v617: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 1915/238269 objects misplaced (0.804%); 4.2 MiB/s, 67 objects/s recovering 2023-10-09T04:31:35.218 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:35.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:35.266 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:31:35.305 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:31:35.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:35.350 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:31:35.401 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:31:35.406 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:35.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:31:35.509 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:31:35.550 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:31:35.589 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:31:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:36 smithi029 ceph-mon[216903]: pgmap v618: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 1915/233601 objects misplaced (0.820%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T04:31:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3907255290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:36 smithi029 ceph-mon[223858]: pgmap v618: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 1915/233601 objects misplaced (0.820%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T04:31:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3907255290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:36 smithi177 ceph-mon[195797]: pgmap v618: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 1915/233601 objects misplaced (0.820%); 3.7 MiB/s, 58 objects/s recovering 2023-10-09T04:31:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3907255290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:36.996 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:37.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:37.101 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:37.161 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:38.186 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:38.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:38.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:38.349 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:38.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:38 smithi029 ceph-mon[216903]: pgmap v619: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1695/229083 objects misplaced (0.740%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:31:38.354 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:38 smithi029 ceph-mon[223858]: pgmap v619: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1695/229083 objects misplaced (0.740%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:31:38.410 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:38 smithi177 ceph-mon[195797]: pgmap v619: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1695/229083 objects misplaced (0.740%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:31:38.532 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:38.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:38.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:38.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:38.685 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:38.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:38.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:38.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:38.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:38.884 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:38.914 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:38.980 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:38.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:39.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:39.125 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:39.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:39.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:39.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:39.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:39.414 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:39.422 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 267 (0%) 2023-10-09T04:31:39.422 INFO:teuthology.orchestra.run.smithi029.stdout:avg 33.375 2023-10-09T04:31:39.422 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 6.04023 -> 6.04023 (expected baseline 5.40399) 2023-10-09T04:31:39.422 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 22 -> 22 pgs (0.659176 -> 0.659176 * mean) 2023-10-09T04:31:39.422 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-09T04:31:39.422 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:31:39.423 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T04:31:39.423 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2023-10-09T04:31:39.423 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:31:39.423 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0251 2023-10-09T04:31:39.423 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.2506 2023-10-09T04:31:39.452 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:31:40.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:31:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:40 smithi029 ceph-mon[223858]: pgmap v620: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 1695/227979 objects misplaced (0.743%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:31:40.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:40 smithi029 ceph-mon[223858]: from='client.55237 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:40 smithi029 ceph-mon[216903]: pgmap v620: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 1695/227979 objects misplaced (0.743%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:31:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:40 smithi029 ceph-mon[216903]: from='client.55237 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:40 smithi177 ceph-mon[195797]: pgmap v620: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 1695/227979 objects misplaced (0.743%); 2.8 MiB/s, 44 objects/s recovering 2023-10-09T04:31:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:40 smithi177 ceph-mon[195797]: from='client.55237 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:31:41.012 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:41.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:41.194 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:41.671 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:41.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:41.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:41.804 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:41.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:41.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:41.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:41.976 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.018 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:42.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:42.160 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:42.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:42.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:42 smithi029 ceph-mon[216903]: pgmap v621: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1695/223950 objects misplaced (0.757%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:31:42.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:42 smithi029 ceph-mon[223858]: pgmap v621: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1695/223950 objects misplaced (0.757%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:31:42.418 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:42.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:42 smithi177 ceph-mon[195797]: pgmap v621: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 1695/223950 objects misplaced (0.757%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T04:31:42.833 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:42.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:42.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:42.988 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:43.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:43.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:31:42] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-10-09T04:31:43.215 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:43.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:43.454 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:43.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:43.601 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:43.680 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:43.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:43.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:43.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:43.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:43.956 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.114 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:44.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:44.262 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:44.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:44.354 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:44.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:44.490 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:31:44.490 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:31:44.495 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:44 smithi177 ceph-mon[195797]: pgmap v622: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1454/218514 objects misplaced (0.665%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:31:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:44.572 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:44.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:44.739 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:44.766 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:44 smithi029 ceph-mon[216903]: pgmap v622: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1454/218514 objects misplaced (0.665%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:31:44.766 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:44.766 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:44.766 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:44 smithi029 ceph-mon[223858]: pgmap v622: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1454/218514 objects misplaced (0.665%); 3.6 MiB/s, 56 objects/s recovering 2023-10-09T04:31:44.766 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:44.767 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:44.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:44.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:44.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:44.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:45.018 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:45.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:45.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:45.204 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:45.302 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:46.557 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:31:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:46 smithi177 ceph-mon[195797]: pgmap v623: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1454/214380 objects misplaced (0.678%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:31:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3333155467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:46 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:46.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:46 smithi029 ceph-mon[216903]: pgmap v623: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1454/214380 objects misplaced (0.678%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:31:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3333155467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:46 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:46 smithi029 ceph-mon[223858]: pgmap v623: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 1454/214380 objects misplaced (0.678%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:31:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3333155467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:46 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:31:47.511 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:31:47.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:31:47.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:47 smithi177 ceph-mon[195797]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:47.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:47.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:31:47.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[223858]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: osdmap e170: 8 total, 8 up, 8 in 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.803 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: failed to encode map e170 with expected crc 2023-10-09T04:31:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:47 smithi029 ceph-mon[216903]: failed to encode map e170 with expected crc 2023-10-09T04:31:48.028 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:48.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:48.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:48.225 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:48 smithi029 ceph-mon[216903]: pgmap v625: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 1186/210174 objects misplaced (0.564%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:48.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:48 smithi029 ceph-mon[223858]: pgmap v625: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 1186/210174 objects misplaced (0.564%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:48 smithi177 ceph-mon[195797]: pgmap v625: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 1186/210174 objects misplaced (0.564%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:49.074 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:49.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:49.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:49.243 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:49.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:49.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:49.332 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:49.348 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:49.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:31:49.503 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:49.667 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:31:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:49.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:31:49.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:49.845 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:50.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:31:50.095 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:50.103 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:31:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:50.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:50.155 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:31:50.202 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:31:50.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:50.259 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:31:50.306 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:31:50.326 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:50.349 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:31:50.393 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:31:50.435 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:31:50.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:50 smithi029 ceph-mon[216903]: pgmap v626: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 1186/208803 objects misplaced (0.568%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:50.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1931294155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:50.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:50 smithi029 ceph-mon[223858]: pgmap v626: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 1186/208803 objects misplaced (0.568%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:50.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1931294155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:50 smithi177 ceph-mon[195797]: pgmap v626: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 1186/208803 objects misplaced (0.568%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1931294155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:31:52.230 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:52.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:52.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:52.336 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:52.362 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:52.406 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:52.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:52.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:52.540 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:52.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:52 smithi177 ceph-mon[195797]: pgmap v627: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 1186/204396 objects misplaced (0.580%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:52.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:52 smithi029 ceph-mon[216903]: pgmap v627: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 1186/204396 objects misplaced (0.580%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:52.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:52 smithi029 ceph-mon[223858]: pgmap v627: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 1186/204396 objects misplaced (0.580%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T04:31:52.818 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:31:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:31:52] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T04:31:53.425 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:53.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:53.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:53.589 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:53.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:53.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:53.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:53.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:53.871 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:54.178 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:54.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:54.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:54.373 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:54.412 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:54.558 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:54.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:35396 9.2 9:4b31cc3b:::benchmark_data_smithi029_299184_object35395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:47.764924+0000", 2023-10-09T04:31:54.572 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 126.797916468, 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34490238499999998, 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 35396 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.764924+0000", 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.764924+0000", 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.764922+0000", 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:54.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.764951+0000", 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9082999999999999e-05 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.764966+0000", 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5218999999999999e-05 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.764974+0000", 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0099999999999995e-06 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.765655+0000", 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00068023499999999995 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.765771+0000", 2023-10-09T04:31:54.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115821 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.765902+0000", 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013163500000000001 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.767266+0000", 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00136418 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:47.770087+0000", 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028202290000000001 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.109700+0000", 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33961356300000001 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.109755+0000", 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5012000000000001e-05 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:48.109826+0000", 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1174000000000007e-05 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.579 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38520 9.8 9:108867f8:::benchmark_data_smithi029_299184_object38519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 124.529622854, 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1949972569999998, 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38520 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.580 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033216+0000", 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033239+0000", 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2768999999999998e-05 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033248+0000", 2023-10-09T04:31:54.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6910000000000003e-06 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033254+0000", 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0399999999999998e-06 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033339+0000", 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5388000000000003e-05 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033348+0000", 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7469999999999994e-06 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022545+0000", 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98919705899999999 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022559+0000", 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3551e-05 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216745+0000", 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.194186607 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216864+0000", 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011896300000000001 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216980+0000", 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116215 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.222890+0000", 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059099820000000003 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.224040+0000", 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001149583 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226281+0000", 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022405580000000001 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226344+0000", 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2959999999999994e-05 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.228215+0000", 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001871295 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38523 9.8 9:186f9d6c:::benchmark_data_smithi029_299184_object38522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 124.529129471, 2023-10-09T04:31:54.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1990103689999998, 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38523 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033710+0000", 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033730+0000", 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0020000000000001e-05 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033742+0000", 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2299e-05 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033748+0000", 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4079999999999997e-06 2023-10-09T04:31:54.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033788+0000", 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0441000000000001e-05 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033795+0000", 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1439999999999997e-06 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022741+0000", 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98894592299999995 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022751+0000", 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4960000000000002e-06 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220959+0000", 2023-10-09T04:31:54.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.198207902 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221125+0000", 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016651700000000001 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221423+0000", 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000297767 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.224239+0000", 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028161560000000002 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226232+0000", 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019932280000000001 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230759+0000", 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045263370000000001 2023-10-09T04:31:54.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230821+0000", 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1879999999999997e-05 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.232721+0000", 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001900723 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.596 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:53418 9.8 9:1b04f29d:::benchmark_data_smithi029_299184_object53417:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:06.602955+0000", 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 107.959885813, 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.437350287, 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 53418 2023-10-09T04:31:54.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.602955+0000", 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.602955+0000", 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.602953+0000", 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.602972+0000", 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8482e-05 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.602983+0000", 2023-10-09T04:31:54.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1224e-05 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.602989+0000", 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9440000000000002e-06 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.603045+0000", 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5719e-05 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.603172+0000", 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127298 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.603289+0000", 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011740500000000001 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.605060+0000", 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017705450000000001 2023-10-09T04:31:54.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.605114+0000", 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4030999999999998e-05 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.906237+0000", 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30112301699999999 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:06.906296+0000", 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8795000000000002e-05 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:07.040305+0000", 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.134009138 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112523 9.8 9:166f8be2:::benchmark_data_smithi029_299184_object14220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.452335599000001, 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.01694084, 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112523 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.604 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110503+0000", 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110510+0000", 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4899999999999997e-06 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110525+0000", 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4864e-05 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110551+0000", 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5894e-05 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110637+0000", 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6575000000000004e-05 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110776+0000", 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013886399999999999 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110903+0000", 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012670300000000001 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.117848+0000", 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069447629999999996 2023-10-09T04:31:54.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.117863+0000", 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5756999999999998e-05 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127354+0000", 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0094912250000001 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127446+0000", 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1144999999999996e-05 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112554 9.8 9:196c9f66:::benchmark_data_smithi029_299184_object14251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.413196908, 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99418602, 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.610 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112554 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149642+0000", 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149649+0000", 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6830000000000008e-06 2023-10-09T04:31:54.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149667+0000", 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7935999999999999e-05 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149676+0000", 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8149999999999995e-06 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149759+0000", 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3293000000000004e-05 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149772+0000", 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.225e-05 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127865+0000", 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.9780934 2023-10-09T04:31:54.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127992+0000", 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127248 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.128124+0000", 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013166 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137717+0000", 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095931829999999999 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137731+0000", 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3898e-05 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143735+0000", 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060035740000000002 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143787+0000", 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2207999999999999e-05 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143830+0000", 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2657000000000002e-05 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112566 9.8 9:1591ae10:::benchmark_data_smithi029_299184_object14263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.407130677000001, 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98828444299999996, 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112566 2023-10-09T04:31:54.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155708+0000", 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155715+0000", 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.737e-06 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155732+0000", 2023-10-09T04:31:54.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6733000000000001e-05 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155740+0000", 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3159999999999997e-06 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155798+0000", 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8369000000000003e-05 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155810+0000", 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1187e-05 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131524+0000", 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97571455100000004 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131668+0000", 2023-10-09T04:31:54.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000144115 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131781+0000", 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011272 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137768+0000", 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059868439999999998 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137815+0000", 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7286000000000001e-05 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143926+0000", 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061107280000000002 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143955+0000", 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9420999999999998e-05 2023-10-09T04:31:54.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143994+0000", 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8894999999999999e-05 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112581 9.8 9:188eb5a8:::benchmark_data_smithi029_299184_object14278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.390004557000001, 2023-10-09T04:31:54.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97130810899999998, 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112581 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:31:54.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172834+0000", 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172841+0000", 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0890000000000002e-06 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172854+0000", 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2677000000000001e-05 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172863+0000", 2023-10-09T04:31:54.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172924+0000", 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1587999999999994e-05 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172935+0000", 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0971e-05 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134673+0000", 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96173812199999997 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134756+0000", 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2958000000000001e-05 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134831+0000", 2023-10-09T04:31:54.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4935e-05 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137804+0000", 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029725469999999999 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137823+0000", 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9485000000000001e-05 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144083+0000", 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062598230000000003 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144109+0000", 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5782000000000002e-05 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144144+0000", 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5181000000000001e-05 2023-10-09T04:31:54.632 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112605 9.8 9:173f4bff:::benchmark_data_smithi029_299184_object14302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.360641046000001, 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95321568300000004, 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112605 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202203+0000", 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0260000000000001e-06 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202215+0000", 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1698e-05 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202222+0000", 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0870000000000001e-06 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202287+0000", 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5141999999999994e-05 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202297+0000", 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0135e-05 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137839+0000", 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.935542184 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137923+0000", 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.365e-05 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.138000+0000", 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6852000000000002e-05 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143877+0000", 2023-10-09T04:31:54.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058773489999999996 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143886+0000", 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9919999999999996e-06 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155344+0000", 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011457606 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155380+0000", 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6705000000000002e-05 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155415+0000", 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4813e-05 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112609 9.8 9:1697f7cc:::benchmark_data_smithi029_299184_object14306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.356991227000002, 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94970479100000005, 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112609 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205848+0000", 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205853+0000", 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2599999999999999e-06 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205861+0000", 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5339999999999999e-06 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205867+0000", 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1129999999999997e-06 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205926+0000", 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8548000000000003e-05 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205937+0000", 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1100999999999999e-05 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140841+0000", 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93490438799999998 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140924+0000", 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2818000000000006e-05 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140995+0000", 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0840000000000006e-05 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144049+0000", 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030538369999999998 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144062+0000", 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2706e-05 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155470+0000", 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011407907 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155514+0000", 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4409999999999997e-05 2023-10-09T04:31:54.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155554+0000", 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0036000000000001e-05 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112659 9.8 9:1ba9566d:::benchmark_data_smithi029_299184_object14356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.291670438000001, 2023-10-09T04:31:54.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.89206917200000002, 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112659 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:31:54.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271169+0000", 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:54.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271174+0000", 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.592e-06 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271186+0000", 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.153e-05 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271194+0000", 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2179999999999999e-06 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271280+0000", 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6273999999999994e-05 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271288+0000", 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7749999999999993e-06 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144186+0000", 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.87289803399999999 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144264+0000", 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8158000000000006e-05 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144352+0000", 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7874000000000006e-05 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155299+0000", 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010946424999999999 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155630+0000", 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033138200000000002 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163143+0000", 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007512627 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163198+0000", 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5139999999999997e-05 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163239+0000", 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1260000000000001e-05 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112683 9.8 9:1e5d2014:::benchmark_data_smithi029_299184_object14380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.260498199000001, 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.868701897, 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.656 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112683 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302341+0000", 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302347+0000", 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5869999999999998e-06 2023-10-09T04:31:54.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302361+0000", 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3838e-05 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302369+0000", 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2539999999999992e-06 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302430+0000", 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0968999999999999e-05 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302446+0000", 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5427e-05 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148056+0000", 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84561038799999999 2023-10-09T04:31:54.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148181+0000", 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012532600000000001 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148295+0000", 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113237 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155432+0000", 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071375379999999997 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163283+0000", 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078503070000000008 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170952+0000", 2023-10-09T04:31:54.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076698299999999999 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170999+0000", 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6375000000000002e-05 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171044+0000", 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5340000000000003e-05 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.663 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112688 9.8 9:1bc999dd:::benchmark_data_smithi029_299184_object14385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.257700639999999, 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86601131600000003, 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.664 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112688 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305139+0000", 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305143+0000", 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9339999999999999e-06 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305155+0000", 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1765e-05 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305161+0000", 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3829999999999997e-06 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305229+0000", 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7902000000000001e-05 2023-10-09T04:31:54.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305244+0000", 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.464e-05 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151650+0000", 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84640602600000003 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151753+0000", 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103168 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151867+0000", 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001144 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163313+0000", 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011445079 2023-10-09T04:31:54.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163377+0000", 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4084000000000005e-05 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171081+0000", 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077041899999999996 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171119+0000", 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8631000000000003e-05 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171151+0000", 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1636e-05 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.671 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112697 9.8 9:12bbffcf:::benchmark_data_smithi029_299184_object14394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.244319375, 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85940739799999999, 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112697 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318520+0000", 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318526+0000", 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7739999999999999e-06 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318544+0000", 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8156e-05 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318553+0000", 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8859999999999993e-06 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318616+0000", 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2434000000000006e-05 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318627+0000", 2023-10-09T04:31:54.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0893000000000001e-05 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155677+0000", 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83705072999999997 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155793+0000", 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115912 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155913+0000", 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012010000000000001 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163366+0000", 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074525160000000002 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163387+0000", 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0649000000000002e-05 2023-10-09T04:31:54.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177849+0000", 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014462773999999999 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177892+0000", 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2336999999999998e-05 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177928+0000", 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6739000000000002e-05 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112698 9.8 9:1730e59e:::benchmark_data_smithi029_299184_object14395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.243648750999998, 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85884108599999998, 2023-10-09T04:31:54.679 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112698 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319191+0000", 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319196+0000", 2023-10-09T04:31:54.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319210+0000", 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4318000000000001e-05 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319217+0000", 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0319999999999998e-06 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319272+0000", 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4486e-05 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319283+0000", 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1447000000000001e-05 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159355+0000", 2023-10-09T04:31:54.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84007240500000002 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159465+0000", 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110021 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159570+0000", 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104728 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163437+0000", 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003866754 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.166866+0000", 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034293549999999998 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177962+0000", 2023-10-09T04:31:54.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011095997999999999 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177995+0000", 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2712999999999997e-05 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178033+0000", 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7809999999999999e-05 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.686 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112742 9.8 9:1d544479:::benchmark_data_smithi029_299184_object14439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.179738016000002, 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.806861312, 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.687 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112742 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383101+0000", 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383107+0000", 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6359999999999999e-06 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383119+0000", 2023-10-09T04:31:54.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2502e-05 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383126+0000", 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3629999999999999e-06 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383193+0000", 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7113999999999995e-05 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383204+0000", 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1276e-05 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163543+0000", 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78033866200000002 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163658+0000", 2023-10-09T04:31:54.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011532299999999999 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163768+0000", 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110164 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170921+0000", 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071524010000000001 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178095+0000", 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071740179999999999 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189883+0000", 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011788378 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189927+0000", 2023-10-09T04:31:54.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4149e-05 2023-10-09T04:31:54.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:54.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189964+0000", 2023-10-09T04:31:54.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6482000000000001e-05 2023-10-09T04:31:54.695 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.695 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.695 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112743 9.8 9:12e2e405:::benchmark_data_smithi029_299184_object14440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.179691090999999, 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.806888253, 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112743 2023-10-09T04:31:54.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383152+0000", 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9610000000000001e-06 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383159+0000", 2023-10-09T04:31:54.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9990000000000002e-06 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383164+0000", 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8269999999999997e-06 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383278+0000", 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114821 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383291+0000", 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2601e-05 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.166912+0000", 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78362119900000005 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.167043+0000", 2023-10-09T04:31:54.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013115800000000001 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.167159+0000", 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115868 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178066+0000", 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010906701 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189514+0000", 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011448309 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189971+0000", 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045658199999999997 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190001+0000", 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0080999999999999e-05 2023-10-09T04:31:54.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190038+0000", 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6575999999999997e-05 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112772 9.8 9:120ab9ea:::benchmark_data_smithi029_299184_object14469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.434122+0000", 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.128718301999999, 2023-10-09T04:31:54.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76100376999999997, 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112772 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434122+0000", 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434122+0000", 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434121+0000", 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434128+0000", 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6760000000000007e-06 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434148+0000", 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9491999999999999e-05 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434156+0000", 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7669999999999992e-06 2023-10-09T04:31:54.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434221+0000", 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4502e-05 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.434229+0000", 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2109999999999998e-06 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171176+0000", 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73694674800000004 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171284+0000", 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010794900000000001 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171411+0000", 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012717300000000001 2023-10-09T04:31:54.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178123+0000", 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067123119999999998 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190043+0000", 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011919574 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195041+0000", 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049975949999999996 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195087+0000", 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6835999999999997e-05 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195126+0000", 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8483999999999999e-05 2023-10-09T04:31:54.710 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112798 9.8 9:11723e6e:::benchmark_data_smithi029_299184_object14495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.494839+0000", 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.068001289000001, 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.70039198000000003, 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.711 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112798 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494839+0000", 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494839+0000", 2023-10-09T04:31:54.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494838+0000", 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494844+0000", 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1199999999999999e-06 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494856+0000", 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2492e-05 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494863+0000", 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8839999999999997e-06 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494933+0000", 2023-10-09T04:31:54.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9536000000000006e-05 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.494945+0000", 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2150999999999999e-05 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.174433+0000", 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.67948806699999997 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.174543+0000", 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109937 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.174646+0000", 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010319600000000001 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178153+0000", 2023-10-09T04:31:54.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035073629999999999 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190050+0000", 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011896636 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195160+0000", 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051102550000000002 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195203+0000", 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2339999999999998e-05 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195231+0000", 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8483000000000002e-05 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112812 9.8 9:1f912d00:::benchmark_data_smithi029_299184_object14509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.545616+0000", 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.017224675000001, 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.64970857000000004, 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112812 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.719 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545616+0000", 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545616+0000", 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545614+0000", 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545620+0000", 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1469999999999998e-06 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545633+0000", 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2612000000000001e-05 2023-10-09T04:31:54.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545639+0000", 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545701+0000", 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1493000000000002e-05 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:31:54.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.545715+0000", 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3774999999999999e-05 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178175+0000", 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.632459891 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178271+0000", 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6464000000000003e-05 2023-10-09T04:31:54.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178364+0000", 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3209999999999997e-05 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189874+0000", 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011510004000000001 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:31:54.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190055+0000", 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000180349 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195266+0000", 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005210892 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195297+0000", 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1569000000000002e-05 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:31:54.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.195324+0000", 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7189e-05 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:31:54.726 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:31:54.741 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:54.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:54.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:54.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:54.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:54.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:54 smithi029 ceph-mon[223858]: pgmap v628: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 955/199029 objects misplaced (0.480%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:31:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:54 smithi029 ceph-mon[216903]: pgmap v628: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 955/199029 objects misplaced (0.480%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:31:55.016 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:54 smithi177 ceph-mon[195797]: pgmap v628: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 955/199029 objects misplaced (0.480%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:31:55.165 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:55.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:55.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:55.336 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:55.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:55.582 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:55.794 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:31:55.823 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:55.869 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:31:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:55.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:31:55.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:31:55.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:55.999 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:56.082 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:56.087 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:56 smithi029 ceph-mon[216903]: pgmap v629: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 955/195021 objects misplaced (0.490%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:31:56.087 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/961388497' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:31:56.087 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:31:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:56.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:56.091 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:56 smithi029 ceph-mon[223858]: pgmap v629: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 955/195021 objects misplaced (0.490%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:31:56.091 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/961388497' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:31:56.092 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:31:56.134 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:31:56.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:56 smithi177 ceph-mon[195797]: pgmap v629: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 955/195021 objects misplaced (0.490%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:31:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/961388497' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:31:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:31:56.308 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:56.762 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:56.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:56.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:31:56.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:56.939 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:57.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:57.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:57 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:31:57.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:57 smithi177 ceph-mon[195797]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:31:57.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:57 smithi177 ceph-mon[195797]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:57 smithi177 ceph-mon[195797]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.109 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:57 smithi177 ceph-mon[195797]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.110 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:57 smithi177 ceph-mon[195797]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.143 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:31:57.216 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:31:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[223858]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[223858]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[223858]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[223858]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[223858]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[216903]: osdmap e171: 8 total, 8 up, 8 in 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[216903]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[216903]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[216903]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:57 smithi029 ceph-mon[216903]: failed to encode map e171 with expected crc 2023-10-09T04:31:57.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:57.443 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:31:57.452 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.497 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:57.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:31:57.656 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:57.938 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:31:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:57.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:31:58.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:58.099 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.150 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:58.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:58 smithi177 ceph-mon[195797]: pgmap v631: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 726/190842 objects misplaced (0.380%); 2.4 MiB/s, 45 objects/s recovering 2023-10-09T04:31:58.317 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:58 smithi029 ceph-mon[216903]: pgmap v631: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 726/190842 objects misplaced (0.380%); 2.4 MiB/s, 45 objects/s recovering 2023-10-09T04:31:58.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:58 smithi029 ceph-mon[223858]: pgmap v631: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 726/190842 objects misplaced (0.380%); 2.4 MiB/s, 45 objects/s recovering 2023-10-09T04:31:58.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:58.565 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.674 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:58.696 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:31:58.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:58.819 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:58.885 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:58.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:58.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:58.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:31:59.011 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:31:59.075 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:59.195 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:31:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:59.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:31:59.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:31:59.362 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:59.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:31:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:59.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:31:59.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:31:59.586 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:59.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:31:59.673 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:59.673 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:31:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:59.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:31:59.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:59.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:31:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:59.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:31:59.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:31:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:31:59.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:31:59.811 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:31:59.889 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:31:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:31:59.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:31:59.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:00.040 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:00.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:00.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:00.381 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:00.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:00.577 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:32:00.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:00.682 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:00 smithi177 ceph-mon[195797]: pgmap v632: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 726/189585 objects misplaced (0.383%); 2.4 MiB/s, 45 objects/s recovering 2023-10-09T04:32:00.682 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3273061593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:00.682 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:00 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:00.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:00.764 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:00.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:00.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:00.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:00 smithi029 ceph-mon[223858]: pgmap v632: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 726/189585 objects misplaced (0.383%); 2.4 MiB/s, 45 objects/s recovering 2023-10-09T04:32:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3273061593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:00 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:00 smithi029 ceph-mon[216903]: pgmap v632: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 726/189585 objects misplaced (0.383%); 2.4 MiB/s, 45 objects/s recovering 2023-10-09T04:32:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3273061593' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:00 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.35373.0:136834 9.4 e172/144 9:23b2fcf4:::benchmark_data_smithi029_299184_object38531:head v 172'8436, mlcod=172'8433)", 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:32:00.921655+0000", 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.004514232, 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045363790000000001, 2023-10-09T04:32:00.926 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921655+0000", 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921655+0000", 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:00.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921654+0000", 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921662+0000", 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1899999999999995e-06 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:00.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921675+0000", 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3199e-05 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921680+0000", 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0529999999999997e-06 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921715+0000", 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4570000000000003e-05 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.921731+0000", 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6528000000000002e-05 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.35373.0:136837 9.4 e172/144 9:20053d5d:::benchmark_data_smithi029_299184_object38534:head v 172'8437, mlcod=172'8433)", 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:32:00.922747+0000", 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0034224199999999998, 2023-10-09T04:32:00.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034870880000000002, 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.922747+0000", 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.922747+0000", 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:00.931 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.922747+0000", 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.922752+0000", 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.716e-06 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:00.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.922766+0000", 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3471e-05 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.922771+0000", 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2059999999999998e-06 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.924478+0000", 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001707421 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:00.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:00.924491+0000", 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2176e-05 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 2 2023-10-09T04:32:00.934 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:32:00.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:01.005 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:01.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:01.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:01.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:01.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:01.239 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:01.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:01.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:01.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:01.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.811 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:01.814 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:01.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:01.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:01.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: osdmap e172: 8 total, 8 up, 8 in 2023-10-09T04:32:01.857 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: failed to encode map e172 with expected crc 2023-10-09T04:32:01.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:32:02.558 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:02.774 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:02.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:02.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:02 smithi029 ceph-mon[216903]: pgmap v634: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 726/185352 objects misplaced (0.392%); 1.1 MiB/s, 28 objects/s recovering 2023-10-09T04:32:02.820 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:32:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:32:02] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T04:32:02.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:02 smithi029 ceph-mon[223858]: pgmap v634: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 726/185352 objects misplaced (0.392%); 1.1 MiB/s, 28 objects/s recovering 2023-10-09T04:32:02.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:02.887 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:03.001 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:03.004 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:02 smithi177 ceph-mon[195797]: pgmap v634: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 726/185352 objects misplaced (0.392%); 1.1 MiB/s, 28 objects/s recovering 2023-10-09T04:32:03.289 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:03.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:03.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:03.443 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:03.569 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:03.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:03.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:03.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:32:03.748 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:03.757 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:32:03.803 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:32:03.830 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:32:03.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:32:03.935 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:32:03.967 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:32:04.005 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:32:04.006 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.006 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:32:04.316 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:04.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:04.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:04.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:04.450 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:04.914 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:04.914 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:04 smithi029 ceph-mon[216903]: pgmap v635: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 327/179949 objects misplaced (0.182%); 1.1 MiB/s, 78 objects/s recovering 2023-10-09T04:32:04.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2061701201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:04 smithi029 ceph-mon[223858]: pgmap v635: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 327/179949 objects misplaced (0.182%); 1.1 MiB/s, 78 objects/s recovering 2023-10-09T04:32:04.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2061701201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:04 smithi177 ceph-mon[195797]: pgmap v635: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 327/179949 objects misplaced (0.182%); 1.1 MiB/s, 78 objects/s recovering 2023-10-09T04:32:04.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2061701201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:04.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:32:05.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:32:05.813 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:05.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:05.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:06.038 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:06.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:06.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:06.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:06.454 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:06.486 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:06.594 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:06.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:06.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:06.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:06 smithi177 ceph-mon[195797]: pgmap v636: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 327/175941 objects misplaced (0.186%); 1.0 MiB/s, 71 objects/s recovering 2023-10-09T04:32:06.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:06 smithi177 ceph-mon[195797]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:32:06.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:06 smithi177 ceph-mon[195797]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:06 smithi177 ceph-mon[195797]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.725 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:06 smithi177 ceph-mon[195797]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.727 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[223858]: pgmap v636: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 327/175941 objects misplaced (0.186%); 1.0 MiB/s, 71 objects/s recovering 2023-10-09T04:32:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[223858]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:32:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[223858]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[223858]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[223858]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[216903]: pgmap v636: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 327/175941 objects misplaced (0.186%); 1.0 MiB/s, 71 objects/s recovering 2023-10-09T04:32:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[216903]: osdmap e173: 8 total, 8 up, 8 in 2023-10-09T04:32:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[216903]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[216903]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:06 smithi029 ceph-mon[216903]: failed to encode map e173 with expected crc 2023-10-09T04:32:06.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:06.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:06.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:07.008 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:07.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:07.065 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:07.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:07.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:07.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:08.024 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:08.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: pgmap v638: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 327/172662 objects misplaced (0.189%); 0 B/s, 49 objects/s recovering 2023-10-09T04:32:08.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:32:08.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.099 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.100 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3132430768' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:32:08.101 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:32:08.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: pgmap v638: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 327/172662 objects misplaced (0.189%); 0 B/s, 49 objects/s recovering 2023-10-09T04:32:08.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:32:08.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3132430768' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: pgmap v638: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 327/172662 objects misplaced (0.189%); 0 B/s, 49 objects/s recovering 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: osdmap e174: 8 total, 8 up, 8 in 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: failed to encode map e174 with expected crc 2023-10-09T04:32:08.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3132430768' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:32:08.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:08.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:08.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:08.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:32:08.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:08.447 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:08.641 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:08.641 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:09.265 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:09.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:09.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:09.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:09.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[216903]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3132430768' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:32:09.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[216903]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:32:09.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[216903]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[216903]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[216903]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[223858]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3132430768' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:32:09.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[223858]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:32:09.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[223858]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[223858]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:09 smithi029 ceph-mon[223858]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.425 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:09.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:09 smithi177 ceph-mon[195797]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3132430768' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:32:09.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:09 smithi177 ceph-mon[195797]: osdmap e175: 8 total, 8 up, 8 in 2023-10-09T04:32:09.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:09 smithi177 ceph-mon[195797]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:09 smithi177 ceph-mon[195797]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:09 smithi177 ceph-mon[195797]: failed to encode map e175 with expected crc 2023-10-09T04:32:09.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:32:09.761 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:09.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:09.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:09.955 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:09.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:32:10.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:10.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:10.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:10.292 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:10.304 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:10.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:10.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:10.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:10.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:10.616 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:10.651 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:10.791 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:10.798 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:10.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:10.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:10.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:10 smithi029 ceph-mon[216903]: pgmap v641: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:10 smithi029 ceph-mon[223858]: pgmap v641: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:10.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:10 smithi177 ceph-mon[195797]: pgmap v641: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:10.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:10.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:10.989 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:11.190 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:11.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:11.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:11.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:11.373 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:11.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:11.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:11.611 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1331546854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:11.611 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:11.641 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:32:11.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:11.744 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1331546854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:11.744 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:11.748 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:11.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:11.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:11.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1331546854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:11.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:32:11.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:12.028 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:12.281 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:12.281 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:12.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:32:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:32:12] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-10-09T04:32:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: pgmap v642: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[223858]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: pgmap v642: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:12.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:12.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:12 smithi029 ceph-mon[216903]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: pgmap v642: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: osdmap e176: 8 total, 8 up, 8 in 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:12 smithi177 ceph-mon[195797]: failed to encode map e176 with expected crc 2023-10-09T04:32:13.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:32:13.597 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:13.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:13.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:13.754 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:13.807 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:13.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:14.014 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:14.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:14.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:14.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:14.147 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:14.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:14 smithi029 ceph-mon[216903]: pgmap v644: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:14.655 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:14 smithi177 ceph-mon[195797]: pgmap v644: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:14 smithi029 ceph-mon[223858]: pgmap v644: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 0 B/s, 54 objects/s recovering 2023-10-09T04:32:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:15.386 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:15.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:15.541 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:15.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:15.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:15.745 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:15.745 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:15.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:15.848 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:16.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:16.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:16.254 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:32:16.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:32:16.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:32:16.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:32:16.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:32:16.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:32:16.401 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:16.408 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:32:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:16.463 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.481 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.482 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:32:16.514 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:16.560 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:16.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[216903]: pgmap v645: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 0 B/s, 42 objects/s recovering 2023-10-09T04:32:16.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2684870310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:16.665 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:16.665 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:16.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:32:16.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:16.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:16.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:16.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:16.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[223858]: pgmap v645: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 0 B/s, 42 objects/s recovering 2023-10-09T04:32:16.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2684870310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:16 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:16.950 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:17.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:16 smithi177 ceph-mon[195797]: pgmap v645: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 0 B/s, 42 objects/s recovering 2023-10-09T04:32:17.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2684870310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:17.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:16 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:17.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:16 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:17.257 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:17.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:32:17.694 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:32:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1816465313' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:32:17.861 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:17.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1816465313' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:32:17.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1816465313' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:32:17.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:17.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:18.034 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:18.275 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:18.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:18.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:18.409 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:18.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: pgmap v646: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T04:32:18.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1816465313' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:32:18.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:32:18.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:18.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:18.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:18.825 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:18.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:18.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:18.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: pgmap v646: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T04:32:18.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1816465313' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:32:18.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:32:18.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: pgmap v646: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1816465313' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: osdmap e177: 8 total, 8 up, 8 in 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: failed to encode map e177 with expected crc 2023-10-09T04:32:18.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:32:18.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:32:18.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:18.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:32:19.027 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:19.111 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:60682 8.3 8:ce9d62de:::benchmark_data_smithi029_252925_object60681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:23:44.509352+0000", 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 514.59886482700006, 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.01123827, 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.112 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 60682 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509352+0000", 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509352+0000", 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509352+0000", 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509405+0000", 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2490000000000001e-05 2023-10-09T04:32:19.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509410+0000", 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4199999999999998e-06 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509413+0000", 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8399999999999999e-06 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509447+0000", 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4516e-05 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509466+0000", 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8768999999999999e-05 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.510299+0000", 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0008327859999999 2023-10-09T04:32:19.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.510665+0000", 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036586600000000001 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 6", 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.510849+0000", 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018468099999999999 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.520416+0000", 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095662820000000006 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.520438+0000", 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2648000000000001e-05 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.520492+0000", 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3640999999999997e-05 2023-10-09T04:32:19.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.520591+0000", 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8758000000000006e-05 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:60687 8.3 8:c5224ef7:::benchmark_data_smithi029_252925_object60686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:23:44.509419+0000", 2023-10-09T04:32:19.119 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 514.59879789900003, 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0138087629999999, 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 60687 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509419+0000", 2023-10-09T04:32:19.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509419+0000", 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509419+0000", 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509486+0000", 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6490999999999997e-05 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509492+0000", 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3589999999999996e-06 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509496+0000", 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.206e-06 2023-10-09T04:32:19.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509531+0000", 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4783999999999998e-05 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509537+0000", 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1129999999999997e-06 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.513132+0000", 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0035947220000001 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.513260+0000", 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127911 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 6", 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.513392+0000", 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013215699999999999 2023-10-09T04:32:19.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.520599+0000", 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0072077089999999996 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.523090+0000", 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002490099 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.523141+0000", 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1048000000000003e-05 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.523228+0000", 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7551000000000003e-05 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:60688 8.3 8:c6f8358f:::benchmark_data_smithi029_252925_object60687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:32:19.126 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:23:44.509504+0000", 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 514.59871283899997, 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.014534813, 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 60688 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509504+0000", 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509504+0000", 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509504+0000", 2023-10-09T04:32:19.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509552+0000", 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8367e-05 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509557+0000", 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9749999999999998e-06 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509560+0000", 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7719999999999999e-06 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509585+0000", 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5236999999999999e-05 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.509589+0000", 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6679999999999999e-06 2023-10-09T04:32:19.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.515543+0000", 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0059542770000001 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.515670+0000", 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126556 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 6", 2023-10-09T04:32:19.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.515817+0000", 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000147553 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.520605+0000", 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004787085 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.523838+0000", 2023-10-09T04:32:19.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003233358 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.523908+0000", 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0542000000000003e-05 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.524039+0000", 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013087400000000001 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.133 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:60700 8.3 8:c25b960d:::benchmark_data_smithi029_252925_object60699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:23:44.518918+0000", 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 514.58929934599996, 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.005312625, 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 60700 2023-10-09T04:32:19.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.518918+0000", 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.518918+0000", 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.518917+0000", 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.518972+0000", 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5071000000000002e-05 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.518987+0000", 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.535e-05 2023-10-09T04:32:19.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.519001+0000", 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4382000000000001e-05 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.519049+0000", 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7759000000000001e-05 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:44.519062+0000", 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2772000000000001e-05 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.517871+0000", 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99880900299999997 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.518013+0000", 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014155 2023-10-09T04:32:19.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 6", 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.518112+0000", 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9036999999999994e-05 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.523255+0000", 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005143346 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.524076+0000", 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082142999999999995 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.524143+0000", 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6985999999999999e-05 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:23:45.524231+0000", 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7258999999999999e-05 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.140 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:88378 8.3 8:cfe2c830:::benchmark_data_smithi029_252925_object88377:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:24:07.903155+0000", 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 491.20506192300002, 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1410075479999999, 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.141 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88378 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903155+0000", 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903155+0000", 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903154+0000", 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903326+0000", 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000171843 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903348+0000", 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2646999999999998e-05 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903358+0000", 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7899999999999994e-06 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903441+0000", 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.25e-05 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903452+0000", 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1263999999999999e-05 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.905916+0000", 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002463724 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.905927+0000", 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1234e-05 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.029179+0000", 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.123252417 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.029340+0000", 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000160598 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.029519+0000", 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017940100000000001 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.037157+0000", 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076376660000000004 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.037173+0000", 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5695000000000001e-05 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.042768+0000", 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055959540000000002 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.042854+0000", 2023-10-09T04:32:19.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5826000000000001e-05 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.044163+0000", 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001308622 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:88395 8.3 8:c3495bb0:::benchmark_data_smithi029_252925_object88394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:24:07.903375+0000", 2023-10-09T04:32:19.149 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 491.20484260400002, 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1530170019999999, 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88395 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903375+0000", 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903375+0000", 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903374+0000", 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903422+0000", 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7802000000000001e-05 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903432+0000", 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0019e-05 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903438+0000", 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7379999999999999e-06 2023-10-09T04:32:19.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903518+0000", 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0234999999999993e-05 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903525+0000", 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7229999999999998e-06 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.905951+0000", 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024260810000000001 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.905959+0000", 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3059999999999998e-06 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.033723+0000", 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1277640630000001 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.033848+0000", 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000125189 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.033982+0000", 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000133719 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.040511+0000", 2023-10-09T04:32:19.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065290519999999996 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.040525+0000", 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3981e-05 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.055415+0000", 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014890143 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.055481+0000", 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5679000000000006e-05 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.056392+0000", 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00091092600000000005 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:88403 8.3 8:cb2deed1:::benchmark_data_smithi029_252925_object88402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:24:07.903448+0000", 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 491.20476903000002, 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1588572150000001, 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88403 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.158 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903448+0000", 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903448+0000", 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903448+0000", 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903574+0000", 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012680499999999999 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903585+0000", 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0871e-05 2023-10-09T04:32:19.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903591+0000", 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1480000000000003e-06 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903697+0000", 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105702 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.903705+0000", 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6599999999999995e-06 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.905994+0000", 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022888140000000001 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:07.906004+0000", 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0434e-05 2023-10-09T04:32:19.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.037205+0000", 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.131201283 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.037382+0000", 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000176811 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:32:19.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.037590+0000", 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020783299999999999 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.040619+0000", 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030288770000000001 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.040835+0000", 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021567600000000001 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.061692+0000", 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020857265 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.061766+0000", 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4472000000000001e-05 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:24:09.062306+0000", 2023-10-09T04:32:19.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00053922300000000002 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216848 8.4 8:211e3bcd:::benchmark_data_smithi029_252925_object100224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.014383+0000", 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.09383389099997, 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.06401448, 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.166 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216848 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014383+0000", 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014383+0000", 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014381+0000", 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014393+0000", 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1449e-05 2023-10-09T04:32:19.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014412+0000", 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8734999999999999e-05 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014420+0000", 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1019999999999997e-06 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014463+0000", 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3296e-05 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014764+0000", 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030136699999999998 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014876+0000", 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111643 2023-10-09T04:32:19.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.021735+0000", 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006858846 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.024857+0000", 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003122437 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047062+0000", 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022204433999999999 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047069+0000", 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0199999999999997e-06 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078305+0000", 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0312359339999999 2023-10-09T04:32:19.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078398+0000", 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3210999999999999e-05 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216849 8.d 8:b99a9c5c:::benchmark_data_smithi029_252925_object100225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.09378110699998, 2023-10-09T04:32:19.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.065392602, 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216849 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:32:19.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014439+0000", 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6399999999999999e-06 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014447+0000", 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4039999999999998e-06 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014451+0000", 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2400000000000001e-06 2023-10-09T04:32:19.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014508+0000", 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6774999999999998e-05 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014640+0000", 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013230199999999999 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:32:19.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014765+0000", 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012486499999999999 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.021125+0000", 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063596449999999997 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.021747+0000", 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062273799999999996 2023-10-09T04:32:19.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.045986+0000", 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024238684999999999 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.045996+0000", 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9280000000000005e-06 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079773+0000", 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.03377703 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079829+0000", 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.588e-05 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.180 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216854 8.4 8:2ca06186:::benchmark_data_smithi029_252925_object100230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.028195+0000", 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.08002194199997, 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.050669147, 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216854 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028195+0000", 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028195+0000", 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028194+0000", 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028200+0000", 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2450000000000003e-06 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028215+0000", 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5193e-05 2023-10-09T04:32:19.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028221+0000", 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0569999999999998e-06 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028262+0000", 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0238999999999999e-05 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028380+0000", 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118136 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028518+0000", 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000137847 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.037151+0000", 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086330499999999998 2023-10-09T04:32:19.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047506+0000", 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010354847 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047513+0000", 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.294e-06 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078789+0000", 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0312761539999999 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078865+0000", 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5489999999999997e-05 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216858 8.d 8:baa8e899:::benchmark_data_smithi029_252925_object100234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.031712+0000", 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.07650552500002, 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0484245670000001, 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216858 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031712+0000", 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031712+0000", 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031710+0000", 2023-10-09T04:32:19.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031718+0000", 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8040000000000002e-06 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031731+0000", 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2768e-05 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031738+0000", 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2490000000000004e-06 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031805+0000", 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6885999999999996e-05 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031928+0000", 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012300399999999999 2023-10-09T04:32:19.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.032043+0000", 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114602 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.037383+0000", 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005340142 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.046091+0000", 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0087078280000000008 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.046097+0000", 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5320000000000003e-06 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.080075+0000", 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.033978201 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.080136+0000", 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1017000000000002e-05 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216872 8.4 8:202ae64d:::benchmark_data_smithi029_252925_object100248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-10-09T04:32:19.194 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.063198+0000", 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.04501924300001, 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.021720041, 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216872 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063198+0000", 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063198+0000", 2023-10-09T04:32:19.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063197+0000", 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063203+0000", 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9900000000000002e-06 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063215+0000", 2023-10-09T04:32:19.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2184999999999999e-05 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063222+0000", 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1380000000000001e-06 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063260+0000", 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7339000000000003e-05 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063265+0000", 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2560000000000002e-06 2023-10-09T04:32:19.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078904+0000", 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0156389729999999 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079001+0000", 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7195999999999996e-05 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4", 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079093+0000", 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1589999999999996e-05 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084300+0000", 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052073559999999998 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084747+0000", 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00044712400000000002 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084810+0000", 2023-10-09T04:32:19.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3211000000000002e-05 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084918+0000", 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107633 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38499 9.9 9:9a8fb78d:::benchmark_data_smithi029_299184_object38498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:32:19.204 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.087444318, 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.179240123, 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38499 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:32:19.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020771+0000", 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020915+0000", 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000143788 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020932+0000", 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.698e-05 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020941+0000", 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2529999999999993e-06 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022592+0000", 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016505980000000001 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022601+0000", 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9099999999999994e-06 2023-10-09T04:32:19.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.199150+0000", 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.17654937 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.200013+0000", 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000862749 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38501 9.9 9:91e7d993:::benchmark_data_smithi029_299184_object38500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.08725227599999, 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1804904860000001, 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38501 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.210 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020964+0000", 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021040+0000", 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5575999999999996e-05 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021060+0000", 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9754000000000001e-05 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021065+0000", 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4999999999999999e-06 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022627+0000", 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015613140000000001 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022638+0000", 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1204e-05 2023-10-09T04:32:19.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201357+0000", 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1787194400000001 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201456+0000", 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8263000000000004e-05 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.214 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38505 9.9 9:9fe8cf27:::benchmark_data_smithi029_299184_object38504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.08714633899999, 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1806352849999999, 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38505 2023-10-09T04:32:19.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021070+0000", 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021146+0000", 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5275e-05 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021154+0000", 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8720000000000002e-06 2023-10-09T04:32:19.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021170+0000", 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6252999999999998e-05 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022657+0000", 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001487588 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022662+0000", 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5549999999999996e-06 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201579+0000", 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1789165530000001 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201706+0000", 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127687 2023-10-09T04:32:19.219 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38509 9.9 9:938e198b:::benchmark_data_smithi029_299184_object38508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.087041871, 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.180647043, 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.220 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38509 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:32:19.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021220+0000", 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5445000000000003e-05 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021249+0000", 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8875000000000001e-05 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021254+0000", 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1209999999999998e-06 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022682+0000", 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014274299999999999 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022687+0000", 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4149999999999998e-06 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201740+0000", 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1790529270000001 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201822+0000", 2023-10-09T04:32:19.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2379000000000005e-05 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38510 9.9 9:98f164af:::benchmark_data_smithi029_299184_object38509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.08695803000001, 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.180751331, 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.225 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38510 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021326+0000", 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6715000000000001e-05 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021334+0000", 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7129999999999994e-06 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021340+0000", 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1510000000000001e-06 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022709+0000", 2023-10-09T04:32:19.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013682690000000001 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022715+0000", 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0990000000000004e-06 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201936+0000", 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1792214670000001 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202011+0000", 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4392000000000002e-05 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38528 9.9 9:91ac7278:::benchmark_data_smithi029_299184_object38527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.08685061400001, 2023-10-09T04:32:19.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.203529793, 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38528 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021366+0000", 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021405+0000", 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8535999999999998e-05 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021412+0000", 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8129999999999996e-06 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021418+0000", 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2499999999999997e-06 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022736+0000", 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001317948 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022741+0000", 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.096e-06 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202061+0000", 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1793201959999999 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202400+0000", 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033863900000000001 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202619+0000", 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000219619 2023-10-09T04:32:19.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.207410+0000", 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047910080000000002 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209061+0000", 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001650257 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.222756+0000", 2023-10-09T04:32:19.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013695182 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.222822+0000", 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6346000000000005e-05 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.224896+0000", 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002074471 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.237 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38539 9.9 9:931725d0:::benchmark_data_smithi029_299184_object38538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.08679473699999, 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2048092829999999, 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.238 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38539 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021422+0000", 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021457+0000", 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5354000000000001e-05 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021465+0000", 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2339999999999997e-06 2023-10-09T04:32:19.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021469+0000", 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3619999999999999e-06 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022985+0000", 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015154610000000001 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022992+0000", 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3250000000000002e-06 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204186+0000", 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.181193653 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204309+0000", 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123767 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204574+0000", 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026451699999999997 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209071+0000", 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044968439999999998 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209400+0000", 2023-10-09T04:32:19.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032915499999999997 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.225011+0000", 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015611534 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.225082+0000", 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0273999999999998e-05 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226232+0000", 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001150264 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38547 9.9 9:9345b2c2:::benchmark_data_smithi029_299184_object38546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:32:19.245 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.08674345599999, 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.206218247, 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38547 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021473+0000", 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:19.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021502+0000", 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8175999999999999e-05 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021508+0000", 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:19.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021513+0000", 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3660000000000002e-06 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.023032+0000", 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015193610000000001 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.023040+0000", 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6899999999999992e-06 2023-10-09T04:32:19.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.205732+0000", 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1826917210000001 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.205900+0000", 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000168913 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.206070+0000", 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016947199999999999 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209441+0000", 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033709899999999999 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209549+0000", 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108433 2023-10-09T04:32:19.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226306+0000", 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016757141 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226351+0000", 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4988999999999999e-05 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:19.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.227692+0000", 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001340651 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:32:19.253 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:19.254 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:19.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:19.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:19.392 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:19.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:19.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:19.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:19.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:19.597 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:19.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:19.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:19.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:19.812 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:32:20.020 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:20.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:20.191 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.283 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:20.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:20.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:20.465 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:20.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:20.659 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:20.678 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:20.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:32:20.729 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:20 smithi177 ceph-mon[195797]: pgmap v648: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:32:20.741 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:20.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:20.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:20.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:21.078 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:20 smithi029 ceph-mon[216903]: pgmap v648: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:32:21.079 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:20 smithi029 ceph-mon[223858]: pgmap v648: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-09T04:32:21.098 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:21.137 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:21.247 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:21.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:21.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:21.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:21.438 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:21.703 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:21.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:21.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:21.905 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:22.372 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:22.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:22.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:22.516 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:22.659 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:32:22.659 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:32:22.660 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:32:22.660 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:32:22.751 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:32:22.881 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:22 smithi177 ceph-mon[195797]: pgmap v649: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:32:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/523057328' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:23.114 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:23.117 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:22 smithi029 ceph-mon[216903]: pgmap v649: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:32:23.117 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/523057328' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:23.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:23.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:23.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:22 smithi029 ceph-mon[223858]: pgmap v649: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-09T04:32:23.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/523057328' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:23.119 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:32:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:32:22] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-10-09T04:32:23.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:23.279 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:23.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/523057328' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: failed to encode map e178 with expected crc 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: failed to encode map e178 with expected crc 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: failed to encode map e178 with expected crc 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: failed to encode map e178 with expected crc 2023-10-09T04:32:23.711 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:23 smithi177 ceph-mon[195797]: failed to encode map e178 with expected crc 2023-10-09T04:32:23.711 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:32:23 smithi177 conmon[174387]: logger=cleanup t=2023-10-09T04:32:23.433156808Z level=info msg="Completed cleanup jobs" duration=248.820646ms 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/523057328' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[216903]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/523057328' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: osdmap e178: 8 total, 8 up, 8 in 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:23 smithi029 ceph-mon[223858]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.025 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:24.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:24.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:24.129 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:24.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:24.210 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:24.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:24.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:24.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:24.524 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:24.633 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:24.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:24.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:24.771 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:24.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:24.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:24.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:24.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:24 smithi029 ceph-mon[216903]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:24 smithi029 ceph-mon[216903]: pgmap v651: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T04:32:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:24 smithi029 ceph-mon[223858]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:24 smithi029 ceph-mon[223858]: pgmap v651: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T04:32:24.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:24 smithi177 ceph-mon[195797]: failed to encode map e178 with expected crc 2023-10-09T04:32:24.976 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:24 smithi177 ceph-mon[195797]: pgmap v651: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-09T04:32:24.991 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:25.362 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:25.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:25.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:25.529 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:25.661 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:25.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:25.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:25.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:25.843 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.065 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:26.108 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:26 smithi029 ceph-mon[216903]: pgmap v652: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 474 op/s 2023-10-09T04:32:26.108 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1057732431' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:32:26.108 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:32:26.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:26 smithi029 ceph-mon[223858]: pgmap v652: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 474 op/s 2023-10-09T04:32:26.109 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1057732431' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:32:26.109 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:32:26.129 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:32:26.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:26.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:32:26.239 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:26.337 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:32:26.337 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:32:26.337 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:32:26.338 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:32:26.338 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:32:26.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:26 smithi177 ceph-mon[195797]: pgmap v652: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 474 op/s 2023-10-09T04:32:26.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1057732431' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:32:26.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:32:26.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:26.548 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.730 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:32:26.730 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:32:26.730 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:32:26.791 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:26.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:26.797 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:32:26.827 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:32:26.884 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:32:26.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:26.918 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:32:26.924 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:26.945 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:32:27.001 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:27.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:27.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:27.139 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.139 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.139 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.139 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.140 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.140 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.140 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.140 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:32:27.149 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:32:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:32:27.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/702616447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:27.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/702616447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:32:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: osdmap e179: 8 total, 8 up, 8 in 2023-10-09T04:32:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: failed to encode map e179 with expected crc 2023-10-09T04:32:27.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/702616447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:27.861 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:28.173 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:28.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:28.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:28.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:32:28.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:28.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:28 smithi029 ceph-mon[216903]: pgmap v654: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T04:32:28.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:28 smithi029 ceph-mon[223858]: pgmap v654: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T04:32:28.388 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:28.458 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:28 smithi177 ceph-mon[195797]: pgmap v654: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-09T04:32:28.513 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:28.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:28.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:28.633 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:28.665 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:28.771 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:28.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:28.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:28.966 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:29.213 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:29.216 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:29.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:29.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:29.381 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:29.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:29.782 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:29.807 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:29.807 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:29.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:29.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:29.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:30.371 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:30.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:30.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:30.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:30.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:32:30.590 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:30.636 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:30 smithi029 ceph-mon[216903]: pgmap v655: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T04:32:30.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:30 smithi029 ceph-mon[223858]: pgmap v655: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T04:32:30.723 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:30.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:30.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:30.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:32:30.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:30 smithi177 ceph-mon[195797]: pgmap v655: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-09T04:32:30.837 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:30.911 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:31.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:31.241 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.295 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:31.321 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:31.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:31.470 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.635 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:31.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:31.811 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:31.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:31.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:31.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:32.110 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:32.235 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:32.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:32.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:32.365 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:32.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:32.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:32.521 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:32.550 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:32.591 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:32.759 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:32 smithi177 ceph-mon[195797]: pgmap v656: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-10-09T04:32:32.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:32.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:32.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:32 smithi029 ceph-mon[216903]: pgmap v656: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-10-09T04:32:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:32 smithi029 ceph-mon[223858]: pgmap v656: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-10-09T04:32:32.820 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:32:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:32:32] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-10-09T04:32:32.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:32.947 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:33.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:33.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:33.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:33.329 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:33.466 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:33.467 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:33.589 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:32:33.604 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:32:33.753 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:33.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:33.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3225866875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:33.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/815712394' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:32:33.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:33 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:32:33.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:33.918 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:33.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3225866875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:33.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/815712394' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:32:33.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:33 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:32:33.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3225866875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:32:33.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/815712394' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:32:33.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:33 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:32:33.997 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:33.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:34.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:34.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:34.129 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:34.204 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:34.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:34.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:34.371 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:34.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:32:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: pgmap v657: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3225866875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:34 smithi177 ceph-mon[195797]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: pgmap v657: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3225866875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[223858]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: pgmap v657: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3225866875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: osdmap e180: 8 total, 8 up, 8 in 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: failed to encode map e180 with expected crc 2023-10-09T04:32:34.835 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:34 smithi029 ceph-mon[216903]: failed to encode map e180 with expected crc 2023-10-09T04:32:35.038 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:35.752 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:35.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:35.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:35.891 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:35.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:35.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:35.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:36.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:36.063 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:32:36.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:36.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:36.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:36.598 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:36 smithi029 ceph-mon[223858]: pgmap v659: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-09T04:32:36.678 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:36.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:36.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:36.687 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:36 smithi177 ceph-mon[195797]: pgmap v659: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-09T04:32:36.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:36.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:36 smithi029 ceph-mon[216903]: pgmap v659: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-09T04:32:36.859 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:37.059 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:37.060 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:37.082 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:37.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:37.128 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:32:37.175 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:32:37.180 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:37.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:37.217 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:32:37.253 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:32:37.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:37.291 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:32:37.344 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:32:37.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:37.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:37.378 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:32:37.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:32:37.500 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:37.548 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:32:37.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:37.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:32:37.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:32:37.623 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.623 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.624 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.624 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.624 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.624 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.624 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.624 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:32:37.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:37.757 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2460686270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:37.759 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2460686270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:37.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2460686270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:37.794 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:37.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:37.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:37.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:32:38.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:38.118 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:38.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:38.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:38.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:38.436 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:38.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:38.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:38.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:38.646 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:32:38.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:38.725 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:38.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:32:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:38 smithi177 ceph-mon[195797]: pgmap v660: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:32:38.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:38 smithi029 ceph-mon[216903]: pgmap v660: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:32:38.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:38 smithi029 ceph-mon[223858]: pgmap v660: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-09T04:32:38.938 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:38.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:39.044 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:39.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:39.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:39.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:39.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:39.503 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:39.915 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:39.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:40.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:40.101 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:40.260 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:40.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:40.366 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:40.443 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:40.490 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:40.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:40.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:40.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:40.615 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:40 smithi177 ceph-mon[195797]: pgmap v661: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T04:32:40.627 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:56911 9.d 9:b7beb3db:::benchmark_data_smithi029_299184_object56910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:10.977844+0000", 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 149.64520855399999, 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29796152799999998, 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.628 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 56911 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977844+0000", 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977844+0000", 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977843+0000", 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977948+0000", 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105351 2023-10-09T04:32:40.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977961+0000", 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2982e-05 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977967+0000", 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.322e-06 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.977990+0000", 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3334999999999999e-05 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.978184+0000", 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019400999999999999 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.978326+0000", 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014164599999999999 2023-10-09T04:32:40.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.980347+0000", 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002021765 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.980927+0000", 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057991800000000001 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.275735+0000", 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.294807869 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.275777+0000", 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2151999999999999e-05 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.275806+0000", 2023-10-09T04:32:40.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8527e-05 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:56914 9.d 9:b78eb913:::benchmark_data_smithi029_299184_object56913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:10.981752+0000", 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 149.641300661, 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30480986399999999, 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.635 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 56914 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981752+0000", 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981752+0000", 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981751+0000", 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981859+0000", 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108002 2023-10-09T04:32:40.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981875+0000", 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6359e-05 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981882+0000", 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4849999999999997e-06 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.981964+0000", 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2208999999999998e-05 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.982088+0000", 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012430699999999999 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.982226+0000", 2023-10-09T04:32:40.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013835 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.984107+0000", 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018803800000000001 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.984794+0000", 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00068732199999999998 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.286464+0000", 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30166952699999999 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.286523+0000", 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9413999999999997e-05 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.286562+0000", 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9002999999999999e-05 2023-10-09T04:32:40.641 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:56919 9.3 9:c32e2060:::benchmark_data_smithi029_299184_object56918:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:10.985922+0000", 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 149.63713067800001, 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29517223599999998, 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.642 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 56919 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.985922+0000", 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.985922+0000", 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.985910+0000", 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999876 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.985963+0000", 2023-10-09T04:32:40.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3267999999999999e-05 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.985979+0000", 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5254e-05 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.985985+0000", 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2689999999999996e-06 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.986039+0000", 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4323999999999997e-05 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.986155+0000", 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115677 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.986277+0000", 2023-10-09T04:32:40.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000121994 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.988365+0000", 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002087879 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:10.988396+0000", 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1343000000000001e-05 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.281014+0000", 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29261783000000002 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.281061+0000", 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7590999999999999e-05 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:11.281094+0000", 2023-10-09T04:32:40.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2981999999999997e-05 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93235 9.3 9:c156807d:::benchmark_data_smithi029_299184_object93234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.968153+0000", 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.654899539, 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36316654599999998, 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.649 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93235 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968153+0000", 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968153+0000", 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968152+0000", 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968195+0000", 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2914000000000003e-05 2023-10-09T04:32:40.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968211+0000", 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5778999999999999e-05 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968217+0000", 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8789999999999997e-06 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968267+0000", 2023-10-09T04:32:40.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9808000000000002e-05 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968418+0000", 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015034400000000001 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968588+0000", 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000170216 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.971752+0000", 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031637530000000001 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.972574+0000", 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082220700000000004 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331231+0000", 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35865675499999999 2023-10-09T04:32:40.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331278+0000", 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7107999999999997e-05 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331320+0000", 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2222999999999999e-05 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93236 9.e 9:76e382f0:::benchmark_data_smithi029_299184_object93235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.968237+0000", 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.654815958, 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35501352800000002, 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93236 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.656 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968237+0000", 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968237+0000", 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968236+0000", 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968271+0000", 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4069000000000002e-05 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968279+0000", 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5439999999999999e-06 2023-10-09T04:32:40.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968286+0000", 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.345e-06 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968360+0000", 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3647999999999997e-05 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968480+0000", 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012005799999999999 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968716+0000", 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023634099999999999 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.978927+0000", 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010210736 2023-10-09T04:32:40.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.984861+0000", 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059335890000000004 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.323189+0000", 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33832865699999998 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.323225+0000", 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5608000000000002e-05 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.323250+0000", 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5418999999999998e-05 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93239 9.e 9:71c36557:::benchmark_data_smithi029_299184_object93238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.968534+0000", 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.654518964, 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35482608900000001, 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93239 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.663 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968534+0000", 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968534+0000", 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968533+0000", 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968577+0000", 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4116000000000002e-05 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968591+0000", 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3814e-05 2023-10-09T04:32:40.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.968597+0000", 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2389999999999999e-06 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.971875+0000", 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032773479999999998 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.972010+0000", 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000134943 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.972156+0000", 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000146112 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.978932+0000", 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067764349999999999 2023-10-09T04:32:40.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.984879+0000", 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059464519999999996 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.323291+0000", 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33841268099999999 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.323325+0000", 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3769999999999997e-05 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.323360+0000", 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4993000000000002e-05 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93242 9.e 9:79889d0b:::benchmark_data_smithi029_299184_object93241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.669 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.969814+0000", 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.65323937700001, 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36777390399999998, 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93242 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969814+0000", 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969814+0000", 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969812+0000", 2023-10-09T04:32:40.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969900+0000", 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8168e-05 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969920+0000", 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9222e-05 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969934+0000", 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4007e-05 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.975648+0000", 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057140100000000003 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.975782+0000", 2023-10-09T04:32:40.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013435500000000001 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.975933+0000", 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015052800000000001 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.979803+0000", 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003870793 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.984893+0000", 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050895549999999999 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.337484+0000", 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35259152700000002 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.337541+0000", 2023-10-09T04:32:40.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6292000000000002e-05 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.337587+0000", 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6628999999999997e-05 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93243 9.d 9:b9b845f6:::benchmark_data_smithi029_299184_object93242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.969941+0000", 2023-10-09T04:32:40.676 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.65311191799999, 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35280185600000002, 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93243 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969941+0000", 2023-10-09T04:32:40.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969941+0000", 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969940+0000", 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:40.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969985+0000", 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4876000000000002e-05 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969993+0000", 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3089999999999996e-06 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.969998+0000", 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7890000000000002e-06 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.970038+0000", 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9626999999999999e-05 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.970129+0000", 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1222000000000002e-05 2023-10-09T04:32:40.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.970213+0000", 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3448000000000005e-05 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977586+0000", 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073738650000000003 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977916+0000", 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032916999999999999 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.322655+0000", 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34473916599999999 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.322698+0000", 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3229000000000002e-05 2023-10-09T04:32:40.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.322743+0000", 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4814000000000002e-05 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93246 9.3 9:c69cba6b:::benchmark_data_smithi029_299184_object93245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.977194+0000", 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.645859371, 2023-10-09T04:32:40.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35425319900000002, 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93246 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977194+0000", 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977194+0000", 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977192+0000", 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977270+0000", 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7953000000000004e-05 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977283+0000", 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2464e-05 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977291+0000", 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1839999999999999e-06 2023-10-09T04:32:40.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977346+0000", 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5009000000000001e-05 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977474+0000", 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000128214 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.977612+0000", 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000137847 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.980449+0000", 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028367819999999999 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.984778+0000", 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043294730000000003 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331363+0000", 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34658511800000003 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331411+0000", 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7079000000000001e-05 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331447+0000", 2023-10-09T04:32:40.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6188e-05 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:93252 9.3 9:cd7cfbbd:::benchmark_data_smithi029_299184_object93251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:30:55.981716+0000", 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 104.641337111, 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.349838129, 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.690 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93252 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981716+0000", 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981716+0000", 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981714+0000", 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981767+0000", 2023-10-09T04:32:40.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2574999999999997e-05 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981780+0000", 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2726e-05 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981787+0000", 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9990000000000002e-06 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981838+0000", 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1337e-05 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.981983+0000", 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014466899999999999 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.982132+0000", 2023-10-09T04:32:40.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000149104 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.985040+0000", 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002907785 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:55.988173+0000", 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031330049999999999 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331476+0000", 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34330386800000001 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331523+0000", 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6888999999999998e-05 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:30:56.331554+0000", 2023-10-09T04:32:40.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0633000000000003e-05 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:114621 9.3 9:cb608724:::benchmark_data_smithi029_299184_object16318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:26.526965+0000", 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 74.096087539999999, 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28879131099999999, 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.697 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 114621 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.526965+0000", 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.526965+0000", 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.526964+0000", 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.526970+0000", 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8810000000000001e-06 2023-10-09T04:32:40.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.526983+0000", 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3141e-05 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.526989+0000", 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7649999999999999e-06 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.527047+0000", 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8421000000000002e-05 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.527213+0000", 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016525099999999999 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.527354+0000", 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014118600000000001 2023-10-09T04:32:40.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.535336+0000", 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079824130000000007 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.597769+0000", 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.062432877999999997 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.815663+0000", 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.217893588 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.815716+0000", 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2834000000000003e-05 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.815757+0000", 2023-10-09T04:32:40.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1066999999999998e-05 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:114673 9.d 9:b46db0f9:::benchmark_data_smithi029_299184_object16370:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:26.846643+0000", 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 73.776409959999995, 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30797777300000001, 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.704 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 114673 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.846643+0000", 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.846643+0000", 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.846642+0000", 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.846648+0000", 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9699999999999996e-06 2023-10-09T04:32:40.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.846659+0000", 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0916e-05 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.846665+0000", 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0039999999999996e-06 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.847693+0000", 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001028347 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.847806+0000", 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113337 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.847877+0000", 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0848999999999999e-05 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.851810+0000", 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039330379999999998 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855825+0000", 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040149610000000001 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.154532+0000", 2023-10-09T04:32:40.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.298706474 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.154580+0000", 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8454000000000001e-05 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.154621+0000", 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0509000000000001e-05 2023-10-09T04:32:40.710 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:114682 9.3 9:c3051922:::benchmark_data_smithi029_299184_object16379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:26.855642+0000", 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 73.767411034000006, 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29603887099999998, 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.711 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 114682 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855642+0000", 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855642+0000", 2023-10-09T04:32:40.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855641+0000", 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855647+0000", 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9859999999999999e-06 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855674+0000", 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7063000000000002e-05 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.855680+0000", 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3210000000000001e-06 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.857810+0000", 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021303619999999998 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.857894+0000", 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3424999999999995e-05 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.857955+0000", 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0986000000000002e-05 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.860872+0000", 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029173580000000001 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.863358+0000", 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00248585 2023-10-09T04:32:40.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.151573+0000", 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28821506000000002 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.151637+0000", 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3688000000000004e-05 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.151681+0000", 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3988e-05 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:114695 9.d 9:b359d8c9:::benchmark_data_smithi029_299184_object16392:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:26.865963+0000", 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 73.757089754999996, 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31096429799999997, 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.718 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 114695 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.865963+0000", 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.865963+0000", 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.865962+0000", 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.865968+0000", 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.446e-06 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.865977+0000", 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0750000000000004e-06 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.865981+0000", 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2250000000000002e-06 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.866029+0000", 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7953e-05 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.866152+0000", 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000122955 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.866261+0000", 2023-10-09T04:32:40.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109565 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.869839+0000", 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035772270000000001 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.869999+0000", 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016055299999999999 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.176834+0000", 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30683516500000002 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.176888+0000", 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3291999999999997e-05 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.176927+0000", 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9848999999999997e-05 2023-10-09T04:32:40.724 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:114696 9.d 9:b3492272:::benchmark_data_smithi029_299184_object16393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:26.870217+0000", 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 73.752835731999994, 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30681349899999999, 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.725 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 114696 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870217+0000", 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870217+0000", 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870216+0000", 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870224+0000", 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8399999999999995e-06 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870236+0000", 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2697000000000001e-05 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870242+0000", 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0689999999999998e-06 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870274+0000", 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.146e-05 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870360+0000", 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6278000000000003e-05 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.870431+0000", 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0928999999999998e-05 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.873093+0000", 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026621990000000001 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.873382+0000", 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028830099999999998 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.176973+0000", 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30359174700000002 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.177007+0000", 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3630000000000002e-05 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.177031+0000", 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3626e-05 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.731 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:114698 9.d 9:bb80a1a2:::benchmark_data_smithi029_299184_object16395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:26.873999+0000", 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 73.749054143999999, 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.307700747, 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.732 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 114698 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.873999+0000", 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.873999+0000", 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.873998+0000", 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.874004+0000", 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3929999999999996e-06 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.874017+0000", 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3294e-05 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.874022+0000", 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9790000000000001e-06 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.874054+0000", 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1612000000000002e-05 2023-10-09T04:32:40.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.874145+0000", 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1500000000000001e-05 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,5", 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.874219+0000", 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3727999999999996e-05 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.876751+0000", 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002531606 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:26.877036+0000", 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000284973 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.181607+0000", 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30457147099999998 2023-10-09T04:32:40.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.181661+0000", 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4030000000000003e-05 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:27.181699+0000", 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8349000000000002e-05 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.738 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:119233 9.3 9:c2baff2f:::benchmark_data_smithi029_299184_object20930:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:33.505979+0000", 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 67.117073727000005, 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31508659500000002, 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.739 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119233 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.505979+0000", 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.505979+0000", 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.505978+0000", 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.505984+0000", 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1380000000000004e-06 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.505995+0000", 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.118e-05 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.506000+0000", 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6199999999999998e-06 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.506043+0000", 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3087000000000003e-05 2023-10-09T04:32:40.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.506133+0000", 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0118999999999996e-05 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.506206+0000", 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3151999999999994e-05 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.509213+0000", 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030071310000000001 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.607211+0000", 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.097997789000000002 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.820971+0000", 2023-10-09T04:32:40.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21375951300000001 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.821025+0000", 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3869000000000002e-05 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:33.821066+0000", 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.121e-05 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.745 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:133468 9.3 9:c9e64f7b:::benchmark_data_smithi029_299184_object35165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:55.693193+0000", 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 44.929859600999997, 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32137877999999998, 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.746 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 133468 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693193+0000", 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693193+0000", 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693192+0000", 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693199+0000", 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2690000000000002e-06 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693211+0000", 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1997999999999999e-05 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693217+0000", 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7620000000000001e-06 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:40.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693271+0000", 2023-10-09T04:32:40.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4632000000000001e-05 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693388+0000", 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116626 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693459+0000", 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1098000000000003e-05 2023-10-09T04:32:40.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.697451+0000", 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039920809999999998 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.701989+0000", 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045373849999999997 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.014476+0000", 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.312487442 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.014529+0000", 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2899999999999998e-05 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.014572+0000", 2023-10-09T04:32:40.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3105000000000001e-05 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:133472 9.3 9:c8cb0e9b:::benchmark_data_smithi029_299184_object35169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:55.693235+0000", 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 44.929817909999997, 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32147968999999998, 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.754 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 133472 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693235+0000", 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693235+0000", 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693235+0000", 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693237+0000", 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4650000000000001e-06 2023-10-09T04:32:40.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693243+0000", 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7910000000000001e-06 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.693247+0000", 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7220000000000001e-06 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.698676+0000", 2023-10-09T04:32:40.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054291610000000001 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.698825+0000", 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014882 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.698969+0000", 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014480100000000001 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.702012+0000", 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003042435 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.702072+0000", 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9984e-05 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.014605+0000", 2023-10-09T04:32:40.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31253355999999999 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.014678+0000", 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2408000000000002e-05 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.014715+0000", 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.693e-05 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.760 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:133478 9.3 9:c2352303:::benchmark_data_smithi029_299184_object35175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e170)", 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:31:55.703339+0000", 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 44.919713569999999, 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31238888100000001, 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 133478 2023-10-09T04:32:40.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703339+0000", 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703339+0000", 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703338+0000", 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703344+0000", 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8599999999999998e-06 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703356+0000", 2023-10-09T04:32:40.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2044e-05 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703361+0000", 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7990000000000001e-06 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703401+0000", 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0068999999999999e-05 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703487+0000", 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6018000000000002e-05 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.703559+0000", 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1766000000000004e-05 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.706320+0000", 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027613640000000001 2023-10-09T04:32:40.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:55.706468+0000", 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014793299999999999 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.015643+0000", 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30917510799999998 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.015693+0000", 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9870999999999997e-05 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:31:56.015728+0000", 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5203000000000002e-05 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:32:40.767 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:32:40.814 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:40.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:40.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:40.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:40 smithi029 ceph-mon[216903]: pgmap v661: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T04:32:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:40 smithi029 ceph-mon[223858]: pgmap v661: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-09T04:32:41.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:41.087 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:41.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:41.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:41.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:41.360 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:41.491 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:41.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:41.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:32:41.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:41.642 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:41.669 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:32:41.776 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2967438080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:32:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:41 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:32:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:41.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:41.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:32:41.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:32:41.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2967438080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:32:41.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:41 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:32:41.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2967438080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:32:41.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:41 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:32:41.947 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:41.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:32:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:41.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:32:42.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:42.137 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:32:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:32:42] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-10-09T04:32:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[223858]: pgmap v662: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-09T04:32:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[223858]: failed to encode map e181 with expected crc 2023-10-09T04:32:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[223858]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[223858]: failed to encode map e181 with expected crc 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[223858]: failed to encode map e181 with expected crc 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[216903]: pgmap v662: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[216903]: failed to encode map e181 with expected crc 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[216903]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[216903]: failed to encode map e181 with expected crc 2023-10-09T04:32:42.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:42 smithi029 ceph-mon[216903]: failed to encode map e181 with expected crc 2023-10-09T04:32:43.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:42 smithi177 ceph-mon[195797]: pgmap v662: 89 pgs: 89 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-09T04:32:43.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:42 smithi177 ceph-mon[195797]: failed to encode map e181 with expected crc 2023-10-09T04:32:43.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:42 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:32:43.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:42 smithi177 ceph-mon[195797]: osdmap e181: 8 total, 8 up, 8 in 2023-10-09T04:32:43.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:42 smithi177 ceph-mon[195797]: failed to encode map e181 with expected crc 2023-10-09T04:32:43.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:42 smithi177 ceph-mon[195797]: failed to encode map e181 with expected crc 2023-10-09T04:32:43.190 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:43.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:43.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:43.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:43.347 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:43.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:43.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:43.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:43.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:32:43.674 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:43.804 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:43.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:43.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:43.956 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:44.003 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:44.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:44.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:44.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:44.056 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:44.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:44.170 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:44.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:44.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:44.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:44.384 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:44.452 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:44.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:44.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:44 smithi177 ceph-mon[195797]: pgmap v664: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T04:32:44.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:44.505 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:44.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:44.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:44 smithi029 ceph-mon[216903]: pgmap v664: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T04:32:44.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:44.592 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:44.592 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:44 smithi029 ceph-mon[223858]: pgmap v664: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-09T04:32:44.592 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:32:44.593 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:44.629 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:44.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:44.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:44.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:44.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:44.870 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:44.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:44.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:45.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:45.156 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:45.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:45.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:45.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:45.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:45.448 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:45.543 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:32:45.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:45.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:45.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:45.705 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/638191422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:45.706 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:45 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:45.707 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:45.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:45.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:45.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/638191422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:45 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/638191422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:45 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:32:45.973 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: pgmap v665: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:46 smithi177 ceph-mon[195797]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: pgmap v665: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[216903]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: pgmap v665: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: osdmap e182: 8 total, 8 up, 8 in 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: failed to encode map e182 with expected crc 2023-10-09T04:32:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:46 smithi029 ceph-mon[223858]: failed to encode map e182 with expected crc 2023-10-09T04:32:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:47 smithi177 ceph-mon[195797]: failed to encode map e182 with expected crc 2023-10-09T04:32:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:47 smithi029 ceph-mon[216903]: failed to encode map e182 with expected crc 2023-10-09T04:32:47.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:47 smithi029 ceph-mon[223858]: failed to encode map e182 with expected crc 2023-10-09T04:32:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:48 smithi177 ceph-mon[195797]: pgmap v667: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T04:32:48.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:48 smithi029 ceph-mon[216903]: pgmap v667: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T04:32:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:48 smithi029 ceph-mon[223858]: pgmap v667: 89 pgs: 89 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-10-09T04:32:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:50 smithi029 ceph-mon[216903]: pgmap v668: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T04:32:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:50 smithi029 ceph-mon[223858]: pgmap v668: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T04:32:51.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:50 smithi177 ceph-mon[195797]: pgmap v668: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-10-09T04:32:52.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:52 smithi029 ceph-mon[216903]: pgmap v669: 89 pgs: 89 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:32:52.825 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:32:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:32:52] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.43.0" 2023-10-09T04:32:52.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:52 smithi029 ceph-mon[223858]: pgmap v669: 89 pgs: 89 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:32:52.917 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:52.935 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:52.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:52.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:53.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:52 smithi177 ceph-mon[195797]: pgmap v669: 89 pgs: 89 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-09T04:32:53.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:53.121 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:53.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:53.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:53.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:53.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:53.397 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:53.671 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:32:53.693 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:53.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:53.712 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:32:53.752 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:32:53.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:32:53.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:53.827 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:32:53.859 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:53.873 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:32:53.910 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:32:53.940 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:32:53.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:53.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:53.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:53.972 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.972 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.972 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.973 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.973 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.973 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.973 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:32:53.973 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:32:54.024 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:32:54.025 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:32:54.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:54.142 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:54.362 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:54.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:54.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:54.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:54.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:54.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:54.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:54 smithi029 ceph-mon[223858]: pgmap v670: 89 pgs: 89 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-09T04:32:54.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2679878218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:54.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:54 smithi029 ceph-mon[216903]: pgmap v670: 89 pgs: 89 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-09T04:32:54.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2679878218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:54.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:54.801 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:54 smithi177 ceph-mon[195797]: pgmap v670: 89 pgs: 89 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-09T04:32:54.801 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2679878218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:32:54.806 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:54.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:54.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:54.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:54.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:32:55.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:55.070 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:55.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:55.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:55.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:32:55.265 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:55.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:32:55.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:55.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:32:55.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:55.555 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:56.351 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:56.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:56.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:56.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:32:56.521 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:56 smithi029 ceph-mon[223858]: pgmap v671: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-09T04:32:56.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:56 smithi029 ceph-mon[216903]: pgmap v671: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-09T04:32:56.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:32:56.819 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:32:56.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:56.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:32:56.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:56.991 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:56 smithi177 ceph-mon[195797]: pgmap v671: 89 pgs: 89 active+clean; 1.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-09T04:32:57.032 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:32:57.091 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:57.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:57.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:32:57.270 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:57.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:32:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:57.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:32:57.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:57.564 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:57.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:57.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:57.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:57.834 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:58.107 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:58.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:58.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:58.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:32:58.274 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:58.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:32:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:58.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:32:58.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:58.594 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:58.594 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:32:58.649 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:32:58.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:58.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:32:58.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:32:58.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:58.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:32:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:58 smithi029 ceph-mon[216903]: pgmap v672: 89 pgs: 89 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T04:32:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/878991890' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:32:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:58 smithi029 ceph-mon[223858]: pgmap v672: 89 pgs: 89 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T04:32:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/878991890' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:32:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:58.948 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:32:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:32:58.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:32:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:58 smithi177 ceph-mon[195797]: pgmap v672: 89 pgs: 89 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-09T04:32:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/878991890' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:32:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:32:59.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:32:59.192 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:32:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/878991890' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:32:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:32:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[216903]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/878991890' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:32:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:32:59 smithi029 ceph-mon[223858]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/878991890' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: osdmap e183: 8 total, 8 up, 8 in 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:32:59 smithi177 ceph-mon[195797]: failed to encode map e183 with expected crc 2023-10-09T04:33:00.207 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:00.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:00.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:00.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:00.358 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:00.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:33:00.397 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:00.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:00.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:00.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:00.591 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:00.624 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:00.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:00.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:00 smithi177 ceph-mon[195797]: pgmap v674: 89 pgs: 89 active+clean; 1.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-10-09T04:33:00.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:00.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:00.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:00.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:00 smithi029 ceph-mon[216903]: pgmap v674: 89 pgs: 89 active+clean; 1.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-10-09T04:33:00.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:00 smithi029 ceph-mon[223858]: pgmap v674: 89 pgs: 89 active+clean; 1.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-10-09T04:33:00.843 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:00.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:00.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:00.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:01.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:01.177 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:01.295 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:01.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:01.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:01.454 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:01.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:01.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:01.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:01.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:01.673 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:33:01.729 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:01.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1020503877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:01.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1020503877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1020503877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: pgmap v675: 89 pgs: 89 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1020503877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[216903]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: pgmap v675: 89 pgs: 89 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1020503877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:02 smithi029 ceph-mon[223858]: failed to encode map e184 with expected crc 2023-10-09T04:33:02.926 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:33:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:33:02] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T04:33:03.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: pgmap v675: 89 pgs: 89 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-09T04:33:03.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:03.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1020503877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:03.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: osdmap e184: 8 total, 8 up, 8 in 2023-10-09T04:33:03.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: failed to encode map e184 with expected crc 2023-10-09T04:33:03.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: failed to encode map e184 with expected crc 2023-10-09T04:33:03.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: failed to encode map e184 with expected crc 2023-10-09T04:33:03.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:02 smithi177 ceph-mon[195797]: failed to encode map e184 with expected crc 2023-10-09T04:33:04.435 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:04.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:04.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:04.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:04.625 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:04.878 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:04 smithi029 ceph-mon[216903]: pgmap v677: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2023-10-09T04:33:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:04.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:04.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:04 smithi029 ceph-mon[223858]: pgmap v677: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2023-10-09T04:33:04.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:05.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:04 smithi177 ceph-mon[195797]: pgmap v677: 89 pgs: 89 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2023-10-09T04:33:05.050 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:05.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:05.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:05.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:05.166 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:05.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:05.292 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:05.343 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:33:05.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:05.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:05.375 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:33:05.425 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:33:05.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:05.472 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:33:05.511 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:33:05.548 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:05.555 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:33:05.612 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:33:05.668 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:33:05.670 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:05.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:05.680 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.681 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:33:05.703 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:33:05.703 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:33:05.703 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:33:05.704 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:33:05.704 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:33:05.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:05.833 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:05.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:05.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:05.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:06.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:06.114 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:06 smithi029 ceph-mon[216903]: pgmap v678: 89 pgs: 89 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 495 op/s 2023-10-09T04:33:06.114 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1138927873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:06.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:06 smithi029 ceph-mon[223858]: pgmap v678: 89 pgs: 89 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 495 op/s 2023-10-09T04:33:06.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1138927873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:06.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:06 smithi177 ceph-mon[195797]: pgmap v678: 89 pgs: 89 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 495 op/s 2023-10-09T04:33:06.118 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1138927873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:06.131 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:06.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:06.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:06.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:06.367 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:06.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:06.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:06.562 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:06.625 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:06.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:33:07.504 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:07.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:07.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:07.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:07.676 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:07.992 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:08.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:08.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:08.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:08.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:08 smithi029 ceph-mon[216903]: pgmap v679: 89 pgs: 89 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:33:08.312 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:08 smithi029 ceph-mon[223858]: pgmap v679: 89 pgs: 89 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:33:08.315 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:08.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:33:08.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:08 smithi177 ceph-mon[195797]: pgmap v679: 89 pgs: 89 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:33:08.784 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:08.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:08.898 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:08.945 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:09.019 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:09.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:09.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:09.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:09.215 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:09.341 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:09.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:09.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:09.549 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:09.574 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:09.678 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:09.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:09.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:09.830 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:09.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:09.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:09.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:10.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:10.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:10.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:10.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:10.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:10.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:10 smithi029 ceph-mon[216903]: pgmap v680: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T04:33:10.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/352275626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:33:10.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:33:10.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:10 smithi029 ceph-mon[223858]: pgmap v680: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T04:33:10.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/352275626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:33:10.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:33:10.164 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:33:10.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:10.311 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:10.399 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:10.403 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:10 smithi177 ceph-mon[195797]: pgmap v680: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-09T04:33:10.403 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/352275626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:33:10.404 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:33:10.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:10.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:10.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:10.560 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:10.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:10.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:10.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:10.788 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:10.875 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:10.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:10.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:10.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:33:10.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:11.036 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:33:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:33:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[216903]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.425 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:11 smithi029 ceph-mon[223858]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:11.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:33:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: osdmap e185: 8 total, 8 up, 8 in 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:11 smithi177 ceph-mon[195797]: failed to encode map e185 with expected crc 2023-10-09T04:33:11.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:11.593 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:11.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:11.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:11.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:11.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:11.836 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:11.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:11.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:11.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:12.047 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:12.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:12.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:12.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:12.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:12.305 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:12.328 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:12 smithi029 ceph-mon[216903]: pgmap v682: 89 pgs: 89 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:33:12.328 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:12 smithi029 ceph-mon[223858]: pgmap v682: 89 pgs: 89 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:33:12.374 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:33:12.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:33:12.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:12.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:12 smithi177 ceph-mon[195797]: pgmap v682: 89 pgs: 89 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-10-09T04:33:12.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:12.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:12.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:12.564 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:12.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:12.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:12.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:12.790 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:12.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:33:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:33:12] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2023-10-09T04:33:13.381 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:13.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:13.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:13.521 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:13.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:13.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:13.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:13.672 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:13.686 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:13.719 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:13.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:13.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:13.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:13.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:13.969 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:14.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:14 smithi029 ceph-mon[216903]: pgmap v683: 89 pgs: 89 active+clean; 972 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T04:33:14.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:14.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2315387081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:14.234 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:33:14.252 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:14.260 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:14.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:14.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:14.380 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:33:14.380 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:33:14.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:88376 8.1 8:88006b61:::benchmark_data_smithi029_252925_object88375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:07.903122+0000", 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 546.47200814300004, 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1049434119999999, 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:14.381 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88376 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903122+0000", 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903122+0000", 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903120+0000", 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903165+0000", 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5729000000000002e-05 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903192+0000", 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6563999999999998e-05 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903204+0000", 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1545e-05 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903284+0000", 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0809999999999994e-05 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903303+0000", 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8732999999999998e-05 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.906977+0000", 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036738930000000001 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.906996+0000", 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8550999999999999e-05 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.007957+0000", 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1009610590000001 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.008065+0000", 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108768 2023-10-09T04:33:14.386 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:88377 8.1 8:833f4176:::benchmark_data_smithi029_252925_object88376:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:07.903224+0000", 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 546.47190599600003, 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1062361759999999, 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.387 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88377 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903224+0000", 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903224+0000", 2023-10-09T04:33:14.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903223+0000", 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903260+0000", 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7178999999999998e-05 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903269+0000", 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0429999999999996e-06 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903276+0000", 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8460000000000002e-06 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903444+0000", 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016738199999999999 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903452+0000", 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5599999999999994e-06 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907372+0000", 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039196220000000002 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907382+0000", 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0662000000000001e-05 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009358+0000", 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.101975511 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009460+0000", 2023-10-09T04:33:14.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102259 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:88383 8.1 8:8ae67f58:::benchmark_data_smithi029_252925_object88382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e119)", 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:24:07.903283+0000", 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 546.47184726299997, 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1166282439999999, 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.393 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88383 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903283+0000", 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903283+0000", 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903282+0000", 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903321+0000", 2023-10-09T04:33:14.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9394999999999998e-05 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903367+0000", 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5163000000000001e-05 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903376+0000", 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6520000000000001e-06 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903555+0000", 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017835799999999999 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.903566+0000", 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1619e-05 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907415+0000", 2023-10-09T04:33:14.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038489359999999999 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:07.907424+0000", 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.884e-06 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009625+0000", 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1022007709999999 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.009778+0000", 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015287800000000001 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.010541+0000", 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000762848 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.014175+0000", 2023-10-09T04:33:14.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036340040000000001 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.016651+0000", 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024762690000000001 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.018480+0000", 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00182884 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.018557+0000", 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6822000000000004e-05 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:24:09.019911+0000", 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013545639999999999 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.401 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:61972 9.9 9:90f3cb0e:::benchmark_data_smithi029_299184_object61971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.529067+0000", 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.84606260199999, 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1577070090000001, 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 61972 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529067+0000", 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529067+0000", 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529066+0000", 2023-10-09T04:33:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529094+0000", 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7983999999999998e-05 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529108+0000", 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3859999999999999e-05 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529116+0000", 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5699999999999993e-06 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529174+0000", 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7540999999999997e-05 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529182+0000", 2023-10-09T04:33:14.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0900000000000005e-06 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.530789+0000", 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016072059999999999 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.530800+0000", 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0720000000000001e-05 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.681698+0000", 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1508984250000001 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682029+0000", 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033048000000000002 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686711+0000", 2023-10-09T04:33:14.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046828160000000002 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686774+0000", 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3068e-05 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:61980 9.9 9:9f2d6e30:::benchmark_data_smithi029_299184_object61979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:33:14.408 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.536369+0000", 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.83876136500001, 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.154441914, 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 61980 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536369+0000", 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536369+0000", 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536367+0000", 2023-10-09T04:33:14.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536393+0000", 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5503999999999998e-05 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536411+0000", 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8919999999999998e-05 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536421+0000", 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2580000000000001e-06 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536486+0000", 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5320000000000005e-05 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536497+0000", 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1276e-05 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682269+0000", 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1457712170000001 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682371+0000", 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102947 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682476+0000", 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010462 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686667+0000", 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041908800000000001 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690531+0000", 2023-10-09T04:33:14.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038639249999999998 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690651+0000", 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011974200000000001 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690710+0000", 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9570000000000001e-05 2023-10-09T04:33:14.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690811+0000", 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100449 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:61982 9.9 9:9c6412a6:::benchmark_data_smithi029_299184_object61981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.542493+0000", 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.832637414, 2023-10-09T04:33:14.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1484788960000001, 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 61982 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542493+0000", 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542493+0000", 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542491+0000", 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542520+0000", 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9142e-05 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542534+0000", 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4059999999999999e-05 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542543+0000", 2023-10-09T04:33:14.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7450000000000001e-06 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542673+0000", 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012984300000000001 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542688+0000", 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5449999999999999e-05 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.684067+0000", 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1413787470000001 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.684210+0000", 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014304300000000001 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.684405+0000", 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019457200000000001 2023-10-09T04:33:14.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690490+0000", 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060849470000000003 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690595+0000", 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105111 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690866+0000", 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00027154099999999998 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690898+0000", 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2054999999999997e-05 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690972+0000", 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3209999999999999e-05 2023-10-09T04:33:14.423 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:62008 9.9 9:9ba51e8c:::benchmark_data_smithi029_299184_object62007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.566286+0000", 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.80884368400001, 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.129050973, 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.424 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62008 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566286+0000", 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566286+0000", 2023-10-09T04:33:14.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566284+0000", 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566318+0000", 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3426000000000002e-05 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566336+0000", 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7604e-05 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566345+0000", 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5100000000000004e-06 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566429+0000", 2023-10-09T04:33:14.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3690999999999995e-05 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566440+0000", 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.114e-05 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686796+0000", 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1203562330000001 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686945+0000", 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000149128 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.687135+0000", 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000189657 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690609+0000", 2023-10-09T04:33:14.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034738770000000002 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695192+0000", 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045837070000000002 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695231+0000", 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8741000000000001e-05 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695266+0000", 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4295000000000003e-05 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695337+0000", 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1872999999999994e-05 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:62039 9.9 9:9b50b364:::benchmark_data_smithi029_299184_object62038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.603585+0000", 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.77154490000001, 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.091967184, 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62039 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.432 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603585+0000", 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603585+0000", 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603583+0000", 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603608+0000", 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4511e-05 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603625+0000", 2023-10-09T04:33:14.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7382000000000001e-05 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603635+0000", 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7990000000000005e-06 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603752+0000", 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011654000000000001 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603767+0000", 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5027e-05 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.688633+0000", 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0848658689999999 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.688747+0000", 2023-10-09T04:33:14.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011479799999999999 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.688908+0000", 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016042399999999999 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695208+0000", 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062998819999999997 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695379+0000", 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017120500000000001 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695422+0000", 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3010999999999999e-05 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695483+0000", 2023-10-09T04:33:14.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0869999999999998e-05 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695552+0000", 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9579000000000006e-05 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:62051 9.9 9:92149584:::benchmark_data_smithi029_299184_object62050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:33:14.439 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.616339+0000", 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.758791277, 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.086473722, 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62051 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616339+0000", 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616339+0000", 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616337+0000", 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:14.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616361+0000", 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4099000000000001e-05 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616375+0000", 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3662000000000001e-05 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616383+0000", 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4579999999999993e-06 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616449+0000", 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5829999999999998e-05 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616462+0000", 2023-10-09T04:33:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2737e-05 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.691033+0000", 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0745713690000001 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.691164+0000", 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130483 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.691296+0000", 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131746 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695568+0000", 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042726090000000001 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702529+0000", 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069606370000000004 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702648+0000", 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011957 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702718+0000", 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9638e-05 2023-10-09T04:33:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702813+0000", 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4545999999999999e-05 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166771 9.b 9:dd78fb9c:::benchmark_data_smithi029_299184_object68468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.077564+0000", 2023-10-09T04:33:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.297565699, 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3448709380000001, 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166771 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077564+0000", 2023-10-09T04:33:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077564+0000", 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077563+0000", 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077570+0000", 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2880000000000004e-06 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077583+0000", 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3049e-05 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077592+0000", 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5779999999999999e-06 2023-10-09T04:33:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077645+0000", 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2834000000000003e-05 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077763+0000", 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118306 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077877+0000", 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114071 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422189+0000", 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3443123450000001 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422208+0000", 2023-10-09T04:33:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8799000000000001e-05 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422325+0000", 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116734 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422381+0000", 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5961000000000002e-05 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422435+0000", 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4348999999999998e-05 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166776 9.5 9:ab9ec60c:::benchmark_data_smithi029_299184_object68473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081143+0000", 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.293986743000001, 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.346525757, 2023-10-09T04:33:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166776 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081143+0000", 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081143+0000", 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081142+0000", 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081149+0000", 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2899999999999997e-06 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081169+0000", 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9813000000000002e-05 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081178+0000", 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8119999999999997e-06 2023-10-09T04:33:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.085605+0000", 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044272540000000003 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.085733+0000", 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012755500000000001 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.085838+0000", 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105432 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427505+0000", 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3416669649999999 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427520+0000", 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5014000000000001e-05 2023-10-09T04:33:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427561+0000", 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1162999999999999e-05 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427622+0000", 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0359000000000003e-05 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427669+0000", 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7559999999999999e-05 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166777 9.b 9:d7522bda:::benchmark_data_smithi029_299184_object68474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081193+0000", 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.293937053000001, 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3414661510000001, 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166777 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081193+0000", 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081193+0000", 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081192+0000", 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081196+0000", 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4259999999999999e-06 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081203+0000", 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.374e-06 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081209+0000", 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0430000000000004e-06 2023-10-09T04:33:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082783+0000", 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015734099999999999 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082897+0000", 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114535 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.083005+0000", 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000107649 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422253+0000", 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.339248386 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422261+0000", 2023-10-09T04:33:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1030000000000002e-06 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422564+0000", 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00030216800000000001 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422617+0000", 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.304e-05 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422659+0000", 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2553999999999999e-05 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166778 9.5 9:ac5d0fe7:::benchmark_data_smithi029_299184_object68475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.293910789999998, 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.357283987, 2023-10-09T04:33:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166778 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081222+0000", 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9979999999999999e-06 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081228+0000", 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3509999999999998e-06 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081233+0000", 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0950000000000003e-06 2023-10-09T04:33:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090131+0000", 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088972760000000008 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090227+0000", 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6126999999999996e-05 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090322+0000", 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5367999999999999e-05 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427723+0000", 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3374010250000001 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427756+0000", 2023-10-09T04:33:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3222999999999997e-05 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.438410+0000", 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010653856999999999 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.438462+0000", 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1761000000000001e-05 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.438503+0000", 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1337000000000001e-05 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166782 9.b 9:de3f3061:::benchmark_data_smithi029_299184_object68479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081559+0000", 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.293570883000001, 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3412489839999999, 2023-10-09T04:33:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166782 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081559+0000", 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081559+0000", 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081558+0000", 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081564+0000", 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.22e-06 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081574+0000", 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.08e-05 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081583+0000", 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2430000000000005e-06 2023-10-09T04:33:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.087862+0000", 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062788369999999998 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.087948+0000", 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6804000000000004e-05 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.088029+0000", 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0216000000000006e-05 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422267+0000", 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3342385990000001 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422497+0000", 2023-10-09T04:33:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022963700000000001 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422721+0000", 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022408899999999999 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422771+0000", 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0076e-05 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422808+0000", 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7255999999999997e-05 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:166781 9.8 e182/167 9:1ce571f8:::benchmark_data_smithi029_299184_object68478:head v 182'10397, mlcod=182'10395)", 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081697+0000", 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.293433017000002, 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3404559030000001, 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081697+0000", 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081697+0000", 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081696+0000", 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081703+0000", 2023-10-09T04:33:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4289999999999998e-06 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081712+0000", 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5210000000000008e-06 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081718+0000", 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1129999999999997e-06 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082774+0000", 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001055865 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082795+0000", 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0959999999999999e-05 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422101+0000", 2023-10-09T04:33:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3393052329999999 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422153+0000", 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2463999999999998e-05 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:166771 9.b e182/163 ondisk, result = 0)", 2023-10-09T04:33:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.086822+0000", 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.288307929999998, 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.335388005, 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086822+0000", 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086822+0000", 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086821+0000", 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086828+0000", 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086840+0000", 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2448e-05 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086848+0000", 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8669999999999994e-06 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422168+0000", 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.335319393 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422180+0000", 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.293e-05 2023-10-09T04:33:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422210+0000", 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9728000000000001e-05 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:166784 9.8 e182/167 9:1040dc07:::benchmark_data_smithi029_299184_object68481:head v 182'10398, mlcod=182'10395)", 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.089440+0000", 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.2856898, 2023-10-09T04:33:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3327767939999999, 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089440+0000", 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089440+0000", 2023-10-09T04:33:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089439+0000", 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089450+0000", 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0657e-05 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089459+0000", 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7839999999999992e-06 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089465+0000", 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0769999999999996e-06 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090074+0000", 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00060908399999999997 2023-10-09T04:33:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090094+0000", 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0406000000000001e-05 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422161+0000", 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3320668019999999 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422217+0000", 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5782000000000002e-05 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166787 9.9 9:9b506fbe:::benchmark_data_smithi029_299184_object68484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.093964+0000", 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.281165756, 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3390893639999999, 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166787 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093964+0000", 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093964+0000", 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093963+0000", 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093969+0000", 2023-10-09T04:33:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0739999999999998e-06 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422069+0000", 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.328100367 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422094+0000", 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4117e-05 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422212+0000", 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118548 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422393+0000", 2023-10-09T04:33:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000181349 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422546+0000", 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000152614 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432118+0000", 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095720019999999996 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432933+0000", 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00081524799999999997 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432942+0000", 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432994+0000", 2023-10-09T04:33:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2791000000000003e-05 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.433054+0000", 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9372e-05 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:166771 9.b e182/163 ondisk, result = 0)", 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.095374+0000", 2023-10-09T04:33:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.279755828999999, 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.327075148, 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095374+0000", 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095374+0000", 2023-10-09T04:33:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095373+0000", 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095379+0000", 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3180000000000003e-06 2023-10-09T04:33:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422081+0000", 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3267017489999999 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422093+0000", 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2282e-05 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422308+0000", 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021445599999999999 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422318+0000", 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7629999999999995e-06 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422449+0000", 2023-10-09T04:33:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131755 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:166777 9.b e182/163 ondisk, result = 0)", 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 26.279331955, 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3264704890000001, 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:33:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095802+0000", 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2590000000000002e-06 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422075+0000", 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3262729289999999 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422092+0000", 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6778000000000001e-05 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422237+0000", 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014545199999999999 2023-10-09T04:33:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422246+0000", 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9369999999999993e-06 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422269+0000", 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2693000000000001e-05 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:14.514 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:33:14.514 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:14.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:14.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:14 smithi177 ceph-mon[195797]: pgmap v683: 89 pgs: 89 active+clean; 972 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T04:33:14.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:14.515 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2315387081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:14.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:14.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:14.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:14 smithi029 ceph-mon[223858]: pgmap v683: 89 pgs: 89 active+clean; 972 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-09T04:33:14.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:14.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2315387081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:14.668 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:15 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:33:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:15 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:33:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2315387081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:15 smithi177 ceph-mon[195797]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:33:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:15 smithi177 ceph-mon[195797]: failed to encode map e186 with expected crc 2023-10-09T04:33:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:15 smithi177 ceph-mon[195797]: failed to encode map e186 with expected crc 2023-10-09T04:33:15.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:33:15.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:33:15.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2315387081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:15.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[216903]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[216903]: failed to encode map e186 with expected crc 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[216903]: failed to encode map e186 with expected crc 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2315387081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[223858]: osdmap e186: 8 total, 8 up, 8 in 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[223858]: failed to encode map e186 with expected crc 2023-10-09T04:33:15.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:15 smithi029 ceph-mon[223858]: failed to encode map e186 with expected crc 2023-10-09T04:33:16.513 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:16 smithi029 ceph-mon[216903]: pgmap v685: 89 pgs: 89 active+clean; 972 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 507 op/s 2023-10-09T04:33:16.513 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:16 smithi029 ceph-mon[223858]: pgmap v685: 89 pgs: 89 active+clean; 972 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 507 op/s 2023-10-09T04:33:16.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:16 smithi177 ceph-mon[195797]: pgmap v685: 89 pgs: 89 active+clean; 972 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 507 op/s 2023-10-09T04:33:16.743 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:16.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:16.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:16.905 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:16.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:16.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:16.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:17.095 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:17.132 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:17.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:17.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:17.307 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:17.382 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:18 smithi029 ceph-mon[216903]: pgmap v686: 89 pgs: 89 active+clean; 811 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-10-09T04:33:18.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:18 smithi029 ceph-mon[223858]: pgmap v686: 89 pgs: 89 active+clean; 811 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-10-09T04:33:18.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:18 smithi177 ceph-mon[195797]: pgmap v686: 89 pgs: 89 active+clean; 811 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-10-09T04:33:18.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:33:18.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:33:18.960 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:18.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:19.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:19.164 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:19.394 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:19.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:19.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:19.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:19.548 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:33:19.587 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:19.587 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:33:19.599 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:19.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:33:19.652 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:33:19.686 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:33:19.731 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:33:19.776 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:33:19.811 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:33:19.835 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:19.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:19.843 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:33:19.853 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.853 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:33:19.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:20.035 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:20.488 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:33:20.504 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:33:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:20 smithi177 ceph-mon[195797]: pgmap v687: 89 pgs: 89 active+clean; 775 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:33:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2937679850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:20.533 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:20 smithi029 ceph-mon[216903]: pgmap v687: 89 pgs: 89 active+clean; 775 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:33:20.533 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2937679850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:20.533 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:20 smithi029 ceph-mon[223858]: pgmap v687: 89 pgs: 89 active+clean; 775 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-09T04:33:20.533 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2937679850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:20.702 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:20.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:20.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:20.851 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:20.899 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:20.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:21.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:21.032 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:21.128 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:21.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:21.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:21.310 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:21.373 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:21.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:21.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:21.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:21 smithi177 ceph-mon[195797]: from='client.45473 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:21.537 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:21.590 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:21.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:21 smithi029 ceph-mon[216903]: from='client.45473 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:21.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:21 smithi029 ceph-mon[223858]: from='client.45473 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:21.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:21.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:21.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:21.849 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:22.053 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:22.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:22.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:22.233 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:22.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:22.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:22 smithi029 ceph-mon[216903]: pgmap v688: 89 pgs: 89 active+clean; 695 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:33:22.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:22 smithi029 ceph-mon[223858]: pgmap v688: 89 pgs: 89 active+clean; 695 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:33:22.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:22.483 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:22.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:22 smithi177 ceph-mon[195797]: pgmap v688: 89 pgs: 89 active+clean; 695 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-09T04:33:22.823 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:33:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:33:22] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.43.0" 2023-10-09T04:33:23.310 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:23.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:23.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:23.447 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:23.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:23.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:23.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:23.749 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:23.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:23.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:23.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:24.026 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:24.056 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:24.094 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":687,"stamp":"2023-10-09T04:33:18.906735+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":812455864,"num_objects":12413,"num_object_clones":0,"num_object_copies":37239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12413,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":184513,"num_write_kb":6293646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9630,"num_bytes_recovered":579276022,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138929,"ondisk_log_size":138929,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6552116,"kb_used_data":2615348,"kb_used_omap":20,"kb_used_meta":3936619,"kb_avail":743409100,"statfs":{"total":767960285184,"available":761250918400,"internally_reserved":0,"allocated":2678116352,"data_stored":2673296922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4031098510},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,3,8,8,6,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-332136448,"num_objects":-5068,"num_object_clones":0,"num_object_copies":-15204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5068,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.765663"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162417+0000","last_change":"2023-10-09T04:27:30.004963+0000","last_active":"2023-10-09T04:33:10.162417+0000","last_peered":"2023-10-09T04:33:10.162417+0000","last_clean":"2023-10-09T04:33:10.162417+0000","last_became_active":"2023-10-09T04:27:30.004581+0000","last_became_peered":"2023-10-09T04:27:30.004581+0000","last_unstale":"2023-10-09T04:33:10.162417+0000","last_undegraded":"2023-10-09T04:33:10.162417+0000","last_fullsized":"2023-10-09T04:33:10.162417+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:40.077805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147705+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:33:10.147705+0000","last_peered":"2023-10-09T04:33:10.147705+0000","last_clean":"2023-10-09T04:33:10.147705+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:33:10.147705+0000","last_undegraded":"2023-10-09T04:33:10.147705+0000","last_fullsized":"2023-10-09T04:33:10.147705+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.142317+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:33:10.142317+0000","last_peered":"2023-10-09T04:33:10.142317+0000","last_clean":"2023-10-09T04:33:10.142317+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:33:10.142317+0000","last_undegraded":"2023-10-09T04:33:10.142317+0000","last_fullsized":"2023-10-09T04:33:10.142317+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162199+0000","last_change":"2023-10-09T04:29:51.054580+0000","last_active":"2023-10-09T04:33:10.162199+0000","last_peered":"2023-10-09T04:33:10.162199+0000","last_clean":"2023-10-09T04:33:10.162199+0000","last_became_active":"2023-10-09T04:29:51.054223+0000","last_became_peered":"2023-10-09T04:29:51.054223+0000","last_unstale":"2023-10-09T04:33:10.162199+0000","last_undegraded":"2023-10-09T04:33:10.162199+0000","last_fullsized":"2023-10-09T04:33:10.162199+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:22.233569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":172,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147186+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:33:10.147186+0000","last_peered":"2023-10-09T04:33:10.147186+0000","last_clean":"2023-10-09T04:33:10.147186+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:33:10.147186+0000","last_undegraded":"2023-10-09T04:33:10.147186+0000","last_fullsized":"2023-10-09T04:33:10.147186+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":144,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.173843+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:33:10.173843+0000","last_peered":"2023-10-09T04:33:10.173843+0000","last_clean":"2023-10-09T04:33:10.173843+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:33:10.173843+0000","last_undegraded":"2023-10-09T04:33:10.173843+0000","last_fullsized":"2023-10-09T04:33:10.173843+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162697+0000","last_change":"2023-10-09T04:29:51.059222+0000","last_active":"2023-10-09T04:33:10.162697+0000","last_peered":"2023-10-09T04:33:10.162697+0000","last_clean":"2023-10-09T04:33:10.162697+0000","last_became_active":"2023-10-09T04:29:51.058771+0000","last_became_peered":"2023-10-09T04:29:51.058771+0000","last_unstale":"2023-10-09T04:33:10.162697+0000","last_undegraded":"2023-10-09T04:33:10.162697+0000","last_fullsized":"2023-10-09T04:33:10.162697+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:34:26.840067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":136,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147261+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:33:10.147261+0000","last_peered":"2023-10-09T04:33:10.147261+0000","last_clean":"2023-10-09T04:33:10.147261+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:33:10.147261+0000","last_undegraded":"2023-10-09T04:33:10.147261+0000","last_fullsized":"2023-10-09T04:33:10.147261+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.164327+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:33:10.164327+0000","last_peered":"2023-10-09T04:33:10.164327+0000","last_clean":"2023-10-09T04:33:10.164327+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:33:10.164327+0000","last_undegraded":"2023-10-09T04:33:10.164327+0000","last_fullsized":"2023-10-09T04:33:10.164327+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":109,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.146899+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:33:10.146899+0000","last_peered":"2023-10-09T04:33:10.146899+0000","last_clean":"2023-10-09T04:33:10.146899+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:33:10.146899+0000","last_undegraded":"2023-10-09T04:33:10.146899+0000","last_fullsized":"2023-10-09T04:33:10.146899+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":148,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163575+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:33:10.163575+0000","last_peered":"2023-10-09T04:33:10.163575+0000","last_clean":"2023-10-09T04:33:10.163575+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:33:10.163575+0000","last_undegraded":"2023-10-09T04:33:10.163575+0000","last_fullsized":"2023-10-09T04:33:10.163575+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.145278+0000","last_change":"2023-10-09T04:27:29.983053+0000","last_active":"2023-10-09T04:33:10.145278+0000","last_peered":"2023-10-09T04:33:10.145278+0000","last_clean":"2023-10-09T04:33:10.145278+0000","last_became_active":"2023-10-09T04:27:29.982725+0000","last_became_peered":"2023-10-09T04:27:29.982725+0000","last_unstale":"2023-10-09T04:33:10.145278+0000","last_undegraded":"2023-10-09T04:33:10.145278+0000","last_fullsized":"2023-10-09T04:33:10.145278+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:33.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.169901+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:33:10.169901+0000","last_peered":"2023-10-09T04:33:10.169901+0000","last_clean":"2023-10-09T04:33:10.169901+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:33:10.169901+0000","last_undegraded":"2023-10-09T04:33:10.169901+0000","last_fullsized":"2023-10-09T04:33:10.169901+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162719+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:33:10.162719+0000","last_peered":"2023-10-09T04:33:10.162719+0000","last_clean":"2023-10-09T04:33:10.162719+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:33:10.162719+0000","last_undegraded":"2023-10-09T04:33:10.162719+0000","last_fullsized":"2023-10-09T04:33:10.162719+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163607+0000","last_change":"2023-10-09T04:27:29.989226+0000","last_active":"2023-10-09T04:33:10.163607+0000","last_peered":"2023-10-09T04:33:10.163607+0000","last_clean":"2023-10-09T04:33:10.163607+0000","last_became_active":"2023-10-09T04:27:29.988867+0000","last_became_peered":"2023-10-09T04:27:29.988867+0000","last_unstale":"2023-10-09T04:33:10.163607+0000","last_undegraded":"2023-10-09T04:33:10.163607+0000","last_fullsized":"2023-10-09T04:33:10.163607+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:15.559571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":136,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.148120+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:33:10.148120+0000","last_peered":"2023-10-09T04:33:10.148120+0000","last_clean":"2023-10-09T04:33:10.148120+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:33:10.148120+0000","last_undegraded":"2023-10-09T04:33:10.148120+0000","last_fullsized":"2023-10-09T04:33:10.148120+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.160721+0000","last_change":"2023-10-09T04:29:51.053986+0000","last_active":"2023-10-09T04:33:10.160721+0000","last_peered":"2023-10-09T04:33:10.160721+0000","last_clean":"2023-10-09T04:33:10.160721+0000","last_became_active":"2023-10-09T04:29:51.053715+0000","last_became_peered":"2023-10-09T04:29:51.053715+0000","last_unstale":"2023-10-09T04:33:10.160721+0000","last_undegraded":"2023-10-09T04:33:10.160721+0000","last_fullsized":"2023-10-09T04:33:10.160721+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:44.445249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.168618+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:33:10.168618+0000","last_peered":"2023-10-09T04:33:10.168618+0000","last_clean":"2023-10-09T04:33:10.168618+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:33:10.168618+0000","last_undegraded":"2023-10-09T04:33:10.168618+0000","last_fullsized":"2023-10-09T04:33:10.168618+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":128,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.166571+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:33:10.166571+0000","last_peered":"2023-10-09T04:33:10.166571+0000","last_clean":"2023-10-09T04:33:10.166571+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:33:10.166571+0000","last_undegraded":"2023-10-09T04:33:10.166571+0000","last_fullsized":"2023-10-09T04:33:10.166571+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162279+0000","last_change":"2023-10-09T04:29:51.059556+0000","last_active":"2023-10-09T04:33:10.162279+0000","last_peered":"2023-10-09T04:33:10.162279+0000","last_clean":"2023-10-09T04:33:10.162279+0000","last_became_active":"2023-10-09T04:29:51.059209+0000","last_became_peered":"2023-10-09T04:29:51.059209+0000","last_unstale":"2023-10-09T04:33:10.162279+0000","last_undegraded":"2023-10-09T04:33:10.162279+0000","last_fullsized":"2023-10-09T04:33:10.162279+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:54.203833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":193,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.169236+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:33:10.169236+0000","last_peered":"2023-10-09T04:33:10.169236+0000","last_clean":"2023-10-09T04:33:10.169236+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:33:10.169236+0000","last_undegraded":"2023-10-09T04:33:10.169236+0000","last_fullsized":"2023-10-09T04:33:10.169236+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":136,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147725+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:33:10.147725+0000","last_peered":"2023-10-09T04:33:10.147725+0000","last_clean":"2023-10-09T04:33:10.147725+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:33:10.147725+0000","last_undegraded":"2023-10-09T04:33:10.147725+0000","last_fullsized":"2023-10-09T04:33:10.147725+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"186'11676","reported_seq":16600,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:17.611263+0000","last_change":"2023-10-09T04:32:07.124101+0000","last_active":"2023-10-09T04:33:17.611263+0000","last_peered":"2023-10-09T04:33:17.611263+0000","last_clean":"2023-10-09T04:33:17.611263+0000","last_became_active":"2023-10-09T04:32:07.103231+0000","last_became_peered":"2023-10-09T04:32:07.103231+0000","last_unstale":"2023-10-09T04:33:17.611263+0000","last_undegraded":"2023-10-09T04:33:17.611263+0000","last_fullsized":"2023-10-09T04:33:17.611263+0000","mapping_epoch":173,"log_start":"152'1800","ondisk_log_start":"152'1800","created":144,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":9876,"log_dups_size":0,"ondisk_log_size":9876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.609581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47316992,"num_objects":722,"num_object_clones":0,"num_object_copies":2166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11676,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2428,"num_bytes_recovered":106168320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":174,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163725+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:33:10.163725+0000","last_peered":"2023-10-09T04:33:10.163725+0000","last_clean":"2023-10-09T04:33:10.163725+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:33:10.163725+0000","last_undegraded":"2023-10-09T04:33:10.163725+0000","last_fullsized":"2023-10-09T04:33:10.163725+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.167885+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:33:10.167885+0000","last_peered":"2023-10-09T04:33:10.167885+0000","last_clean":"2023-10-09T04:33:10.167885+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:33:10.167885+0000","last_undegraded":"2023-10-09T04:33:10.167885+0000","last_fullsized":"2023-10-09T04:33:10.167885+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"186'11344","reported_seq":11396,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:16.583978+0000","last_change":"2023-10-09T04:29:24.154510+0000","last_active":"2023-10-09T04:33:16.583978+0000","last_peered":"2023-10-09T04:33:16.583978+0000","last_clean":"2023-10-09T04:33:16.583978+0000","last_became_active":"2023-10-09T04:29:24.154250+0000","last_became_peered":"2023-10-09T04:29:24.154250+0000","last_unstale":"2023-10-09T04:33:16.583978+0000","last_undegraded":"2023-10-09T04:33:16.583978+0000","last_fullsized":"2023-10-09T04:33:16.583978+0000","mapping_epoch":144,"log_start":"161'4200","ondisk_log_start":"161'4200","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7144,"log_dups_size":0,"ondisk_log_size":7144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:37.323214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49545216,"num_objects":756,"num_object_clones":0,"num_object_copies":2268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11344,"num_write_kb":387200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163704+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:33:10.163704+0000","last_peered":"2023-10-09T04:33:10.163704+0000","last_clean":"2023-10-09T04:33:10.163704+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:33:10.163704+0000","last_undegraded":"2023-10-09T04:33:10.163704+0000","last_fullsized":"2023-10-09T04:33:10.163704+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162411+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:33:10.162411+0000","last_peered":"2023-10-09T04:33:10.162411+0000","last_clean":"2023-10-09T04:33:10.162411+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:33:10.162411+0000","last_undegraded":"2023-10-09T04:33:10.162411+0000","last_fullsized":"2023-10-09T04:33:10.162411+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"186'11431","reported_seq":11480,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.937724+0000","last_change":"2023-10-09T04:29:24.161991+0000","last_active":"2023-10-09T04:33:14.937724+0000","last_peered":"2023-10-09T04:33:14.937724+0000","last_clean":"2023-10-09T04:33:14.937724+0000","last_became_active":"2023-10-09T04:29:24.161712+0000","last_became_peered":"2023-10-09T04:29:24.161712+0000","last_unstale":"2023-10-09T04:33:14.937724+0000","last_undegraded":"2023-10-09T04:33:14.937724+0000","last_fullsized":"2023-10-09T04:33:14.937724+0000","mapping_epoch":144,"log_start":"152'1400","ondisk_log_start":"152'1400","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10031,"log_dups_size":0,"ondisk_log_size":10031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:52.360661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54984704,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11431,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":129,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163413+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:33:10.163413+0000","last_peered":"2023-10-09T04:33:10.163413+0000","last_clean":"2023-10-09T04:33:10.163413+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:33:10.163413+0000","last_undegraded":"2023-10-09T04:33:10.163413+0000","last_fullsized":"2023-10-09T04:33:10.163413+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.170255+0000","last_change":"2023-10-09T04:27:30.005025+0000","last_active":"2023-10-09T04:33:10.170255+0000","last_peered":"2023-10-09T04:33:10.170255+0000","last_clean":"2023-10-09T04:33:10.170255+0000","last_became_active":"2023-10-09T04:27:30.004692+0000","last_became_peered":"2023-10-09T04:27:30.004692+0000","last_unstale":"2023-10-09T04:33:10.170255+0000","last_undegraded":"2023-10-09T04:33:10.170255+0000","last_fullsized":"2023-10-09T04:33:10.170255+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:10.737398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"186'11666","reported_seq":11716,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.776909+0000","last_change":"2023-10-09T04:29:24.685975+0000","last_active":"2023-10-09T04:33:14.776909+0000","last_peered":"2023-10-09T04:33:14.776909+0000","last_clean":"2023-10-09T04:33:14.776909+0000","last_became_active":"2023-10-09T04:29:24.685702+0000","last_became_peered":"2023-10-09T04:29:24.685702+0000","last_unstale":"2023-10-09T04:33:14.776909+0000","last_undegraded":"2023-10-09T04:33:14.776909+0000","last_fullsized":"2023-10-09T04:33:14.776909+0000","mapping_epoch":144,"log_start":"152'1600","ondisk_log_start":"152'1600","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10066,"log_dups_size":0,"ondisk_log_size":10066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:57.821949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11666,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.164200+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:33:10.164200+0000","last_peered":"2023-10-09T04:33:10.164200+0000","last_clean":"2023-10-09T04:33:10.164200+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:33:10.164200+0000","last_undegraded":"2023-10-09T04:33:10.164200+0000","last_fullsized":"2023-10-09T04:33:10.164200+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162742+0000","last_change":"2023-10-09T04:29:51.056784+0000","last_active":"2023-10-09T04:33:10.162742+0000","last_peered":"2023-10-09T04:33:10.162742+0000","last_clean":"2023-10-09T04:33:10.162742+0000","last_became_active":"2023-10-09T04:29:51.056272+0000","last_became_peered":"2023-10-09T04:29:51.056272+0000","last_unstale":"2023-10-09T04:33:10.162742+0000","last_undegraded":"2023-10-09T04:33:10.162742+0000","last_fullsized":"2023-10-09T04:33:10.162742+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:31.908683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"186'11337","reported_seq":11388,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:16.328222+0000","last_change":"2023-10-09T04:29:24.541340+0000","last_active":"2023-10-09T04:33:16.328222+0000","last_peered":"2023-10-09T04:33:16.328222+0000","last_clean":"2023-10-09T04:33:16.328222+0000","last_became_active":"2023-10-09T04:29:24.540863+0000","last_became_peered":"2023-10-09T04:29:24.540863+0000","last_unstale":"2023-10-09T04:33:16.328222+0000","last_undegraded":"2023-10-09T04:33:16.328222+0000","last_fullsized":"2023-10-09T04:33:16.328222+0000","mapping_epoch":144,"log_start":"159'3600","ondisk_log_start":"159'3600","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:14.095614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47513600,"num_objects":725,"num_object_clones":0,"num_object_copies":2175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11337,"num_write_kb":385984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.151396+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:33:10.151396+0000","last_peered":"2023-10-09T04:33:10.151396+0000","last_clean":"2023-10-09T04:33:10.151396+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:33:10.151396+0000","last_undegraded":"2023-10-09T04:33:10.151396+0000","last_fullsized":"2023-10-09T04:33:10.151396+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.169785+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:33:10.169785+0000","last_peered":"2023-10-09T04:33:10.169785+0000","last_clean":"2023-10-09T04:33:10.169785+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:33:10.169785+0000","last_undegraded":"2023-10-09T04:33:10.169785+0000","last_fullsized":"2023-10-09T04:33:10.169785+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"186'11392","reported_seq":11444,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:16.547006+0000","last_change":"2023-10-09T04:29:24.685606+0000","last_active":"2023-10-09T04:33:16.547006+0000","last_peered":"2023-10-09T04:33:16.547006+0000","last_clean":"2023-10-09T04:33:16.547006+0000","last_became_active":"2023-10-09T04:29:24.685115+0000","last_became_peered":"2023-10-09T04:29:24.685115+0000","last_unstale":"2023-10-09T04:33:16.547006+0000","last_undegraded":"2023-10-09T04:33:16.547006+0000","last_fullsized":"2023-10-09T04:33:16.547006+0000","mapping_epoch":144,"log_start":"161'4200","ondisk_log_start":"161'4200","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7192,"log_dups_size":0,"ondisk_log_size":7192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:33.550656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46137344,"num_objects":704,"num_object_clones":0,"num_object_copies":2112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11392,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.170183+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:33:10.170183+0000","last_peered":"2023-10-09T04:33:10.170183+0000","last_clean":"2023-10-09T04:33:10.170183+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:33:10.170183+0000","last_undegraded":"2023-10-09T04:33:10.170183+0000","last_fullsized":"2023-10-09T04:33:10.170183+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.169883+0000","last_change":"2023-10-09T04:27:29.982774+0000","last_active":"2023-10-09T04:33:10.169883+0000","last_peered":"2023-10-09T04:33:10.169883+0000","last_clean":"2023-10-09T04:33:10.169883+0000","last_became_active":"2023-10-09T04:27:29.982507+0000","last_became_peered":"2023-10-09T04:27:29.982507+0000","last_unstale":"2023-10-09T04:33:10.169883+0000","last_undegraded":"2023-10-09T04:33:10.169883+0000","last_fullsized":"2023-10-09T04:33:10.169883+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:14.613302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"186'11589","reported_seq":11638,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.890667+0000","last_change":"2023-10-09T04:29:24.158352+0000","last_active":"2023-10-09T04:33:14.890667+0000","last_peered":"2023-10-09T04:33:14.890667+0000","last_clean":"2023-10-09T04:33:14.890667+0000","last_became_active":"2023-10-09T04:29:24.158010+0000","last_became_peered":"2023-10-09T04:29:24.158010+0000","last_unstale":"2023-10-09T04:33:14.890667+0000","last_undegraded":"2023-10-09T04:33:14.890667+0000","last_fullsized":"2023-10-09T04:33:14.890667+0000","mapping_epoch":144,"log_start":"159'3800","ondisk_log_start":"159'3800","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7789,"log_dups_size":0,"ondisk_log_size":7789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:24.852154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54329344,"num_objects":829,"num_object_clones":0,"num_object_copies":2487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11589,"num_write_kb":397376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162050+0000","last_change":"2023-10-09T04:27:30.010756+0000","last_active":"2023-10-09T04:33:10.162050+0000","last_peered":"2023-10-09T04:33:10.162050+0000","last_clean":"2023-10-09T04:33:10.162050+0000","last_became_active":"2023-10-09T04:27:30.010403+0000","last_became_peered":"2023-10-09T04:27:30.010403+0000","last_unstale":"2023-10-09T04:33:10.162050+0000","last_undegraded":"2023-10-09T04:33:10.162050+0000","last_fullsized":"2023-10-09T04:33:10.162050+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:43:10.557818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.170540+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:33:10.170540+0000","last_peered":"2023-10-09T04:33:10.170540+0000","last_clean":"2023-10-09T04:33:10.170540+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:33:10.170540+0000","last_undegraded":"2023-10-09T04:33:10.170540+0000","last_fullsized":"2023-10-09T04:33:10.170540+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"186'11435","reported_seq":11486,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:16.349202+0000","last_change":"2023-10-09T04:29:24.685470+0000","last_active":"2023-10-09T04:33:16.349202+0000","last_peered":"2023-10-09T04:33:16.349202+0000","last_clean":"2023-10-09T04:33:16.349202+0000","last_became_active":"2023-10-09T04:29:24.685144+0000","last_became_peered":"2023-10-09T04:29:24.685144+0000","last_unstale":"2023-10-09T04:33:16.349202+0000","last_undegraded":"2023-10-09T04:33:16.349202+0000","last_fullsized":"2023-10-09T04:33:16.349202+0000","mapping_epoch":144,"log_start":"159'3700","ondisk_log_start":"159'3700","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:43.630951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51052544,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11435,"num_write_kb":390848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147022+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:33:10.147022+0000","last_peered":"2023-10-09T04:33:10.147022+0000","last_clean":"2023-10-09T04:33:10.147022+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:33:10.147022+0000","last_undegraded":"2023-10-09T04:33:10.147022+0000","last_fullsized":"2023-10-09T04:33:10.147022+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.142656+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:33:10.142656+0000","last_peered":"2023-10-09T04:33:10.142656+0000","last_clean":"2023-10-09T04:33:10.142656+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:33:10.142656+0000","last_undegraded":"2023-10-09T04:33:10.142656+0000","last_fullsized":"2023-10-09T04:33:10.142656+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"186'11684","reported_seq":11734,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.911198+0000","last_change":"2023-10-09T04:29:24.161371+0000","last_active":"2023-10-09T04:33:14.911198+0000","last_peered":"2023-10-09T04:33:14.911198+0000","last_clean":"2023-10-09T04:33:14.911198+0000","last_became_active":"2023-10-09T04:29:24.161102+0000","last_became_peered":"2023-10-09T04:29:24.161102+0000","last_unstale":"2023-10-09T04:33:14.911198+0000","last_undegraded":"2023-10-09T04:33:14.911198+0000","last_fullsized":"2023-10-09T04:33:14.911198+0000","mapping_epoch":144,"log_start":"152'1600","ondisk_log_start":"152'1600","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:31.094244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54394880,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11684,"num_write_kb":400448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":159,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.143573+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:33:10.143573+0000","last_peered":"2023-10-09T04:33:10.143573+0000","last_clean":"2023-10-09T04:33:10.143573+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:33:10.143573+0000","last_undegraded":"2023-10-09T04:33:10.143573+0000","last_fullsized":"2023-10-09T04:33:10.143573+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.168661+0000","last_change":"2023-10-09T04:27:57.993021+0000","last_active":"2023-10-09T04:33:10.168661+0000","last_peered":"2023-10-09T04:33:10.168661+0000","last_clean":"2023-10-09T04:33:10.168661+0000","last_became_active":"2023-10-09T04:27:57.992486+0000","last_became_peered":"2023-10-09T04:27:57.992486+0000","last_unstale":"2023-10-09T04:33:10.168661+0000","last_undegraded":"2023-10-09T04:33:10.168661+0000","last_fullsized":"2023-10-09T04:33:10.168661+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:56.130811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.145358+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:33:10.145358+0000","last_peered":"2023-10-09T04:33:10.145358+0000","last_clean":"2023-10-09T04:33:10.145358+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:33:10.145358+0000","last_undegraded":"2023-10-09T04:33:10.145358+0000","last_fullsized":"2023-10-09T04:33:10.145358+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"186'11564","reported_seq":16446,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:17.394698+0000","last_change":"2023-10-09T04:30:44.045543+0000","last_active":"2023-10-09T04:33:17.394698+0000","last_peered":"2023-10-09T04:33:17.394698+0000","last_clean":"2023-10-09T04:33:17.394698+0000","last_became_active":"2023-10-09T04:30:17.679175+0000","last_became_peered":"2023-10-09T04:30:17.679175+0000","last_unstale":"2023-10-09T04:33:17.394698+0000","last_undegraded":"2023-10-09T04:33:17.394698+0000","last_fullsized":"2023-10-09T04:33:17.394698+0000","mapping_epoch":160,"log_start":"152'1500","ondisk_log_start":"152'1500","created":144,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10064,"log_dups_size":0,"ondisk_log_size":10064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:56.902852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11564,"num_write_kb":391296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2407,"num_bytes_recovered":157745152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.164194+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:33:10.164194+0000","last_peered":"2023-10-09T04:33:10.164194+0000","last_clean":"2023-10-09T04:33:10.164194+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:33:10.164194+0000","last_undegraded":"2023-10-09T04:33:10.164194+0000","last_fullsized":"2023-10-09T04:33:10.164194+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":167,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162673+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:33:10.162673+0000","last_peered":"2023-10-09T04:33:10.162673+0000","last_clean":"2023-10-09T04:33:10.162673+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:33:10.162673+0000","last_undegraded":"2023-10-09T04:33:10.162673+0000","last_fullsized":"2023-10-09T04:33:10.162673+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.143535+0000","last_change":"2023-10-09T04:27:57.991197+0000","last_active":"2023-10-09T04:33:10.143535+0000","last_peered":"2023-10-09T04:33:10.143535+0000","last_clean":"2023-10-09T04:33:10.143535+0000","last_became_active":"2023-10-09T04:27:57.990548+0000","last_became_peered":"2023-10-09T04:27:57.990548+0000","last_unstale":"2023-10-09T04:33:10.143535+0000","last_undegraded":"2023-10-09T04:33:10.143535+0000","last_fullsized":"2023-10-09T04:33:10.143535+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:05.799943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"186'11405","reported_seq":11455,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.917373+0000","last_change":"2023-10-09T04:29:24.161931+0000","last_active":"2023-10-09T04:33:14.917373+0000","last_peered":"2023-10-09T04:33:14.917373+0000","last_clean":"2023-10-09T04:33:14.917373+0000","last_became_active":"2023-10-09T04:29:24.161582+0000","last_became_peered":"2023-10-09T04:29:24.161582+0000","last_unstale":"2023-10-09T04:33:14.917373+0000","last_undegraded":"2023-10-09T04:33:14.917373+0000","last_fullsized":"2023-10-09T04:33:14.917373+0000","mapping_epoch":144,"log_start":"152'1400","ondisk_log_start":"152'1400","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10005,"log_dups_size":0,"ondisk_log_size":10005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.147516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51052544,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11405,"num_write_kb":389888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.146829+0000","last_change":"2023-10-09T04:29:51.233999+0000","last_active":"2023-10-09T04:33:10.146829+0000","last_peered":"2023-10-09T04:33:10.146829+0000","last_clean":"2023-10-09T04:33:10.146829+0000","last_became_active":"2023-10-09T04:29:51.233578+0000","last_became_peered":"2023-10-09T04:29:51.233578+0000","last_unstale":"2023-10-09T04:33:10.146829+0000","last_undegraded":"2023-10-09T04:33:10.146829+0000","last_fullsized":"2023-10-09T04:33:10.146829+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:44.172048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.142242+0000","last_change":"2023-10-09T04:27:29.982732+0000","last_active":"2023-10-09T04:33:10.142242+0000","last_peered":"2023-10-09T04:33:10.142242+0000","last_clean":"2023-10-09T04:33:10.142242+0000","last_became_active":"2023-10-09T04:27:29.982275+0000","last_became_peered":"2023-10-09T04:27:29.982275+0000","last_unstale":"2023-10-09T04:33:10.142242+0000","last_undegraded":"2023-10-09T04:33:10.142242+0000","last_fullsized":"2023-10-09T04:33:10.142242+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:43.712715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.142721+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:33:10.142721+0000","last_peered":"2023-10-09T04:33:10.142721+0000","last_clean":"2023-10-09T04:33:10.142721+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:33:10.142721+0000","last_undegraded":"2023-10-09T04:33:10.142721+0000","last_fullsized":"2023-10-09T04:33:10.142721+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"186'11436","reported_seq":16227,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:16.340763+0000","last_change":"2023-10-09T04:31:24.127748+0000","last_active":"2023-10-09T04:33:16.340763+0000","last_peered":"2023-10-09T04:33:16.340763+0000","last_clean":"2023-10-09T04:33:16.340763+0000","last_became_active":"2023-10-09T04:31:24.115397+0000","last_became_peered":"2023-10-09T04:31:24.115397+0000","last_unstale":"2023-10-09T04:33:16.340763+0000","last_undegraded":"2023-10-09T04:33:16.340763+0000","last_fullsized":"2023-10-09T04:33:16.340763+0000","mapping_epoch":167,"log_start":"159'3700","ondisk_log_start":"159'3700","created":144,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7736,"log_dups_size":0,"ondisk_log_size":7736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:25.364174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51511296,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":391104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2357,"num_bytes_recovered":154468352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.164720+0000","last_change":"2023-10-09T04:27:29.988702+0000","last_active":"2023-10-09T04:33:10.164720+0000","last_peered":"2023-10-09T04:33:10.164720+0000","last_clean":"2023-10-09T04:33:10.164720+0000","last_became_active":"2023-10-09T04:27:29.988390+0000","last_became_peered":"2023-10-09T04:27:29.988390+0000","last_unstale":"2023-10-09T04:33:10.164720+0000","last_undegraded":"2023-10-09T04:33:10.164720+0000","last_fullsized":"2023-10-09T04:33:10.164720+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:33.897326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":578,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163071+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:33:10.163071+0000","last_peered":"2023-10-09T04:33:10.163071+0000","last_clean":"2023-10-09T04:33:10.163071+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:33:10.163071+0000","last_undegraded":"2023-10-09T04:33:10.163071+0000","last_fullsized":"2023-10-09T04:33:10.163071+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":108,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147590+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:33:10.147590+0000","last_peered":"2023-10-09T04:33:10.147590+0000","last_clean":"2023-10-09T04:33:10.147590+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:33:10.147590+0000","last_undegraded":"2023-10-09T04:33:10.147590+0000","last_fullsized":"2023-10-09T04:33:10.147590+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":127,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.173473+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:33:10.173473+0000","last_peered":"2023-10-09T04:33:10.173473+0000","last_clean":"2023-10-09T04:33:10.173473+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:33:10.173473+0000","last_undegraded":"2023-10-09T04:33:10.173473+0000","last_fullsized":"2023-10-09T04:33:10.173473+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"186'11504","reported_seq":16391,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:17.482788+0000","last_change":"2023-10-09T04:30:45.687596+0000","last_active":"2023-10-09T04:33:17.482788+0000","last_peered":"2023-10-09T04:33:17.482788+0000","last_clean":"2023-10-09T04:33:17.482788+0000","last_became_active":"2023-10-09T04:30:45.682622+0000","last_became_peered":"2023-10-09T04:30:45.682622+0000","last_unstale":"2023-10-09T04:33:17.482788+0000","last_undegraded":"2023-10-09T04:33:17.482788+0000","last_fullsized":"2023-10-09T04:33:17.482788+0000","mapping_epoch":163,"log_start":"152'1500","ondisk_log_start":"152'1500","created":144,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10004,"log_dups_size":0,"ondisk_log_size":10004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:10.606752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43384832,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11504,"num_write_kb":389312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2410,"num_bytes_recovered":157941760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":186,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162031+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:33:10.162031+0000","last_peered":"2023-10-09T04:33:10.162031+0000","last_clean":"2023-10-09T04:33:10.162031+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:33:10.162031+0000","last_undegraded":"2023-10-09T04:33:10.162031+0000","last_fullsized":"2023-10-09T04:33:10.162031+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.158080+0000","last_change":"2023-10-09T04:29:51.054503+0000","last_active":"2023-10-09T04:33:10.158080+0000","last_peered":"2023-10-09T04:33:10.158080+0000","last_clean":"2023-10-09T04:33:10.158080+0000","last_became_active":"2023-10-09T04:29:51.054130+0000","last_became_peered":"2023-10-09T04:29:51.054130+0000","last_unstale":"2023-10-09T04:33:10.158080+0000","last_undegraded":"2023-10-09T04:33:10.158080+0000","last_fullsized":"2023-10-09T04:33:10.158080+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:33.217600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162358+0000","last_change":"2023-10-09T04:29:51.235611+0000","last_active":"2023-10-09T04:33:10.162358+0000","last_peered":"2023-10-09T04:33:10.162358+0000","last_clean":"2023-10-09T04:33:10.162358+0000","last_became_active":"2023-10-09T04:29:51.235272+0000","last_became_peered":"2023-10-09T04:29:51.235272+0000","last_unstale":"2023-10-09T04:33:10.162358+0000","last_undegraded":"2023-10-09T04:33:10.162358+0000","last_fullsized":"2023-10-09T04:33:10.162358+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:41.701077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"186'11686","reported_seq":11735,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.916948+0000","last_change":"2023-10-09T04:29:24.160813+0000","last_active":"2023-10-09T04:33:14.916948+0000","last_peered":"2023-10-09T04:33:14.916948+0000","last_clean":"2023-10-09T04:33:14.916948+0000","last_became_active":"2023-10-09T04:29:24.160543+0000","last_became_peered":"2023-10-09T04:29:24.160543+0000","last_unstale":"2023-10-09T04:33:14.916948+0000","last_undegraded":"2023-10-09T04:33:14.916948+0000","last_fullsized":"2023-10-09T04:33:14.916948+0000","mapping_epoch":144,"log_start":"160'3900","ondisk_log_start":"160'3900","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:57.034286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58327040,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11686,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":169,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.163534+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:33:10.163534+0000","last_peered":"2023-10-09T04:33:10.163534+0000","last_clean":"2023-10-09T04:33:10.163534+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:33:10.163534+0000","last_undegraded":"2023-10-09T04:33:10.163534+0000","last_fullsized":"2023-10-09T04:33:10.163534+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.143494+0000","last_change":"2023-10-09T04:27:29.981956+0000","last_active":"2023-10-09T04:33:10.143494+0000","last_peered":"2023-10-09T04:33:10.143494+0000","last_clean":"2023-10-09T04:33:10.143494+0000","last_became_active":"2023-10-09T04:27:29.981663+0000","last_became_peered":"2023-10-09T04:27:29.981663+0000","last_unstale":"2023-10-09T04:33:10.143494+0000","last_undegraded":"2023-10-09T04:33:10.143494+0000","last_fullsized":"2023-10-09T04:33:10.143494+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:47.639333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.166205+0000","last_change":"2023-10-09T04:29:51.056593+0000","last_active":"2023-10-09T04:33:10.166205+0000","last_peered":"2023-10-09T04:33:10.166205+0000","last_clean":"2023-10-09T04:33:10.166205+0000","last_became_active":"2023-10-09T04:29:51.055758+0000","last_became_peered":"2023-10-09T04:29:51.055758+0000","last_unstale":"2023-10-09T04:33:10.166205+0000","last_undegraded":"2023-10-09T04:33:10.166205+0000","last_fullsized":"2023-10-09T04:33:10.166205+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:42.366511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"186'11509","reported_seq":11559,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.930933+0000","last_change":"2023-10-09T04:29:24.160706+0000","last_active":"2023-10-09T04:33:14.930933+0000","last_peered":"2023-10-09T04:33:14.930933+0000","last_clean":"2023-10-09T04:33:14.930933+0000","last_became_active":"2023-10-09T04:29:24.160380+0000","last_became_peered":"2023-10-09T04:29:24.160380+0000","last_unstale":"2023-10-09T04:33:14.930933+0000","last_undegraded":"2023-10-09T04:33:14.930933+0000","last_fullsized":"2023-10-09T04:33:14.930933+0000","mapping_epoch":144,"log_start":"159'3800","ondisk_log_start":"159'3800","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:06.208544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51904536,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11509,"num_write_kb":393601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147842+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:33:10.147842+0000","last_peered":"2023-10-09T04:33:10.147842+0000","last_clean":"2023-10-09T04:33:10.147842+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:33:10.147842+0000","last_undegraded":"2023-10-09T04:33:10.147842+0000","last_fullsized":"2023-10-09T04:33:10.147842+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.142385+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:33:10.142385+0000","last_peered":"2023-10-09T04:33:10.142385+0000","last_clean":"2023-10-09T04:33:10.142385+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:33:10.142385+0000","last_undegraded":"2023-10-09T04:33:10.142385+0000","last_fullsized":"2023-10-09T04:33:10.142385+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147107+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:33:10.147107+0000","last_peered":"2023-10-09T04:33:10.147107+0000","last_clean":"2023-10-09T04:33:10.147107+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:33:10.147107+0000","last_undegraded":"2023-10-09T04:33:10.147107+0000","last_fullsized":"2023-10-09T04:33:10.147107+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"186'11555","reported_seq":11606,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:16.314422+0000","last_change":"2023-10-09T04:29:24.159100+0000","last_active":"2023-10-09T04:33:16.314422+0000","last_peered":"2023-10-09T04:33:16.314422+0000","last_clean":"2023-10-09T04:33:16.314422+0000","last_became_active":"2023-10-09T04:29:24.158495+0000","last_became_peered":"2023-10-09T04:29:24.158495+0000","last_unstale":"2023-10-09T04:33:16.314422+0000","last_undegraded":"2023-10-09T04:33:16.314422+0000","last_fullsized":"2023-10-09T04:33:16.314422+0000","mapping_epoch":144,"log_start":"159'3800","ondisk_log_start":"159'3800","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7755,"log_dups_size":0,"ondisk_log_size":7755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:31.573022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52101120,"num_objects":795,"num_object_clones":0,"num_object_copies":2385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11555,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.173770+0000","last_change":"2023-10-09T04:27:30.000853+0000","last_active":"2023-10-09T04:33:10.173770+0000","last_peered":"2023-10-09T04:33:10.173770+0000","last_clean":"2023-10-09T04:33:10.173770+0000","last_became_active":"2023-10-09T04:27:30.000545+0000","last_became_peered":"2023-10-09T04:27:30.000545+0000","last_unstale":"2023-10-09T04:33:10.173770+0000","last_undegraded":"2023-10-09T04:33:10.173770+0000","last_fullsized":"2023-10-09T04:33:10.173770+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:47.506034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":48,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162447+0000","last_change":"2023-10-09T04:29:51.062083+0000","last_active":"2023-10-09T04:33:10.162447+0000","last_peered":"2023-10-09T04:33:10.162447+0000","last_clean":"2023-10-09T04:33:10.162447+0000","last_became_active":"2023-10-09T04:29:51.055667+0000","last_became_peered":"2023-10-09T04:29:51.055667+0000","last_unstale":"2023-10-09T04:33:10.162447+0000","last_undegraded":"2023-10-09T04:33:10.162447+0000","last_fullsized":"2023-10-09T04:33:10.162447+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:08.504718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":129,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162267+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:33:10.162267+0000","last_peered":"2023-10-09T04:33:10.162267+0000","last_clean":"2023-10-09T04:33:10.162267+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:33:10.162267+0000","last_undegraded":"2023-10-09T04:33:10.162267+0000","last_fullsized":"2023-10-09T04:33:10.162267+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.166027+0000","last_change":"2023-10-09T04:29:51.057053+0000","last_active":"2023-10-09T04:33:10.166027+0000","last_peered":"2023-10-09T04:33:10.166027+0000","last_clean":"2023-10-09T04:33:10.166027+0000","last_became_active":"2023-10-09T04:29:51.056398+0000","last_became_peered":"2023-10-09T04:29:51.056398+0000","last_unstale":"2023-10-09T04:33:10.166027+0000","last_undegraded":"2023-10-09T04:33:10.166027+0000","last_fullsized":"2023-10-09T04:33:10.166027+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:08:11.102317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":157,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.149029+0000","last_change":"2023-10-09T04:27:57.989106+0000","last_active":"2023-10-09T04:33:10.149029+0000","last_peered":"2023-10-09T04:33:10.149029+0000","last_clean":"2023-10-09T04:33:10.149029+0000","last_became_active":"2023-10-09T04:27:57.988778+0000","last_became_peered":"2023-10-09T04:27:57.988778+0000","last_unstale":"2023-10-09T04:33:10.149029+0000","last_undegraded":"2023-10-09T04:33:10.149029+0000","last_fullsized":"2023-10-09T04:33:10.149029+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:33.473019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":148,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.164138+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:33:10.164138+0000","last_peered":"2023-10-09T04:33:10.164138+0000","last_clean":"2023-10-09T04:33:10.164138+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:33:10.164138+0000","last_undegraded":"2023-10-09T04:33:10.164138+0000","last_fullsized":"2023-10-09T04:33:10.164138+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162916+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:33:10.162916+0000","last_peered":"2023-10-09T04:33:10.162916+0000","last_clean":"2023-10-09T04:33:10.162916+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:33:10.162916+0000","last_undegraded":"2023-10-09T04:33:10.162916+0000","last_fullsized":"2023-10-09T04:33:10.162916+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":131,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.168683+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:33:10.168683+0000","last_peered":"2023-10-09T04:33:10.168683+0000","last_clean":"2023-10-09T04:33:10.168683+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:33:10.168683+0000","last_undegraded":"2023-10-09T04:33:10.168683+0000","last_fullsized":"2023-10-09T04:33:10.168683+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":183,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162551+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:33:10.162551+0000","last_peered":"2023-10-09T04:33:10.162551+0000","last_clean":"2023-10-09T04:33:10.162551+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:33:10.162551+0000","last_undegraded":"2023-10-09T04:33:10.162551+0000","last_fullsized":"2023-10-09T04:33:10.162551+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.147740+0000","last_change":"2023-10-09T04:27:57.989240+0000","last_active":"2023-10-09T04:33:10.147740+0000","last_peered":"2023-10-09T04:33:10.147740+0000","last_clean":"2023-10-09T04:33:10.147740+0000","last_became_active":"2023-10-09T04:27:57.988856+0000","last_became_peered":"2023-10-09T04:27:57.988856+0000","last_unstale":"2023-10-09T04:33:10.147740+0000","last_undegraded":"2023-10-09T04:33:10.147740+0000","last_fullsized":"2023-10-09T04:33:10.147740+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:52.380796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.146835+0000","last_change":"2023-10-09T04:29:51.233809+0000","last_active":"2023-10-09T04:33:10.146835+0000","last_peered":"2023-10-09T04:33:10.146835+0000","last_clean":"2023-10-09T04:33:10.146835+0000","last_became_active":"2023-10-09T04:29:51.233460+0000","last_became_peered":"2023-10-09T04:29:51.233460+0000","last_unstale":"2023-10-09T04:33:10.146835+0000","last_undegraded":"2023-10-09T04:33:10.146835+0000","last_fullsized":"2023-10-09T04:33:10.146835+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:33.172831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":50,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162835+0000","last_change":"2023-10-09T04:29:51.238602+0000","last_active":"2023-10-09T04:33:10.162835+0000","last_peered":"2023-10-09T04:33:10.162835+0000","last_clean":"2023-10-09T04:33:10.162835+0000","last_became_active":"2023-10-09T04:29:51.234688+0000","last_became_peered":"2023-10-09T04:29:51.234688+0000","last_unstale":"2023-10-09T04:33:10.162835+0000","last_undegraded":"2023-10-09T04:33:10.162835+0000","last_fullsized":"2023-10-09T04:33:10.162835+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:58.339304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-09T04:33:10.162033+0000","last_change":"2023-10-09T04:27:29.981763+0000","last_active":"2023-10-09T04:33:10.162033+0000","last_peered":"2023-10-09T04:33:10.162033+0000","last_clean":"2023-10-09T04:33:10.162033+0000","last_became_active":"2023-10-09T04:27:29.981401+0000","last_became_peered":"2023-10-09T04:27:29.981401+0000","last_unstale":"2023-10-09T04:33:10.162033+0000","last_undegraded":"2023-10-09T04:33:10.162033+0000","last_fullsized":"2023-10-09T04:33:10.162033+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:35.242521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":811859992,"num_objects":12389,"num_object_clones":0,"num_object_copies":37167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12389,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":184213,"num_write_kb":6291201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9602,"num_bytes_recovered":576323584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2584883200,"data_stored":2584870984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138713,"ondisk_log_size":138713,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234316,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1403712,"kb_used_data":496824,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":92341440,"statfs":{"total":95995035648,"available":94557634560,"internally_reserved":0,"allocated":508747776,"data_stored":508226227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,6,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:17 2023","interfaces":[{"interface":"back","average":{"1min":21.751000000000001,"5min":5.7110000000000003,"15min":5.2320000000000002},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.22900000000000001},"max":{"1min":186.446,"5min":186.446,"15min":629.22500000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":22.292999999999999,"5min":5.8220000000000001,"15min":5.0519999999999996},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":186.429,"5min":186.429,"15min":629.43100000000004},"last":0.78000000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:32:17 2023","interfaces":[{"interface":"back","average":{"1min":17.155999999999999,"5min":8.0150000000000006,"15min":4.1790000000000003},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":186.74799999999999,"5min":224.13499999999999,"15min":224.13499999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":17.141999999999999,"5min":7.8769999999999998,"15min":4.1859999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.25700000000000001},"max":{"1min":186.67500000000001,"5min":224.203,"15min":224.203},"last":0.53500000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.306,"15min":0.78800000000000003},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.890000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.339,"15min":0.80900000000000005},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":10.784000000000001,"5min":10.784000000000001,"15min":10.784000000000001},"last":0.439}]},{"osd":3,"last update":"Mon Oct 9 04:33:04 2023","interfaces":[{"interface":"back","average":{"1min":10.27,"5min":12.497999999999999,"15min":4.5309999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":110.50700000000001,"5min":249.989,"15min":249.989},"last":0.46000000000000002},{"interface":"front","average":{"1min":10.083,"5min":12.624000000000001,"15min":4.5940000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":112.54600000000001,"5min":250.21700000000001,"15min":250.21700000000001},"last":0.80900000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:32:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":1.133,"15min":1.1419999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23799999999999999},"max":{"1min":10.869,"5min":10.869,"15min":10.869},"last":0.745},{"interface":"front","average":{"1min":1.431,"5min":0.76500000000000001,"15min":0.89100000000000001},"min":{"1min":0.34999999999999998,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":10.613,"5min":10.613,"15min":10.613},"last":0.40600000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:32:35 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":0.80900000000000005,"15min":0.93700000000000006},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":4.3099999999999996,"5min":5.2599999999999998,"15min":8.8610000000000007},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.992,"5min":1.175,"15min":1.115},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":10.895,"5min":10.895,"15min":10.895},"last":1.486}]},{"osd":6,"last update":"Mon Oct 9 04:32:35 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.95799999999999996,"15min":1.1319999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":4.5220000000000002,"5min":5.2350000000000003,"15min":12.007},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.0760000000000001,"15min":1.1419999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.31900000000000001},"max":{"1min":10.917,"5min":10.917,"15min":10.917},"last":5.6509999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495950,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":452148,"kb_used_data":154796,"kb_used_omap":0,"kb_used_meta":297280,"kb_avail":93293004,"statfs":{"total":95995035648,"available":95532036096,"internally_reserved":0,"allocated":158511104,"data_stored":158009334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304414720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":2.6859999999999999,"15min":2.218},"min":{"1min":0.29899999999999999,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":8.9469999999999992,"5min":105.631,"15min":105.631},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.5379999999999998,"15min":2.6019999999999999},"min":{"1min":0.26900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":7.4329999999999998,"5min":104.61799999999999,"15min":104.61799999999999},"last":0.314}]},{"osd":1,"last update":"Mon Oct 9 04:32:17 2023","interfaces":[{"interface":"back","average":{"1min":7.0999999999999996,"5min":5.1369999999999996,"15min":3.2549999999999999},"min":{"1min":0.28499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":86.879999999999995,"5min":185.74000000000001,"15min":185.74000000000001},"last":1.2070000000000001},{"interface":"front","average":{"1min":7.1070000000000002,"5min":5.5970000000000004,"15min":3.4870000000000001},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":86.918999999999997,"5min":189.03999999999999,"15min":189.03999999999999},"last":1.343}]},{"osd":2,"last update":"Mon Oct 9 04:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.629,"5min":1.454,"15min":0.86599999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.6079999999999997,"5min":5.7880000000000003,"15min":5.7880000000000003},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.089,"15min":0.97099999999999997},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.891,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":5.976}]},{"osd":3,"last update":"Mon Oct 9 04:33:07 2023","interfaces":[{"interface":"back","average":{"1min":4.548,"5min":2.0219999999999998,"15min":1.1000000000000001},"min":{"1min":0.47199999999999998,"5min":0.314,"15min":0.314},"max":{"1min":59.392000000000003,"5min":59.392000000000003,"15min":59.392000000000003},"last":0.372},{"interface":"front","average":{"1min":5.0979999999999999,"5min":2.1419999999999999,"15min":1.093},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":59.203000000000003,"5min":59.203000000000003,"15min":59.203000000000003},"last":6.1280000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:32:23 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.002,"15min":1.359},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":8.0950000000000006,"5min":8.0950000000000006,"15min":61.622},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.3160000000000001,"5min":0.99399999999999999,"15min":1.3380000000000001},"min":{"1min":0.31,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":8.9920000000000009,"5min":8.9920000000000009,"15min":62.040999999999997},"last":0.41799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:32:23 2023","interfaces":[{"interface":"back","average":{"1min":7.7489999999999997,"5min":2.2130000000000001,"15min":1.6499999999999999},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.28100000000000003},"max":{"1min":81.087000000000003,"5min":81.087000000000003,"15min":81.087000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":6.8449999999999998,"5min":2.2629999999999999,"15min":1.7010000000000001},"min":{"1min":0.372,"5min":0.34000000000000002,"15min":0.27400000000000002},"max":{"1min":81.706999999999994,"5min":81.706999999999994,"15min":81.706999999999994},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:32:53 2023","interfaces":[{"interface":"back","average":{"1min":5.4610000000000003,"5min":1.819,"15min":1.6160000000000001},"min":{"1min":0.36899999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":81.674999999999997,"5min":81.674999999999997,"15min":81.674999999999997},"last":0.505},{"interface":"front","average":{"1min":6.3360000000000003,"5min":1.958,"15min":1.762},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":81.055000000000007,"5min":81.055000000000007,"15min":81.055000000000007},"last":0.438}]}]},{"osd":1,"up_from":118,"seq":506806141040,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928692,"kb_used_data":543212,"kb_used_omap":4,"kb_used_meta":385467,"kb_avail":92816460,"statfs":{"total":95995035648,"available":95044055040,"internally_reserved":0,"allocated":556249088,"data_stored":555512213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":394719118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":3.0329999999999999,"15min":3.0750000000000002},"min":{"1min":0.21199999999999999,"5min":0.20899999999999999,"15min":0.19600000000000001},"max":{"1min":12.323,"5min":89.153000000000006,"15min":89.153000000000006},"last":0.70499999999999996},{"interface":"front","average":{"1min":3.5939999999999999,"5min":3.347,"15min":3.3050000000000002},"min":{"1min":0.29899999999999999,"5min":0.253,"15min":0.24399999999999999},"max":{"1min":12.766,"5min":88.988,"15min":88.988},"last":0.78800000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":1.631,"15min":0.77400000000000002},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.872,"5min":10.872,"15min":10.872},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.1259999999999999,"15min":0.59999999999999998},"min":{"1min":0.251,"5min":0.218,"15min":0.218},"max":{"1min":6.2590000000000003,"5min":8.048,"15min":8.048},"last":1.897}]},{"osd":3,"last update":"Mon Oct 9 04:33:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":4.3239999999999998,"15min":1.706},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.29,"5min":195.035,"15min":195.035},"last":0.374},{"interface":"front","average":{"1min":2.4039999999999999,"5min":4.6820000000000004,"15min":1.8300000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.3929999999999998,"5min":195.465,"15min":195.465},"last":3.254}]},{"osd":4,"last update":"Mon Oct 9 04:32:21 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":1.6839999999999999,"15min":2.8450000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":5.9009999999999998,"5min":12.537000000000001,"15min":107.795},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.1299999999999999,"15min":1.651},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":6.1369999999999996,"5min":6.1369999999999996,"15min":109.167},"last":0.52900000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:32:21 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":1.5569999999999999,"15min":2.4009999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33700000000000002},"max":{"1min":8.9909999999999997,"5min":8.9909999999999997,"15min":12.414999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.714,"15min":2.673},"min":{"1min":0.42299999999999999,"5min":0.36799999999999999,"15min":0.34899999999999998},"max":{"1min":5.851,"5min":12.605,"15min":13.933999999999999},"last":2.9740000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:32:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.4490000000000001,"15min":2.2530000000000001},"min":{"1min":0.45900000000000002,"5min":0.41299999999999998,"15min":0.36599999999999999},"max":{"1min":6.2530000000000001,"5min":8.3770000000000007,"15min":13.388},"last":1.5580000000000001},{"interface":"front","average":{"1min":2.3210000000000002,"5min":1.7410000000000001,"15min":2.766},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.35099999999999998},"max":{"1min":5.9569999999999999,"5min":12.632999999999999,"15min":13.882},"last":1.5}]},{"osd":7,"last update":"Mon Oct 9 04:32:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":1.8420000000000001,"15min":2.8500000000000001},"min":{"1min":0.46999999999999997,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":5.931,"5min":12.656000000000001,"15min":13.907},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.4580000000000002,"5min":1.272,"15min":1.597},"min":{"1min":0.437,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":6.1799999999999997,"5min":6.1799999999999997,"15min":7.8239999999999998},"last":0.496}]}]},{"osd":0,"up_from":91,"seq":390842024097,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":595380,"kb_used_data":182060,"kb_used_omap":5,"kb_used_meta":413306,"kb_avail":93149772,"statfs":{"total":95995035648,"available":95385366528,"internally_reserved":0,"allocated":186429440,"data_stored":185692825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":423225978},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:32:16 2023","interfaces":[{"interface":"back","average":{"1min":3.5539999999999998,"5min":5.2030000000000003,"15min":4.1139999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.192},"max":{"1min":12.974,"5min":167.92699999999999,"15min":167.92699999999999},"last":1.1879999999999999},{"interface":"front","average":{"1min":4.3049999999999997,"5min":5.1900000000000004,"15min":3.7650000000000001},"min":{"1min":0.224,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":12.643000000000001,"5min":169.78999999999999,"15min":169.78999999999999},"last":3.3380000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":3.0899999999999999,"5min":2.6960000000000002,"15min":1.151},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":12.962,"5min":107.548,"15min":107.548},"last":9.0640000000000001},{"interface":"front","average":{"1min":2.6309999999999998,"5min":2.3370000000000002,"15min":1.0149999999999999},"min":{"1min":0.247,"5min":0.22,"15min":0.22},"max":{"1min":8.7739999999999991,"5min":107.35899999999999,"15min":107.35899999999999},"last":1.147}]},{"osd":3,"last update":"Mon Oct 9 04:33:07 2023","interfaces":[{"interface":"back","average":{"1min":2.29,"5min":4.923,"15min":1.895},"min":{"1min":0.27300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.6969999999999992,"5min":213.28399999999999,"15min":213.28399999999999},"last":4.0030000000000001},{"interface":"front","average":{"1min":3.1000000000000001,"5min":5.0039999999999996,"15min":1.9359999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.895,"5min":212.27000000000001,"15min":212.27000000000001},"last":4.0910000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:32:26 2023","interfaces":[{"interface":"back","average":{"1min":3.7629999999999999,"5min":1.6719999999999999,"15min":1.708},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":10.717000000000001,"5min":10.717000000000001,"15min":10.73},"last":1.355},{"interface":"front","average":{"1min":4.3719999999999999,"5min":1.6339999999999999,"15min":1.7709999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":15.941000000000001,"5min":15.941000000000001,"15min":15.941000000000001},"last":2.645}]},{"osd":5,"last update":"Mon Oct 9 04:32:26 2023","interfaces":[{"interface":"back","average":{"1min":4.0759999999999996,"5min":1.5289999999999999,"15min":1.827},"min":{"1min":0.33900000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":12.807,"5min":12.807,"15min":12.807},"last":2.6949999999999998},{"interface":"front","average":{"1min":2.1280000000000001,"5min":1.1679999999999999,"15min":1.7290000000000001},"min":{"1min":0.36299999999999999,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":4.7000000000000002,"5min":5.306,"15min":14.144},"last":0.47599999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:32:26 2023","interfaces":[{"interface":"back","average":{"1min":4.758,"5min":1.782,"15min":2.0169999999999999},"min":{"1min":0.40899999999999997,"5min":0.39500000000000002,"15min":0.35499999999999998},"max":{"1min":16.369,"5min":16.369,"15min":16.369},"last":2.7290000000000001},{"interface":"front","average":{"1min":2.528,"5min":1.25,"15min":1.9710000000000001},"min":{"1min":0.433,"5min":0.41199999999999998,"15min":0.378},"max":{"1min":6.6109999999999998,"5min":6.6109999999999998,"15min":14.090999999999999},"last":0.442}]},{"osd":7,"last update":"Mon Oct 9 04:32:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":1.268,"15min":1.8680000000000001},"min":{"1min":0.48299999999999998,"5min":0.42199999999999999,"15min":0.40699999999999997},"max":{"1min":6.3529999999999998,"5min":6.3529999999999998,"15min":11.890000000000001},"last":3.1280000000000001},{"interface":"front","average":{"1min":3.7229999999999999,"5min":1.6739999999999999,"15min":2.0179999999999998},"min":{"1min":0.41099999999999998,"5min":0.39200000000000002,"15min":0.35399999999999998},"max":{"1min":9.157,"5min":9.157,"15min":11.176},"last":2.5209999999999999}]}]},{"osd":2,"up_from":135,"seq":579820585032,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408228,"kb_used_data":181340,"kb_used_omap":7,"kb_used_meta":226872,"kb_avail":93336924,"statfs":{"total":95995035648,"available":95577010176,"internally_reserved":0,"allocated":185692160,"data_stored":185029288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":232317009},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":4.2229999999999999,"15min":1.625},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.1240000000000006,"5min":129.465,"15min":129.465},"last":0.192},{"interface":"front","average":{"1min":2.5939999999999999,"5min":4.1970000000000001,"15min":1.6419999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.019,"5min":129.51599999999999,"15min":129.51599999999999},"last":0.26300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":3.2669999999999999,"15min":1.3069999999999999},"min":{"1min":0.21299999999999999,"5min":0.193,"15min":0.193},"max":{"1min":41.055,"5min":54.542999999999999,"15min":54.542999999999999},"last":0.215},{"interface":"front","average":{"1min":4.0979999999999999,"5min":3.028,"15min":1.238},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":39.984000000000002,"5min":51.579999999999998,"15min":51.579999999999998},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:33:14 2023","interfaces":[{"interface":"back","average":{"1min":18.420999999999999,"5min":5.8620000000000001,"15min":2.2040000000000002},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":147.655,"5min":147.655,"15min":147.655},"last":0.39300000000000002},{"interface":"front","average":{"1min":17.818000000000001,"5min":5.9180000000000001,"15min":2.2629999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":147.607,"5min":147.607,"15min":147.607},"last":0.438}]},{"osd":4,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.502,"15min":0.90100000000000002},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.3849999999999998,"5min":12.765000000000001,"15min":12.765000000000001},"last":3.2850000000000001},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.0960000000000001,"15min":0.77400000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.5439999999999996,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":3.3149999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.4750000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":21.533999999999999,"5min":21.533999999999999,"15min":21.533999999999999},"last":6.327},{"interface":"front","average":{"1min":3.0600000000000001,"5min":1.7490000000000001,"15min":0.97799999999999998},"min":{"1min":0.41099999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":21.716000000000001,"5min":21.716000000000001,"15min":21.716000000000001},"last":5.7530000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.129,"15min":0.80100000000000005},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8959999999999999,"5min":9.2460000000000004,"15min":9.2460000000000004},"last":3.2599999999999998},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.48,"15min":0.91800000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":10.140000000000001,"5min":11.641999999999999,"15min":11.641999999999999},"last":3.234}]},{"osd":7,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.784,"15min":1.012},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":25.835999999999999,"5min":25.835999999999999,"15min":25.835999999999999},"last":1.8049999999999999},{"interface":"front","average":{"1min":3.9430000000000001,"5min":1.839,"15min":1.0289999999999999},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":26.416,"5min":26.416,"15min":26.416},"last":1.7250000000000001}]}]},{"osd":3,"up_from":142,"seq":609885356098,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619088,"kb_used_data":404616,"kb_used_omap":2,"kb_used_meta":214461,"kb_avail":93126064,"statfs":{"total":95995035648,"available":95361089536,"internally_reserved":0,"allocated":414326784,"data_stored":413678985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":219608117},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":7.7629999999999999,"5min":2.8010000000000002,"15min":1.359},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":91.634,"5min":91.634,"15min":91.634},"last":0.28999999999999998},{"interface":"front","average":{"1min":7.0110000000000001,"5min":2.8599999999999999,"15min":1.222},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":91.646000000000001,"5min":91.646000000000001,"15min":91.646000000000001},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":8.202,"5min":5.0190000000000001,"15min":2.6549999999999998},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":119.00700000000001,"5min":180.02500000000001,"15min":180.02500000000001},"last":13.617000000000001},{"interface":"front","average":{"1min":9.4480000000000004,"5min":5.641,"15min":2.375},"min":{"1min":0.29499999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":122.977,"5min":184.00899999999999,"15min":184.00899999999999},"last":9.8650000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":8.7810000000000006,"5min":3.105,"15min":1.3080000000000001},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":118.238,"5min":118.238,"15min":118.238},"last":0.38700000000000001},{"interface":"front","average":{"1min":9.141,"5min":3.1579999999999999,"15min":2.0619999999999998},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":117.64400000000001,"5min":117.64400000000001,"15min":117.64400000000001},"last":0.31}]},{"osd":4,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.4910000000000001,"15min":1.698},"min":{"1min":0.29999999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7679999999999998,"5min":23.526,"15min":23.526},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.2040000000000002,"5min":1.621,"15min":1.1950000000000001},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":9.1440000000000001,"5min":9.1440000000000001,"15min":9.1440000000000001},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":6.2169999999999996,"5min":2.456,"15min":1.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":57.475000000000001,"5min":57.475000000000001,"15min":57.475000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":6.4089999999999998,"5min":2.71,"15min":1.3600000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":57.651000000000003,"5min":57.651000000000003,"15min":57.651000000000003},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8740000000000001,"15min":1.0640000000000001},"min":{"1min":0.38,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.3000000000000007,"5min":10.119999999999999,"15min":10.119999999999999},"last":0.436},{"interface":"front","average":{"1min":2.532,"5min":1.8420000000000001,"15min":1.798},"min":{"1min":0.38,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.7270000000000003,"5min":23.59,"15min":23.59},"last":0.52300000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":6.907,"5min":2.6800000000000002,"15min":2.0649999999999999},"min":{"1min":0.56399999999999995,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":57.399000000000001,"5min":57.399000000000001,"15min":57.399000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":7.1980000000000004,"5min":2.8420000000000001,"15min":1.609},"min":{"1min":0.47699999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":57.447000000000003,"5min":57.447000000000003,"15min":57.447000000000003},"last":0.59199999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019224,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":768564,"kb_used_data":170604,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":92976588,"statfs":{"total":95995035648,"available":95208026112,"internally_reserved":0,"allocated":174698496,"data_stored":174188597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:29 2023","interfaces":[{"interface":"back","average":{"1min":17.399999999999999,"5min":5.0990000000000002,"15min":3.0569999999999999},"min":{"1min":0.29499999999999998,"5min":0.23000000000000001,"15min":0.20499999999999999},"max":{"1min":219.577,"5min":219.577,"15min":219.577},"last":0.23799999999999999},{"interface":"front","average":{"1min":16.393000000000001,"5min":4.3230000000000004,"15min":2.9860000000000002},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.247},"max":{"1min":218.96000000000001,"5min":218.96000000000001,"15min":218.96000000000001},"last":0.27100000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:32:18 2023","interfaces":[{"interface":"back","average":{"1min":22.423999999999999,"5min":8.3800000000000008,"15min":4.2110000000000003},"min":{"1min":0.27300000000000002,"5min":0.246,"15min":0.23400000000000001},"max":{"1min":142.74799999999999,"5min":142.74799999999999,"15min":142.74799999999999},"last":1.7110000000000001},{"interface":"front","average":{"1min":22.966000000000001,"5min":8.7400000000000002,"15min":4.516},"min":{"1min":0.30299999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":139.09700000000001,"5min":139.09700000000001,"15min":139.09700000000001},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4889999999999999,"5min":2.5459999999999998,"15min":1.21},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.9100000000000001,"5min":50.972999999999999,"15min":50.972999999999999},"last":2.0049999999999999},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.7170000000000001,"15min":0.95199999999999996},"min":{"1min":0.32100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.335,"5min":39.792000000000002,"15min":39.792000000000002},"last":0.42199999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:33:01 2023","interfaces":[{"interface":"back","average":{"1min":17.379000000000001,"5min":10.377000000000001,"15min":3.8759999999999999},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":209.79599999999999,"5min":262.524,"15min":262.524},"last":6.9699999999999998},{"interface":"front","average":{"1min":18.452000000000002,"5min":10.991,"15min":4.0819999999999999},"min":{"1min":0.38,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":209.68799999999999,"5min":262.55099999999999,"15min":262.55099999999999},"last":9.8119999999999994}]},{"osd":5,"last update":"Mon Oct 9 04:33:01 2023","interfaces":[{"interface":"back","average":{"1min":5.9009999999999998,"5min":1.9239999999999999,"15min":1.2809999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23499999999999999},"max":{"1min":73.510000000000005,"5min":73.510000000000005,"15min":73.510000000000005},"last":0.29699999999999999},{"interface":"front","average":{"1min":6.1360000000000001,"5min":2.415,"15min":1.452},"min":{"1min":0.28799999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":73.828000000000003,"5min":73.828000000000003,"15min":73.828000000000003},"last":0.32100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:32:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":1.6699999999999999,"15min":1.258},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":6.3780000000000001,"5min":52.801000000000002,"15min":52.801000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.581,"15min":1.242},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":9.1359999999999992,"5min":51.064999999999998,"15min":51.064999999999998},"last":0.38400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:32:50 2023","interfaces":[{"interface":"back","average":{"1min":6.0860000000000003,"5min":1.946,"15min":1.3700000000000001},"min":{"1min":0.39900000000000002,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":76.283000000000001,"5min":76.283000000000001,"15min":76.283000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":6.3959999999999999,"5min":2.0449999999999999,"15min":1.4390000000000001},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.29199999999999998},"max":{"1min":76.259,"5min":76.259,"15min":76.259},"last":0.44500000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757588,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1376304,"kb_used_data":481896,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":92368848,"statfs":{"total":95995035648,"available":94585700352,"internally_reserved":0,"allocated":493461504,"data_stored":492959453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:22 2023","interfaces":[{"interface":"back","average":{"1min":6.5359999999999996,"5min":4.4660000000000002,"15min":2.7869999999999999},"min":{"1min":0.318,"5min":0.22,"15min":0.22},"max":{"1min":81.619,"5min":127.108,"15min":127.108},"last":0.42799999999999999},{"interface":"front","average":{"1min":7.1980000000000004,"5min":4.4720000000000004,"15min":2.5190000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":81.724000000000004,"5min":128.036,"15min":128.036},"last":0.29999999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:32:22 2023","interfaces":[{"interface":"back","average":{"1min":6.6360000000000001,"5min":4.923,"15min":2.927},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":109.95699999999999,"5min":109.95699999999999,"15min":109.95699999999999},"last":5.8529999999999998},{"interface":"front","average":{"1min":7.8920000000000003,"5min":6.7939999999999996,"15min":3.6760000000000002},"min":{"1min":0.28299999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":110.542,"5min":116.04000000000001,"15min":116.04000000000001},"last":0.81599999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:32:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5230000000000001,"5min":3.2589999999999999,"15min":1.4630000000000001},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.5310000000000006,"5min":119.14,"15min":119.14},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.3160000000000001,"5min":2.8679999999999999,"15min":1.3380000000000001},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":5.0910000000000002,"5min":118.81100000000001,"15min":118.81100000000001},"last":2.286}]},{"osd":3,"last update":"Mon Oct 9 04:33:14 2023","interfaces":[{"interface":"back","average":{"1min":19.989999999999998,"5min":8.5280000000000005,"15min":3.2650000000000001},"min":{"1min":0.39200000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":259.053,"5min":259.053,"15min":259.053},"last":7.4199999999999999},{"interface":"front","average":{"1min":19.395,"5min":8.4280000000000008,"15min":3.2639999999999998},"min":{"1min":0.436,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":259.108,"5min":259.108,"15min":259.108},"last":7.5720000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:32:58 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.754,"15min":0.91900000000000004},"min":{"1min":0.309,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":7.2699999999999996,"5min":7.2699999999999996,"15min":9.641},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.145,"15min":1.0369999999999999},"min":{"1min":0.30199999999999999,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":10.645,"5min":10.645,"15min":10.645},"last":0.502}]},{"osd":6,"last update":"Mon Oct 9 04:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":0.90500000000000003,"15min":1.012},"min":{"1min":0.317,"5min":0.316,"15min":0.26000000000000001},"max":{"1min":11.192,"5min":11.192,"15min":11.192},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.5269999999999999,"5min":0.95899999999999996,"15min":1.115},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.0599999999999996,"5min":6.0599999999999996,"15min":14.334},"last":0.47099999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:32:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.254,"15min":1.2470000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.27400000000000002},"max":{"1min":8.1470000000000002,"5min":8.1470000000000002,"15min":9.9000000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.117,"15min":1.133},"min":{"1min":0.40300000000000002,"5min":0.35999999999999999,"15min":0.33000000000000002},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":11.315},"last":0.48899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":182059008,"data_stored":182059008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":544149504,"data_stored":544145432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":174718976,"data_stored":174718976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":405995520,"data_stored":405995520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":169082880,"data_stored":169082880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":475533312,"data_stored":475529240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":152961024,"data_stored":152961024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":480382976,"data_stored":480378904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:33:24.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:33:24.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:24.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:24.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:33:24.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:24.437 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:24.598 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:24.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:24.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:24.807 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:24 smithi177 ceph-mon[195797]: pgmap v689: 89 pgs: 89 active+clean; 593 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:33:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:25.131 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:25.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:25.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:25.140 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:24 smithi029 ceph-mon[223858]: pgmap v689: 89 pgs: 89 active+clean; 593 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:33:25.140 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:25.140 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:25.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:24 smithi029 ceph-mon[216903]: pgmap v689: 89 pgs: 89 active+clean; 593 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-09T04:33:25.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:25.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:25.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:25.259 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:25.259 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:25.289 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:25.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:25.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:25.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:25.571 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:25.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:25.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:25.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:25.800 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:25.863 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:25.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:25.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:26.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:26.069 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:26.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:26 smithi029 ceph-mon[216903]: pgmap v690: 89 pgs: 89 active+clean; 593 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:33:26.355 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:26 smithi029 ceph-mon[223858]: pgmap v690: 89 pgs: 89 active+clean; 593 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:33:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:26 smithi177 ceph-mon[195797]: pgmap v690: 89 pgs: 89 active+clean; 593 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 567 op/s 2023-10-09T04:33:26.643 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:26.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:26.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:26.804 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:26.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:26.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:27.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:27.115 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:27.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:27.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:27.470 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:27.499 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:27.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:33:27.549 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:27.588 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:27.658 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:27.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:27.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:27.825 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:27.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:27.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:28.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:28.103 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:28.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:28 smithi177 ceph-mon[195797]: pgmap v691: 89 pgs: 89 active+clean; 431 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-09T04:33:28.317 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:28.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:28 smithi029 ceph-mon[223858]: pgmap v691: 89 pgs: 89 active+clean; 431 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-09T04:33:28.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:28 smithi029 ceph-mon[216903]: pgmap v691: 89 pgs: 89 active+clean; 431 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-09T04:33:28.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:28.473 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:28.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:28.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:28.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:28.808 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:29.020 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:29.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:29.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:29.164 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:33:29.194 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:29.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3983424577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:33:29.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:29.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3983424577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:33:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3983424577' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:33:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: pgmap v692: 89 pgs: 89 active+clean; 394 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3983424577' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[216903]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: pgmap v692: 89 pgs: 89 active+clean; 394 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3983424577' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:30 smithi029 ceph-mon[223858]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: pgmap v692: 89 pgs: 89 active+clean; 394 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 555 op/s 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3983424577' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: osdmap e187: 8 total, 8 up, 8 in 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: failed to encode map e187 with expected crc 2023-10-09T04:33:30.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:30 smithi177 ceph-mon[195797]: failed to encode map e187 with expected crc 2023-10-09T04:33:32.004 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:32.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:32.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:32.094 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:32.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:32.199 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:32.306 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:32 smithi029 ceph-mon[216903]: pgmap v694: 89 pgs: 89 active+clean; 297 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T04:33:32.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:32 smithi029 ceph-mon[223858]: pgmap v694: 89 pgs: 89 active+clean; 297 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T04:33:32.370 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:32.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:32.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:32.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:32 smithi177 ceph-mon[195797]: pgmap v694: 89 pgs: 89 active+clean; 297 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-09T04:33:32.551 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:32.649 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:32.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:32.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:32.807 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:32.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:32.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:33.013 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:33:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:33:32] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-10-09T04:33:33.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:33.094 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:33.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:33.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:33.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:33.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:33.383 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:33.982 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:33:33.983 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:33:33.983 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:33:33.986 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:33:34.032 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:33:34.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:33:34.377 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:33:34.377 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:33:34.377 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.378 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:33:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:34 smithi029 ceph-mon[216903]: pgmap v695: 89 pgs: 89 active+clean; 184 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:33:34.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1177617996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:34 smithi029 ceph-mon[223858]: pgmap v695: 89 pgs: 89 active+clean; 184 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:33:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1177617996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:34.487 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:34.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:34 smithi177 ceph-mon[195797]: pgmap v695: 89 pgs: 89 active+clean; 184 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:33:34.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1177617996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:34.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:34.708 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:35.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:35.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:35.081 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:33:35.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:35.151 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:35.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:35.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:35.272 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:33:35.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:35.369 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:35.642 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 98300 objects 2023-10-09T04:33:35.642 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :152.545 2023-10-09T04:33:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:35.937 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:35.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:36.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:36.065 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:36.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:36.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:36.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:36.240 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:36.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:36.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:36.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:36.409 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:36 smithi177 ceph-mon[195797]: pgmap v696: 89 pgs: 89 active+clean; 184 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:33:36.410 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:36 smithi177 ceph-mon[195797]: from='client.45485 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:36.412 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:36.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:36 smithi029 ceph-mon[216903]: pgmap v696: 89 pgs: 89 active+clean; 184 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:33:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:36 smithi029 ceph-mon[216903]: from='client.45485 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:36 smithi029 ceph-mon[223858]: pgmap v696: 89 pgs: 89 active+clean; 184 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-10-09T04:33:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:36 smithi029 ceph-mon[223858]: from='client.45485 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:33:36.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:36.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:36.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:36.588 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:36.873 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-09T04:33:36.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-09T04:33:36.958 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:36.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:37.014 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":696,"stamp":"2023-10-09T04:33:34.916066+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":193140664,"num_objects":2963,"num_object_clones":0,"num_object_copies":8889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2963,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":193963,"num_write_kb":6293646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9630,"num_bytes_recovered":579276022,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139479,"ondisk_log_size":139479,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4586824,"kb_used_data":658120,"kb_used_omap":20,"kb_used_meta":3928427,"kb_avail":745374392,"statfs":{"total":767960285184,"available":763263377408,"internally_reserved":0,"allocated":673914880,"data_stored":669082283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4022709902},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-428408832,"num_objects":-6537,"num_object_clones":0,"num_object_copies":-19611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6537,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003946"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":166,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258391+0000","last_change":"2023-10-09T04:27:30.004963+0000","last_active":"2023-10-09T04:33:14.258391+0000","last_peered":"2023-10-09T04:33:14.258391+0000","last_clean":"2023-10-09T04:33:14.258391+0000","last_became_active":"2023-10-09T04:27:30.004581+0000","last_became_peered":"2023-10-09T04:27:30.004581+0000","last_unstale":"2023-10-09T04:33:14.258391+0000","last_undegraded":"2023-10-09T04:33:14.258391+0000","last_fullsized":"2023-10-09T04:33:14.258391+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:40.077805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.255328+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:33:14.255328+0000","last_peered":"2023-10-09T04:33:14.255328+0000","last_clean":"2023-10-09T04:33:14.255328+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:33:14.255328+0000","last_undegraded":"2023-10-09T04:33:14.255328+0000","last_fullsized":"2023-10-09T04:33:14.255328+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":150,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251488+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:33:14.251488+0000","last_peered":"2023-10-09T04:33:14.251488+0000","last_clean":"2023-10-09T04:33:14.251488+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:33:14.251488+0000","last_undegraded":"2023-10-09T04:33:14.251488+0000","last_fullsized":"2023-10-09T04:33:14.251488+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.242406+0000","last_change":"2023-10-09T04:29:51.054580+0000","last_active":"2023-10-09T04:33:14.242406+0000","last_peered":"2023-10-09T04:33:14.242406+0000","last_clean":"2023-10-09T04:33:14.242406+0000","last_became_active":"2023-10-09T04:29:51.054223+0000","last_became_peered":"2023-10-09T04:29:51.054223+0000","last_unstale":"2023-10-09T04:33:14.242406+0000","last_undegraded":"2023-10-09T04:33:14.242406+0000","last_fullsized":"2023-10-09T04:33:14.242406+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:28:22.233569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":173,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.254358+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:33:14.254358+0000","last_peered":"2023-10-09T04:33:14.254358+0000","last_clean":"2023-10-09T04:33:14.254358+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:33:14.254358+0000","last_undegraded":"2023-10-09T04:33:14.254358+0000","last_fullsized":"2023-10-09T04:33:14.254358+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.256001+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:33:14.256001+0000","last_peered":"2023-10-09T04:33:14.256001+0000","last_clean":"2023-10-09T04:33:14.256001+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:33:14.256001+0000","last_undegraded":"2023-10-09T04:33:14.256001+0000","last_fullsized":"2023-10-09T04:33:14.256001+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251857+0000","last_change":"2023-10-09T04:29:51.059222+0000","last_active":"2023-10-09T04:33:14.251857+0000","last_peered":"2023-10-09T04:33:14.251857+0000","last_clean":"2023-10-09T04:33:14.251857+0000","last_became_active":"2023-10-09T04:29:51.058771+0000","last_became_peered":"2023-10-09T04:29:51.058771+0000","last_unstale":"2023-10-09T04:33:14.251857+0000","last_undegraded":"2023-10-09T04:33:14.251857+0000","last_fullsized":"2023-10-09T04:33:14.251857+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:34:26.840067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.254432+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:33:14.254432+0000","last_peered":"2023-10-09T04:33:14.254432+0000","last_clean":"2023-10-09T04:33:14.254432+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:33:14.254432+0000","last_undegraded":"2023-10-09T04:33:14.254432+0000","last_fullsized":"2023-10-09T04:33:14.254432+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235256+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:33:14.235256+0000","last_peered":"2023-10-09T04:33:14.235256+0000","last_clean":"2023-10-09T04:33:14.235256+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:33:14.235256+0000","last_undegraded":"2023-10-09T04:33:14.235256+0000","last_fullsized":"2023-10-09T04:33:14.235256+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":110,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.253437+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:33:14.253437+0000","last_peered":"2023-10-09T04:33:14.253437+0000","last_clean":"2023-10-09T04:33:14.253437+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:33:14.253437+0000","last_undegraded":"2023-10-09T04:33:14.253437+0000","last_fullsized":"2023-10-09T04:33:14.253437+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":149,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.249646+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:33:14.249646+0000","last_peered":"2023-10-09T04:33:14.249646+0000","last_clean":"2023-10-09T04:33:14.249646+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:33:14.249646+0000","last_undegraded":"2023-10-09T04:33:14.249646+0000","last_fullsized":"2023-10-09T04:33:14.249646+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:26.616932+0000","last_change":"2023-10-09T04:27:29.983053+0000","last_active":"2023-10-09T04:33:26.616932+0000","last_peered":"2023-10-09T04:33:26.616932+0000","last_clean":"2023-10-09T04:33:26.616932+0000","last_became_active":"2023-10-09T04:27:29.982725+0000","last_became_peered":"2023-10-09T04:27:29.982725+0000","last_unstale":"2023-10-09T04:33:26.616932+0000","last_undegraded":"2023-10-09T04:33:26.616932+0000","last_fullsized":"2023-10-09T04:33:26.616932+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:33.931626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":153,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258834+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:33:14.258834+0000","last_peered":"2023-10-09T04:33:14.258834+0000","last_clean":"2023-10-09T04:33:14.258834+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:33:14.258834+0000","last_undegraded":"2023-10-09T04:33:14.258834+0000","last_fullsized":"2023-10-09T04:33:14.258834+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258572+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:33:14.258572+0000","last_peered":"2023-10-09T04:33:14.258572+0000","last_clean":"2023-10-09T04:33:14.258572+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:33:14.258572+0000","last_undegraded":"2023-10-09T04:33:14.258572+0000","last_fullsized":"2023-10-09T04:33:14.258572+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":148,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.249338+0000","last_change":"2023-10-09T04:27:29.989226+0000","last_active":"2023-10-09T04:33:14.249338+0000","last_peered":"2023-10-09T04:33:14.249338+0000","last_clean":"2023-10-09T04:33:14.249338+0000","last_became_active":"2023-10-09T04:27:29.988867+0000","last_became_peered":"2023-10-09T04:27:29.988867+0000","last_unstale":"2023-10-09T04:33:14.249338+0000","last_undegraded":"2023-10-09T04:33:14.249338+0000","last_fullsized":"2023-10-09T04:33:14.249338+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:15.559571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.259233+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:33:14.259233+0000","last_peered":"2023-10-09T04:33:14.259233+0000","last_clean":"2023-10-09T04:33:14.259233+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:33:14.259233+0000","last_undegraded":"2023-10-09T04:33:14.259233+0000","last_fullsized":"2023-10-09T04:33:14.259233+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.242654+0000","last_change":"2023-10-09T04:29:51.053986+0000","last_active":"2023-10-09T04:33:14.242654+0000","last_peered":"2023-10-09T04:33:14.242654+0000","last_clean":"2023-10-09T04:33:14.242654+0000","last_became_active":"2023-10-09T04:29:51.053715+0000","last_became_peered":"2023-10-09T04:29:51.053715+0000","last_unstale":"2023-10-09T04:33:14.242654+0000","last_undegraded":"2023-10-09T04:33:14.242654+0000","last_fullsized":"2023-10-09T04:33:14.242654+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:44.445249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":146,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235397+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:33:14.235397+0000","last_peered":"2023-10-09T04:33:14.235397+0000","last_clean":"2023-10-09T04:33:14.235397+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:33:14.235397+0000","last_undegraded":"2023-10-09T04:33:14.235397+0000","last_fullsized":"2023-10-09T04:33:14.235397+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":129,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.256973+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:33:14.256973+0000","last_peered":"2023-10-09T04:33:14.256973+0000","last_clean":"2023-10-09T04:33:14.256973+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:33:14.256973+0000","last_undegraded":"2023-10-09T04:33:14.256973+0000","last_fullsized":"2023-10-09T04:33:14.256973+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.242476+0000","last_change":"2023-10-09T04:29:51.059556+0000","last_active":"2023-10-09T04:33:14.242476+0000","last_peered":"2023-10-09T04:33:14.242476+0000","last_clean":"2023-10-09T04:33:14.242476+0000","last_became_active":"2023-10-09T04:29:51.059209+0000","last_became_peered":"2023-10-09T04:29:51.059209+0000","last_unstale":"2023-10-09T04:33:14.242476+0000","last_undegraded":"2023-10-09T04:33:14.242476+0000","last_fullsized":"2023-10-09T04:33:14.242476+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:42:54.203833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":194,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.255222+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:33:14.255222+0000","last_peered":"2023-10-09T04:33:14.255222+0000","last_clean":"2023-10-09T04:33:14.255222+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:33:14.255222+0000","last_undegraded":"2023-10-09T04:33:14.255222+0000","last_fullsized":"2023-10-09T04:33:14.255222+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":137,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.252658+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:33:14.252658+0000","last_peered":"2023-10-09T04:33:14.252658+0000","last_clean":"2023-10-09T04:33:14.252658+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:33:14.252658+0000","last_undegraded":"2023-10-09T04:33:14.252658+0000","last_fullsized":"2023-10-09T04:33:14.252658+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"187'12274","reported_seq":17198,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:32.552232+0000","last_change":"2023-10-09T04:32:07.124101+0000","last_active":"2023-10-09T04:33:32.552232+0000","last_peered":"2023-10-09T04:33:32.552232+0000","last_clean":"2023-10-09T04:33:32.552232+0000","last_became_active":"2023-10-09T04:32:07.103231+0000","last_became_peered":"2023-10-09T04:32:07.103231+0000","last_unstale":"2023-10-09T04:33:32.552232+0000","last_undegraded":"2023-10-09T04:33:32.552232+0000","last_fullsized":"2023-10-09T04:33:32.552232+0000","mapping_epoch":173,"log_start":"152'2200","ondisk_log_start":"152'2200","created":144,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:31.609581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8126464,"num_objects":124,"num_object_clones":0,"num_object_copies":372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12274,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2428,"num_bytes_recovered":106168320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":175,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.249021+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:33:14.249021+0000","last_peered":"2023-10-09T04:33:14.249021+0000","last_clean":"2023-10-09T04:33:14.249021+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:33:14.249021+0000","last_undegraded":"2023-10-09T04:33:14.249021+0000","last_fullsized":"2023-10-09T04:33:14.249021+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.234934+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:33:14.234934+0000","last_peered":"2023-10-09T04:33:14.234934+0000","last_clean":"2023-10-09T04:33:14.234934+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:33:14.234934+0000","last_undegraded":"2023-10-09T04:33:14.234934+0000","last_fullsized":"2023-10-09T04:33:14.234934+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"187'11942","reported_seq":11995,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:31.613615+0000","last_change":"2023-10-09T04:29:24.154510+0000","last_active":"2023-10-09T04:33:31.613615+0000","last_peered":"2023-10-09T04:33:31.613615+0000","last_clean":"2023-10-09T04:33:31.613615+0000","last_became_active":"2023-10-09T04:29:24.154250+0000","last_became_peered":"2023-10-09T04:29:24.154250+0000","last_unstale":"2023-10-09T04:33:31.613615+0000","last_undegraded":"2023-10-09T04:33:31.613615+0000","last_fullsized":"2023-10-09T04:33:31.613615+0000","mapping_epoch":144,"log_start":"162'4700","ondisk_log_start":"162'4700","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7242,"log_dups_size":0,"ondisk_log_size":7242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:43:37.323214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10354688,"num_objects":158,"num_object_clones":0,"num_object_copies":474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11942,"num_write_kb":387200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":148,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.249063+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:33:14.249063+0000","last_peered":"2023-10-09T04:33:14.249063+0000","last_clean":"2023-10-09T04:33:14.249063+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:33:14.249063+0000","last_undegraded":"2023-10-09T04:33:14.249063+0000","last_fullsized":"2023-10-09T04:33:14.249063+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":164,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258403+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:33:14.258403+0000","last_peered":"2023-10-09T04:33:14.258403+0000","last_clean":"2023-10-09T04:33:14.258403+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:33:14.258403+0000","last_undegraded":"2023-10-09T04:33:14.258403+0000","last_fullsized":"2023-10-09T04:33:14.258403+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"187'12017","reported_seq":12068,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.953890+0000","last_change":"2023-10-09T04:29:24.161991+0000","last_active":"2023-10-09T04:33:29.953890+0000","last_peered":"2023-10-09T04:33:29.953890+0000","last_clean":"2023-10-09T04:33:29.953890+0000","last_became_active":"2023-10-09T04:29:24.161712+0000","last_became_peered":"2023-10-09T04:29:24.161712+0000","last_unstale":"2023-10-09T04:33:29.953890+0000","last_undegraded":"2023-10-09T04:33:29.953890+0000","last_fullsized":"2023-10-09T04:33:29.953890+0000","mapping_epoch":144,"log_start":"152'2000","ondisk_log_start":"152'2000","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:15:52.360661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12017,"num_write_kb":392640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":130,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.249556+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:33:14.249556+0000","last_peered":"2023-10-09T04:33:14.249556+0000","last_clean":"2023-10-09T04:33:14.249556+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:33:14.249556+0000","last_undegraded":"2023-10-09T04:33:14.249556+0000","last_fullsized":"2023-10-09T04:33:14.249556+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.255529+0000","last_change":"2023-10-09T04:27:30.005025+0000","last_active":"2023-10-09T04:33:14.255529+0000","last_peered":"2023-10-09T04:33:14.255529+0000","last_clean":"2023-10-09T04:33:14.255529+0000","last_became_active":"2023-10-09T04:27:30.004692+0000","last_became_peered":"2023-10-09T04:27:30.004692+0000","last_unstale":"2023-10-09T04:33:14.255529+0000","last_undegraded":"2023-10-09T04:33:14.255529+0000","last_fullsized":"2023-10-09T04:33:14.255529+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:10.737398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"187'12280","reported_seq":12331,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.949186+0000","last_change":"2023-10-09T04:29:24.685975+0000","last_active":"2023-10-09T04:33:29.949186+0000","last_peered":"2023-10-09T04:33:29.949186+0000","last_clean":"2023-10-09T04:33:29.949186+0000","last_became_active":"2023-10-09T04:29:24.685702+0000","last_became_peered":"2023-10-09T04:29:24.685702+0000","last_unstale":"2023-10-09T04:33:29.949186+0000","last_undegraded":"2023-10-09T04:33:29.949186+0000","last_fullsized":"2023-10-09T04:33:29.949186+0000","mapping_epoch":144,"log_start":"152'2200","ondisk_log_start":"152'2200","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:57.821949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14548992,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12280,"num_write_kb":400064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235135+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:33:14.235135+0000","last_peered":"2023-10-09T04:33:14.235135+0000","last_clean":"2023-10-09T04:33:14.235135+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:33:14.235135+0000","last_undegraded":"2023-10-09T04:33:14.235135+0000","last_fullsized":"2023-10-09T04:33:14.235135+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.252796+0000","last_change":"2023-10-09T04:29:51.056784+0000","last_active":"2023-10-09T04:33:14.252796+0000","last_peered":"2023-10-09T04:33:14.252796+0000","last_clean":"2023-10-09T04:33:14.252796+0000","last_became_active":"2023-10-09T04:29:51.056272+0000","last_became_peered":"2023-10-09T04:29:51.056272+0000","last_unstale":"2023-10-09T04:33:14.252796+0000","last_undegraded":"2023-10-09T04:33:14.252796+0000","last_fullsized":"2023-10-09T04:33:14.252796+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:33:31.908683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"187'11891","reported_seq":11943,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:31.495726+0000","last_change":"2023-10-09T04:29:24.541340+0000","last_active":"2023-10-09T04:33:31.495726+0000","last_peered":"2023-10-09T04:33:31.495726+0000","last_clean":"2023-10-09T04:33:31.495726+0000","last_became_active":"2023-10-09T04:29:24.540863+0000","last_became_peered":"2023-10-09T04:29:24.540863+0000","last_unstale":"2023-10-09T04:33:31.495726+0000","last_undegraded":"2023-10-09T04:33:31.495726+0000","last_fullsized":"2023-10-09T04:33:31.495726+0000","mapping_epoch":144,"log_start":"161'4100","ondisk_log_start":"161'4100","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:14.095614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11206656,"num_objects":171,"num_object_clones":0,"num_object_copies":513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11891,"num_write_kb":385984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":181,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.253205+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:33:14.253205+0000","last_peered":"2023-10-09T04:33:14.253205+0000","last_clean":"2023-10-09T04:33:14.253205+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:33:14.253205+0000","last_undegraded":"2023-10-09T04:33:14.253205+0000","last_fullsized":"2023-10-09T04:33:14.253205+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258796+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:33:14.258796+0000","last_peered":"2023-10-09T04:33:14.258796+0000","last_clean":"2023-10-09T04:33:14.258796+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:33:14.258796+0000","last_undegraded":"2023-10-09T04:33:14.258796+0000","last_fullsized":"2023-10-09T04:33:14.258796+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"187'11918","reported_seq":11972,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:31.587641+0000","last_change":"2023-10-09T04:29:24.685606+0000","last_active":"2023-10-09T04:33:31.587641+0000","last_peered":"2023-10-09T04:33:31.587641+0000","last_clean":"2023-10-09T04:33:31.587641+0000","last_became_active":"2023-10-09T04:29:24.685115+0000","last_became_peered":"2023-10-09T04:29:24.685115+0000","last_unstale":"2023-10-09T04:33:31.587641+0000","last_undegraded":"2023-10-09T04:33:31.587641+0000","last_fullsized":"2023-10-09T04:33:31.587641+0000","mapping_epoch":144,"log_start":"162'4700","ondisk_log_start":"162'4700","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7218,"log_dups_size":0,"ondisk_log_size":7218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:16:33.550656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11665408,"num_objects":178,"num_object_clones":0,"num_object_copies":534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11918,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":146,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.255453+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:33:14.255453+0000","last_peered":"2023-10-09T04:33:14.255453+0000","last_clean":"2023-10-09T04:33:14.255453+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:33:14.255453+0000","last_undegraded":"2023-10-09T04:33:14.255453+0000","last_fullsized":"2023-10-09T04:33:14.255453+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":164,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258643+0000","last_change":"2023-10-09T04:27:29.982774+0000","last_active":"2023-10-09T04:33:14.258643+0000","last_peered":"2023-10-09T04:33:14.258643+0000","last_clean":"2023-10-09T04:33:14.258643+0000","last_became_active":"2023-10-09T04:27:29.982507+0000","last_became_peered":"2023-10-09T04:27:29.982507+0000","last_unstale":"2023-10-09T04:33:14.258643+0000","last_undegraded":"2023-10-09T04:33:14.258643+0000","last_fullsized":"2023-10-09T04:33:14.258643+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:46:14.613302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"187'12191","reported_seq":12241,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.937785+0000","last_change":"2023-10-09T04:29:24.158352+0000","last_active":"2023-10-09T04:33:29.937785+0000","last_peered":"2023-10-09T04:33:29.937785+0000","last_clean":"2023-10-09T04:33:29.937785+0000","last_became_active":"2023-10-09T04:29:24.158010+0000","last_became_peered":"2023-10-09T04:29:24.158010+0000","last_unstale":"2023-10-09T04:33:29.937785+0000","last_undegraded":"2023-10-09T04:33:29.937785+0000","last_fullsized":"2023-10-09T04:33:29.937785+0000","mapping_epoch":144,"log_start":"161'4400","ondisk_log_start":"161'4400","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:24.852154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12191,"num_write_kb":397376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":148,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258249+0000","last_change":"2023-10-09T04:27:30.010756+0000","last_active":"2023-10-09T04:33:14.258249+0000","last_peered":"2023-10-09T04:33:14.258249+0000","last_clean":"2023-10-09T04:33:14.258249+0000","last_became_active":"2023-10-09T04:27:30.010403+0000","last_became_peered":"2023-10-09T04:27:30.010403+0000","last_unstale":"2023-10-09T04:33:14.258249+0000","last_undegraded":"2023-10-09T04:33:14.258249+0000","last_fullsized":"2023-10-09T04:33:14.258249+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:43:10.557818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.259088+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:33:14.259088+0000","last_peered":"2023-10-09T04:33:14.259088+0000","last_clean":"2023-10-09T04:33:14.259088+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:33:14.259088+0000","last_undegraded":"2023-10-09T04:33:14.259088+0000","last_fullsized":"2023-10-09T04:33:14.259088+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"187'12052","reported_seq":12104,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:31.232542+0000","last_change":"2023-10-09T04:29:24.685470+0000","last_active":"2023-10-09T04:33:31.232542+0000","last_peered":"2023-10-09T04:33:31.232542+0000","last_clean":"2023-10-09T04:33:31.232542+0000","last_became_active":"2023-10-09T04:29:24.685144+0000","last_became_peered":"2023-10-09T04:29:24.685144+0000","last_unstale":"2023-10-09T04:33:31.232542+0000","last_undegraded":"2023-10-09T04:33:31.232542+0000","last_fullsized":"2023-10-09T04:33:31.232542+0000","mapping_epoch":144,"log_start":"161'4300","ondisk_log_start":"161'4300","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7752,"log_dups_size":0,"ondisk_log_size":7752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:20:43.630951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12052,"num_write_kb":390848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":138,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.254187+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:33:14.254187+0000","last_peered":"2023-10-09T04:33:14.254187+0000","last_clean":"2023-10-09T04:33:14.254187+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:33:14.254187+0000","last_undegraded":"2023-10-09T04:33:14.254187+0000","last_fullsized":"2023-10-09T04:33:14.254187+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.250978+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:33:14.250978+0000","last_peered":"2023-10-09T04:33:14.250978+0000","last_clean":"2023-10-09T04:33:14.250978+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:33:14.250978+0000","last_undegraded":"2023-10-09T04:33:14.250978+0000","last_fullsized":"2023-10-09T04:33:14.250978+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"187'12280","reported_seq":12331,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.950322+0000","last_change":"2023-10-09T04:29:24.161371+0000","last_active":"2023-10-09T04:33:29.950322+0000","last_peered":"2023-10-09T04:33:29.950322+0000","last_clean":"2023-10-09T04:33:29.950322+0000","last_became_active":"2023-10-09T04:29:24.161102+0000","last_became_peered":"2023-10-09T04:29:24.161102+0000","last_unstale":"2023-10-09T04:33:29.950322+0000","last_undegraded":"2023-10-09T04:33:29.950322+0000","last_fullsized":"2023-10-09T04:33:29.950322+0000","mapping_epoch":144,"log_start":"152'2200","ondisk_log_start":"152'2200","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:15:31.094244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15335424,"num_objects":234,"num_object_clones":0,"num_object_copies":702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12280,"num_write_kb":400448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":160,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251516+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:33:14.251516+0000","last_peered":"2023-10-09T04:33:14.251516+0000","last_clean":"2023-10-09T04:33:14.251516+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:33:14.251516+0000","last_undegraded":"2023-10-09T04:33:14.251516+0000","last_fullsized":"2023-10-09T04:33:14.251516+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":164,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235305+0000","last_change":"2023-10-09T04:27:57.993021+0000","last_active":"2023-10-09T04:33:14.235305+0000","last_peered":"2023-10-09T04:33:14.235305+0000","last_clean":"2023-10-09T04:33:14.235305+0000","last_became_active":"2023-10-09T04:27:57.992486+0000","last_became_peered":"2023-10-09T04:27:57.992486+0000","last_unstale":"2023-10-09T04:33:14.235305+0000","last_undegraded":"2023-10-09T04:33:14.235305+0000","last_fullsized":"2023-10-09T04:33:14.235305+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:56:56.130811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.250892+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:33:14.250892+0000","last_peered":"2023-10-09T04:33:14.250892+0000","last_clean":"2023-10-09T04:33:14.250892+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:33:14.250892+0000","last_undegraded":"2023-10-09T04:33:14.250892+0000","last_fullsized":"2023-10-09T04:33:14.250892+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"187'12116","reported_seq":16999,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:32.524533+0000","last_change":"2023-10-09T04:30:44.045543+0000","last_active":"2023-10-09T04:33:32.524533+0000","last_peered":"2023-10-09T04:33:32.524533+0000","last_clean":"2023-10-09T04:33:32.524533+0000","last_became_active":"2023-10-09T04:30:17.679175+0000","last_became_peered":"2023-10-09T04:30:17.679175+0000","last_unstale":"2023-10-09T04:33:32.524533+0000","last_undegraded":"2023-10-09T04:33:32.524533+0000","last_fullsized":"2023-10-09T04:33:32.524533+0000","mapping_epoch":160,"log_start":"152'2100","ondisk_log_start":"152'2100","created":144,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10016,"log_dups_size":0,"ondisk_log_size":10016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:56.902852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7340032,"num_objects":112,"num_object_clones":0,"num_object_copies":336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12116,"num_write_kb":391296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2407,"num_bytes_recovered":157745152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235167+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:33:14.235167+0000","last_peered":"2023-10-09T04:33:14.235167+0000","last_clean":"2023-10-09T04:33:14.235167+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:33:14.235167+0000","last_undegraded":"2023-10-09T04:33:14.235167+0000","last_fullsized":"2023-10-09T04:33:14.235167+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":168,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258447+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:33:14.258447+0000","last_peered":"2023-10-09T04:33:14.258447+0000","last_clean":"2023-10-09T04:33:14.258447+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:33:14.258447+0000","last_undegraded":"2023-10-09T04:33:14.258447+0000","last_fullsized":"2023-10-09T04:33:14.258447+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":166,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251495+0000","last_change":"2023-10-09T04:27:57.991197+0000","last_active":"2023-10-09T04:33:14.251495+0000","last_peered":"2023-10-09T04:33:14.251495+0000","last_clean":"2023-10-09T04:33:14.251495+0000","last_became_active":"2023-10-09T04:27:57.990548+0000","last_became_peered":"2023-10-09T04:27:57.990548+0000","last_unstale":"2023-10-09T04:33:14.251495+0000","last_undegraded":"2023-10-09T04:33:14.251495+0000","last_fullsized":"2023-10-09T04:33:14.251495+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:50:05.799943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"187'11991","reported_seq":12042,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.902203+0000","last_change":"2023-10-09T04:29:24.161931+0000","last_active":"2023-10-09T04:33:29.902203+0000","last_peered":"2023-10-09T04:33:29.902203+0000","last_clean":"2023-10-09T04:33:29.902203+0000","last_became_active":"2023-10-09T04:29:24.161582+0000","last_became_peered":"2023-10-09T04:29:24.161582+0000","last_unstale":"2023-10-09T04:33:29.902203+0000","last_undegraded":"2023-10-09T04:33:29.902203+0000","last_fullsized":"2023-10-09T04:33:29.902203+0000","mapping_epoch":144,"log_start":"152'1900","ondisk_log_start":"152'1900","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10091,"log_dups_size":0,"ondisk_log_size":10091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.147516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12648448,"num_objects":193,"num_object_clones":0,"num_object_copies":579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11991,"num_write_kb":389888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":154,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.228025+0000","last_change":"2023-10-09T04:29:51.233999+0000","last_active":"2023-10-09T04:33:14.228025+0000","last_peered":"2023-10-09T04:33:14.228025+0000","last_clean":"2023-10-09T04:33:14.228025+0000","last_became_active":"2023-10-09T04:29:51.233578+0000","last_became_peered":"2023-10-09T04:29:51.233578+0000","last_unstale":"2023-10-09T04:33:14.228025+0000","last_undegraded":"2023-10-09T04:33:14.228025+0000","last_fullsized":"2023-10-09T04:33:14.228025+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:44.172048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.250715+0000","last_change":"2023-10-09T04:27:29.982732+0000","last_active":"2023-10-09T04:33:14.250715+0000","last_peered":"2023-10-09T04:33:14.250715+0000","last_clean":"2023-10-09T04:33:14.250715+0000","last_became_active":"2023-10-09T04:27:29.982275+0000","last_became_peered":"2023-10-09T04:27:29.982275+0000","last_unstale":"2023-10-09T04:33:14.250715+0000","last_undegraded":"2023-10-09T04:33:14.250715+0000","last_fullsized":"2023-10-09T04:33:14.250715+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:43.712715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251052+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:33:14.251052+0000","last_peered":"2023-10-09T04:33:14.251052+0000","last_clean":"2023-10-09T04:33:14.251052+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:33:14.251052+0000","last_undegraded":"2023-10-09T04:33:14.251052+0000","last_fullsized":"2023-10-09T04:33:14.251052+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"187'12045","reported_seq":16838,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:31.506648+0000","last_change":"2023-10-09T04:31:24.127748+0000","last_active":"2023-10-09T04:33:31.506648+0000","last_peered":"2023-10-09T04:33:31.506648+0000","last_clean":"2023-10-09T04:33:31.506648+0000","last_became_active":"2023-10-09T04:31:24.115397+0000","last_became_peered":"2023-10-09T04:31:24.115397+0000","last_unstale":"2023-10-09T04:33:31.506648+0000","last_undegraded":"2023-10-09T04:33:31.506648+0000","last_fullsized":"2023-10-09T04:33:31.506648+0000","mapping_epoch":167,"log_start":"161'4300","ondisk_log_start":"161'4300","created":144,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:25.364174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11599872,"num_objects":177,"num_object_clones":0,"num_object_copies":531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12045,"num_write_kb":391104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2357,"num_bytes_recovered":154468352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235222+0000","last_change":"2023-10-09T04:27:29.988702+0000","last_active":"2023-10-09T04:33:14.235222+0000","last_peered":"2023-10-09T04:33:14.235222+0000","last_clean":"2023-10-09T04:33:14.235222+0000","last_became_active":"2023-10-09T04:27:29.988390+0000","last_became_peered":"2023-10-09T04:27:29.988390+0000","last_unstale":"2023-10-09T04:33:14.235222+0000","last_undegraded":"2023-10-09T04:33:14.235222+0000","last_fullsized":"2023-10-09T04:33:14.235222+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:45:33.897326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":579,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.248670+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:33:14.248670+0000","last_peered":"2023-10-09T04:33:14.248670+0000","last_clean":"2023-10-09T04:33:14.248670+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:33:14.248670+0000","last_undegraded":"2023-10-09T04:33:14.248670+0000","last_fullsized":"2023-10-09T04:33:14.248670+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":109,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.257540+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:33:14.257540+0000","last_peered":"2023-10-09T04:33:14.257540+0000","last_clean":"2023-10-09T04:33:14.257540+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:33:14.257540+0000","last_undegraded":"2023-10-09T04:33:14.257540+0000","last_fullsized":"2023-10-09T04:33:14.257540+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":128,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.255805+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:33:14.255805+0000","last_peered":"2023-10-09T04:33:14.255805+0000","last_clean":"2023-10-09T04:33:14.255805+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:33:14.255805+0000","last_undegraded":"2023-10-09T04:33:14.255805+0000","last_fullsized":"2023-10-09T04:33:14.255805+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"187'12068","reported_seq":16955,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:32.591847+0000","last_change":"2023-10-09T04:30:45.687596+0000","last_active":"2023-10-09T04:33:32.591847+0000","last_peered":"2023-10-09T04:33:32.591847+0000","last_clean":"2023-10-09T04:33:32.591847+0000","last_became_active":"2023-10-09T04:30:45.682622+0000","last_became_peered":"2023-10-09T04:30:45.682622+0000","last_unstale":"2023-10-09T04:33:32.591847+0000","last_undegraded":"2023-10-09T04:33:32.591847+0000","last_fullsized":"2023-10-09T04:33:32.591847+0000","mapping_epoch":163,"log_start":"152'2000","ondisk_log_start":"152'2000","created":144,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":10068,"log_dups_size":0,"ondisk_log_size":10068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:10.606752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6422528,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12068,"num_write_kb":389312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2410,"num_bytes_recovered":157941760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":187,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.248660+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:33:14.248660+0000","last_peered":"2023-10-09T04:33:14.248660+0000","last_clean":"2023-10-09T04:33:14.248660+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:33:14.248660+0000","last_undegraded":"2023-10-09T04:33:14.248660+0000","last_fullsized":"2023-10-09T04:33:14.248660+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.242823+0000","last_change":"2023-10-09T04:29:51.054503+0000","last_active":"2023-10-09T04:33:14.242823+0000","last_peered":"2023-10-09T04:33:14.242823+0000","last_clean":"2023-10-09T04:33:14.242823+0000","last_became_active":"2023-10-09T04:29:51.054130+0000","last_became_peered":"2023-10-09T04:29:51.054130+0000","last_unstale":"2023-10-09T04:33:14.242823+0000","last_undegraded":"2023-10-09T04:33:14.242823+0000","last_fullsized":"2023-10-09T04:33:14.242823+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:54:33.217600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.242539+0000","last_change":"2023-10-09T04:29:51.235611+0000","last_active":"2023-10-09T04:33:14.242539+0000","last_peered":"2023-10-09T04:33:14.242539+0000","last_clean":"2023-10-09T04:33:14.242539+0000","last_became_active":"2023-10-09T04:29:51.235272+0000","last_became_peered":"2023-10-09T04:29:51.235272+0000","last_unstale":"2023-10-09T04:33:14.242539+0000","last_undegraded":"2023-10-09T04:33:14.242539+0000","last_fullsized":"2023-10-09T04:33:14.242539+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:21:41.701077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"187'12331","reported_seq":12381,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.923973+0000","last_change":"2023-10-09T04:29:24.160813+0000","last_active":"2023-10-09T04:33:29.923973+0000","last_peered":"2023-10-09T04:33:29.923973+0000","last_clean":"2023-10-09T04:33:29.923973+0000","last_became_active":"2023-10-09T04:29:24.160543+0000","last_became_peered":"2023-10-09T04:29:24.160543+0000","last_unstale":"2023-10-09T04:33:29.923973+0000","last_undegraded":"2023-10-09T04:33:29.923973+0000","last_fullsized":"2023-10-09T04:33:29.923973+0000","mapping_epoch":144,"log_start":"161'4600","ondisk_log_start":"161'4600","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:58:57.034286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16056320,"num_objects":245,"num_object_clones":0,"num_object_copies":735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12331,"num_write_kb":402432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":170,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.249597+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:33:14.249597+0000","last_peered":"2023-10-09T04:33:14.249597+0000","last_clean":"2023-10-09T04:33:14.249597+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:33:14.249597+0000","last_undegraded":"2023-10-09T04:33:14.249597+0000","last_fullsized":"2023-10-09T04:33:14.249597+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251405+0000","last_change":"2023-10-09T04:27:29.981956+0000","last_active":"2023-10-09T04:33:14.251405+0000","last_peered":"2023-10-09T04:33:14.251405+0000","last_clean":"2023-10-09T04:33:14.251405+0000","last_became_active":"2023-10-09T04:27:29.981663+0000","last_became_peered":"2023-10-09T04:27:29.981663+0000","last_unstale":"2023-10-09T04:33:14.251405+0000","last_undegraded":"2023-10-09T04:33:14.251405+0000","last_fullsized":"2023-10-09T04:33:14.251405+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:47.639333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258326+0000","last_change":"2023-10-09T04:29:51.056593+0000","last_active":"2023-10-09T04:33:14.258326+0000","last_peered":"2023-10-09T04:33:14.258326+0000","last_clean":"2023-10-09T04:33:14.258326+0000","last_became_active":"2023-10-09T04:29:51.055758+0000","last_became_peered":"2023-10-09T04:29:51.055758+0000","last_unstale":"2023-10-09T04:33:14.258326+0000","last_undegraded":"2023-10-09T04:33:14.258326+0000","last_fullsized":"2023-10-09T04:33:14.258326+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:42.366511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"187'12082","reported_seq":12133,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:29.949859+0000","last_change":"2023-10-09T04:29:24.160706+0000","last_active":"2023-10-09T04:33:29.949859+0000","last_peered":"2023-10-09T04:33:29.949859+0000","last_clean":"2023-10-09T04:33:29.949859+0000","last_became_active":"2023-10-09T04:29:24.160380+0000","last_became_peered":"2023-10-09T04:29:24.160380+0000","last_unstale":"2023-10-09T04:33:29.949859+0000","last_undegraded":"2023-10-09T04:33:29.949859+0000","last_fullsized":"2023-10-09T04:33:29.949859+0000","mapping_epoch":144,"log_start":"161'4300","ondisk_log_start":"161'4300","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:40:06.208544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14352408,"num_objects":220,"num_object_clones":0,"num_object_copies":660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":220,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12082,"num_write_kb":393601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":181,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.252750+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:33:14.252750+0000","last_peered":"2023-10-09T04:33:14.252750+0000","last_clean":"2023-10-09T04:33:14.252750+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:33:14.252750+0000","last_undegraded":"2023-10-09T04:33:14.252750+0000","last_fullsized":"2023-10-09T04:33:14.252750+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":136,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251154+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:33:14.251154+0000","last_peered":"2023-10-09T04:33:14.251154+0000","last_clean":"2023-10-09T04:33:14.251154+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:33:14.251154+0000","last_undegraded":"2023-10-09T04:33:14.251154+0000","last_fullsized":"2023-10-09T04:33:14.251154+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":170,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.254274+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:33:14.254274+0000","last_peered":"2023-10-09T04:33:14.254274+0000","last_clean":"2023-10-09T04:33:14.254274+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:33:14.254274+0000","last_undegraded":"2023-10-09T04:33:14.254274+0000","last_fullsized":"2023-10-09T04:33:14.254274+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"187'12185","reported_seq":12237,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-10-09T04:33:31.495480+0000","last_change":"2023-10-09T04:29:24.159100+0000","last_active":"2023-10-09T04:33:31.495480+0000","last_peered":"2023-10-09T04:33:31.495480+0000","last_clean":"2023-10-09T04:33:31.495480+0000","last_became_active":"2023-10-09T04:29:24.158495+0000","last_became_peered":"2023-10-09T04:29:24.158495+0000","last_unstale":"2023-10-09T04:33:31.495480+0000","last_undegraded":"2023-10-09T04:33:31.495480+0000","last_fullsized":"2023-10-09T04:33:31.495480+0000","mapping_epoch":144,"log_start":"161'4400","ondisk_log_start":"161'4400","created":144,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:29:23.136965+0000","last_clean_scrub_stamp":"2023-10-09T04:29:23.136965+0000","objects_scrubbed":0,"log_size":7785,"log_dups_size":0,"ondisk_log_size":7785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:31.573022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10813440,"num_objects":165,"num_object_clones":0,"num_object_copies":495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12185,"num_write_kb":395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":150,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.255977+0000","last_change":"2023-10-09T04:27:30.000853+0000","last_active":"2023-10-09T04:33:14.255977+0000","last_peered":"2023-10-09T04:33:14.255977+0000","last_clean":"2023-10-09T04:33:14.255977+0000","last_became_active":"2023-10-09T04:27:30.000545+0000","last_became_peered":"2023-10-09T04:27:30.000545+0000","last_unstale":"2023-10-09T04:33:14.255977+0000","last_undegraded":"2023-10-09T04:33:14.255977+0000","last_fullsized":"2023-10-09T04:33:14.255977+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:53:47.506034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":49,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.242615+0000","last_change":"2023-10-09T04:29:51.062083+0000","last_active":"2023-10-09T04:33:14.242615+0000","last_peered":"2023-10-09T04:33:14.242615+0000","last_clean":"2023-10-09T04:33:14.242615+0000","last_became_active":"2023-10-09T04:29:51.055667+0000","last_became_peered":"2023-10-09T04:29:51.055667+0000","last_unstale":"2023-10-09T04:33:14.242615+0000","last_undegraded":"2023-10-09T04:33:14.242615+0000","last_fullsized":"2023-10-09T04:33:14.242615+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:09:08.504718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":130,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258325+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:33:14.258325+0000","last_peered":"2023-10-09T04:33:14.258325+0000","last_clean":"2023-10-09T04:33:14.258325+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:33:14.258325+0000","last_undegraded":"2023-10-09T04:33:14.258325+0000","last_fullsized":"2023-10-09T04:33:14.258325+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":45,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258198+0000","last_change":"2023-10-09T04:29:51.057053+0000","last_active":"2023-10-09T04:33:14.258198+0000","last_peered":"2023-10-09T04:33:14.258198+0000","last_clean":"2023-10-09T04:33:14.258198+0000","last_became_active":"2023-10-09T04:29:51.056398+0000","last_became_peered":"2023-10-09T04:29:51.056398+0000","last_unstale":"2023-10-09T04:33:14.258198+0000","last_undegraded":"2023-10-09T04:33:14.258198+0000","last_fullsized":"2023-10-09T04:33:14.258198+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:08:11.102317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":158,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.228352+0000","last_change":"2023-10-09T04:27:57.989106+0000","last_active":"2023-10-09T04:33:14.228352+0000","last_peered":"2023-10-09T04:33:14.228352+0000","last_clean":"2023-10-09T04:33:14.228352+0000","last_became_active":"2023-10-09T04:27:57.988778+0000","last_became_peered":"2023-10-09T04:27:57.988778+0000","last_unstale":"2023-10-09T04:33:14.228352+0000","last_undegraded":"2023-10-09T04:33:14.228352+0000","last_fullsized":"2023-10-09T04:33:14.228352+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:16:33.473019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":149,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235082+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:33:14.235082+0000","last_peered":"2023-10-09T04:33:14.235082+0000","last_clean":"2023-10-09T04:33:14.235082+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:33:14.235082+0000","last_undegraded":"2023-10-09T04:33:14.235082+0000","last_fullsized":"2023-10-09T04:33:14.235082+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":164,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258658+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:33:14.258658+0000","last_peered":"2023-10-09T04:33:14.258658+0000","last_clean":"2023-10-09T04:33:14.258658+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:33:14.258658+0000","last_undegraded":"2023-10-09T04:33:14.258658+0000","last_fullsized":"2023-10-09T04:33:14.258658+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":132,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.235355+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:33:14.235355+0000","last_peered":"2023-10-09T04:33:14.235355+0000","last_clean":"2023-10-09T04:33:14.235355+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:33:14.235355+0000","last_undegraded":"2023-10-09T04:33:14.235355+0000","last_fullsized":"2023-10-09T04:33:14.235355+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258516+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:33:14.258516+0000","last_peered":"2023-10-09T04:33:14.258516+0000","last_clean":"2023-10-09T04:33:14.258516+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:33:14.258516+0000","last_undegraded":"2023-10-09T04:33:14.258516+0000","last_fullsized":"2023-10-09T04:33:14.258516+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":156,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.252595+0000","last_change":"2023-10-09T04:27:57.989240+0000","last_active":"2023-10-09T04:33:14.252595+0000","last_peered":"2023-10-09T04:33:14.252595+0000","last_clean":"2023-10-09T04:33:14.252595+0000","last_became_active":"2023-10-09T04:27:57.988856+0000","last_became_peered":"2023-10-09T04:27:57.988856+0000","last_unstale":"2023-10-09T04:33:14.252595+0000","last_undegraded":"2023-10-09T04:33:14.252595+0000","last_fullsized":"2023-10-09T04:33:14.252595+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:40:52.380796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.253356+0000","last_change":"2023-10-09T04:29:51.233809+0000","last_active":"2023-10-09T04:33:14.253356+0000","last_peered":"2023-10-09T04:33:14.253356+0000","last_clean":"2023-10-09T04:33:14.253356+0000","last_became_active":"2023-10-09T04:29:51.233460+0000","last_became_peered":"2023-10-09T04:29:51.233460+0000","last_unstale":"2023-10-09T04:33:14.253356+0000","last_undegraded":"2023-10-09T04:33:14.253356+0000","last_fullsized":"2023-10-09T04:33:14.253356+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:33.172831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":51,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.251991+0000","last_change":"2023-10-09T04:29:51.238602+0000","last_active":"2023-10-09T04:33:14.251991+0000","last_peered":"2023-10-09T04:33:14.251991+0000","last_clean":"2023-10-09T04:33:14.251991+0000","last_became_active":"2023-10-09T04:29:51.234688+0000","last_became_peered":"2023-10-09T04:29:51.234688+0000","last_unstale":"2023-10-09T04:33:14.251991+0000","last_undegraded":"2023-10-09T04:33:14.251991+0000","last_fullsized":"2023-10-09T04:33:14.251991+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:58.339304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":157,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-09T04:33:14.258235+0000","last_change":"2023-10-09T04:27:29.981763+0000","last_active":"2023-10-09T04:33:14.258235+0000","last_peered":"2023-10-09T04:33:14.258235+0000","last_clean":"2023-10-09T04:33:14.258235+0000","last_became_active":"2023-10-09T04:27:29.981401+0000","last_became_peered":"2023-10-09T04:27:29.981401+0000","last_unstale":"2023-10-09T04:33:14.258235+0000","last_undegraded":"2023-10-09T04:33:14.258235+0000","last_fullsized":"2023-10-09T04:33:14.258235+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:35.242521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":192544792,"num_objects":2939,"num_object_clones":0,"num_object_copies":8817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2939,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193663,"num_write_kb":6291201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9602,"num_bytes_recovered":576323584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":590753792,"data_stored":590741576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139263,"ondisk_log_size":139263,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234319,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064328,"kb_used_data":153344,"kb_used_omap":0,"kb_used_meta":910976,"kb_avail":92680824,"statfs":{"total":95995035648,"available":94905163776,"internally_reserved":0,"allocated":157024256,"data_stored":156502499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932839424},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":5.9249999999999998,"15min":5.2309999999999999},"min":{"1min":0.247,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":7.8129999999999997,"5min":186.446,"15min":629.22500000000002},"last":111.72199999999999},{"interface":"front","average":{"1min":2.6000000000000001,"5min":6.1550000000000002,"15min":5.1219999999999999},"min":{"1min":0.312,"5min":0.245,"15min":0.245},"max":{"1min":8.5850000000000009,"5min":186.429,"15min":629.43100000000004},"last":111.76600000000001}]},{"osd":1,"last update":"Mon Oct 9 04:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":8.2720000000000002,"15min":4.2190000000000003},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.7789999999999999,"5min":224.13499999999999,"15min":224.13499999999999},"last":0.434},{"interface":"front","average":{"1min":3.0289999999999999,"5min":8.3119999999999994,"15min":4.2699999999999996},"min":{"1min":0.32900000000000001,"5min":0.27200000000000002,"15min":0.25700000000000001},"max":{"1min":9.5749999999999993,"5min":224.203,"15min":224.203},"last":0.39900000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.306,"15min":0.78800000000000003},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.890000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.5059999999999998,"5min":1.339,"15min":0.80900000000000005},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":10.784000000000001,"5min":10.784000000000001,"15min":10.784000000000001},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:33:04 2023","interfaces":[{"interface":"back","average":{"1min":10.27,"5min":12.497999999999999,"15min":4.5309999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":110.50700000000001,"5min":249.989,"15min":249.989},"last":133.13499999999999},{"interface":"front","average":{"1min":10.083,"5min":12.624000000000001,"15min":4.5940000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":112.54600000000001,"5min":250.21700000000001,"15min":250.21700000000001},"last":133.35900000000001}]},{"osd":4,"last update":"Mon Oct 9 04:32:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3370000000000002,"5min":1.133,"15min":1.1419999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23799999999999999},"max":{"1min":10.869,"5min":10.869,"15min":10.869},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.431,"5min":0.76500000000000001,"15min":0.89100000000000001},"min":{"1min":0.34999999999999998,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":10.613,"5min":10.613,"15min":10.613},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Oct 9 04:32:35 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":0.80900000000000005,"15min":0.93700000000000006},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":4.3099999999999996,"5min":5.2599999999999998,"15min":8.8610000000000007},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.992,"5min":1.175,"15min":1.115},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":10.895,"5min":10.895,"15min":10.895},"last":0.47799999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:32:35 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":0.95799999999999996,"15min":1.1319999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":4.5220000000000002,"5min":5.2350000000000003,"15min":12.007},"last":0.501},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.0760000000000001,"15min":1.1419999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.31900000000000001},"max":{"1min":10.917,"5min":10.917,"15min":10.917},"last":0.45200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495954,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":314620,"kb_used_data":9140,"kb_used_omap":0,"kb_used_meta":305472,"kb_avail":93430532,"statfs":{"total":95995035648,"available":95672864768,"internally_reserved":0,"allocated":9359360,"data_stored":8857382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312803328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:23 2023","interfaces":[{"interface":"back","average":{"1min":65.793999999999997,"5min":15.589,"15min":6.4669999999999996},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":1284.521,"5min":1284.521,"15min":1284.521},"last":0.28299999999999997},{"interface":"front","average":{"1min":65.739999999999995,"5min":15.513999999999999,"15min":6.7889999999999997},"min":{"1min":0.314,"5min":0.253,"15min":0.253},"max":{"1min":1284.4690000000001,"5min":1284.4690000000001,"15min":1284.4690000000001},"last":1.048}]},{"osd":1,"last update":"Mon Oct 9 04:33:18 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":5.4320000000000004,"15min":3.2839999999999998},"min":{"1min":0.32600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.7699999999999996,"5min":185.74000000000001,"15min":185.74000000000001},"last":2.552},{"interface":"front","average":{"1min":1.7949999999999999,"5min":5.4299999999999997,"15min":3.4420000000000002},"min":{"1min":0.26500000000000001,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":5.915,"5min":189.03999999999999,"15min":189.03999999999999},"last":0.36199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.629,"5min":1.454,"15min":0.86599999999999999},"min":{"1min":0.375,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.6079999999999997,"5min":5.7880000000000003,"15min":5.7880000000000003},"last":2.5760000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.089,"15min":0.97099999999999997},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.891,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":0.32100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:33:07 2023","interfaces":[{"interface":"back","average":{"1min":4.548,"5min":2.0219999999999998,"15min":1.1000000000000001},"min":{"1min":0.47199999999999998,"5min":0.314,"15min":0.314},"max":{"1min":59.392000000000003,"5min":59.392000000000003,"15min":59.392000000000003},"last":0.435},{"interface":"front","average":{"1min":5.0979999999999999,"5min":2.1419999999999999,"15min":1.093},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":59.203000000000003,"5min":59.203000000000003,"15min":59.203000000000003},"last":4.5640000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:33:23 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.1459999999999999,"15min":1.135},"min":{"1min":0.30399999999999999,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":5.4699999999999998,"5min":8.0950000000000006,"15min":12.743},"last":2.5190000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.82599999999999996,"15min":1.085},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":4.3159999999999998,"5min":8.9920000000000009,"15min":12.028},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:33:23 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":2.3959999999999999,"15min":1.599},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":5.5030000000000001,"5min":81.087000000000003,"15min":81.087000000000003},"last":2.6269999999999998},{"interface":"front","average":{"1min":1.4330000000000001,"5min":2.2290000000000001,"15min":1.6699999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.27400000000000002},"max":{"1min":5.8319999999999999,"5min":81.706999999999994,"15min":81.706999999999994},"last":0.40999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:32:53 2023","interfaces":[{"interface":"back","average":{"1min":5.4610000000000003,"5min":1.819,"15min":1.6160000000000001},"min":{"1min":0.36899999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":81.674999999999997,"5min":81.674999999999997,"15min":81.674999999999997},"last":0.55500000000000005},{"interface":"front","average":{"1min":6.3360000000000003,"5min":1.958,"15min":1.762},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":81.055000000000007,"5min":81.055000000000007,"15min":81.055000000000007},"last":2.6030000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141043,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":573760,"kb_used_data":188152,"kb_used_omap":4,"kb_used_meta":385467,"kb_avail":93171392,"statfs":{"total":95995035648,"available":95407505408,"internally_reserved":0,"allocated":192667648,"data_stored":191926800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":394719118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:25 2023","interfaces":[{"interface":"back","average":{"1min":8.1989999999999998,"5min":4.4109999999999996,"15min":3.4140000000000001},"min":{"1min":0.23999999999999999,"5min":0.20899999999999999,"15min":0.19600000000000001},"max":{"1min":97.308000000000007,"5min":97.308000000000007,"15min":97.308000000000007},"last":0.18099999999999999},{"interface":"front","average":{"1min":9.2750000000000004,"5min":4.9710000000000001,"15min":3.7080000000000002},"min":{"1min":0.314,"5min":0.253,"15min":0.24399999999999999},"max":{"1min":97.055999999999997,"5min":97.055999999999997,"15min":97.055999999999997},"last":5.5519999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:32:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":1.631,"15min":0.77400000000000002},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.872,"5min":10.872,"15min":10.872},"last":2.8879999999999999},{"interface":"front","average":{"1min":1.6779999999999999,"5min":1.1259999999999999,"15min":0.59999999999999998},"min":{"1min":0.251,"5min":0.218,"15min":0.218},"max":{"1min":6.2590000000000003,"5min":8.048,"15min":8.048},"last":7.8159999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:33:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":4.3239999999999998,"15min":1.706},"min":{"1min":0.30399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.29,"5min":195.035,"15min":195.035},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.4039999999999999,"5min":4.6820000000000004,"15min":1.8300000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.3929999999999998,"5min":195.465,"15min":195.465},"last":2.9180000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:33:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":1.627,"15min":2.7440000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.7890000000000001,"5min":12.537000000000001,"15min":107.795},"last":7.8799999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.23,"15min":1.653},"min":{"1min":0.38700000000000001,"5min":0.33100000000000002,"15min":0.32700000000000001},"max":{"1min":4.6639999999999997,"5min":6.1369999999999996,"15min":109.167},"last":0.28799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:33:25 2023","interfaces":[{"interface":"back","average":{"1min":3.2759999999999998,"5min":2.0409999999999999,"15min":2.4369999999999998},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.33700000000000002},"max":{"1min":10.186999999999999,"5min":10.186999999999999,"15min":12.414999999999999},"last":3.335},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9390000000000001,"15min":2.6400000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.061999999999999,"5min":12.605,"15min":13.933999999999999},"last":7.944}]},{"osd":6,"last update":"Mon Oct 9 04:33:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8220000000000001,"5min":1.877,"15min":2.27},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.36599999999999999},"max":{"1min":8.2799999999999994,"5min":8.3770000000000007,"15min":13.388},"last":5.5199999999999996},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.847,"15min":2.6899999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":8.5860000000000003,"5min":12.632999999999999,"15min":13.882},"last":12.596}]},{"osd":7,"last update":"Mon Oct 9 04:33:25 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":1.8839999999999999,"15min":2.7589999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":4.944,"5min":12.656000000000001,"15min":13.907},"last":8.4160000000000004},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.4350000000000001,"15min":1.595},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9050000000000002,"5min":6.1799999999999997,"15min":7.8239999999999998},"last":4.2050000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024100,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438204,"kb_used_data":33076,"kb_used_omap":5,"kb_used_meta":405114,"kb_avail":93306948,"statfs":{"total":95995035648,"available":95546314752,"internally_reserved":0,"allocated":33869824,"data_stored":33133001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":414837370},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:33:17 2023","interfaces":[{"interface":"back","average":{"1min":5.0490000000000004,"5min":6.0330000000000004,"15min":4.2060000000000004},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":65.173000000000002,"5min":167.92699999999999,"15min":167.92699999999999},"last":2.4569999999999999},{"interface":"front","average":{"1min":6.0330000000000004,"5min":6.2759999999999998,"15min":3.9700000000000002},"min":{"1min":0.33700000000000002,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":66.620000000000005,"5min":169.78999999999999,"15min":169.78999999999999},"last":0.48599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:32:38 2023","interfaces":[{"interface":"back","average":{"1min":3.0899999999999999,"5min":2.6960000000000002,"15min":1.151},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":12.962,"5min":107.548,"15min":107.548},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.6309999999999998,"5min":2.3370000000000002,"15min":1.0149999999999999},"min":{"1min":0.247,"5min":0.22,"15min":0.22},"max":{"1min":8.7739999999999991,"5min":107.35899999999999,"15min":107.35899999999999},"last":0.94399999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:33:07 2023","interfaces":[{"interface":"back","average":{"1min":2.29,"5min":4.923,"15min":1.895},"min":{"1min":0.27300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.6969999999999992,"5min":213.28399999999999,"15min":213.28399999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":3.1000000000000001,"5min":5.0039999999999996,"15min":1.9359999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":10.895,"5min":212.27000000000001,"15min":212.27000000000001},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:33:30 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.8,"15min":1.708},"min":{"1min":0.34399999999999997,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":6.1550000000000002,"5min":10.717000000000001,"15min":10.73},"last":0.84099999999999997},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.0310000000000001,"15min":1.8320000000000001},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":10.303000000000001,"5min":15.941000000000001,"15min":15.941000000000001},"last":0.55600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:33:30 2023","interfaces":[{"interface":"back","average":{"1min":3.5779999999999998,"5min":2.117,"15min":1.9510000000000001},"min":{"1min":0.433,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":12.013999999999999,"5min":12.807,"15min":12.807},"last":1.073},{"interface":"front","average":{"1min":2.718,"5min":1.581,"15min":1.778},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.308},"max":{"1min":8.4589999999999996,"5min":8.4589999999999996,"15min":14.144},"last":0.35599999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.1120000000000001,"15min":2.0760000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.35499999999999998},"max":{"1min":10.351000000000001,"5min":16.369,"15min":16.369},"last":1.0409999999999999},{"interface":"front","average":{"1min":2.165,"5min":1.55,"15min":1.954},"min":{"1min":0.442,"5min":0.41199999999999998,"15min":0.378},"max":{"1min":8.4930000000000003,"5min":8.4930000000000003,"15min":14.090999999999999},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6629999999999998,"5min":1.6319999999999999,"15min":1.909},"min":{"1min":0.52800000000000002,"5min":0.42199999999999999,"15min":0.40699999999999997},"max":{"1min":8.5229999999999997,"5min":8.5229999999999997,"15min":11.890000000000001},"last":0.433},{"interface":"front","average":{"1min":2.4670000000000001,"5min":1.9970000000000001,"15min":2.0649999999999999},"min":{"1min":0.40400000000000003,"5min":0.39200000000000002,"15min":0.35399999999999998},"max":{"1min":9.7110000000000003,"5min":9.7110000000000003,"15min":11.176},"last":0.877}]}]},{"osd":2,"up_from":135,"seq":579820585035,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":247920,"kb_used_data":29160,"kb_used_omap":7,"kb_used_meta":218680,"kb_avail":93497232,"statfs":{"total":95995035648,"available":95741165568,"internally_reserved":0,"allocated":29859840,"data_stored":29192995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":223928401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6859999999999999,"5min":4.2229999999999999,"15min":1.625},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":9.1240000000000006,"5min":129.465,"15min":129.465},"last":6.4390000000000001},{"interface":"front","average":{"1min":2.5939999999999999,"5min":4.1970000000000001,"15min":1.6419999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":10.019,"5min":129.51599999999999,"15min":129.51599999999999},"last":4.8620000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":4.641,"5min":3.2669999999999999,"15min":1.3069999999999999},"min":{"1min":0.21299999999999999,"5min":0.193,"15min":0.193},"max":{"1min":41.055,"5min":54.542999999999999,"15min":54.542999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":4.0979999999999999,"5min":3.028,"15min":1.238},"min":{"1min":0.26800000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":39.984000000000002,"5min":51.579999999999998,"15min":51.579999999999998},"last":0.39900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:33:14 2023","interfaces":[{"interface":"back","average":{"1min":18.420999999999999,"5min":5.8620000000000001,"15min":2.2040000000000002},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":147.655,"5min":147.655,"15min":147.655},"last":3.5600000000000001},{"interface":"front","average":{"1min":17.818000000000001,"5min":5.9180000000000001,"15min":2.2629999999999999},"min":{"1min":0.27400000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":147.607,"5min":147.607,"15min":147.607},"last":1.5860000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.502,"15min":0.90100000000000002},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.3849999999999998,"5min":12.765000000000001,"15min":12.765000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":1.0960000000000001,"15min":0.77400000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.5439999999999996,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.41399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":3.4359999999999999,"5min":1.4750000000000001,"15min":0.877},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":21.533999999999999,"5min":21.533999999999999,"15min":21.533999999999999},"last":1.776},{"interface":"front","average":{"1min":3.0600000000000001,"5min":1.7490000000000001,"15min":0.97799999999999998},"min":{"1min":0.41099999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":21.716000000000001,"5min":21.716000000000001,"15min":21.716000000000001},"last":2.8620000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.129,"15min":0.80100000000000005},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8959999999999999,"5min":9.2460000000000004,"15min":9.2460000000000004},"last":6.4000000000000004},{"interface":"front","average":{"1min":2.2050000000000001,"5min":1.48,"15min":0.91800000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":10.140000000000001,"5min":11.641999999999999,"15min":11.641999999999999},"last":2.0379999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:32:44 2023","interfaces":[{"interface":"back","average":{"1min":4.3390000000000004,"5min":1.784,"15min":1.012},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":25.835999999999999,"5min":25.835999999999999,"15min":25.835999999999999},"last":1.03},{"interface":"front","average":{"1min":3.9430000000000001,"5min":1.839,"15min":1.0289999999999999},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":26.416,"5min":26.416,"15min":26.416},"last":0.35899999999999999}]}]},{"osd":3,"up_from":142,"seq":609885356101,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306268,"kb_used_data":91796,"kb_used_omap":2,"kb_used_meta":214461,"kb_avail":93438884,"statfs":{"total":95995035648,"available":95681417216,"internally_reserved":0,"allocated":93999104,"data_stored":93347332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":219608117},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":7.7629999999999999,"5min":2.8010000000000002,"15min":1.359},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":91.634,"5min":91.634,"15min":91.634},"last":0.76800000000000002},{"interface":"front","average":{"1min":7.0110000000000001,"5min":2.8599999999999999,"15min":1.222},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":91.646000000000001,"5min":91.646000000000001,"15min":91.646000000000001},"last":0.78700000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":8.202,"5min":5.0190000000000001,"15min":2.6549999999999998},"min":{"1min":0.22,"5min":0.192,"15min":0.192},"max":{"1min":119.00700000000001,"5min":180.02500000000001,"15min":180.02500000000001},"last":0.80100000000000005},{"interface":"front","average":{"1min":9.4480000000000004,"5min":5.641,"15min":2.375},"min":{"1min":0.29499999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":122.977,"5min":184.00899999999999,"15min":184.00899999999999},"last":0.67800000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":8.7810000000000006,"5min":3.105,"15min":1.3080000000000001},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":118.238,"5min":118.238,"15min":118.238},"last":0.71299999999999997},{"interface":"front","average":{"1min":9.141,"5min":3.1579999999999999,"15min":2.0619999999999998},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":117.64400000000001,"5min":117.64400000000001,"15min":117.64400000000001},"last":0.69599999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":1.4910000000000001,"15min":1.698},"min":{"1min":0.29999999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7679999999999998,"5min":23.526,"15min":23.526},"last":5.7839999999999998},{"interface":"front","average":{"1min":2.2040000000000002,"5min":1.621,"15min":1.1950000000000001},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":9.1440000000000001,"5min":9.1440000000000001,"15min":9.1440000000000001},"last":5.8209999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":6.2169999999999996,"5min":2.456,"15min":1.4399999999999999},"min":{"1min":0.36199999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":57.475000000000001,"5min":57.475000000000001,"15min":57.475000000000001},"last":11.18},{"interface":"front","average":{"1min":6.4089999999999998,"5min":2.71,"15min":1.3600000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":57.651000000000003,"5min":57.651000000000003,"15min":57.651000000000003},"last":11.191000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8740000000000001,"15min":1.0640000000000001},"min":{"1min":0.38,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.3000000000000007,"5min":10.119999999999999,"15min":10.119999999999999},"last":4.2450000000000001},{"interface":"front","average":{"1min":2.532,"5min":1.8420000000000001,"15min":1.798},"min":{"1min":0.38,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.7270000000000003,"5min":23.59,"15min":23.59},"last":5.6429999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:33:00 2023","interfaces":[{"interface":"back","average":{"1min":6.907,"5min":2.6800000000000002,"15min":2.0649999999999999},"min":{"1min":0.56399999999999995,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":57.399000000000001,"5min":57.399000000000001,"15min":57.399000000000001},"last":4.5990000000000002},{"interface":"front","average":{"1min":7.1980000000000004,"5min":2.8420000000000001,"15min":1.609},"min":{"1min":0.47699999999999998,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":57.447000000000003,"5min":57.447000000000003,"15min":57.447000000000003},"last":4.585}]}]},{"osd":4,"up_from":30,"seq":128849019228,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":605316,"kb_used_data":11452,"kb_used_omap":0,"kb_used_meta":593856,"kb_avail":93139836,"statfs":{"total":95995035648,"available":95375192064,"internally_reserved":0,"allocated":11726848,"data_stored":11216533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608108544},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:30 2023","interfaces":[{"interface":"back","average":{"1min":3.5979999999999999,"5min":5.718,"15min":3.1749999999999998},"min":{"1min":0.23799999999999999,"5min":0.23000000000000001,"15min":0.20499999999999999},"max":{"1min":35.145000000000003,"5min":219.577,"15min":219.577},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.9409999999999998,"5min":4.9690000000000003,"15min":3.0779999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.247},"max":{"1min":35.326000000000001,"5min":218.96000000000001,"15min":218.96000000000001},"last":1.365}]},{"osd":1,"last update":"Mon Oct 9 04:33:21 2023","interfaces":[{"interface":"back","average":{"1min":4.2939999999999996,"5min":9.0980000000000008,"15min":4.367},"min":{"1min":0.27900000000000003,"5min":0.246,"15min":0.23400000000000001},"max":{"1min":35.634999999999998,"5min":142.74799999999999,"15min":142.74799999999999},"last":5.0549999999999997},{"interface":"front","average":{"1min":4.1760000000000002,"5min":9.3610000000000007,"15min":4.6360000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26700000000000002},"max":{"1min":35.533999999999999,"5min":139.09700000000001,"15min":139.09700000000001},"last":4.226}]},{"osd":2,"last update":"Mon Oct 9 04:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4889999999999999,"5min":2.5459999999999998,"15min":1.21},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.9100000000000001,"5min":50.972999999999999,"15min":50.972999999999999},"last":4.1829999999999998},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.7170000000000001,"15min":0.95199999999999996},"min":{"1min":0.32100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.335,"5min":39.792000000000002,"15min":39.792000000000002},"last":3.6240000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:33:01 2023","interfaces":[{"interface":"back","average":{"1min":17.379000000000001,"5min":10.377000000000001,"15min":3.8759999999999999},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":209.79599999999999,"5min":262.524,"15min":262.524},"last":0.47999999999999998},{"interface":"front","average":{"1min":18.452000000000002,"5min":10.991,"15min":4.0819999999999999},"min":{"1min":0.38,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":209.68799999999999,"5min":262.55099999999999,"15min":262.55099999999999},"last":5.0170000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:33:01 2023","interfaces":[{"interface":"back","average":{"1min":5.9009999999999998,"5min":1.9239999999999999,"15min":1.2809999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23499999999999999},"max":{"1min":73.510000000000005,"5min":73.510000000000005,"15min":73.510000000000005},"last":0.29999999999999999},{"interface":"front","average":{"1min":6.1360000000000001,"5min":2.415,"15min":1.452},"min":{"1min":0.28799999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":73.828000000000003,"5min":73.828000000000003,"15min":73.828000000000003},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.9650000000000001,"15min":1.3360000000000001},"min":{"1min":0.307,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":13.374000000000001,"5min":52.801000000000002,"15min":52.801000000000002},"last":0.79900000000000004},{"interface":"front","average":{"1min":2.2829999999999999,"5min":1.917,"15min":1.3220000000000001},"min":{"1min":0.32500000000000001,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":7.1619999999999999,"5min":51.064999999999998,"15min":51.064999999999998},"last":7.1619999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:32:50 2023","interfaces":[{"interface":"back","average":{"1min":6.0860000000000003,"5min":1.946,"15min":1.3700000000000001},"min":{"1min":0.39900000000000002,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":76.283000000000001,"5min":76.283000000000001,"15min":76.283000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":6.3959999999999999,"5min":2.0449999999999999,"15min":1.4390000000000001},"min":{"1min":0.35899999999999999,"5min":0.33100000000000002,"15min":0.29199999999999998},"max":{"1min":76.259,"5min":76.259,"15min":76.259},"last":0.83099999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757591,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1036408,"kb_used_data":142000,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":92708744,"statfs":{"total":95995035648,"available":94933753856,"internally_reserved":0,"allocated":145408000,"data_stored":144905741,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:33:24 2023","interfaces":[{"interface":"back","average":{"1min":4.9660000000000002,"5min":5.2309999999999999,"15min":2.992},"min":{"1min":0.27500000000000002,"5min":0.22,"15min":0.22},"max":{"1min":18.164999999999999,"5min":127.108,"15min":127.108},"last":4.1660000000000004},{"interface":"front","average":{"1min":2.3769999999999998,"5min":4.7549999999999999,"15min":2.5939999999999999},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":14.581,"5min":128.036,"15min":128.036},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 04:33:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":5.2599999999999998,"15min":2.972},"min":{"1min":0.30499999999999999,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":14.222,"5min":109.95699999999999,"15min":109.95699999999999},"last":1.667},{"interface":"front","average":{"1min":3.746,"5min":5.9660000000000002,"15min":3.7930000000000001},"min":{"1min":0.503,"5min":0.28299999999999997,"15min":0.27600000000000002},"max":{"1min":14.117000000000001,"5min":110.542,"15min":116.04000000000001},"last":1.3520000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:32:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5230000000000001,"5min":3.2589999999999999,"15min":1.4630000000000001},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.5310000000000006,"5min":119.14,"15min":119.14},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.3160000000000001,"5min":2.8679999999999999,"15min":1.3380000000000001},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":5.0910000000000002,"5min":118.81100000000001,"15min":118.81100000000001},"last":0.64900000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:33:14 2023","interfaces":[{"interface":"back","average":{"1min":19.989999999999998,"5min":8.5280000000000005,"15min":3.2650000000000001},"min":{"1min":0.39200000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":259.053,"5min":259.053,"15min":259.053},"last":0.41799999999999998},{"interface":"front","average":{"1min":19.395,"5min":8.4280000000000008,"15min":3.2639999999999998},"min":{"1min":0.436,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":259.108,"5min":259.108,"15min":259.108},"last":1.881}]},{"osd":4,"last update":"Mon Oct 9 04:32:58 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.754,"15min":0.91900000000000004},"min":{"1min":0.309,"5min":0.248,"15min":0.23499999999999999},"max":{"1min":7.2699999999999996,"5min":7.2699999999999996,"15min":9.641},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.145,"15min":1.0369999999999999},"min":{"1min":0.30199999999999999,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":10.645,"5min":10.645,"15min":10.645},"last":1.327}]},{"osd":6,"last update":"Mon Oct 9 04:33:27 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.0680000000000001,"15min":1.0720000000000001},"min":{"1min":0.34699999999999998,"5min":0.316,"15min":0.26000000000000001},"max":{"1min":6.6580000000000004,"5min":11.192,"15min":11.192},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.149,"15min":1.131},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.5090000000000003,"5min":6.0599999999999996,"15min":14.334},"last":0.32100000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:32:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.254,"15min":1.2470000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.27400000000000002},"max":{"1min":8.1470000000000002,"5min":8.1470000000000002,"15min":9.9000000000000004},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.117,"15min":1.133},"min":{"1min":0.40300000000000002,"5min":0.35999999999999999,"15min":0.33000000000000002},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":11.315},"last":0.48299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":30146560,"data_stored":30146560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":166662144,"data_stored":166658072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":27000832,"data_stored":27000832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":91226112,"data_stored":91226112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":7340032,"data_stored":7340032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":130093056,"data_stored":130088984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":6094848,"data_stored":6094848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":132190208,"data_stored":132186136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:33:37.020 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:33:37.021 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:33:37.021 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-09T04:33:37.021 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:33:37.021 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:33:37.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-09T04:33:37.062 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:37.096 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:37.171 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:37.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:37.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:37.277 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:37.308 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:37.336 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:37.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:37.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:37.380 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:33:37.380 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:33:37.381 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:33:37.381 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:33:37.381 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:33:37.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:37.496 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:37.749 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:37.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:37.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:37.880 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[216903]: pgmap v697: 89 pgs: 89 active+clean; 33 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-09T04:33:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[223858]: pgmap v697: 89 pgs: 89 active+clean; 33 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:37 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:37 smithi177 ceph-mon[195797]: pgmap v697: 89 pgs: 89 active+clean; 33 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-09T04:33:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:33:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:33:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:37 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:38.447 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:38.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:33:38.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:33:38.582 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:38.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:38.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:38.729 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:38.732 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:39.499 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:39.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:39.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:39.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:39.635 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:39.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:39.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:39.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:39.823 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:39 smithi029 ceph-mon[216903]: pgmap v698: 89 pgs: 89 active+clean; 12 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T04:33:40.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:39 smithi029 ceph-mon[223858]: pgmap v698: 89 pgs: 89 active+clean; 12 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T04:33:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:39 smithi177 ceph-mon[195797]: pgmap v698: 89 pgs: 89 active+clean; 12 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-09T04:33:40.528 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:40.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:40.588 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:40.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:40.663 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:40.713 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:40.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:40.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:40.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:40.851 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:41.397 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:41.606 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:41.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:41.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:41.745 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:41.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:41.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:41.899 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:41.933 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:42.128 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:33:42.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:42 smithi029 ceph-mon[216903]: pgmap v699: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-10-09T04:33:42.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1896531246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:42 smithi029 ceph-mon[223858]: pgmap v699: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-10-09T04:33:42.334 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1896531246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.514 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:42.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:42.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:42 smithi177 ceph-mon[195797]: pgmap v699: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-10-09T04:33:42.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1896531246' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:33:42.619 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:42.651 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:42.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:42.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:42.803 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:42.835 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:42.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:33:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:33:42] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-10-09T04:33:43.146 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_4' removed 2023-10-09T04:33:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1896531246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:33:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/417349474' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:33:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:33:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[216903]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1896531246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/417349474' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:43 smithi029 ceph-mon[223858]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1896531246' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: osdmap e188: 8 total, 8 up, 8 in 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/417349474' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: failed to encode map e188 with expected crc 2023-10-09T04:33:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:43 smithi177 ceph-mon[195797]: failed to encode map e188 with expected crc 2023-10-09T04:33:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: pgmap v701: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 293 op/s 2023-10-09T04:33:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:33:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T04:33:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[223858]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: pgmap v701: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 293 op/s 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:44 smithi029 ceph-mon[216903]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: pgmap v701: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 293 op/s 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: osdmap e189: 8 total, 8 up, 8 in 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:44 smithi177 ceph-mon[195797]: failed to encode map e189 with expected crc 2023-10-09T04:33:44.529 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:33:44.529 INFO:teuthology.orchestra.run.smithi029.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:33:44.529 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:33:44.778 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:33:44.779 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:33:44.779 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-09T04:33:44.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-09T04:33:45.095 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:45.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:45.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:45.231 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:45.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:45.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:45 smithi029 ceph-mon[216903]: failed to encode map e189 with expected crc 2023-10-09T04:33:45.372 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:45 smithi029 ceph-mon[223858]: failed to encode map e189 with expected crc 2023-10-09T04:33:45.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:45.412 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:45 smithi177 ceph-mon[195797]: failed to encode map e189 with expected crc 2023-10-09T04:33:45.415 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:45.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:45.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:45.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:45.596 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:46.060 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:46.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:46.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:46.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:46.197 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:46.253 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:46.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:46.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:46.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:46 smithi029 ceph-mon[216903]: pgmap v703: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:46.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:46 smithi029 ceph-mon[223858]: pgmap v703: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:46.387 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:46.450 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:46.505 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:33:46.505 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175266 8.7 8:e7a13831:::benchmark_data_smithi029_252925_object58642:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.954098+0000", 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.548373454, 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.332614636, 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.506 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175266 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954098+0000", 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954098+0000", 2023-10-09T04:33:46.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954097+0000", 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954102+0000", 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5439999999999998e-06 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954115+0000", 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2418999999999999e-05 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954123+0000", 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.729e-06 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954229+0000", 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010627 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954239+0000", 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8169999999999994e-06 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.286609+0000", 2023-10-09T04:33:46.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3323706710000001 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.286713+0000", 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103776 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175267 8.7 8:e1f96042:::benchmark_data_smithi029_252925_object58643:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.954134+0000", 2023-10-09T04:33:46.511 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.54833819300001, 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.373225146, 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175267 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954134+0000", 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954134+0000", 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954133+0000", 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954136+0000", 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6749999999999998e-06 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954142+0000", 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8980000000000001e-06 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954146+0000", 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5730000000000001e-06 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954257+0000", 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110103 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954263+0000", 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2790000000000004e-06 2023-10-09T04:33:46.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.286988+0000", 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3327250900000001 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.287111+0000", 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000122929 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327254+0000", 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040143021000000001 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327359+0000", 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104993 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.517 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175270 8.7 8:e7030a3f:::benchmark_data_smithi029_252925_object58646:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.954150+0000", 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.54832178599997, 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.347536466, 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175270 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954150+0000", 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954150+0000", 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954150+0000", 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954152+0000", 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9379999999999999e-06 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954156+0000", 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5519999999999998e-06 2023-10-09T04:33:46.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954161+0000", 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2690000000000001e-06 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954281+0000", 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011993 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954287+0000", 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0460000000000002e-06 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.287148+0000", 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3328615450000001 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.287253+0000", 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010454300000000001 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.287744+0000", 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049103299999999999 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301478+0000", 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013734807 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301543+0000", 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4214000000000006e-05 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301587+0000", 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4097999999999997e-05 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301628+0000", 2023-10-09T04:33:46.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1319000000000002e-05 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301687+0000", 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8451e-05 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175275 8.7 8:e148bad1:::benchmark_data_smithi029_252925_object58651:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.525 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.954164+0000", 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.54830807399998, 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3477493149999999, 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175275 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954164+0000", 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954164+0000", 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954164+0000", 2023-10-09T04:33:46.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954165+0000", 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.866e-06 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954170+0000", 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4689999999999999e-06 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954174+0000", 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4009999999999999e-06 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954304+0000", 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000129763 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954310+0000", 2023-10-09T04:33:46.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0499999999999997e-06 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.291804+0000", 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.337494403 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.291962+0000", 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015713700000000001 2023-10-09T04:33:46.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.292104+0000", 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142118 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301486+0000", 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.009381927 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301783+0000", 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000297406 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301818+0000", 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4687000000000002e-05 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301846+0000", 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8391999999999999e-05 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301913+0000", 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6983999999999994e-05 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175279 8.7 8:ec25d703:::benchmark_data_smithi029_252925_object58655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.954177+0000", 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.54829452400003, 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3618701200000001, 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175279 2023-10-09T04:33:46.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954177+0000", 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954177+0000", 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954177+0000", 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954179+0000", 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.007e-06 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954184+0000", 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7520000000000004e-06 2023-10-09T04:33:46.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954188+0000", 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9339999999999999e-06 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954332+0000", 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014406999999999999 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.954338+0000", 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2899999999999999e-06 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.294563+0000", 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3402250570000001 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.294667+0000", 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103719 2023-10-09T04:33:46.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.294745+0000", 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8381999999999996e-05 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301494+0000", 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067490620000000001 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301982+0000", 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00048755100000000002 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.315873+0000", 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013890883 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.315936+0000", 2023-10-09T04:33:46.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3047999999999997e-05 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.316047+0000", 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000111623 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175284 8.7 8:e65b7e16:::benchmark_data_smithi029_252925_object58660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.541 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.963138+0000", 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.53933422799997, 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3530974019999999, 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175284 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.963138+0000", 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.963138+0000", 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.963137+0000", 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.963141+0000", 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4470000000000002e-06 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.963145+0000", 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9130000000000002e-06 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.963149+0000", 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2949999999999998e-06 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.964075+0000", 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00092584500000000001 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.964084+0000", 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4530000000000008e-06 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.299083+0000", 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3349994350000001 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.299165+0000", 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1448999999999999e-05 2023-10-09T04:33:46.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.299252+0000", 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6607000000000006e-05 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.301925+0000", 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026737010000000001 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.315829+0000", 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013904178 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.316091+0000", 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026186999999999998 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.316138+0000", 2023-10-09T04:33:46.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6687999999999998e-05 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.316235+0000", 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7095000000000005e-05 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175306 8.7 8:e18e626a:::benchmark_data_smithi029_252925_object58682:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:05.996699+0000", 2023-10-09T04:33:46.549 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.50577246900002, 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.329975935, 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175306 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.996699+0000", 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.996699+0000", 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.996698+0000", 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:46.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.996705+0000", 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0600000000000002e-06 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.996719+0000", 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3560000000000001e-05 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.996725+0000", 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4489999999999996e-06 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.997763+0000", 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010375149999999999 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:05.997774+0000", 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0961000000000001e-05 2023-10-09T04:33:46.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.302025+0000", 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3042516630000001 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.302159+0000", 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000133864 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.302258+0000", 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9108e-05 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.316280+0000", 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014022140000000001 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.317077+0000", 2023-10-09T04:33:46.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00079697600000000002 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326505+0000", 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094279640000000005 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326564+0000", 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8248000000000002e-05 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326675+0000", 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000111852 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175320 8.7 8:e74477c4:::benchmark_data_smithi029_252925_object58696:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.024130+0000", 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.478341833, 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.302768701, 2023-10-09T04:33:46.557 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175320 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024130+0000", 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024130+0000", 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024129+0000", 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024135+0000", 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1369999999999999e-06 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024142+0000", 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1060000000000002e-06 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024145+0000", 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1669999999999999e-06 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024198+0000", 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2544999999999999e-05 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.024208+0000", 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0438000000000001e-05 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.305454+0000", 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2812459139999999 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.305571+0000", 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011701600000000001 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.305645+0000", 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4189000000000004e-05 2023-10-09T04:33:46.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.316321+0000", 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010675318 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.317084+0000", 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00076343299999999999 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326745+0000", 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096608600000000003 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326799+0000", 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3755999999999998e-05 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326899+0000", 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9914999999999995e-05 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175327 8.7 8:e1c97dd2:::benchmark_data_smithi029_252925_object58703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.032137+0000", 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.47033468299998, 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.295414405, 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.565 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175327 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032137+0000", 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032137+0000", 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032136+0000", 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032141+0000", 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4e-06 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032151+0000", 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0710000000000001e-06 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032156+0000", 2023-10-09T04:33:46.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6890000000000001e-06 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032228+0000", 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1264000000000001e-05 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.032240+0000", 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2272e-05 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.308510+0000", 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2762702500000001 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.308640+0000", 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130078 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.308756+0000", 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115591 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.317068+0000", 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083120030000000001 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.320717+0000", 2023-10-09T04:33:46.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036492349999999998 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327406+0000", 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066893439999999998 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327455+0000", 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8952000000000002e-05 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327552+0000", 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6397000000000006e-05 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175330 8.7 8:e6fd0488:::benchmark_data_smithi029_252925_object58706:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.040037+0000", 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.462435051, 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.301290761, 2023-10-09T04:33:46.573 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175330 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040037+0000", 2023-10-09T04:33:46.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040037+0000", 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040035+0000", 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040043+0000", 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6590000000000007e-06 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040059+0000", 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6625000000000001e-05 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040069+0000", 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5070000000000006e-06 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040150+0000", 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.106e-05 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040161+0000", 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1127e-05 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.312408+0000", 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2722472970000001 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.312549+0000", 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014119800000000001 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.312705+0000", 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000155537 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.320728+0000", 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080227600000000003 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.334462+0000", 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013734201999999999 2023-10-09T04:33:46.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341187+0000", 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067248300000000002 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341236+0000", 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9033999999999998e-05 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341328+0000", 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1805000000000006e-05 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175332 8.7 8:e8bc86ea:::benchmark_data_smithi029_252925_object58708:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.040082+0000", 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.46238946400001, 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.301534435, 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.581 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175332 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040082+0000", 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040082+0000", 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040082+0000", 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040086+0000", 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4960000000000001e-06 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040094+0000", 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.473e-06 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040099+0000", 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0849999999999996e-06 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040183+0000", 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3594e-05 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.040190+0000", 2023-10-09T04:33:46.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0979999999999996e-06 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.317110+0000", 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2769202879999999 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.317243+0000", 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000133208 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.317377+0000", 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013360299999999999 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326916+0000", 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095390140000000002 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.334499+0000", 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075831020000000004 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341483+0000", 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069843370000000002 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341529+0000", 2023-10-09T04:33:46.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5754999999999998e-05 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341617+0000", 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7764999999999997e-05 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.589 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175414 8.7 8:e05c8032:::benchmark_data_smithi029_252925_object58790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.190338+0000", 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.312133657, 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.151453952, 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175414 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190338+0000", 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190338+0000", 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190337+0000", 2023-10-09T04:33:46.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190342+0000", 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5949999999999998e-06 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190354+0000", 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1562999999999999e-05 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190362+0000", 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0069999999999997e-06 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190426+0000", 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4061999999999998e-05 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.190441+0000", 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5011000000000001e-05 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.320763+0000", 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.130322072 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.320885+0000", 2023-10-09T04:33:46.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012217200000000001 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.320971+0000", 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5263e-05 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.326926+0000", 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059558359999999999 2023-10-09T04:33:46.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341412+0000", 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014485174999999999 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341659+0000", 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024752500000000001 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341711+0000", 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1962999999999997e-05 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341792+0000", 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1048000000000001e-05 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175418 8.7 8:ed2a329d:::benchmark_data_smithi029_252925_object58794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.197364+0000", 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.30510822299999, 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1528473800000001, 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.598 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175418 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197364+0000", 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197364+0000", 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197362+0000", 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197371+0000", 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1449999999999997e-06 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197390+0000", 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8924999999999999e-05 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197400+0000", 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8600000000000005e-06 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197466+0000", 2023-10-09T04:33:46.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6523e-05 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.197479+0000", 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2310000000000001e-05 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.323684+0000", 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.126205052 2023-10-09T04:33:46.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.323796+0000", 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112243 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.323873+0000", 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7170999999999997e-05 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341358+0000", 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017485305 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341850+0000", 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049209600000000003 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.350126+0000", 2023-10-09T04:33:46.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082760190000000008 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.350171+0000", 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4020000000000002e-05 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.350211+0000", 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0540999999999997e-05 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.605 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175422 8.7 8:e34810b7:::benchmark_data_smithi029_252925_object58798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.203997+0000", 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.29847475100001, 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.154213817, 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.606 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175422 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.203997+0000", 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.203997+0000", 2023-10-09T04:33:46.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.203996+0000", 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.204003+0000", 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3409999999999997e-06 2023-10-09T04:33:46.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.204032+0000", 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8541e-05 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.204039+0000", 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.294e-06 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.204102+0000", 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2632000000000006e-05 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.204116+0000", 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4651000000000001e-05 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327604+0000", 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1234882479999999 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327738+0000", 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013399500000000001 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.327885+0000", 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000146274 2023-10-09T04:33:46.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341422+0000", 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013537192999999999 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.350245+0000", 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088229169999999996 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.358125+0000", 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078800580000000005 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.358171+0000", 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6110999999999999e-05 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.358211+0000", 2023-10-09T04:33:46.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9966999999999999e-05 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:175443 8.7 8:ea39c2c5:::benchmark_data_smithi029_252925_object58819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:26:06.258793+0000", 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 460.24367845299997, 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0995180099999999, 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.614 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 175443 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258793+0000", 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258793+0000", 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258791+0000", 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258800+0000", 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3100000000000001e-06 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258815+0000", 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5071e-05 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258822+0000", 2023-10-09T04:33:46.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0759999999999997e-06 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258860+0000", 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8006000000000002e-05 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:06.258870+0000", 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0237999999999999e-05 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.331488+0000", 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.072617554 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.331595+0000", 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010699 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:33:46.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.331715+0000", 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120185 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.341427+0000", 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097125159999999992 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.350278+0000", 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088505159999999992 2023-10-09T04:33:46.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.358243+0000", 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007965415 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.358280+0000", 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7258999999999997e-05 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.621 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:26:07.358311+0000", 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1003000000000001e-05 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216847 8.a 8:5fc75d23:::benchmark_data_smithi029_252925_object100223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 400.48784164300002, 2023-10-09T04:33:46.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.090995154, 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216847 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014633+0000", 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7219999999999999e-06 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014641+0000", 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3270000000000001e-06 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014646+0000", 2023-10-09T04:33:46.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9479999999999999e-06 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019056+0000", 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044100279999999999 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019186+0000", 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130605 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019311+0000", 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124444 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.028477+0000", 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091663030000000006 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.031031+0000", 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025535000000000002 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.041136+0000", 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010105124 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.041146+0000", 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0193e-05 2023-10-09T04:33:46.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.105547+0000", 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0644010289999999 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.105625+0000", 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8319999999999996e-05 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216860 8.1 8:8d7bc4ee:::benchmark_data_smithi029_252925_object100236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.036509+0000", 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 400.46596329499999, 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0727470320000001, 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216860 2023-10-09T04:33:46.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036509+0000", 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036509+0000", 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036507+0000", 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036515+0000", 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3799999999999994e-06 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036532+0000", 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6906e-05 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036540+0000", 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4490000000000002e-06 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.046000+0000", 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094603259999999998 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.046015+0000", 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4802e-05 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.083499+0000", 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0374842280000001 2023-10-09T04:33:46.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.083617+0000", 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011813 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0", 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.083749+0000", 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013148000000000001 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.102159+0000", 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018410037000000001 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.109121+0000", 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069619340000000004 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.109171+0000", 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0247999999999997e-05 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.109256+0000", 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4887000000000002e-05 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.637 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38515 9.5 9:a1f72eec:::benchmark_data_smithi029_299184_object38514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020059+0000", 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 235.48241256399999, 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.199031669, 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38515 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.638 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020059+0000", 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020059+0000", 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020057+0000", 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020093+0000", 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5815000000000002e-05 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020111+0000", 2023-10-09T04:33:46.640 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7827999999999999e-05 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020123+0000", 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1863e-05 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020175+0000", 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1601000000000003e-05 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020186+0000", 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1617000000000001e-05 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206129+0000", 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1859429779999999 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.207366+0000", 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012366600000000001 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.207499+0000", 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013340100000000001 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212549+0000", 2023-10-09T04:33:46.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050496259999999998 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212591+0000", 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1692e-05 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212644+0000", 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3046e-05 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217457+0000", 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048135089999999997 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217527+0000", 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9851000000000006e-05 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219091+0000", 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015640490000000001 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38517 9.5 9:a2f1fa15:::benchmark_data_smithi029_299184_object38516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020134+0000", 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 235.48233825700001, 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2006594580000001, 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.646 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38517 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020134+0000", 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020134+0000", 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020133+0000", 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020153+0000", 2023-10-09T04:33:46.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0139000000000001e-05 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020159+0000", 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6080000000000003e-06 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020165+0000", 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6210000000000001e-06 2023-10-09T04:33:46.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020214+0000", 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9135999999999998e-05 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020221+0000", 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1980000000000004e-06 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210594+0000", 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1903735600000001 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210757+0000", 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000162552 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210947+0000", 2023-10-09T04:33:46.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190216 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212633+0000", 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001685913 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212764+0000", 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013039099999999999 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217187+0000", 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044233149999999997 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219145+0000", 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019578719999999998 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219205+0000", 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9941000000000001e-05 2023-10-09T04:33:46.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220793+0000", 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015884250000000001 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38536 9.5 9:ad9b3f04:::benchmark_data_smithi029_299184_object38535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:33:46.654 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 235.48230367400001, 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2106301479999999, 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38536 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:33:46.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020185+0000", 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7113999999999999e-05 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020189+0000", 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.245e-06 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020193+0000", 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3909999999999998e-06 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020248+0000", 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5114000000000001e-05 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020256+0000", 2023-10-09T04:33:46.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4859999999999997e-06 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.214300+0000", 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1940435709999999 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.214417+0000", 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011676900000000001 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.214586+0000", 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000169545 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216587+0000", 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020009870000000001 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.225062+0000", 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084753099999999998 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.225143+0000", 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0411e-05 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.229153+0000", 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040097960000000004 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.229213+0000", 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0350000000000003e-05 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230798+0000", 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015853530000000001 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.662 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:33:46.663 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:33:46.663 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:33:46.663 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:46 smithi177 ceph-mon[195797]: pgmap v703: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:46.759 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:33:46.779 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:33:46.797 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:33:46.818 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:33:46.839 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:33:46.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:33:46.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:33:46.897 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:33:46.899 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.899 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.899 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.899 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.900 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.900 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.900 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.900 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:33:46.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:33:47.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2804214818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:47.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2804214818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2804214818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:47.998 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:48.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:48.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:48.135 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:48.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:48.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:48.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:48.315 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:48.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:48 smithi029 ceph-mon[216903]: pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:48.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:48 smithi029 ceph-mon[223858]: pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:48.360 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:48.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:48.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:48.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:48 smithi177 ceph-mon[195797]: pgmap v704: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:48.492 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:48.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:48.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:48.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:48.665 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:48.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:48.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:48.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:48.838 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:48.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:48.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:48.987 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:49.013 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:49.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:49.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:49.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:49.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:33:49.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:49.191 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:49.215 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:49.586 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:49.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:49.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:49.719 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:49.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:49.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:49.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:49.897 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:49.939 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:49.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:49.982 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:50.045 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:50.070 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:50.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:50.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:50.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:50.250 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:50.281 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_5' created 2023-10-09T04:33:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:50 smithi177 ceph-mon[195797]: pgmap v705: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3450280751' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:33:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:33:50.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:33:50.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:50 smithi029 ceph-mon[216903]: pgmap v705: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:50.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3450280751' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:33:50.654 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:33:50.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:50 smithi029 ceph-mon[223858]: pgmap v705: 73 pgs: 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:50.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3450280751' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:33:50.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-09T04:33:50.657 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:50.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:50.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:50.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:50.788 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:51.035 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:51.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:51.092 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:33:51.093 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:33:51.093 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:33:51.093 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-09T04:33:51.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T04:33:51.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:51.171 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:51.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:51.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:51.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:51.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:51.356 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:51 smithi177 ceph-mon[195797]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:51 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:33:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:51 smithi177 ceph-mon[195797]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T04:33:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:51 smithi177 ceph-mon[195797]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:51 smithi177 ceph-mon[195797]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:51 smithi177 ceph-mon[195797]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T04:33:51.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[223858]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:33:51.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[223858]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T04:33:51.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[223858]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[223858]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[223858]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T04:33:51.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[216903]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-09T04:33:51.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[216903]: osdmap e190: 8 total, 8 up, 8 in 2023-10-09T04:33:51.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[216903]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[216903]: failed to encode map e190 with expected crc 2023-10-09T04:33:51.539 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:51 smithi029 ceph-mon[216903]: osdmap e191: 8 total, 8 up, 8 in 2023-10-09T04:33:51.896 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:51.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:52.001 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:52.031 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:52.079 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:52.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:52.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:52.220 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:52.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: pgmap v707: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:52.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1432805025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: pgmap v707: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.525 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T04:33:52.525 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1432805025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:52.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-09T04:33:52.730 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:52.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:52.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: pgmap v707: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:52.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: failed to encode map e191 with expected crc 2023-10-09T04:33:52.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: osdmap e192: 8 total, 8 up, 8 in 2023-10-09T04:33:52.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1432805025' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:33:52.782 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:52.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:52.861 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:52.909 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:52.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:53.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:53.044 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:53.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:53.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:53.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:53.129 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:33:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:33:52] "GET /metrics HTTP/1.1" 200 34031 "" "Prometheus/2.43.0" 2023-10-09T04:33:53.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:53.227 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:53.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:53.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:53.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:53.277 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:33:53.291 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:53 smithi177 ceph-mon[195797]: failed to encode map e192 with expected crc 2023-10-09T04:33:53.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:33:53.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[223858]: failed to encode map e192 with expected crc 2023-10-09T04:33:53.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:53.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1432805025' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:53.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[223858]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T04:33:53.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[216903]: failed to encode map e192 with expected crc 2023-10-09T04:33:53.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:53.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1432805025' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:53.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:53 smithi029 ceph-mon[216903]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T04:33:53.415 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:53.762 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:33:53.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:53.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:33:53.775 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:53 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:33:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1432805025' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:33:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:53 smithi177 ceph-mon[195797]: osdmap e193: 8 total, 8 up, 8 in 2023-10-09T04:33:53.827 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:33:53.827 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:33:53.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:33:53.901 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:53.950 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:33:53.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:53.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:33:54.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:54.085 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:54.129 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:54.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:54.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:54.267 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:54.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:54 smithi029 ceph-mon[216903]: pgmap v710: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:54.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:54 smithi029 ceph-mon[216903]: failed to encode map e193 with expected crc 2023-10-09T04:33:54.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:54 smithi029 ceph-mon[223858]: pgmap v710: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:54.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:54 smithi029 ceph-mon[223858]: failed to encode map e193 with expected crc 2023-10-09T04:33:54.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:54 smithi177 ceph-mon[195797]: pgmap v710: 89 pgs: 13 unknown, 76 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:54.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:54 smithi177 ceph-mon[195797]: failed to encode map e193 with expected crc 2023-10-09T04:33:55.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:55.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:55.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:55.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:33:55.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:33:55.477 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:55.503 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:55.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:33:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:55.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:33:55.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:33:55.688 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:55.732 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:33:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:55.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:33:55.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:55.868 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:56 smithi029 ceph-mon[216903]: pgmap v712: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:56 smithi029 ceph-mon[223858]: pgmap v712: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:56.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:56 smithi177 ceph-mon[195797]: pgmap v712: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:57.169 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:33:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:57.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:33:57.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:33:57.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:33:57.579 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:57.885 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:33:57.886 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:33:57.898 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:33:57.920 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:33:57.940 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:33:57.960 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:33:57.979 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:33:57.998 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:33:57.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:33:57.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:33:57.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:33:57.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:33:57.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:33:57.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:33:58.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:33:58.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:33:58.281 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:33:58.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:58.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:33:58.337 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:33:58.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:58.415 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:58 smithi029 ceph-mon[216903]: pgmap v713: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2373855111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:58 smithi029 ceph-mon[223858]: pgmap v713: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2373855111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:58.729 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:58.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:58 smithi177 ceph-mon[195797]: pgmap v713: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:33:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2373855111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:33:58.835 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:33:58.866 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:58.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:33:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:58.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:33:59.019 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:33:59.048 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:33:59.308 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T04:33:59.660 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:59.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:33:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2822912236' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:59.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:59.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:33:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2822912236' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:33:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:33:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2822912236' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:33:59.826 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:33:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:33:59.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:33:59.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:33:59.963 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:00.312 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: pgmap v714: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: failed to encode map e194 with expected crc 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2822912236' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: failed to encode map e194 with expected crc 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4283613947' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:34:00.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: pgmap v714: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: failed to encode map e194 with expected crc 2023-10-09T04:34:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2822912236' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:34:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:34:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: failed to encode map e194 with expected crc 2023-10-09T04:34:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4283613947' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:34:00.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:00 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:34:00.765 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:00.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: pgmap v714: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: failed to encode map e194 with expected crc 2023-10-09T04:34:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2822912236' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:34:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: osdmap e194: 8 total, 8 up, 8 in 2023-10-09T04:34:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: failed to encode map e194 with expected crc 2023-10-09T04:34:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4283613947' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:34:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:00 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:34:00.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:00.900 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:00.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:00.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:01.056 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:01.083 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:01.127 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:01.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:01.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:01.266 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[216903]: failed to encode map e194 with expected crc 2023-10-09T04:34:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[216903]: failed to encode map e195 with expected crc 2023-10-09T04:34:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:34:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[216903]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:34:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[216903]: failed to encode map e195 with expected crc 2023-10-09T04:34:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[216903]: failed to encode map e194 with expected crc 2023-10-09T04:34:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[223858]: failed to encode map e194 with expected crc 2023-10-09T04:34:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[223858]: failed to encode map e195 with expected crc 2023-10-09T04:34:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:34:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[223858]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:34:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[223858]: failed to encode map e195 with expected crc 2023-10-09T04:34:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:01 smithi029 ceph-mon[223858]: failed to encode map e194 with expected crc 2023-10-09T04:34:01.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:34:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:01 smithi177 ceph-mon[195797]: failed to encode map e194 with expected crc 2023-10-09T04:34:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:01 smithi177 ceph-mon[195797]: failed to encode map e195 with expected crc 2023-10-09T04:34:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:01 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:34:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:01 smithi177 ceph-mon[195797]: osdmap e195: 8 total, 8 up, 8 in 2023-10-09T04:34:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:01 smithi177 ceph-mon[195797]: failed to encode map e195 with expected crc 2023-10-09T04:34:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:01 smithi177 ceph-mon[195797]: failed to encode map e194 with expected crc 2023-10-09T04:34:01.889 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:01.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:01.988 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:01.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:02.024 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:02.073 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:02.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:02.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:02.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:02.210 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:02.263 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:02.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:02.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:02.398 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:02.422 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:02.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:02 smithi029 ceph-mon[216903]: failed to encode map e195 with expected crc 2023-10-09T04:34:02.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:02 smithi029 ceph-mon[216903]: pgmap v717: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:02.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:02 smithi029 ceph-mon[223858]: failed to encode map e195 with expected crc 2023-10-09T04:34:02.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:02 smithi029 ceph-mon[223858]: pgmap v717: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:02.599 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:02.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:02.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:02.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:02 smithi177 ceph-mon[195797]: failed to encode map e195 with expected crc 2023-10-09T04:34:02.733 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:02 smithi177 ceph-mon[195797]: pgmap v717: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:02.736 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:02.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:02.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:02.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:02.920 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:02.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:34:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:34:02] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T04:34:02.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:02.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:03.012 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-09T04:34:03.015 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:34:03.015 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:34:03.063 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:34:03.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:34:03.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_344401 2023-10-09T04:34:03.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:34:03.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:34:03.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:03.127 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:34:03.127 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:34:03.127 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:34:03.127 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:34:03.127 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:34:03.133 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:03.353 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:03.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:03.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:03.511 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:03.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:03.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:03.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:03.745 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:03.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:03.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:03.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:04.026 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:04.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3714 3698 231.113 231.125 0.00474844 0.00430863 2023-10-09T04:34:04.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:04.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:04.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:04.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:04.245 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:04 smithi029 ceph-mon[216903]: pgmap v718: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:04 smithi029 ceph-mon[223858]: pgmap v718: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:04 smithi177 ceph-mon[195797]: pgmap v718: 89 pgs: 89 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-10-09T04:34:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:05.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6040 6024 188.236 145.375 0.00294055 0.00530398 2023-10-09T04:34:05.729 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:34:05.729 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:34:05.729 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:34:05.730 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.023741 2023-10-09T04:34:05.730 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.023740927042878335 2023-10-09T04:34:05.975 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:05.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:06.064 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 9145 9129 190.142 194.062 0.00282403 0.00525149 2023-10-09T04:34:06.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:06.132 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:06.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:06.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:06.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:06.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:06 smithi029 ceph-mon[216903]: pgmap v719: 89 pgs: 89 active+clean; 170 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s wr, 338 op/s 2023-10-09T04:34:06.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:06 smithi029 ceph-mon[223858]: pgmap v719: 89 pgs: 89 active+clean; 170 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s wr, 338 op/s 2023-10-09T04:34:06.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:06.386 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:06 smithi177 ceph-mon[195797]: pgmap v719: 89 pgs: 89 active+clean; 170 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s wr, 338 op/s 2023-10-09T04:34:07.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 11761 11745 183.479 163.5 0.00406784 0.00544337 2023-10-09T04:34:08.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 14556 14540 181.717 174.688 0.00188865 0.0054189 2023-10-09T04:34:08.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:08 smithi029 ceph-mon[223858]: pgmap v720: 89 pgs: 89 active+clean; 278 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s wr, 555 op/s 2023-10-09T04:34:08.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:08 smithi029 ceph-mon[216903]: pgmap v720: 89 pgs: 89 active+clean; 278 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s wr, 555 op/s 2023-10-09T04:34:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:08 smithi177 ceph-mon[195797]: pgmap v720: 89 pgs: 89 active+clean; 278 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s wr, 555 op/s 2023-10-09T04:34:09.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 16650 16634 173.242 130.875 0.00156647 0.00574643 2023-10-09T04:34:09.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:34:09.738 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:09.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:09.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:09.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:09.887 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:09.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:09.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:10.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 18631 18615 166.18 123.812 0.00337601 0.00601098 2023-10-09T04:34:10.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:10.139 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:10.318 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:10.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:10.367 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:10.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:10.519 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:10.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:10.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:10 smithi029 ceph-mon[216903]: pgmap v721: 89 pgs: 89 active+clean; 495 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-09T04:34:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:10 smithi029 ceph-mon[223858]: pgmap v721: 89 pgs: 89 active+clean; 495 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-09T04:34:10.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:10 smithi177 ceph-mon[195797]: pgmap v721: 89 pgs: 89 active+clean; 495 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-09T04:34:10.783 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:10.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:10.800 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:34:10.837 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:34:10.879 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:34:10.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:10.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:34:10.949 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:10.974 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:34:11.010 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:34:11.052 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:34:11.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 20568 20552 160.539 121.062 0.00177652 0.00620954 2023-10-09T04:34:11.070 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:34:11.359 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2743959607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:11.359 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:11.360 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2743959607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:11.360 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:11.360 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2672791382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:11.377 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:34:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2672791382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2743959607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2672791382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:12.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 22742 22726 157.798 135.875 0.00708882 0.00633186 2023-10-09T04:34:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: pgmap v722: 89 pgs: 89 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:34:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:34:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:34:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:34:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[216903]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: pgmap v722: 89 pgs: 89 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:34:12.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:12 smithi029 ceph-mon[223858]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: pgmap v722: 89 pgs: 89 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-09T04:34:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:34:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:34:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: osdmap e196: 8 total, 8 up, 8 in 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:12 smithi177 ceph-mon[195797]: failed to encode map e196 with expected crc 2023-10-09T04:34:13.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 25173 25157 157.21 151.938 0.00619883 0.00635539 2023-10-09T04:34:13.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:34:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:34:12] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-10-09T04:34:13.272 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:13.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:13.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:13.507 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:13.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:13.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:13.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:13.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:13.688 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:13.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:13.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:13.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:13.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:13.880 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:13.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:13.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:14.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:14.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 26717 26701 151.691 96.5 0.00838838 0.00658773 2023-10-09T04:34:14.096 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:14 smithi177 ceph-mon[195797]: pgmap v724: 89 pgs: 89 active+clean; 1.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:34:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:14 smithi029 ceph-mon[216903]: pgmap v724: 89 pgs: 89 active+clean; 1.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:34:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:14 smithi029 ceph-mon[223858]: pgmap v724: 89 pgs: 89 active+clean; 1.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2023-10-09T04:34:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:15.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 28176 28160 146.648 91.1875 0.0021145 0.00680976 2023-10-09T04:34:16.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 29392 29376 141.213 76 0.00370526 0.00707605 2023-10-09T04:34:16.505 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:16 smithi029 ceph-mon[223858]: pgmap v725: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:34:16.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:16 smithi177 ceph-mon[195797]: pgmap v725: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:34:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:16 smithi029 ceph-mon[216903]: pgmap v725: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:34:16.928 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:16.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:17.035 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:17.065 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 30954 30938 138.099 97.625 0.00456382 0.00723695 2023-10-09T04:34:17.095 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:17.237 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:17.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:17.343 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:17.423 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:17.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:17.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:17.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:17.625 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:17.687 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:18.066 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 32877 32861 136.901 120.188 0.00748942 0.00729967 2023-10-09T04:34:18.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:34:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:18 smithi029 ceph-mon[216903]: pgmap v726: 89 pgs: 89 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T04:34:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:18 smithi029 ceph-mon[223858]: pgmap v726: 89 pgs: 89 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T04:34:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:18 smithi177 ceph-mon[195797]: pgmap v726: 89 pgs: 89 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-10-09T04:34:19.066 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 33964 33948 132.589 67.9375 0.009577 0.00753717 2023-10-09T04:34:19.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:34:19.478 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:19.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:19.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:19.653 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:19.724 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:19.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:19.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:19.831 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:19.911 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:20.012 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:20.072 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:20.072 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 35552 35536 130.604 99.25 0.00647646 0.00764825 2023-10-09T04:34:20.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:20.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:20.192 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:20.282 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:20.353 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:34:20.353 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:34:20.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:20 smithi029 ceph-mon[216903]: pgmap v727: 89 pgs: 89 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T04:34:20.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1360787587' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.023740927042878335}]: dispatch 2023-10-09T04:34:20.532 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:20.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:20.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:20.690 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:20 smithi177 ceph-mon[195797]: pgmap v727: 89 pgs: 89 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T04:34:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1360787587' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.023740927042878335}]: dispatch 2023-10-09T04:34:20.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:20.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:20.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:20.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:20 smithi029 ceph-mon[223858]: pgmap v727: 89 pgs: 89 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-10-09T04:34:20.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1360787587' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.023740927042878335}]: dispatch 2023-10-09T04:34:20.906 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:21.069 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 37262 37246 129.286 106.875 0.00716127 0.00773032 2023-10-09T04:34:21.174 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:21.267 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.0 primary-affinity to 0.0237409 (815552) 2023-10-09T04:34:22.069 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 37675 37659 123.841 25.8125 0.00224828 0.00773517 2023-10-09T04:34:22.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:22 smithi177 ceph-mon[195797]: pgmap v728: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:34:22.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:22 smithi177 ceph-mon[195797]: failed to encode map e197 with expected crc 2023-10-09T04:34:22.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1360787587' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.023740927042878335}]': finished 2023-10-09T04:34:22.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:22 smithi177 ceph-mon[195797]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:34:22.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[223858]: pgmap v728: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:34:22.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[223858]: failed to encode map e197 with expected crc 2023-10-09T04:34:22.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1360787587' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.023740927042878335}]': finished 2023-10-09T04:34:22.630 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[223858]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:34:22.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[216903]: pgmap v728: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:34:22.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[216903]: failed to encode map e197 with expected crc 2023-10-09T04:34:22.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1360787587' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.023740927042878335}]': finished 2023-10-09T04:34:22.631 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:22 smithi029 ceph-mon[216903]: osdmap e197: 8 total, 8 up, 8 in 2023-10-09T04:34:22.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:34:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:34:22] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2023-10-09T04:34:23.070 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:34:23.070226+0000 min lat: 0.00100934 max lat: 1.02723 avg lat: 0.00829339 2023-10-09T04:34:23.071 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:34:23.071 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 38585 38569 120.492 56.875 0.011549 0.00829339 2023-10-09T04:34:23.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:23 smithi177 ceph-mon[195797]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.535 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:23.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:23.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:23.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:34:23.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.541 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[216903]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: osdmap e198: 8 total, 8 up, 8 in 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.542 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:23 smithi029 ceph-mon[223858]: failed to encode map e198 with expected crc 2023-10-09T04:34:23.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:23.704 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:23.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:23.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:23.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:23.952 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:24.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:24.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:24.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:24.069 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 40363 40347 120.046 111.125 0.00257721 0.00830746 2023-10-09T04:34:24.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:24.179 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:24.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:24 smithi029 ceph-mon[216903]: pgmap v731: 89 pgs: 2 peering, 87 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:34:24.523 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:24 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:34:24.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:24 smithi029 ceph-mon[223858]: pgmap v731: 89 pgs: 2 peering, 87 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:34:24.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:24 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:34:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:24 smithi177 ceph-mon[195797]: pgmap v731: 89 pgs: 2 peering, 87 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:34:24.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:24 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:34:24.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:34:24.843 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:24.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:24.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:24.994 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:25.070 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 41623 41607 118.166 78.75 0.0126548 0.00845695 2023-10-09T04:34:25.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:25.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:25.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:25.256 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:25.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:25.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:25.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:25.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:25.518 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:25.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:25.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:25.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:25.808 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:26.070 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 43125 43109 117.11 93.875 0.00276344 0.00850839 2023-10-09T04:34:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:34:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:34:26.122 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:34:26.147 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:26.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:26.177 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.178 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.179 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.179 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.179 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.179 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.179 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:34:26.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:26.303 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:26 smithi029 ceph-mon[216903]: pgmap v732: 89 pgs: 2 peering, 87 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/289619820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:26.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:26 smithi029 ceph-mon[223858]: pgmap v732: 89 pgs: 2 peering, 87 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/289619820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:26.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:26.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:26 smithi177 ceph-mon[195797]: pgmap v732: 89 pgs: 2 peering, 87 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/289619820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:26.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:26.606 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:27.070 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 44433 44417 115.637 81.75 0.0083917 0.00864214 2023-10-09T04:34:27.377 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:27.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:27.483 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:27.567 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:27 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:34:27.567 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:27 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:34:27.568 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:27 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:34:27.568 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:27 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:34:27.570 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:27.669 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:27.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:27.698 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:27.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:27 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:34:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:27 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:34:27.826 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:27.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:27.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:28.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:28.066 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:28.071 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 45806 45790 114.444 85.8125 0.00978675 0.00873364 2023-10-09T04:34:28.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:28.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:28.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:28.300 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:28.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:28.407 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:28 smithi177 ceph-mon[195797]: pgmap v733: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:28.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:28.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:28.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:28.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:28 smithi029 ceph-mon[216903]: pgmap v733: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:28.565 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:28 smithi029 ceph-mon[223858]: pgmap v733: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:28.580 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:28.680 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:28.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:28.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:28.846 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:28.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:28.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:29.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:29.072 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 47358 47342 113.767 97 0.0108134 0.00878464 2023-10-09T04:34:29.117 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:29.204 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:29.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:29.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:29.330 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:34:29.339 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:29.384 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:29.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:29.386 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1919289484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:29.386 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:29 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:29.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:29.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1919289484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:29.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:29 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:29.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:29.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:29.514 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:29.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:34:29.559 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:29.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:29.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:29.736 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1919289484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:29 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:34:29.940 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:34:29.941 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:34:29.941 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-09T04:34:29.941 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:34:29.941 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:34:29.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-09T04:34:30.072 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 48334 48318 111.813 61 0.0128918 0.00893779 2023-10-09T04:34:30.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: pgmap v734: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:30.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:30.329 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[216903]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: pgmap v734: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:34:30.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:34:30.331 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.331 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.331 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.331 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:30 smithi029 ceph-mon[223858]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: pgmap v734: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: osdmap e199: 8 total, 8 up, 8 in 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: failed to encode map e199 with expected crc 2023-10-09T04:34:30.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:30 smithi177 ceph-mon[195797]: failed to encode map e199 with expected crc 2023-10-09T04:34:31.072 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 49358 49342 110.105 64 0.00721248 0.0090776 2023-10-09T04:34:31.078 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:31.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:31.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:31.249 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:31.351 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:31.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:31.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:31.515 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:31.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:31.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:31.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:31.631 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:31.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:31.712 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:31.732 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:50258 10.7 e199/190 10:ee7713cd:::benchmark_data_smithi029_344401_object50257:head v 199'3091, mlcod=199'3088)", 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:34:31.729901+0000", 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0025234469999999998, 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00254551, 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.729901+0000", 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.729901+0000", 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:31.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.729900+0000", 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.729942+0000", 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1508000000000003e-05 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.729958+0000", 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6552e-05 2023-10-09T04:34:31.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.729967+0000", 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4589999999999998e-06 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.732087+0000", 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021206189999999998 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:31.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:34:31.732113+0000", 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5466e-05 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 1 2023-10-09T04:34:31.737 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:34:31.748 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:31.838 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:31.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:31.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.006 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:32.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 50693 50677 109.182 83.4375 0.00704268 0.00915394 2023-10-09T04:34:32.408 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:32.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:32.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:32.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:32.552 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:32.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:32.597 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:32 smithi177 ceph-mon[195797]: pgmap v736: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 15 KiB/s, 0 objects/s recovering 2023-10-09T04:34:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:32.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:32 smithi029 ceph-mon[216903]: pgmap v736: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 15 KiB/s, 0 objects/s recovering 2023-10-09T04:34:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:32 smithi029 ceph-mon[223858]: pgmap v736: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 15 KiB/s, 0 objects/s recovering 2023-10-09T04:34:32.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:32.756 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:33.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 51876 51860 108.007 73.9375 0.00786534 0.00925371 2023-10-09T04:34:33.133 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:33.138 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:34:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:34:32] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T04:34:33.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:33.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:33.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:34:33.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:33.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:34:33.366 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:34.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 53095 53079 106.981 76.1875 0.00630342 0.00934266 2023-10-09T04:34:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:34 smithi029 ceph-mon[216903]: pgmap v737: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T04:34:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:34.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:34 smithi029 ceph-mon[223858]: pgmap v737: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T04:34:34.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:34 smithi177 ceph-mon[195797]: pgmap v737: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T04:34:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:34.819 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:34.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:34.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:34.993 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:35.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 54224 54208 105.843 70.5625 0.0132106 0.00944329 2023-10-09T04:34:36.040 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:36.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:36.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 55413 55397 104.887 74.3125 0.00547103 0.00952961 2023-10-09T04:34:36.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:36.197 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:36.227 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:36.263 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:36.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:36.354 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:36.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:36.452 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:36 smithi177 ceph-mon[195797]: pgmap v738: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:34:36.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:36 smithi029 ceph-mon[223858]: pgmap v738: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:34:36.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:36 smithi029 ceph-mon[216903]: pgmap v738: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:34:36.781 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:36.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:36.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:36.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:36.946 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:37.006 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:34:37.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:37.042 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:34:37.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:37.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:37.073 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 56708 56692 104.183 80.9375 0.0198889 0.00959403 2023-10-09T04:34:37.079 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:34:37.119 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:34:37.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:37.158 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:34:37.181 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:37.198 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:34:37.231 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:34:37.246 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:37.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:37.250 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:34:37.259 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.259 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:34:37.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:37.400 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:37.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:37.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:37.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:37.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1729908568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:37.552 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:37.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1729908568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:37.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1729908568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:37.595 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:37.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:37.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:37.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:37.876 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:37.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:37.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:37.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:38.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 57847 57831 103.24 71.1875 0.00457832 0.009668 2023-10-09T04:34:38.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:38.123 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:38 smithi029 ceph-mon[223858]: pgmap v739: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:34:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:38 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:38.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:38 smithi029 ceph-mon[216903]: pgmap v739: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:34:38.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:38 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:38 smithi177 ceph-mon[195797]: pgmap v739: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:34:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:38 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:34:39.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 59060 59044 102.478 75.8125 0.0112885 0.00975315 2023-10-09T04:34:39.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:39.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:39.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:39.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:40.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 59890 59874 101.11 51.875 0.00868428 0.00988577 2023-10-09T04:34:40.439 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:34:40.439 INFO:teuthology.orchestra.run.smithi029.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:34:40.439 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:34:40.470 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:40.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:40.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:40.609 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:40.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:40 smithi029 ceph-mon[216903]: pgmap v740: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:34:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:40 smithi029 ceph-mon[223858]: pgmap v740: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:34:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:40 smithi177 ceph-mon[195797]: pgmap v740: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:34:41.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 60653 60637 99.7045 47.6875 0.00320733 0.00993643 2023-10-09T04:34:41.666 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:41.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:41.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:41.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:41.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:41.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:34:41.803 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:42.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 61786 61770 98.9636 70.8125 0.0203511 0.0100997 2023-10-09T04:34:42.351 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:34:42.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:42.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:34:42.459 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:42.515 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:42.667 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:42 smithi029 ceph-mon[223858]: pgmap v741: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T04:34:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:42 smithi029 ceph-mon[216903]: pgmap v741: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T04:34:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:34:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:34:42] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T04:34:43.014 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:43.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:43.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:42 smithi177 ceph-mon[195797]: pgmap v741: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-10-09T04:34:43.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:34:43.074947+0000 min lat: 0.00100934 max lat: 1.02723 avg lat: 0.0101538 2023-10-09T04:34:43.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:34:43.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 63037 63021 98.4441 78.1875 0.0185349 0.0101538 2023-10-09T04:34:43.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:43.170 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:43.273 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:34:43.274 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:34:43.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:43.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:43.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:43.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:43.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:34:43.468 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:43.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:43.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:43.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:43.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:43.777 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:44.064 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:34:44.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 64165 64149 97.7625 70.5 0.0214145 0.0102232 2023-10-09T04:34:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[216903]: pgmap v742: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:34:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/743141348' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:44.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[223858]: pgmap v742: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:34:44.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/743141348' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:44.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:44.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:44 smithi177 ceph-mon[195797]: pgmap v742: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-09T04:34:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/743141348' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:34:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:44.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:45.075 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 65102 65086 96.8292 58.5625 0.0110905 0.0103224 2023-10-09T04:34:45.152 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:45.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:45.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/743141348' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[216903]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:45.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:45.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/743141348' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:45 smithi029 ceph-mon[223858]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/743141348' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: osdmap e200: 8 total, 8 up, 8 in 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.321 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.322 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.322 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:45 smithi177 ceph-mon[195797]: failed to encode map e200 with expected crc 2023-10-09T04:34:45.322 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:45.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:45.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:45.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:45.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:45.529 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:46.075 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 66257 66241 96.2561 72.1875 0.0105055 0.010385 2023-10-09T04:34:46.117 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:46.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:46.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.281 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:46.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:46.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:46.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:46.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:46 smithi029 ceph-mon[216903]: pgmap v744: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T04:34:46.379 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:46 smithi029 ceph-mon[223858]: pgmap v744: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T04:34:46.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:34:46.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:46.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:46 smithi177 ceph-mon[195797]: pgmap v744: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-09T04:34:46.540 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:46.674 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:46.688 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:46.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:46.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:46.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:46.845 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:46.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:46.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:47.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:47.074 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 67367 67351 95.6451 69.375 0.028247 0.0104501 2023-10-09T04:34:47.125 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:47.500 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:47.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:47.582 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:34:47.582 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:34:47.582 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-09T04:34:47.583 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T04:34:47.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T04:34:47.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:47.669 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:47.783 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:47.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:47.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:47.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:47.922 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:34:47 smithi029 systemd[1]: Stopping Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:34:47.971 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:48.075 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 68599 68583 95.2307 77 0.00693901 0.0104963 2023-10-09T04:34:48.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:34:48.647 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:48 smithi029 ceph-mon[223858]: pgmap v745: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:34:48.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:48 smithi029 ceph-mon[216903]: pgmap v745: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:34:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:48 smithi177 ceph-mon[195797]: pgmap v745: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:34:48.954 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:48.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:48.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:49.007 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:49.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:49.076 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 69593 69577 94.5087 62.125 0.00738601 0.0105753 2023-10-09T04:34:49.115 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:49.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:49.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:49.291 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:49.365 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:49.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:49.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:49.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:49.620 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:49.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:49.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:49.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:49.803 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:49.899 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:50.076 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 70656 70640 93.9114 66.4375 0.0134809 0.0106438 2023-10-09T04:34:50.125 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:34:49 smithi029 conmon[267369]: 2023-10-09T04:34:49.948+0000 7f0aee820700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:34:50.125 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:34:49 smithi029 conmon[267369]: 2023-10-09T04:34:49.948+0000 7f0aee820700 -1 osd.2 200 *** Got signal Terminated *** 2023-10-09T04:34:50.126 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:34:49 smithi029 conmon[267369]: 2023-10-09T04:34:49.948+0000 7f0aee820700 -1 osd.2 200 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:34:50.159 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:50.323 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:34:50.349 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:34:50.356 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:50.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:50.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:50 smithi029 ceph-mon[216903]: pgmap v746: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:34:50.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:50 smithi029 ceph-mon[216903]: osd.2 marked itself down and dead 2023-10-09T04:34:50.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/836046868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:50.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:50 smithi029 ceph-mon[223858]: pgmap v746: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:34:50.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:50 smithi029 ceph-mon[223858]: osd.2 marked itself down and dead 2023-10-09T04:34:50.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/836046868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:50.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:50.485 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:50.527 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:50.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:50.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:50.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:50.668 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:50.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:50 smithi177 ceph-mon[195797]: pgmap v746: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-09T04:34:50.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:50 smithi177 ceph-mon[195797]: osd.2 marked itself down and dead 2023-10-09T04:34:50.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/836046868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:34:50.955 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:34:50.978 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:34:51.001 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:34:51.021 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:34:51.040 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:34:51.040 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:34:51.076 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 70909 70893 92.2846 15.8125 0.00291282 0.010662 2023-10-09T04:34:51.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:34:51.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[216903]: failed to encode map e201 with expected crc 2023-10-09T04:34:51.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[216903]: osdmap e201: 8 total, 7 up, 8 in 2023-10-09T04:34:51.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[216903]: failed to encode map e201 with expected crc 2023-10-09T04:34:51.616 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:34:51.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[223858]: failed to encode map e201 with expected crc 2023-10-09T04:34:51.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[223858]: osdmap e201: 8 total, 7 up, 8 in 2023-10-09T04:34:51.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:51 smithi029 ceph-mon[223858]: failed to encode map e201 with expected crc 2023-10-09T04:34:51.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:51 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:34:51.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:51 smithi177 ceph-mon[195797]: failed to encode map e201 with expected crc 2023-10-09T04:34:51.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:51 smithi177 ceph-mon[195797]: osdmap e201: 8 total, 7 up, 8 in 2023-10-09T04:34:51.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:51 smithi177 ceph-mon[195797]: failed to encode map e201 with expected crc 2023-10-09T04:34:52.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 71627 71611 91.315 44.875 0.00829744 0.0109467 2023-10-09T04:34:52.314 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: pgmap v748: 89 pgs: 5 stale+active+clean, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: osdmap e202: 8 total, 7 up, 8 in 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[216903]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.315 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:34:51 smithi029 bash[351716]: 0906042e125e7a8c588047bfe07393fee94dfeb9f3eed513575e6477fde04342 2023-10-09T04:34:52.458 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:52.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:52.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:34:52.630 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: pgmap v748: 89 pgs: 5 stale+active+clean, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:34:52.630 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.630 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: osdmap e202: 8 total, 7 up, 8 in 2023-10-09T04:34:52.630 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.630 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.631 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.631 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.631 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.631 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:52 smithi177 ceph-mon[195797]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: pgmap v748: 89 pgs: 5 stale+active+clean, 84 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-09T04:34:52.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: osdmap e202: 8 total, 7 up, 8 in 2023-10-09T04:34:52.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:52 smithi029 ceph-mon[223858]: failed to encode map e202 with expected crc 2023-10-09T04:34:52.638 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:52.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:34:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:52.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:34:52.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:52.880 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:52.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:34:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:34:52] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.43.0" 2023-10-09T04:34:52.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:52.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:52.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:53.038 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:53.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 72780 72764 90.9299 72.0625 0.00815796 0.0109925 2023-10-09T04:34:53.095 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:53.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:53.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:53.290 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:53.348 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:53.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:53.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:53.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:53.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:53 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 4506/211062 objects degraded (2.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:34:53.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:34:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:34:53.614 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:34:53.630 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:53.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:53 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 4506/211062 objects degraded (2.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:34:53.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:53 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 4506/211062 objects degraded (2.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:34:53.747 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:34:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:53.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:34:53.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:53.932 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:54.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:54.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:54.077 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 73990 73974 90.6297 75.625 0.00341093 0.0110173 2023-10-09T04:34:54.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:54.192 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:54.368 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:54 smithi177 ceph-mon[195797]: pgmap v750: 89 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 9 active+undersized, 77 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 4506/211062 objects degraded (2.135%) 2023-10-09T04:34:54.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:54 smithi029 ceph-mon[216903]: pgmap v750: 89 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 9 active+undersized, 77 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 4506/211062 objects degraded (2.135%) 2023-10-09T04:34:54.538 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:54 smithi029 ceph-mon[223858]: pgmap v750: 89 pgs: 1 active+undersized+degraded, 2 stale+active+clean, 9 active+undersized, 77 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 4506/211062 objects degraded (2.135%) 2023-10-09T04:34:54.817 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:54.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:54.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:34:54.977 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:55.074 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:34:55.078 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 75019 75003 90.1225 64.3125 0.00865462 0.0110915 2023-10-09T04:34:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:55.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38520 9.8 9:108867f8:::benchmark_data_smithi029_299184_object38519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:34:55.157 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.11969404600001, 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1949972569999998, 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38520 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033216+0000", 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033239+0000", 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2768999999999998e-05 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033248+0000", 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6910000000000003e-06 2023-10-09T04:34:55.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033254+0000", 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0399999999999998e-06 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033339+0000", 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5388000000000003e-05 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033348+0000", 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7469999999999994e-06 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022545+0000", 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98919705899999999 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022559+0000", 2023-10-09T04:34:55.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3551e-05 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216745+0000", 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.194186607 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216864+0000", 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011896300000000001 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216980+0000", 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116215 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.222890+0000", 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059099820000000003 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.224040+0000", 2023-10-09T04:34:55.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001149583 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226281+0000", 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022405580000000001 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226344+0000", 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2959999999999994e-05 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.228215+0000", 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001871295 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38523 9.8 9:186f9d6c:::benchmark_data_smithi029_299184_object38522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.11920066300002, 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1990103689999998, 2023-10-09T04:34:55.166 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38523 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033710+0000", 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033730+0000", 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0020000000000001e-05 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033742+0000", 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2299e-05 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033748+0000", 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4079999999999997e-06 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033788+0000", 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0441000000000001e-05 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033795+0000", 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1439999999999997e-06 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022741+0000", 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98894592299999995 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022751+0000", 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4960000000000002e-06 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220959+0000", 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.198207902 2023-10-09T04:34:55.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221125+0000", 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016651700000000001 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221423+0000", 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000297767 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.224239+0000", 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028161560000000002 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226232+0000", 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019932280000000001 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230759+0000", 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045263370000000001 2023-10-09T04:34:55.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230821+0000", 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1879999999999997e-05 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.232721+0000", 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001900723 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112523 9.8 9:166f8be2:::benchmark_data_smithi029_299184_object14220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 212.04240679099999, 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.01694084, 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112523 2023-10-09T04:34:55.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110503+0000", 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110510+0000", 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4899999999999997e-06 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110525+0000", 2023-10-09T04:34:55.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4864e-05 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110551+0000", 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5894e-05 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110637+0000", 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6575000000000004e-05 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110776+0000", 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013886399999999999 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110903+0000", 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012670300000000001 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.117848+0000", 2023-10-09T04:34:55.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069447629999999996 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.117863+0000", 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5756999999999998e-05 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127354+0000", 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0094912250000001 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127446+0000", 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1144999999999996e-05 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112554 9.8 9:196c9f66:::benchmark_data_smithi029_299184_object14251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 212.00326810000001, 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99418602, 2023-10-09T04:34:55.181 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112554 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:34:55.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149642+0000", 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149649+0000", 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6830000000000008e-06 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149667+0000", 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7935999999999999e-05 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149676+0000", 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8149999999999995e-06 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149759+0000", 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3293000000000004e-05 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149772+0000", 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.225e-05 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127865+0000", 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.9780934 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127992+0000", 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127248 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.128124+0000", 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013166 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137717+0000", 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095931829999999999 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137731+0000", 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3898e-05 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143735+0000", 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060035740000000002 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143787+0000", 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2207999999999999e-05 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143830+0000", 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2657000000000002e-05 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.188 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112566 9.8 9:1591ae10:::benchmark_data_smithi029_299184_object14263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.99720186900001, 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98828444299999996, 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.189 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112566 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:34:55.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155708+0000", 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155715+0000", 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.737e-06 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155732+0000", 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6733000000000001e-05 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155740+0000", 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3159999999999997e-06 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155798+0000", 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8369000000000003e-05 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155810+0000", 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1187e-05 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131524+0000", 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97571455100000004 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131668+0000", 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000144115 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131781+0000", 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011272 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137768+0000", 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059868439999999998 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137815+0000", 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7286000000000001e-05 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143926+0000", 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061107280000000002 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143955+0000", 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9420999999999998e-05 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143994+0000", 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8894999999999999e-05 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112581 9.8 9:188eb5a8:::benchmark_data_smithi029_299184_object14278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.98007574900001, 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97130810899999998, 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112581 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.197 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172834+0000", 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172841+0000", 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0890000000000002e-06 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172854+0000", 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2677000000000001e-05 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172863+0000", 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172924+0000", 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1587999999999994e-05 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172935+0000", 2023-10-09T04:34:55.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0971e-05 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134673+0000", 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96173812199999997 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134756+0000", 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2958000000000001e-05 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134831+0000", 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4935e-05 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137804+0000", 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029725469999999999 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137823+0000", 2023-10-09T04:34:55.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9485000000000001e-05 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144083+0000", 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062598230000000003 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144109+0000", 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5782000000000002e-05 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144144+0000", 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5181000000000001e-05 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112605 9.8 9:173f4bff:::benchmark_data_smithi029_299184_object14302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.95071223799999, 2023-10-09T04:34:55.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95321568300000004, 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112605 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202203+0000", 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0260000000000001e-06 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202215+0000", 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1698e-05 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202222+0000", 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0870000000000001e-06 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202287+0000", 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5141999999999994e-05 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202297+0000", 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0135e-05 2023-10-09T04:34:55.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137839+0000", 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.935542184 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137923+0000", 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.365e-05 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.138000+0000", 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6852000000000002e-05 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143877+0000", 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058773489999999996 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143886+0000", 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9919999999999996e-06 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155344+0000", 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011457606 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155380+0000", 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6705000000000002e-05 2023-10-09T04:34:55.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155415+0000", 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4813e-05 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112609 9.8 9:1697f7cc:::benchmark_data_smithi029_299184_object14306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.94706241899999, 2023-10-09T04:34:55.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94970479100000005, 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112609 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:34:55.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205848+0000", 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205853+0000", 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2599999999999999e-06 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205861+0000", 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5339999999999999e-06 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205867+0000", 2023-10-09T04:34:55.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1129999999999997e-06 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205926+0000", 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8548000000000003e-05 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205937+0000", 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1100999999999999e-05 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140841+0000", 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93490438799999998 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140924+0000", 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2818000000000006e-05 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140995+0000", 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0840000000000006e-05 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144049+0000", 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030538369999999998 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144062+0000", 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2706e-05 2023-10-09T04:34:55.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155470+0000", 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011407907 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155514+0000", 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4409999999999997e-05 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155554+0000", 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0036000000000001e-05 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112659 9.8 9:1ba9566d:::benchmark_data_smithi029_299184_object14356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.88174162999999, 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.89206917200000002, 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.220 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112659 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271169+0000", 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271174+0000", 2023-10-09T04:34:55.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.592e-06 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271186+0000", 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.153e-05 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271194+0000", 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2179999999999999e-06 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271280+0000", 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6273999999999994e-05 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271288+0000", 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7749999999999993e-06 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144186+0000", 2023-10-09T04:34:55.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.87289803399999999 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144264+0000", 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8158000000000006e-05 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144352+0000", 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7874000000000006e-05 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155299+0000", 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010946424999999999 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155630+0000", 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033138200000000002 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163143+0000", 2023-10-09T04:34:55.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007512627 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163198+0000", 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5139999999999997e-05 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163239+0000", 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1260000000000001e-05 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.227 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112683 9.8 9:1e5d2014:::benchmark_data_smithi029_299184_object14380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.85056939099999, 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.868701897, 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.228 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112683 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:34:55.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302341+0000", 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302347+0000", 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5869999999999998e-06 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302361+0000", 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3838e-05 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302369+0000", 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2539999999999992e-06 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302430+0000", 2023-10-09T04:34:55.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0968999999999999e-05 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302446+0000", 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5427e-05 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148056+0000", 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84561038799999999 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148181+0000", 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012532600000000001 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148295+0000", 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113237 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155432+0000", 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071375379999999997 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163283+0000", 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078503070000000008 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170952+0000", 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076698299999999999 2023-10-09T04:34:55.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170999+0000", 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6375000000000002e-05 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171044+0000", 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5340000000000003e-05 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112688 9.8 9:1bc999dd:::benchmark_data_smithi029_299184_object14385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.84777183200001, 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86601131600000003, 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112688 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.236 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305139+0000", 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305143+0000", 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9339999999999999e-06 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305155+0000", 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1765e-05 2023-10-09T04:34:55.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305161+0000", 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3829999999999997e-06 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305229+0000", 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7902000000000001e-05 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305244+0000", 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.464e-05 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151650+0000", 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84640602600000003 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151753+0000", 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103168 2023-10-09T04:34:55.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151867+0000", 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001144 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163313+0000", 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011445079 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163377+0000", 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4084000000000005e-05 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171081+0000", 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077041899999999996 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171119+0000", 2023-10-09T04:34:55.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8631000000000003e-05 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171151+0000", 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1636e-05 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112697 9.8 9:12bbffcf:::benchmark_data_smithi029_299184_object14394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:34:55.243 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.83439056700001, 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85940739799999999, 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112697 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318520+0000", 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:55.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318526+0000", 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7739999999999999e-06 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318544+0000", 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8156e-05 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318553+0000", 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8859999999999993e-06 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318616+0000", 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2434000000000006e-05 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318627+0000", 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0893000000000001e-05 2023-10-09T04:34:55.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155677+0000", 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83705072999999997 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155793+0000", 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115912 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155913+0000", 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012010000000000001 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163366+0000", 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074525160000000002 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163387+0000", 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0649000000000002e-05 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177849+0000", 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014462773999999999 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177892+0000", 2023-10-09T04:34:55.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2336999999999998e-05 2023-10-09T04:34:55.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177928+0000", 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6739000000000002e-05 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112698 9.8 9:1730e59e:::benchmark_data_smithi029_299184_object14395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.83371994300001, 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85884108599999998, 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112698 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.254 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319191+0000", 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319196+0000", 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319210+0000", 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4318000000000001e-05 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319217+0000", 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0319999999999998e-06 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319272+0000", 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4486e-05 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319283+0000", 2023-10-09T04:34:55.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1447000000000001e-05 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159355+0000", 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84007240500000002 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159465+0000", 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110021 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159570+0000", 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104728 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163437+0000", 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003866754 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.166866+0000", 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034293549999999998 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177962+0000", 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011095997999999999 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177995+0000", 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2712999999999997e-05 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178033+0000", 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7809999999999999e-05 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112742 9.8 9:1d544479:::benchmark_data_smithi029_299184_object14439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:34:55.261 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.769809208, 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.806861312, 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112742 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:34:55.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383101+0000", 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383107+0000", 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6359999999999999e-06 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383119+0000", 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2502e-05 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383126+0000", 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3629999999999999e-06 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383193+0000", 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7113999999999995e-05 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383204+0000", 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1276e-05 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163543+0000", 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78033866200000002 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163658+0000", 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011532299999999999 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163768+0000", 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110164 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170921+0000", 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071524010000000001 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178095+0000", 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071740179999999999 2023-10-09T04:34:55.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189883+0000", 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011788378 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189927+0000", 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4149e-05 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189964+0000", 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6482000000000001e-05 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112743 9.8 9:12e2e405:::benchmark_data_smithi029_299184_object14440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 211.76976228300001, 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.806888253, 2023-10-09T04:34:55.269 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112743 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383152+0000", 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9610000000000001e-06 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383159+0000", 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9990000000000002e-06 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383164+0000", 2023-10-09T04:34:55.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8269999999999997e-06 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383278+0000", 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114821 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383291+0000", 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2601e-05 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.166912+0000", 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78362119900000005 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.167043+0000", 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013115800000000001 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.167159+0000", 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115868 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178066+0000", 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010906701 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189514+0000", 2023-10-09T04:34:55.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011448309 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189971+0000", 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045658199999999997 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190001+0000", 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0080999999999999e-05 2023-10-09T04:34:55.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190038+0000", 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6575999999999997e-05 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166781 9.8 9:1ce571f8:::benchmark_data_smithi029_299184_object68478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:34:55.277 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 127.07157909199999, 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3412964679999999, 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166781 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:34:55.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081336+0000", 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5390000000000001e-06 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081346+0000", 2023-10-09T04:34:55.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1819999999999995e-06 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081351+0000", 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081392+0000", 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.121e-05 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081516+0000", 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012359900000000001 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081633+0000", 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011679399999999999 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.092967+0000", 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011334426 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106870+0000", 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013902978999999999 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422497+0000", 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3156270430000001 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422563+0000", 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5446999999999999e-05 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422629+0000", 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6181999999999998e-05 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166784 9.8 9:1040dc07:::benchmark_data_smithi029_299184_object68481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.089057+0000", 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 127.063854244, 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.333752869, 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166784 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.285 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089057+0000", 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089057+0000", 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089056+0000", 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089062+0000", 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3899999999999998e-06 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089079+0000", 2023-10-09T04:34:55.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6588000000000001e-05 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089087+0000", 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2099999999999993e-06 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089131+0000", 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3968999999999999e-05 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089249+0000", 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011747300000000001 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089362+0000", 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113409 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093090+0000", 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037278599999999999 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106901+0000", 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013810835 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422722+0000", 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.315821562 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422766+0000", 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3812e-05 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422810+0000", 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4121e-05 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166793 9.8 9:1d211ece:::benchmark_data_smithi029_299184_object68490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.102951+0000", 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 127.04996104200001, 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.331321475, 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166793 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102951+0000", 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102951+0000", 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102949+0000", 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102957+0000", 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5089999999999996e-06 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102972+0000", 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5571000000000001e-05 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102980+0000", 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7440000000000007e-06 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.103061+0000", 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1113999999999996e-05 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.103210+0000", 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014861599999999999 2023-10-09T04:34:55.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.103351+0000", 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141543 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106931+0000", 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035802289999999999 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106940+0000", 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3780000000000001e-06 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434173+0000", 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3272329950000001 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434229+0000", 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5723999999999997e-05 2023-10-09T04:34:55.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434272+0000", 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3520999999999999e-05 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166821 9.8 9:1a502545:::benchmark_data_smithi029_299184_object68518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.151353+0000", 2023-10-09T04:34:55.298 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 127.00155870499999, 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2830994099999999, 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166821 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151353+0000", 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151353+0000", 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151352+0000", 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151357+0000", 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4159999999999997e-06 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151370+0000", 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.326e-05 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151378+0000", 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1380000000000007e-06 2023-10-09T04:34:55.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151414+0000", 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.557e-05 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151500+0000", 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6058999999999997e-05 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:34:55.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151570+0000", 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0357000000000004e-05 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.154437+0000", 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028668280000000001 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.155093+0000", 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00065607000000000003 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434364+0000", 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.279271429 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434411+0000", 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7063e-05 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434452+0000", 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0942999999999997e-05 2023-10-09T04:34:55.304 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166947 9.2 9:463488c9:::benchmark_data_smithi029_299184_object68644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:49.574918+0000", 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 125.577993936, 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.791293099, 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:34:55.305 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166947 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574918+0000", 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:34:55.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574918+0000", 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574917+0000", 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574920+0000", 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2789999999999999e-06 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574927+0000", 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5230000000000001e-06 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574932+0000", 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4349999999999996e-06 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.574983+0000", 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0345e-05 2023-10-09T04:34:55.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.575090+0000", 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010738699999999999 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.575175+0000", 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5117000000000005e-05 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.582659+0000", 2023-10-09T04:34:55.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074840009999999997 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.584908+0000", 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022486889999999999 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:50.366110+0000", 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78120184299999995 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:50.366166+0000", 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6573000000000002e-05 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:50.366211+0000", 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4493999999999998e-05 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:34:55.311 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:34:55.312 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:34:55.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:34:55.408 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:55.614 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:34:56.078 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 76131 76115 89.7333 69.5 0.00597628 0.0111316 2023-10-09T04:34:56.305 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:34:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:56.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:34:56.411 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:56 smithi029 ceph-mon[223858]: pgmap v751: 89 pgs: 3 active+undersized+degraded, 2 stale+active+clean, 15 active+undersized, 69 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13940/218421 objects degraded (6.382%) 2023-10-09T04:34:56.412 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2698771077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:56.412 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:56.414 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:34:56.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:56.472 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:56 smithi029 ceph-mon[216903]: pgmap v751: 89 pgs: 3 active+undersized+degraded, 2 stale+active+clean, 15 active+undersized, 69 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13940/218421 objects degraded (6.382%) 2023-10-09T04:34:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2698771077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:56 smithi177 ceph-mon[195797]: pgmap v751: 89 pgs: 3 active+undersized+degraded, 2 stale+active+clean, 15 active+undersized, 69 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13940/218421 objects degraded (6.382%) 2023-10-09T04:34:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2698771077' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:56.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:34:57.078 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 77150 77134 89.2509 63.6875 0.0174202 0.0112 2023-10-09T04:34:57.517 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:57.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:57.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:34:57.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:57.659 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:34:57.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:57.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: osdmap e203: 8 total, 7 up, 8 in 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: osdmap e203: 8 total, 7 up, 8 in 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:57.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:57.688 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:34:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:34:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: osdmap e203: 8 total, 7 up, 8 in 2023-10-09T04:34:57.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: failed to encode map e203 with expected crc 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:57.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:57.821 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:57.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:57.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:58.025 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:58.078 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 78274 78258 88.9054 70.25 0.00746628 0.0112435 2023-10-09T04:34:58.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:58.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:58.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:34:58.303 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:34:58.338 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:58.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:34:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:58.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:34:58.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:34:58.622 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:58.687 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:34:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:58.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:34:58.791 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:58.820 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:34:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:58 smithi177 ceph-mon[195797]: pgmap v753: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 28518/225051 objects degraded (12.672%) 2023-10-09T04:34:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:58.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:58.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:34:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[223858]: pgmap v753: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 28518/225051 objects degraded (12.672%) 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[216903]: pgmap v753: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 28518/225051 objects degraded (12.672%) 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:34:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:34:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:34:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:34:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:34:59.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 78985 78969 88.1114 44.4375 0.00316967 0.0113323 2023-10-09T04:34:59.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:34:59.553 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:34:59.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:34:59.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:34:59.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:34:59.715 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:00.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 80020 80004 87.7005 64.6875 0.0185603 0.0113983 2023-10-09T04:35:00.200 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:00.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:00.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T04:35:00.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:00.398 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:35:00.399 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T04:35:00.420 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:00.554 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:00.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:00.657 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:00.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:00.732 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:00.832 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:00.867 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:00.889 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:00.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:00 smithi029 ceph-mon[216903]: pgmap v754: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 28518/225051 objects degraded (12.672%) 2023-10-09T04:35:00.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 28518/225051 objects degraded (12.672%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:00.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:00.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:00 smithi029 ceph-mon[223858]: pgmap v754: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 28518/225051 objects degraded (12.672%) 2023-10-09T04:35:00.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 28518/225051 objects degraded (12.672%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:00.903 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:00.945 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:00.982 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:00.999 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:01.027 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:01.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:00 smithi177 ceph-mon[195797]: pgmap v754: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 28518/225051 objects degraded (12.672%) 2023-10-09T04:35:01.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 28518/225051 objects degraded (12.672%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:01.074 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:01.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 81181 81165 87.4394 72.5625 0.00938109 0.0114314 2023-10-09T04:35:01.101 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:01.101 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.101 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.102 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:01.102 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.102 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.102 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.102 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.102 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:35:01.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:01.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:01.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:01.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:01.333 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:01.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1332113407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1332113407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1332113407' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:02.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 82095 82079 86.9256 57.125 0.00274808 0.0114879 2023-10-09T04:35:02.424 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:02.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:02.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:02.442 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:02.442 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:02.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:02.572 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:02 smithi177 ceph-mon[195797]: pgmap v755: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s; 30044/238353 objects degraded (12.605%) 2023-10-09T04:35:02.965 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:02 smithi029 ceph-mon[216903]: pgmap v755: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s; 30044/238353 objects degraded (12.605%) 2023-10-09T04:35:02.965 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:35:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:35:02] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T04:35:02.965 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:02 smithi029 ceph-mon[223858]: pgmap v755: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s; 30044/238353 objects degraded (12.605%) 2023-10-09T04:35:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:03.024 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:03.034 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:03.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:03.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:35:03.079707+0000 min lat: 0.00100934 max lat: 1.04904 avg lat: 0.0115318 2023-10-09T04:35:03.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:35:03.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 83072 83056 86.4945 61.0625 0.00868122 0.0115318 2023-10-09T04:35:03.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:03.174 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:03.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:03.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:03.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:03.406 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:03.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:35:03.811 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:03.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:03.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:03.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:03.973 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:04.042 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:04.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:04.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 84061 84045 86.0898 61.8125 0.00359596 0.0116114 2023-10-09T04:35:04.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:04.231 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:04.332 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:04.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:04.446 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:04.524 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:04.526 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:04.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:04.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:04.706 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:35:04.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:04 smithi177 ceph-mon[195797]: pgmap v756: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 30345/240066 objects degraded (12.640%) 2023-10-09T04:35:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1219940213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:04.856 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:04.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:05.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:05.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:05.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:35:05.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:04 smithi029 ceph-mon[216903]: pgmap v756: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 30345/240066 objects degraded (12.640%) 2023-10-09T04:35:05.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1219940213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:05.067 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:05.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:04 smithi029 ceph-mon[223858]: pgmap v756: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 30345/240066 objects degraded (12.640%) 2023-10-09T04:35:05.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1219940213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:05.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:05.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 85127 85111 85.7758 66.625 0.0120587 0.0116534 2023-10-09T04:35:05.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:05.148 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:05.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:05.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:05.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:05.399 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T04:35:05.399 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:35:05.400 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:35:05.400 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:35:05.400 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:35:05.400 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:35:05.410 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1219940213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: osdmap e204: 8 total, 7 up, 8 in 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 30345/240066 objects degraded (12.640%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:05.919 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:05.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:05.995 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1219940213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: osdmap e204: 8 total, 7 up, 8 in 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.997 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 30345/240066 objects degraded (12.640%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:05.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:35:05.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1219940213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:05.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: osdmap e204: 8 total, 7 up, 8 in 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: failed to encode map e204 with expected crc 2023-10-09T04:35:05.999 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 30345/240066 objects degraded (12.640%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:06.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:06.074 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:06.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 85920 85904 85.201 49.5625 0.00605396 0.0117315 2023-10-09T04:35:06.156 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:06.159 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:06.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:06.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:06.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:06.346 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:06.444 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:06.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:06.508 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:06.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:06.614 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:06.715 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:06 smithi177 ceph-mon[195797]: pgmap v758: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 30959/248679 objects degraded (12.449%) 2023-10-09T04:35:06.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:06 smithi029 ceph-mon[216903]: pgmap v758: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 30959/248679 objects degraded (12.449%) 2023-10-09T04:35:06.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:06 smithi029 ceph-mon[223858]: pgmap v758: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 30959/248679 objects degraded (12.449%) 2023-10-09T04:35:06.893 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:06.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:06.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:06.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:07.030 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:07.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 86967 86951 84.8921 65.4375 0.00567982 0.0117754 2023-10-09T04:35:07.097 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:07.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:07.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:07.191 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:07.191 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:07.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:07.232 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:08.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:35:08.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 87782 87766 84.3697 50.9375 0.012644 0.0118486 2023-10-09T04:35:08.299 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:08.331 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:08.365 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:08.404 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:08.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:08.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:08.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:08.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:08.502 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:08.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:08.535 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.535 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.535 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:08.535 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.535 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.536 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.536 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.536 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:35:08.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:08.570 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:08.580 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:08 smithi029 ceph-mon[216903]: pgmap v759: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 32226/255237 objects degraded (12.626%) 2023-10-09T04:35:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/595918503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:08 smithi029 ceph-mon[223858]: pgmap v759: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 32226/255237 objects degraded (12.626%) 2023-10-09T04:35:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/595918503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:09.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:09.017 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:08 smithi177 ceph-mon[195797]: pgmap v759: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 32226/255237 objects degraded (12.626%) 2023-10-09T04:35:09.017 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/595918503' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:09.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:09.079 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 88657 88641 83.92 54.6875 0.00329007 0.0118928 2023-10-09T04:35:09.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:35:09.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:09.186 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:09.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:09.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:09.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:09.429 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:09.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:09.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:09.603 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:09.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:09.718 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:35:09.718 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:35:09.718 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:35:09.744 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:09.812 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:09.825 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:09.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:09.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:10.018 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:10.081 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 89479 89463 83.4323 51.375 0.00790088 0.0119811 2023-10-09T04:35:10.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:10.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:10.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:10.906 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:10.973 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:10.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:10.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:10 smithi029 ceph-mon[216903]: pgmap v760: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 32226/255237 objects degraded (12.626%) 2023-10-09T04:35:10.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 32226/255237 objects degraded (12.626%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:10.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:10 smithi029 ceph-mon[223858]: pgmap v760: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 32226/255237 objects degraded (12.626%) 2023-10-09T04:35:10.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 32226/255237 objects degraded (12.626%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:10.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:11.011 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:10 smithi177 ceph-mon[195797]: pgmap v760: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 32226/255237 objects degraded (12.626%) 2023-10-09T04:35:11.011 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 32226/255237 objects degraded (12.626%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:11.081 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 90305 90289 82.9646 51.625 0.0102849 0.0120478 2023-10-09T04:35:11.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:11.156 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:11.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:35:11.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:11.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:11.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:11.775 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:11.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:11.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:11.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:12.054 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:12.081 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 91372 91356 82.7287 66.6875 0.0130998 0.012083 2023-10-09T04:35:12.213 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:12.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:12.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:12.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:12.517 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:12.750 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:35:12.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:12.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:12.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:12.863 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:12.956 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:12 smithi177 ceph-mon[195797]: pgmap v761: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 33528/266718 objects degraded (12.571%) 2023-10-09T04:35:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3078258886' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:13.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:12 smithi029 ceph-mon[216903]: pgmap v761: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 33528/266718 objects degraded (12.571%) 2023-10-09T04:35:13.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3078258886' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:13.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:13.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:12 smithi029 ceph-mon[223858]: pgmap v761: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s; 33528/266718 objects degraded (12.571%) 2023-10-09T04:35:13.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3078258886' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:13.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:13.046 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:35:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:35:12] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216848 8.4 8:211e3bcd:::benchmark_data_smithi029_252925_object100224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.014383+0000", 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 487.04412845600001, 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.06401448, 2023-10-09T04:35:13.065 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216848 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014383+0000", 2023-10-09T04:35:13.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014383+0000", 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014381+0000", 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:13.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014393+0000", 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1449e-05 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014412+0000", 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8734999999999999e-05 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014420+0000", 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1019999999999997e-06 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014463+0000", 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3296e-05 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014764+0000", 2023-10-09T04:35:13.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030136699999999998 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014876+0000", 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111643 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.021735+0000", 2023-10-09T04:35:13.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006858846 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.024857+0000", 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003122437 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047062+0000", 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022204433999999999 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047069+0000", 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0199999999999997e-06 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078305+0000", 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0312359339999999 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078398+0000", 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3210999999999999e-05 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216849 8.d 8:b99a9c5c:::benchmark_data_smithi029_252925_object100225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 487.04407567200002, 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.065392602, 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216849 2023-10-09T04:35:13.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014436+0000", 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014439+0000", 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6399999999999999e-06 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014447+0000", 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4039999999999998e-06 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014451+0000", 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2400000000000001e-06 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014508+0000", 2023-10-09T04:35:13.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6774999999999998e-05 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014640+0000", 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013230199999999999 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.014765+0000", 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012486499999999999 2023-10-09T04:35:13.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.021125+0000", 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063596449999999997 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.021747+0000", 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062273799999999996 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.045986+0000", 2023-10-09T04:35:13.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024238684999999999 2023-10-09T04:35:13.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.045996+0000", 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9280000000000005e-06 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079773+0000", 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.03377703 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079829+0000", 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.588e-05 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216854 8.4 8:2ca06186:::benchmark_data_smithi029_252925_object100230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.028195+0000", 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 487.03031650700001, 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.050669147, 2023-10-09T04:35:13.083 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216854 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028195+0000", 2023-10-09T04:35:13.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028195+0000", 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028194+0000", 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028200+0000", 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2450000000000003e-06 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028215+0000", 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5193e-05 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028221+0000", 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0569999999999998e-06 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028262+0000", 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0238999999999999e-05 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028380+0000", 2023-10-09T04:35:13.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118136 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.028518+0000", 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000137847 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.037151+0000", 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086330499999999998 2023-10-09T04:35:13.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047506+0000", 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010354847 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.047513+0000", 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.294e-06 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078789+0000", 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0312761539999999 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078865+0000", 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5489999999999997e-05 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216858 8.d 8:baa8e899:::benchmark_data_smithi029_252925_object100234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.031712+0000", 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 487.02680008999999, 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0484245670000001, 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216858 2023-10-09T04:35:13.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031712+0000", 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031712+0000", 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031710+0000", 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031718+0000", 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8040000000000002e-06 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031731+0000", 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2768e-05 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031738+0000", 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2490000000000004e-06 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031805+0000", 2023-10-09T04:35:13.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6885999999999996e-05 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.031928+0000", 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012300399999999999 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.032043+0000", 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114602 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.037383+0000", 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005340142 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.046091+0000", 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0087078280000000008 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.046097+0000", 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5320000000000003e-06 2023-10-09T04:35:13.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.080075+0000", 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.033978201 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.080136+0000", 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1017000000000002e-05 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.097 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.34629.0:216872 8.4 8:202ae64d:::benchmark_data_smithi029_252925_object100248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:27:06.063198+0000", 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 486.99531380799999, 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.021720041, 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.34629", 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:35:13.098 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 216872 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063198+0000", 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063198+0000", 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063197+0000", 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063203+0000", 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9900000000000002e-06 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063215+0000", 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2184999999999999e-05 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063222+0000", 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1380000000000001e-06 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063260+0000", 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7339000000000003e-05 2023-10-09T04:35:13.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:06.063265+0000", 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2560000000000002e-06 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.078904+0000", 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0156389729999999 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079001+0000", 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7195999999999996e-05 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4", 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.079093+0000", 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1589999999999996e-05 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084300+0000", 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052073559999999998 2023-10-09T04:35:13.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084747+0000", 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00044712400000000002 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084810+0000", 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3211000000000002e-05 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:27:07.084918+0000", 2023-10-09T04:35:13.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107633 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38499 9.9 9:9a8fb78d:::benchmark_data_smithi029_299184_object38498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03773888299997, 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.179240123, 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.105 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38499 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020771+0000", 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020915+0000", 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000143788 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020932+0000", 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.698e-05 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020941+0000", 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2529999999999993e-06 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022592+0000", 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016505980000000001 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022601+0000", 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9099999999999994e-06 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.199150+0000", 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.17654937 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.200013+0000", 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000862749 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38501 9.9 9:91e7d993:::benchmark_data_smithi029_299184_object38500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03754684099999, 2023-10-09T04:35:13.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1804904860000001, 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38501 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020964+0000", 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021040+0000", 2023-10-09T04:35:13.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5575999999999996e-05 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021060+0000", 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9754000000000001e-05 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021065+0000", 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4999999999999999e-06 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022627+0000", 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015613140000000001 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022638+0000", 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1204e-05 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201357+0000", 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1787194400000001 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201456+0000", 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8263000000000004e-05 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.115 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38505 9.9 9:9fe8cf27:::benchmark_data_smithi029_299184_object38504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03744090399999, 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1806352849999999, 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38505 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021070+0000", 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021146+0000", 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5275e-05 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021154+0000", 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8720000000000002e-06 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021170+0000", 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6252999999999998e-05 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022657+0000", 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001487588 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022662+0000", 2023-10-09T04:35:13.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5549999999999996e-06 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201579+0000", 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1789165530000001 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201706+0000", 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127687 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.120 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38509 9.9 9:938e198b:::benchmark_data_smithi029_299184_object38508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03733643599998, 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.180647043, 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38509 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.121 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021220+0000", 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5445000000000003e-05 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021249+0000", 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8875000000000001e-05 2023-10-09T04:35:13.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021254+0000", 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1209999999999998e-06 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022682+0000", 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014274299999999999 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022687+0000", 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4149999999999998e-06 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201740+0000", 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1790529270000001 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201822+0000", 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2379000000000005e-05 2023-10-09T04:35:13.125 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38510 9.9 9:98f164af:::benchmark_data_smithi029_299184_object38509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03725259499998, 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.180751331, 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.126 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38510 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021326+0000", 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6715000000000001e-05 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021334+0000", 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7129999999999994e-06 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021340+0000", 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1510000000000001e-06 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022709+0000", 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013682690000000001 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022715+0000", 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0990000000000004e-06 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201936+0000", 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1792214670000001 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202011+0000", 2023-10-09T04:35:13.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4392000000000002e-05 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38528 9.9 9:91ac7278:::benchmark_data_smithi029_299184_object38527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03714517899999, 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.203529793, 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38528 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:35:13.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021366+0000", 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021405+0000", 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8535999999999998e-05 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021412+0000", 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8129999999999996e-06 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021418+0000", 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2499999999999997e-06 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022736+0000", 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001317948 2023-10-09T04:35:13.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022741+0000", 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.096e-06 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202061+0000", 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1793201959999999 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202400+0000", 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033863900000000001 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202619+0000", 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000219619 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.207410+0000", 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047910080000000002 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209061+0000", 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001650257 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.222756+0000", 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013695182 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.222822+0000", 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6346000000000005e-05 2023-10-09T04:35:13.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.224896+0000", 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002074471 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38539 9.9 9:931725d0:::benchmark_data_smithi029_299184_object38538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:35:13.138 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.03708930200003, 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2048092829999999, 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38539 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:35:13.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021422+0000", 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021457+0000", 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5354000000000001e-05 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021465+0000", 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2339999999999997e-06 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021469+0000", 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3619999999999999e-06 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022985+0000", 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015154610000000001 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022992+0000", 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3250000000000002e-06 2023-10-09T04:35:13.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204186+0000", 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.181193653 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204309+0000", 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123767 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:35:13.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204574+0000", 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026451699999999997 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209071+0000", 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044968439999999998 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209400+0000", 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032915499999999997 2023-10-09T04:35:13.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.225011+0000", 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015611534 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.225082+0000", 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0273999999999998e-05 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226232+0000", 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001150264 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38547 9.9 9:9345b2c2:::benchmark_data_smithi029_299184_object38546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 322.037038021, 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.206218247, 2023-10-09T04:35:13.146 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38547 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021473+0000", 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021502+0000", 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8175999999999999e-05 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021508+0000", 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021513+0000", 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3660000000000002e-06 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.023032+0000", 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015193610000000001 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.023040+0000", 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6899999999999992e-06 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.205732+0000", 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1826917210000001 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.205900+0000", 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000168913 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.206070+0000", 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016947199999999999 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209441+0000", 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033709899999999999 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209549+0000", 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108433 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226306+0000", 2023-10-09T04:35:13.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016757141 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226351+0000", 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4988999999999999e-05 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.227692+0000", 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001340651 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.153 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:47657 10.9 10:9a4d341d:::benchmark_data_smithi029_344401_object47656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:34:29.261744+0000", 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 43.796768176, 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33510193500000002, 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.154 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 47657 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261744+0000", 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261744+0000", 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261739+0000", 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999957 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261790+0000", 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0769000000000001e-05 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261807+0000", 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6840000000000001e-05 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261815+0000", 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0299999999999994e-06 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261877+0000", 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1622000000000001e-05 2023-10-09T04:35:13.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261981+0000", 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000104526 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.262071+0000", 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9870000000000005e-05 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.264281+0000", 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022103489999999999 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.265935+0000", 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001654097 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596749+0000", 2023-10-09T04:35:13.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.330813093 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596804+0000", 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5828999999999997e-05 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596846+0000", 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1291999999999997e-05 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.160 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88629 10.3 10:c811f6cb:::benchmark_data_smithi029_344401_object88628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.948379+0000", 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.1101326440000001, 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27891803100000001, 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88629 2023-10-09T04:35:13.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948379+0000", 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948379+0000", 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948378+0000", 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948410+0000", 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2598000000000002e-05 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948421+0000", 2023-10-09T04:35:13.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0132000000000001e-05 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948426+0000", 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7479999999999999e-06 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948590+0000", 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016400299999999999 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948722+0000", 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013193100000000001 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948924+0000", 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000202108 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952152+0000", 2023-10-09T04:35:13.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032279940000000001 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.226908+0000", 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27475596800000002 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.227191+0000", 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028242099999999998 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.227297+0000", 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106526 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88633 10.3 10:c172d3cd:::benchmark_data_smithi029_344401_object88632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.952469+0000", 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.1060426320000003, 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28261339000000002, 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.167 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88633 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952469+0000", 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952469+0000", 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952468+0000", 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952583+0000", 2023-10-09T04:35:13.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114634 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952599+0000", 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6863000000000001e-05 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952623+0000", 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3331000000000001e-05 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952692+0000", 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9302999999999997e-05 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952822+0000", 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000130302 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952961+0000", 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013819700000000001 2023-10-09T04:35:13.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.956280+0000", 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033191800000000001 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.234943+0000", 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27866359000000002 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.234995+0000", 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2012999999999998e-05 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.235083+0000", 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7232999999999996e-05 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88640 10.3 10:c9aaa04f:::benchmark_data_smithi029_344401_object88639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:13.173 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.961126+0000", 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.0973855559999999, 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27617952299999998, 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88640 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961126+0000", 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961126+0000", 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961125+0000", 2023-10-09T04:35:13.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961206+0000", 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0918e-05 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961238+0000", 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2308999999999998e-05 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961245+0000", 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3010000000000003e-06 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961301+0000", 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6542000000000002e-05 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961462+0000", 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016058000000000001 2023-10-09T04:35:13.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961570+0000", 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108497 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.964012+0000", 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024415130000000002 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.237138+0000", 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27312640999999999 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.237201+0000", 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3275000000000007e-05 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.237306+0000", 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010450000000000001 2023-10-09T04:35:13.179 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88647 10.3 10:cc5e9e4d:::benchmark_data_smithi029_344401_object88646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.976906+0000", 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.0816055489999998, 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.260601576, 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.180 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88647 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.976906+0000", 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.976906+0000", 2023-10-09T04:35:13.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.976905+0000", 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.976940+0000", 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4742e-05 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.976955+0000", 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5871000000000001e-05 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.976962+0000", 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6209999999999998e-06 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977191+0000", 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00022886799999999999 2023-10-09T04:35:13.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977327+0000", 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000136363 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977463+0000", 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013536399999999999 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983188+0000", 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005725127 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.237369+0000", 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25418148499999998 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.237421+0000", 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2058999999999997e-05 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.237508+0000", 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6640999999999999e-05 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88650 10.3 10:c9bc8160:::benchmark_data_smithi029_344401_object88649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.979435+0000", 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.0790767250000002, 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.264476773, 2023-10-09T04:35:13.186 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88650 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979435+0000", 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979435+0000", 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979433+0000", 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979486+0000", 2023-10-09T04:35:13.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3000000000000001e-05 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979505+0000", 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8817e-05 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979511+0000", 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1410000000000002e-06 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.980114+0000", 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00060228899999999999 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.980234+0000", 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120141 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.980365+0000", 2023-10-09T04:35:13.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131333 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983356+0000", 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029913499999999998 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.243776+0000", 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.260419228 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.243828+0000", 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2061999999999997e-05 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.243912+0000", 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4315999999999997e-05 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88657 10.3 10:cc315a2c:::benchmark_data_smithi029_344401_object88656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.990753+0000", 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.0677593090000004, 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.253320184, 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88657 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:13.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990753+0000", 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990753+0000", 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:13.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990751+0000", 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990805+0000", 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4191999999999999e-05 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990820+0000", 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4912999999999999e-05 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990826+0000", 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6149999999999996e-06 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990868+0000", 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2029e-05 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:13.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.990971+0000", 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103156 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.991073+0000", 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010247300000000001 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.993464+0000", 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023908570000000001 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.243952+0000", 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.250487246 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.243996+0000", 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4608999999999999e-05 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244073+0000", 2023-10-09T04:35:13.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6602999999999998e-05 2023-10-09T04:35:13.199 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.199 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.199 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.199 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:13.199 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:13.199 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:35:13.200 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 92374 92358 82.4414 62.625 0.00909518 0.012125 2023-10-09T04:35:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:13.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:13.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:13.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:13.375 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:13.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:13.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:13.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:13.624 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:13.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:13.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:13.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:13.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:13.808 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:13.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:13.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:13.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:13.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:13.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: failed to encode map e205 with expected crc 2023-10-09T04:35:13.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:13.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: osdmap e205: 8 total, 7 up, 8 in 2023-10-09T04:35:13.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: failed to encode map e205 with expected crc 2023-10-09T04:35:13.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: failed to encode map e205 with expected crc 2023-10-09T04:35:13.932 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: failed to encode map e205 with expected crc 2023-10-09T04:35:13.932 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: failed to encode map e205 with expected crc 2023-10-09T04:35:13.932 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:14.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:14.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 93146 93130 81.9598 48.25 0.00413423 0.0121845 2023-10-09T04:35:14.131 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: osdmap e205: 8 total, 7 up, 8 in 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: osdmap e205: 8 total, 7 up, 8 in 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: failed to encode map e205 with expected crc 2023-10-09T04:35:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:14.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:14.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:14.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:35:14.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:14.677 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:14.773 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:14 smithi177 ceph-mon[195797]: pgmap v763: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s; 33813/268419 objects degraded (12.597%) 2023-10-09T04:35:14.796 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:14.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:14.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:14.907 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:14 smithi029 ceph-mon[223858]: pgmap v763: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s; 33813/268419 objects degraded (12.597%) 2023-10-09T04:35:14.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:14 smithi029 ceph-mon[216903]: pgmap v763: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s; 33813/268419 objects degraded (12.597%) 2023-10-09T04:35:14.964 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:15.061 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:15.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:15.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 94046 94030 81.6027 56.25 0.0211751 0.0122494 2023-10-09T04:35:15.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:15.180 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:15.190 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:15.206 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:15.231 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:15.239 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:15.260 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:15.290 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:15.321 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.355 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.356 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.356 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:35:15.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:15.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:15.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:15.570 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:15.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:15.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/301655825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:15.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 33813/268419 objects degraded (12.597%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:15.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:15.819 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:15.952 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T04:35:15.952 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:35:15.953 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T04:35:15.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:35:16.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:16.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:16.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 15 94889 94874 81.2075 52.75 0.0281793 0.0123091 2023-10-09T04:35:16.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:16.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/301655825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:16.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 33813/268419 objects degraded (12.597%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/301655825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 33813/268419 objects degraded (12.597%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:16.179 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:16.406 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:16.409 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:16.410 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:16.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:16.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:16.512 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:16.550 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:16.583 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:16.601 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:35:16.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:16.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:16.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:16.844 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:16 smithi177 ceph-mon[195797]: pgmap v764: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s; 34419/276189 objects degraded (12.462%) 2023-10-09T04:35:16.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:16.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:16 smithi029 ceph-mon[216903]: pgmap v764: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s; 34419/276189 objects degraded (12.462%) 2023-10-09T04:35:16.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:16 smithi029 ceph-mon[223858]: pgmap v764: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s; 34419/276189 objects degraded (12.462%) 2023-10-09T04:35:17.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 95927 95911 80.9859 64.8125 0.0191203 0.0123434 2023-10-09T04:35:17.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:17.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:17.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:17.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:17.314 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:17.485 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:17.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:17.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:17.619 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:17.701 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:17.701 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:17.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:17.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:17.835 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:17.914 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:18.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:18.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:18.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 96678 96662 80.5319 46.9375 0.008698 0.012413 2023-10-09T04:35:18.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:18.147 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:18.158 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:18.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:18.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:18.302 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:18.334 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:18.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:35:18.552 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:18.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:18.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:18.722 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:18.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:35:18.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:18.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:18.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:19.009 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:18 smithi177 ceph-mon[195797]: pgmap v765: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s; 35563/282114 objects degraded (12.606%) 2023-10-09T04:35:19.012 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:19.067 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:35:19.074 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:35:19.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 97427 97411 80.0883 46.8125 0.0166695 0.0124815 2023-10-09T04:35:19.106 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:18 smithi029 ceph-mon[216903]: pgmap v765: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s; 35563/282114 objects degraded (12.606%) 2023-10-09T04:35:19.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:18 smithi029 ceph-mon[223858]: pgmap v765: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s; 35563/282114 objects degraded (12.606%) 2023-10-09T04:35:19.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:19.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:19.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:19.288 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:19.355 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:19.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:19.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:19.479 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:19.498 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:19.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:19.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:19.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:19.675 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:19.734 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:19.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:19.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:19.888 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:20.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 98068 98052 79.5684 40.0625 0.00698215 0.0125566 2023-10-09T04:35:20.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:20.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:20.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:20.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:20.280 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:20.600 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:20.603 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":766,"stamp":"2023-10-09T04:35:18.948737+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6161897376,"num_objects":94038,"num_object_clones":0,"num_object_copies":282114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35563,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94038,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":94314,"num_write_kb":6019341,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":3083510,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94230,"ondisk_log_size":94230,"up":242,"acting":242,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22172180,"kb_used_data":17528148,"kb_used_omap":20,"kb_used_meta":4643947,"kb_avail":727789036,"statfs":{"total":767960285184,"available":745255972864,"internally_reserved":0,"allocated":17948823552,"data_stored":17943744753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4755402382},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,3,1,3,1,3,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":587137024,"num_objects":8959,"num_object_clones":0,"num_object_copies":26877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3337,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8959,"num_write_kb":573376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002956"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":191,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.712525+0000","last_change":"2023-10-09T04:34:51.322569+0000","last_active":"2023-10-09T04:35:04.712525+0000","last_peered":"2023-10-09T04:35:04.712525+0000","last_clean":"2023-10-09T04:34:44.061742+0000","last_became_active":"2023-10-09T04:34:51.322569+0000","last_became_peered":"2023-10-09T04:34:51.322569+0000","last_unstale":"2023-10-09T04:35:04.712525+0000","last_undegraded":"2023-10-09T04:35:04.712525+0000","last_fullsized":"2023-10-09T04:34:51.319415+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:46.193676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693457+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:35:04.693457+0000","last_peered":"2023-10-09T04:35:04.693457+0000","last_clean":"2023-10-09T04:35:04.693457+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:35:04.693457+0000","last_undegraded":"2023-10-09T04:35:04.693457+0000","last_fullsized":"2023-10-09T04:35:04.693457+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692024+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:35:04.692024+0000","last_peered":"2023-10-09T04:35:04.692024+0000","last_clean":"2023-10-09T04:35:04.692024+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:35:04.692024+0000","last_undegraded":"2023-10-09T04:35:04.692024+0000","last_fullsized":"2023-10-09T04:35:04.692024+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":118,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712383+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:35:04.712383+0000","last_peered":"2023-10-09T04:35:04.712383+0000","last_clean":"2023-10-09T04:35:04.712383+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:35:04.712383+0000","last_undegraded":"2023-10-09T04:35:04.712383+0000","last_fullsized":"2023-10-09T04:35:04.712383+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":191,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:11.566425+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:35:11.566425+0000","last_peered":"2023-10-09T04:35:11.566425+0000","last_clean":"2023-10-09T04:35:11.566425+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:35:11.566425+0000","last_undegraded":"2023-10-09T04:35:11.566425+0000","last_fullsized":"2023-10-09T04:35:11.566425+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.707847+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:35:04.707847+0000","last_peered":"2023-10-09T04:35:04.707847+0000","last_clean":"2023-10-09T04:35:04.707847+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:35:04.707847+0000","last_undegraded":"2023-10-09T04:35:04.707847+0000","last_fullsized":"2023-10-09T04:35:04.707847+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":117,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709371+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:35:04.709371+0000","last_peered":"2023-10-09T04:35:04.709371+0000","last_clean":"2023-10-09T04:35:04.709371+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:35:04.709371+0000","last_undegraded":"2023-10-09T04:35:04.709371+0000","last_fullsized":"2023-10-09T04:35:04.709371+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692356+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:35:04.692356+0000","last_peered":"2023-10-09T04:35:04.692356+0000","last_clean":"2023-10-09T04:35:04.692356+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:35:04.692356+0000","last_undegraded":"2023-10-09T04:35:04.692356+0000","last_fullsized":"2023-10-09T04:35:04.692356+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712213+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:35:04.712213+0000","last_peered":"2023-10-09T04:35:04.712213+0000","last_clean":"2023-10-09T04:35:04.712213+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:35:04.712213+0000","last_undegraded":"2023-10-09T04:35:04.712213+0000","last_fullsized":"2023-10-09T04:35:04.712213+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":128,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692647+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:35:04.692647+0000","last_peered":"2023-10-09T04:35:04.692647+0000","last_clean":"2023-10-09T04:35:04.692647+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:35:04.692647+0000","last_undegraded":"2023-10-09T04:35:04.692647+0000","last_fullsized":"2023-10-09T04:35:04.692647+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712060+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:35:04.712060+0000","last_peered":"2023-10-09T04:35:04.712060+0000","last_clean":"2023-10-09T04:35:04.712060+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:35:04.712060+0000","last_undegraded":"2023-10-09T04:35:04.712060+0000","last_fullsized":"2023-10-09T04:35:04.712060+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":175,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.692110+0000","last_change":"2023-10-09T04:34:51.322394+0000","last_active":"2023-10-09T04:35:04.692110+0000","last_peered":"2023-10-09T04:35:04.692110+0000","last_clean":"2023-10-09T04:34:44.053596+0000","last_became_active":"2023-10-09T04:34:51.322394+0000","last_became_peered":"2023-10-09T04:34:51.322394+0000","last_unstale":"2023-10-09T04:35:04.692110+0000","last_undegraded":"2023-10-09T04:35:04.692110+0000","last_fullsized":"2023-10-09T04:34:51.313456+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:55.868058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.714639+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:35:04.714639+0000","last_peered":"2023-10-09T04:35:04.714639+0000","last_clean":"2023-10-09T04:35:04.714639+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:35:04.714639+0000","last_undegraded":"2023-10-09T04:35:04.714639+0000","last_fullsized":"2023-10-09T04:35:04.714639+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712305+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:35:04.712305+0000","last_peered":"2023-10-09T04:35:04.712305+0000","last_clean":"2023-10-09T04:35:04.712305+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:35:04.712305+0000","last_undegraded":"2023-10-09T04:35:04.712305+0000","last_fullsized":"2023-10-09T04:35:04.712305+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":173,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:04.711717+0000","last_change":"2023-10-09T04:34:51.334626+0000","last_active":"2023-10-09T04:35:04.711717+0000","last_peered":"2023-10-09T04:35:04.711717+0000","last_clean":"2023-10-09T04:34:44.064651+0000","last_became_active":"2023-10-09T04:34:51.334626+0000","last_became_peered":"2023-10-09T04:34:51.334626+0000","last_unstale":"2023-10-09T04:35:04.711717+0000","last_undegraded":"2023-10-09T04:34:51.326465+0000","last_fullsized":"2023-10-09T04:34:51.326345+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:10.974139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692463+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:35:04.692463+0000","last_peered":"2023-10-09T04:35:04.692463+0000","last_clean":"2023-10-09T04:35:04.692463+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:35:04.692463+0000","last_undegraded":"2023-10-09T04:35:04.692463+0000","last_fullsized":"2023-10-09T04:35:04.692463+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":117,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711428+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:35:04.711428+0000","last_peered":"2023-10-09T04:35:04.711428+0000","last_clean":"2023-10-09T04:35:04.711428+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:35:04.711428+0000","last_undegraded":"2023-10-09T04:35:04.711428+0000","last_fullsized":"2023-10-09T04:35:04.711428+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709583+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:35:04.709583+0000","last_peered":"2023-10-09T04:35:04.709583+0000","last_clean":"2023-10-09T04:35:04.709583+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:35:04.709583+0000","last_undegraded":"2023-10-09T04:35:04.709583+0000","last_fullsized":"2023-10-09T04:35:04.709583+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":147,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.714158+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:35:04.714158+0000","last_peered":"2023-10-09T04:35:04.714158+0000","last_clean":"2023-10-09T04:35:04.714158+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:35:04.714158+0000","last_undegraded":"2023-10-09T04:35:04.714158+0000","last_fullsized":"2023-10-09T04:35:04.714158+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":116,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692297+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:35:04.692297+0000","last_peered":"2023-10-09T04:35:04.692297+0000","last_clean":"2023-10-09T04:35:04.692297+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:35:04.692297+0000","last_undegraded":"2023-10-09T04:35:04.692297+0000","last_fullsized":"2023-10-09T04:35:04.692297+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":211,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.707177+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:35:04.707177+0000","last_peered":"2023-10-09T04:35:04.707177+0000","last_clean":"2023-10-09T04:35:04.707177+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:35:04.707177+0000","last_undegraded":"2023-10-09T04:35:04.707177+0000","last_fullsized":"2023-10-09T04:35:04.707177+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:11.566620+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:35:11.566620+0000","last_peered":"2023-10-09T04:35:11.566620+0000","last_clean":"2023-10-09T04:35:11.566620+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:35:11.566620+0000","last_undegraded":"2023-10-09T04:35:11.566620+0000","last_fullsized":"2023-10-09T04:35:11.566620+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"205'5700","reported_seq":5723,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.974479+0000","last_change":"2023-10-09T04:33:51.283670+0000","last_active":"2023-10-09T04:35:14.974479+0000","last_peered":"2023-10-09T04:35:14.974479+0000","last_clean":"2023-10-09T04:35:14.974479+0000","last_became_active":"2023-10-09T04:33:51.283338+0000","last_became_peered":"2023-10-09T04:33:51.283338+0000","last_unstale":"2023-10-09T04:35:14.974479+0000","last_undegraded":"2023-10-09T04:35:14.974479+0000","last_fullsized":"2023-10-09T04:35:14.974479+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5700,"log_dups_size":0,"ondisk_log_size":5700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:57.393546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373555200,"num_objects":5700,"num_object_clones":0,"num_object_copies":17100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5700,"num_write_kb":364800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":193,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711935+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:35:04.711935+0000","last_peered":"2023-10-09T04:35:04.711935+0000","last_clean":"2023-10-09T04:35:04.711935+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:35:04.711935+0000","last_undegraded":"2023-10-09T04:35:04.711935+0000","last_fullsized":"2023-10-09T04:35:04.711935+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":147,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709151+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:35:04.709151+0000","last_peered":"2023-10-09T04:35:04.709151+0000","last_clean":"2023-10-09T04:35:04.709151+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:35:04.709151+0000","last_undegraded":"2023-10-09T04:35:04.709151+0000","last_fullsized":"2023-10-09T04:35:04.709151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"205'5808","reported_seq":5829,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.744229+0000","last_change":"2023-10-09T04:33:51.451657+0000","last_active":"2023-10-09T04:35:14.744229+0000","last_peered":"2023-10-09T04:35:14.744229+0000","last_clean":"2023-10-09T04:35:14.744229+0000","last_became_active":"2023-10-09T04:33:51.451300+0000","last_became_peered":"2023-10-09T04:33:51.451300+0000","last_unstale":"2023-10-09T04:35:14.744229+0000","last_undegraded":"2023-10-09T04:35:14.744229+0000","last_fullsized":"2023-10-09T04:35:14.744229+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:02.431444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380633088,"num_objects":5808,"num_object_clones":0,"num_object_copies":17424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5808,"num_write_kb":371712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":166,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712310+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:35:04.712310+0000","last_peered":"2023-10-09T04:35:04.712310+0000","last_clean":"2023-10-09T04:35:04.712310+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:35:04.712310+0000","last_undegraded":"2023-10-09T04:35:04.712310+0000","last_fullsized":"2023-10-09T04:35:04.712310+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.718173+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:35:04.718173+0000","last_peered":"2023-10-09T04:35:04.718173+0000","last_clean":"2023-10-09T04:35:04.718173+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:35:04.718173+0000","last_undegraded":"2023-10-09T04:35:04.718173+0000","last_fullsized":"2023-10-09T04:35:04.718173+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"205'5967","reported_seq":6003,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.754286+0000","last_change":"2023-10-09T04:34:22.281046+0000","last_active":"2023-10-09T04:35:14.754286+0000","last_peered":"2023-10-09T04:35:14.754286+0000","last_clean":"2023-10-09T04:35:14.754286+0000","last_became_active":"2023-10-09T04:34:22.265736+0000","last_became_peered":"2023-10-09T04:34:22.265736+0000","last_unstale":"2023-10-09T04:35:14.754286+0000","last_undegraded":"2023-10-09T04:35:14.754286+0000","last_fullsized":"2023-10-09T04:35:14.754286+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5967,"log_dups_size":0,"ondisk_log_size":5967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:23.231373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391053312,"num_objects":5967,"num_object_clones":0,"num_object_copies":17901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5967,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":148,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711898+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:35:04.711898+0000","last_peered":"2023-10-09T04:35:04.711898+0000","last_clean":"2023-10-09T04:35:04.711898+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:35:04.711898+0000","last_undegraded":"2023-10-09T04:35:04.711898+0000","last_fullsized":"2023-10-09T04:35:04.711898+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.708522+0000","last_change":"2023-10-09T04:34:51.328996+0000","last_active":"2023-10-09T04:35:04.708522+0000","last_peered":"2023-10-09T04:35:04.708522+0000","last_clean":"2023-10-09T04:34:44.068074+0000","last_became_active":"2023-10-09T04:34:51.328996+0000","last_became_peered":"2023-10-09T04:34:51.328996+0000","last_unstale":"2023-10-09T04:35:04.708522+0000","last_undegraded":"2023-10-09T04:35:04.708522+0000","last_fullsized":"2023-10-09T04:34:51.321084+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:56.215110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"205'6013","reported_seq":6040,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:16.541209+0000","last_change":"2023-10-09T04:34:51.318540+0000","last_active":"2023-10-09T04:35:16.541209+0000","last_peered":"2023-10-09T04:35:16.541209+0000","last_clean":"2023-10-09T04:34:50.268593+0000","last_became_active":"2023-10-09T04:34:51.318540+0000","last_became_peered":"2023-10-09T04:34:51.318540+0000","last_unstale":"2023-10-09T04:35:16.541209+0000","last_undegraded":"2023-10-09T04:34:51.315308+0000","last_fullsized":"2023-10-09T04:34:51.315200+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6013,"log_dups_size":0,"ondisk_log_size":6013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.444866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394067968,"num_objects":6013,"num_object_clones":0,"num_object_copies":18039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6013,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6013,"num_write_kb":384832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":6013}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709441+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:35:04.709441+0000","last_peered":"2023-10-09T04:35:04.709441+0000","last_clean":"2023-10-09T04:35:04.709441+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:35:04.709441+0000","last_undegraded":"2023-10-09T04:35:04.709441+0000","last_fullsized":"2023-10-09T04:35:04.709441+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":62,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709867+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:35:04.709867+0000","last_peered":"2023-10-09T04:35:04.709867+0000","last_clean":"2023-10-09T04:35:04.709867+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:35:04.709867+0000","last_undegraded":"2023-10-09T04:35:04.709867+0000","last_fullsized":"2023-10-09T04:35:04.709867+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.2","version":"205'6059","reported_seq":6083,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:16.609280+0000","last_change":"2023-10-09T04:33:51.279607+0000","last_active":"2023-10-09T04:35:16.609280+0000","last_peered":"2023-10-09T04:35:16.609280+0000","last_clean":"2023-10-09T04:35:16.609280+0000","last_became_active":"2023-10-09T04:33:51.279269+0000","last_became_peered":"2023-10-09T04:33:51.279269+0000","last_unstale":"2023-10-09T04:35:16.609280+0000","last_undegraded":"2023-10-09T04:35:16.609280+0000","last_fullsized":"2023-10-09T04:35:16.609280+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6059,"log_dups_size":0,"ondisk_log_size":6059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:05.128040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397082624,"num_objects":6059,"num_object_clones":0,"num_object_copies":18177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6059,"num_write_kb":387776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":199,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693351+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:35:04.693351+0000","last_peered":"2023-10-09T04:35:04.693351+0000","last_clean":"2023-10-09T04:35:04.693351+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:35:04.693351+0000","last_undegraded":"2023-10-09T04:35:04.693351+0000","last_fullsized":"2023-10-09T04:35:04.693351+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.714551+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:35:04.714551+0000","last_peered":"2023-10-09T04:35:04.714551+0000","last_clean":"2023-10-09T04:35:04.714551+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:35:04.714551+0000","last_undegraded":"2023-10-09T04:35:04.714551+0000","last_fullsized":"2023-10-09T04:35:04.714551+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"205'5840","reported_seq":5869,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:14.955297+0000","last_change":"2023-10-09T04:34:51.328975+0000","last_active":"2023-10-09T04:35:14.955297+0000","last_peered":"2023-10-09T04:35:14.955297+0000","last_clean":"2023-10-09T04:34:50.301002+0000","last_became_active":"2023-10-09T04:34:51.328975+0000","last_became_peered":"2023-10-09T04:34:51.328975+0000","last_unstale":"2023-10-09T04:35:14.955297+0000","last_undegraded":"2023-10-09T04:34:51.324496+0000","last_fullsized":"2023-10-09T04:34:51.324385+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:53.048540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382730240,"num_objects":5840,"num_object_clones":0,"num_object_copies":17520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5840,"num_write_kb":373760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":5840}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":163,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.707584+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:35:04.707584+0000","last_peered":"2023-10-09T04:35:04.707584+0000","last_clean":"2023-10-09T04:35:04.707584+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:35:04.707584+0000","last_undegraded":"2023-10-09T04:35:04.707584+0000","last_fullsized":"2023-10-09T04:35:04.707584+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":189,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.712854+0000","last_change":"2023-10-09T04:34:51.322935+0000","last_active":"2023-10-09T04:35:04.712854+0000","last_peered":"2023-10-09T04:35:04.712854+0000","last_clean":"2023-10-09T04:34:44.060875+0000","last_became_active":"2023-10-09T04:34:51.322935+0000","last_became_peered":"2023-10-09T04:34:51.322935+0000","last_unstale":"2023-10-09T04:35:04.712854+0000","last_undegraded":"2023-10-09T04:35:04.712854+0000","last_fullsized":"2023-10-09T04:34:51.319826+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:49.238888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"205'5817","reported_seq":5839,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.713159+0000","last_change":"2023-10-09T04:33:51.280550+0000","last_active":"2023-10-09T04:35:14.713159+0000","last_peered":"2023-10-09T04:35:14.713159+0000","last_clean":"2023-10-09T04:35:14.713159+0000","last_became_active":"2023-10-09T04:33:51.280262+0000","last_became_peered":"2023-10-09T04:33:51.280262+0000","last_unstale":"2023-10-09T04:35:14.713159+0000","last_undegraded":"2023-10-09T04:35:14.713159+0000","last_fullsized":"2023-10-09T04:35:14.713159+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5817,"log_dups_size":0,"ondisk_log_size":5817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:29.056390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381222912,"num_objects":5817,"num_object_clones":0,"num_object_copies":17451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5817,"num_write_kb":372288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.717924+0000","last_change":"2023-10-09T04:34:51.333617+0000","last_active":"2023-10-09T04:35:04.717924+0000","last_peered":"2023-10-09T04:35:04.717924+0000","last_clean":"2023-10-09T04:34:44.060999+0000","last_became_active":"2023-10-09T04:34:51.333617+0000","last_became_peered":"2023-10-09T04:34:51.333617+0000","last_unstale":"2023-10-09T04:35:04.717924+0000","last_undegraded":"2023-10-09T04:35:04.717924+0000","last_fullsized":"2023-10-09T04:34:51.322409+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:09.264091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711912+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:35:04.711912+0000","last_peered":"2023-10-09T04:35:04.711912+0000","last_clean":"2023-10-09T04:35:04.711912+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:35:04.711912+0000","last_undegraded":"2023-10-09T04:35:04.711912+0000","last_fullsized":"2023-10-09T04:35:04.711912+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"205'5823","reported_seq":5846,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.803982+0000","last_change":"2023-10-09T04:33:51.281183+0000","last_active":"2023-10-09T04:35:14.803982+0000","last_peered":"2023-10-09T04:35:14.803982+0000","last_clean":"2023-10-09T04:35:14.803982+0000","last_became_active":"2023-10-09T04:33:51.280920+0000","last_became_peered":"2023-10-09T04:33:51.280920+0000","last_unstale":"2023-10-09T04:35:14.803982+0000","last_undegraded":"2023-10-09T04:35:14.803982+0000","last_fullsized":"2023-10-09T04:35:14.803982+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:15.815193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381616128,"num_objects":5823,"num_object_clones":0,"num_object_copies":17469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5823,"num_write_kb":372672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":156,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692116+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:35:04.692116+0000","last_peered":"2023-10-09T04:35:04.692116+0000","last_clean":"2023-10-09T04:35:04.692116+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:35:04.692116+0000","last_undegraded":"2023-10-09T04:35:04.692116+0000","last_fullsized":"2023-10-09T04:35:04.692116+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693342+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:35:04.693342+0000","last_peered":"2023-10-09T04:35:04.693342+0000","last_clean":"2023-10-09T04:35:04.693342+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:35:04.693342+0000","last_undegraded":"2023-10-09T04:35:04.693342+0000","last_fullsized":"2023-10-09T04:35:04.693342+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"205'5803","reported_seq":5826,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.939109+0000","last_change":"2023-10-09T04:33:51.281811+0000","last_active":"2023-10-09T04:35:14.939109+0000","last_peered":"2023-10-09T04:35:14.939109+0000","last_clean":"2023-10-09T04:35:14.939109+0000","last_became_active":"2023-10-09T04:33:51.281495+0000","last_became_peered":"2023-10-09T04:33:51.281495+0000","last_unstale":"2023-10-09T04:35:14.939109+0000","last_undegraded":"2023-10-09T04:35:14.939109+0000","last_fullsized":"2023-10-09T04:35:14.939109+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:53.315124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380305408,"num_objects":5803,"num_object_clones":0,"num_object_copies":17409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5803,"num_write_kb":371392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":179,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692574+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:35:04.692574+0000","last_peered":"2023-10-09T04:35:04.692574+0000","last_clean":"2023-10-09T04:35:04.692574+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:35:04.692574+0000","last_undegraded":"2023-10-09T04:35:04.692574+0000","last_fullsized":"2023-10-09T04:35:04.692574+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":189,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.709676+0000","last_change":"2023-10-09T04:34:51.340476+0000","last_active":"2023-10-09T04:35:04.709676+0000","last_peered":"2023-10-09T04:35:04.709676+0000","last_clean":"2023-10-09T04:34:44.071691+0000","last_became_active":"2023-10-09T04:34:51.340476+0000","last_became_peered":"2023-10-09T04:34:51.340476+0000","last_unstale":"2023-10-09T04:35:04.709676+0000","last_undegraded":"2023-10-09T04:35:04.709676+0000","last_fullsized":"2023-10-09T04:34:51.323270+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:07.997213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692225+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:35:04.692225+0000","last_peered":"2023-10-09T04:35:04.692225+0000","last_clean":"2023-10-09T04:35:04.692225+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:35:04.692225+0000","last_undegraded":"2023-10-09T04:35:04.692225+0000","last_fullsized":"2023-10-09T04:35:04.692225+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"205'5870","reported_seq":5891,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.749621+0000","last_change":"2023-10-09T04:33:51.277983+0000","last_active":"2023-10-09T04:35:14.749621+0000","last_peered":"2023-10-09T04:35:14.749621+0000","last_clean":"2023-10-09T04:35:14.749621+0000","last_became_active":"2023-10-09T04:33:51.277555+0000","last_became_peered":"2023-10-09T04:33:51.277555+0000","last_unstale":"2023-10-09T04:35:14.749621+0000","last_undegraded":"2023-10-09T04:35:14.749621+0000","last_fullsized":"2023-10-09T04:35:14.749621+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5870,"log_dups_size":0,"ondisk_log_size":5870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:59.460010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384696320,"num_objects":5870,"num_object_clones":0,"num_object_copies":17610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5870,"num_write_kb":375680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.710054+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:35:04.710054+0000","last_peered":"2023-10-09T04:35:04.710054+0000","last_clean":"2023-10-09T04:35:04.710054+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:35:04.710054+0000","last_undegraded":"2023-10-09T04:35:04.710054+0000","last_fullsized":"2023-10-09T04:35:04.710054+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":186,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711896+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:35:04.711896+0000","last_peered":"2023-10-09T04:35:04.711896+0000","last_clean":"2023-10-09T04:35:04.711896+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:35:04.711896+0000","last_undegraded":"2023-10-09T04:35:04.711896+0000","last_fullsized":"2023-10-09T04:35:04.711896+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":191,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.692462+0000","last_change":"2023-10-09T04:34:51.316666+0000","last_active":"2023-10-09T04:35:04.692462+0000","last_peered":"2023-10-09T04:35:04.692462+0000","last_clean":"2023-10-09T04:34:44.053522+0000","last_became_active":"2023-10-09T04:34:51.316666+0000","last_became_peered":"2023-10-09T04:34:51.316666+0000","last_unstale":"2023-10-09T04:35:04.692462+0000","last_undegraded":"2023-10-09T04:35:04.692462+0000","last_fullsized":"2023-10-09T04:34:51.313759+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:10.205902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"205'5991","reported_seq":6020,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:14.950664+0000","last_change":"2023-10-09T04:34:51.330190+0000","last_active":"2023-10-09T04:35:14.950664+0000","last_peered":"2023-10-09T04:35:14.950664+0000","last_clean":"2023-10-09T04:34:50.296861+0000","last_became_active":"2023-10-09T04:34:51.330190+0000","last_became_peered":"2023-10-09T04:34:51.330190+0000","last_unstale":"2023-10-09T04:35:14.950664+0000","last_undegraded":"2023-10-09T04:34:51.323489+0000","last_fullsized":"2023-10-09T04:34:51.323336+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5991,"log_dups_size":0,"ondisk_log_size":5991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:59.712251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392626176,"num_objects":5991,"num_object_clones":0,"num_object_copies":17973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5991,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5991,"num_write_kb":383424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":5991}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":123,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.692783+0000","last_change":"2023-10-09T04:34:51.318540+0000","last_active":"2023-10-09T04:35:04.692783+0000","last_peered":"2023-10-09T04:35:04.692783+0000","last_clean":"2023-10-09T04:29:34.560919+0000","last_became_active":"2023-10-09T04:34:51.318540+0000","last_became_peered":"2023-10-09T04:34:51.318540+0000","last_unstale":"2023-10-09T04:35:04.692783+0000","last_undegraded":"2023-10-09T04:35:04.692783+0000","last_fullsized":"2023-10-09T04:34:51.314590+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:41.112095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":176,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.691919+0000","last_change":"2023-10-09T04:34:51.321100+0000","last_active":"2023-10-09T04:35:04.691919+0000","last_peered":"2023-10-09T04:35:04.691919+0000","last_clean":"2023-10-09T04:34:44.053288+0000","last_became_active":"2023-10-09T04:34:51.321100+0000","last_became_peered":"2023-10-09T04:34:51.321100+0000","last_unstale":"2023-10-09T04:35:04.691919+0000","last_undegraded":"2023-10-09T04:35:04.691919+0000","last_fullsized":"2023-10-09T04:34:51.313128+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:11.901708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":174,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693416+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:35:04.693416+0000","last_peered":"2023-10-09T04:35:04.693416+0000","last_clean":"2023-10-09T04:35:04.693416+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:35:04.693416+0000","last_undegraded":"2023-10-09T04:35:04.693416+0000","last_fullsized":"2023-10-09T04:35:04.693416+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"205'5732","reported_seq":5754,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.745509+0000","last_change":"2023-10-09T04:33:51.277098+0000","last_active":"2023-10-09T04:35:14.745509+0000","last_peered":"2023-10-09T04:35:14.745509+0000","last_clean":"2023-10-09T04:35:14.745509+0000","last_became_active":"2023-10-09T04:33:51.276788+0000","last_became_peered":"2023-10-09T04:33:51.276788+0000","last_unstale":"2023-10-09T04:35:14.745509+0000","last_undegraded":"2023-10-09T04:35:14.745509+0000","last_fullsized":"2023-10-09T04:35:14.745509+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5732,"log_dups_size":0,"ondisk_log_size":5732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:20.524012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375652352,"num_objects":5732,"num_object_clones":0,"num_object_copies":17196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5732,"num_write_kb":366848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":176,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.709946+0000","last_change":"2023-10-09T04:34:51.334572+0000","last_active":"2023-10-09T04:35:04.709946+0000","last_peered":"2023-10-09T04:35:04.709946+0000","last_clean":"2023-10-09T04:34:44.071648+0000","last_became_active":"2023-10-09T04:34:51.334572+0000","last_became_peered":"2023-10-09T04:34:51.334572+0000","last_unstale":"2023-10-09T04:35:04.709946+0000","last_undegraded":"2023-10-09T04:35:04.709946+0000","last_fullsized":"2023-10-09T04:34:51.324871+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:03.116911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":597,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711476+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:35:04.711476+0000","last_peered":"2023-10-09T04:35:04.711476+0000","last_clean":"2023-10-09T04:35:04.711476+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:35:04.711476+0000","last_undegraded":"2023-10-09T04:35:04.711476+0000","last_fullsized":"2023-10-09T04:35:04.711476+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693005+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:35:04.693005+0000","last_peered":"2023-10-09T04:35:04.693005+0000","last_clean":"2023-10-09T04:35:04.693005+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:35:04.693005+0000","last_undegraded":"2023-10-09T04:35:04.693005+0000","last_fullsized":"2023-10-09T04:35:04.693005+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":145,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.710755+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:35:04.710755+0000","last_peered":"2023-10-09T04:35:04.710755+0000","last_clean":"2023-10-09T04:35:04.710755+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:35:04.710755+0000","last_undegraded":"2023-10-09T04:35:04.710755+0000","last_fullsized":"2023-10-09T04:35:04.710755+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"205'5953","reported_seq":5992,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:14.947027+0000","last_change":"2023-10-09T04:34:51.327114+0000","last_active":"2023-10-09T04:35:14.947027+0000","last_peered":"2023-10-09T04:35:14.947027+0000","last_clean":"2023-10-09T04:34:50.268727+0000","last_became_active":"2023-10-09T04:34:51.327114+0000","last_became_peered":"2023-10-09T04:34:51.327114+0000","last_unstale":"2023-10-09T04:35:14.947027+0000","last_undegraded":"2023-10-09T04:34:51.325672+0000","last_fullsized":"2023-10-09T04:34:51.325531+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:06.884579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390135808,"num_objects":5953,"num_object_clones":0,"num_object_copies":17859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5953,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5953,"num_write_kb":380992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":5953}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":205,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711391+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:35:04.711391+0000","last_peered":"2023-10-09T04:35:04.711391+0000","last_clean":"2023-10-09T04:35:04.711391+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:35:04.711391+0000","last_undegraded":"2023-10-09T04:35:04.711391+0000","last_fullsized":"2023-10-09T04:35:04.711391+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":117,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.718299+0000","last_change":"2023-10-09T04:34:22.266723+0000","last_active":"2023-10-09T04:35:04.718299+0000","last_peered":"2023-10-09T04:35:04.718299+0000","last_clean":"2023-10-09T04:35:04.718299+0000","last_became_active":"2023-10-09T04:34:22.266261+0000","last_became_peered":"2023-10-09T04:34:22.266261+0000","last_unstale":"2023-10-09T04:35:04.718299+0000","last_undegraded":"2023-10-09T04:35:04.718299+0000","last_fullsized":"2023-10-09T04:35:04.718299+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:54.792194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":69,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.697941+0000","last_change":"2023-10-09T04:34:51.318703+0000","last_active":"2023-10-09T04:35:04.697941+0000","last_peered":"2023-10-09T04:35:04.697941+0000","last_clean":"2023-10-09T04:34:44.061887+0000","last_became_active":"2023-10-09T04:34:51.318703+0000","last_became_peered":"2023-10-09T04:34:51.318703+0000","last_unstale":"2023-10-09T04:35:04.697941+0000","last_undegraded":"2023-10-09T04:35:04.697941+0000","last_fullsized":"2023-10-09T04:34:51.317343+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:36.687606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"205'5876","reported_seq":5898,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-10-09T04:35:14.765475+0000","last_change":"2023-10-09T04:33:51.282958+0000","last_active":"2023-10-09T04:35:14.765475+0000","last_peered":"2023-10-09T04:35:14.765475+0000","last_clean":"2023-10-09T04:35:14.765475+0000","last_became_active":"2023-10-09T04:33:51.282685+0000","last_became_peered":"2023-10-09T04:33:51.282685+0000","last_unstale":"2023-10-09T04:35:14.765475+0000","last_undegraded":"2023-10-09T04:35:14.765475+0000","last_fullsized":"2023-10-09T04:35:14.765475+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5876,"log_dups_size":0,"ondisk_log_size":5876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:49.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385089536,"num_objects":5876,"num_object_clones":0,"num_object_copies":17628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5876,"num_write_kb":376064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711976+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:35:04.711976+0000","last_peered":"2023-10-09T04:35:04.711976+0000","last_clean":"2023-10-09T04:35:04.711976+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:35:04.711976+0000","last_undegraded":"2023-10-09T04:35:04.711976+0000","last_fullsized":"2023-10-09T04:35:04.711976+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":176,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.692338+0000","last_change":"2023-10-09T04:34:51.325423+0000","last_active":"2023-10-09T04:35:04.692338+0000","last_peered":"2023-10-09T04:35:04.692338+0000","last_clean":"2023-10-09T04:34:44.053445+0000","last_became_active":"2023-10-09T04:34:51.325423+0000","last_became_peered":"2023-10-09T04:34:51.325423+0000","last_unstale":"2023-10-09T04:35:04.692338+0000","last_undegraded":"2023-10-09T04:35:04.692338+0000","last_fullsized":"2023-10-09T04:34:51.313428+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.954931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":49,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.710053+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:35:04.710053+0000","last_peered":"2023-10-09T04:35:04.710053+0000","last_clean":"2023-10-09T04:35:04.710053+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:35:04.710053+0000","last_undegraded":"2023-10-09T04:35:04.710053+0000","last_fullsized":"2023-10-09T04:35:04.710053+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"205'5809","reported_seq":5838,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:14.798520+0000","last_change":"2023-10-09T04:34:51.328209+0000","last_active":"2023-10-09T04:35:14.798520+0000","last_peered":"2023-10-09T04:35:14.798520+0000","last_clean":"2023-10-09T04:34:50.302298+0000","last_became_active":"2023-10-09T04:34:51.328209+0000","last_became_peered":"2023-10-09T04:34:51.328209+0000","last_unstale":"2023-10-09T04:35:14.798520+0000","last_undegraded":"2023-10-09T04:34:51.321394+0000","last_fullsized":"2023-10-09T04:34:51.321276+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:47.025638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380698624,"num_objects":5809,"num_object_clones":0,"num_object_copies":17427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5809,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":5809}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":199,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693726+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:35:04.693726+0000","last_peered":"2023-10-09T04:35:04.693726+0000","last_clean":"2023-10-09T04:35:04.693726+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:35:04.693726+0000","last_undegraded":"2023-10-09T04:35:04.693726+0000","last_fullsized":"2023-10-09T04:35:04.693726+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692973+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:35:04.692973+0000","last_peered":"2023-10-09T04:35:04.692973+0000","last_clean":"2023-10-09T04:35:04.692973+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:35:04.692973+0000","last_undegraded":"2023-10-09T04:35:04.692973+0000","last_fullsized":"2023-10-09T04:35:04.692973+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.692198+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:35:04.692198+0000","last_peered":"2023-10-09T04:35:04.692198+0000","last_clean":"2023-10-09T04:35:04.692198+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:35:04.692198+0000","last_undegraded":"2023-10-09T04:35:04.692198+0000","last_fullsized":"2023-10-09T04:35:04.692198+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"205'5953","reported_seq":5981,"reported_epoch":205,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:14.757765+0000","last_change":"2023-10-09T04:34:51.325469+0000","last_active":"2023-10-09T04:35:14.757765+0000","last_peered":"2023-10-09T04:35:14.757765+0000","last_clean":"2023-10-09T04:34:50.287068+0000","last_became_active":"2023-10-09T04:34:51.325469+0000","last_became_peered":"2023-10-09T04:34:51.325469+0000","last_unstale":"2023-10-09T04:35:14.757765+0000","last_undegraded":"2023-10-09T04:34:51.321417+0000","last_fullsized":"2023-10-09T04:34:51.321273+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:06.267427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390135808,"num_objects":5953,"num_object_clones":0,"num_object_copies":17859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5953,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5953,"num_write_kb":380992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":5953}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":174,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.707645+0000","last_change":"2023-10-09T04:34:51.331849+0000","last_active":"2023-10-09T04:35:04.707645+0000","last_peered":"2023-10-09T04:35:04.707645+0000","last_clean":"2023-10-09T04:34:44.065555+0000","last_became_active":"2023-10-09T04:34:51.331849+0000","last_became_peered":"2023-10-09T04:34:51.331849+0000","last_unstale":"2023-10-09T04:35:04.707645+0000","last_undegraded":"2023-10-09T04:35:04.707645+0000","last_fullsized":"2023-10-09T04:34:51.321695+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:09.071914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":58,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.693137+0000","last_change":"2023-10-09T04:34:22.266652+0000","last_active":"2023-10-09T04:35:04.693137+0000","last_peered":"2023-10-09T04:35:04.693137+0000","last_clean":"2023-10-09T04:35:04.693137+0000","last_became_active":"2023-10-09T04:34:22.266267+0000","last_became_peered":"2023-10-09T04:34:22.266267+0000","last_unstale":"2023-10-09T04:35:04.693137+0000","last_undegraded":"2023-10-09T04:35:04.693137+0000","last_fullsized":"2023-10-09T04:35:04.693137+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:50.646309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":148,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.718041+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:35:04.718041+0000","last_peered":"2023-10-09T04:35:04.718041+0000","last_clean":"2023-10-09T04:35:04.718041+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:35:04.718041+0000","last_undegraded":"2023-10-09T04:35:04.718041+0000","last_fullsized":"2023-10-09T04:35:04.718041+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":45,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.711811+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:35:04.711811+0000","last_peered":"2023-10-09T04:35:04.711811+0000","last_clean":"2023-10-09T04:35:04.711811+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:35:04.711811+0000","last_undegraded":"2023-10-09T04:35:04.711811+0000","last_fullsized":"2023-10-09T04:35:04.711811+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.692203+0000","last_change":"2023-10-09T04:34:51.316709+0000","last_active":"2023-10-09T04:35:04.692203+0000","last_peered":"2023-10-09T04:35:04.692203+0000","last_clean":"2023-10-09T04:27:36.073787+0000","last_became_active":"2023-10-09T04:34:51.316709+0000","last_became_peered":"2023-10-09T04:34:51.316709+0000","last_unstale":"2023-10-09T04:35:04.692203+0000","last_undegraded":"2023-10-09T04:35:04.692203+0000","last_fullsized":"2023-10-09T04:34:51.313259+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:43.330782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709312+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:35:04.709312+0000","last_peered":"2023-10-09T04:35:04.709312+0000","last_clean":"2023-10-09T04:35:04.709312+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:35:04.709312+0000","last_undegraded":"2023-10-09T04:35:04.709312+0000","last_fullsized":"2023-10-09T04:35:04.709312+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712069+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:35:04.712069+0000","last_peered":"2023-10-09T04:35:04.712069+0000","last_clean":"2023-10-09T04:35:04.712069+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:35:04.712069+0000","last_undegraded":"2023-10-09T04:35:04.712069+0000","last_fullsized":"2023-10-09T04:35:04.712069+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":150,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.709792+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:35:04.709792+0000","last_peered":"2023-10-09T04:35:04.709792+0000","last_clean":"2023-10-09T04:35:04.709792+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:35:04.709792+0000","last_undegraded":"2023-10-09T04:35:04.709792+0000","last_fullsized":"2023-10-09T04:35:04.709792+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-10-09T04:35:04.712723+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:35:04.712723+0000","last_peered":"2023-10-09T04:35:04.712723+0000","last_clean":"2023-10-09T04:35:04.712723+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:35:04.712723+0000","last_undegraded":"2023-10-09T04:35:04.712723+0000","last_fullsized":"2023-10-09T04:35:04.712723+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":180,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.692119+0000","last_change":"2023-10-09T04:34:51.321322+0000","last_active":"2023-10-09T04:35:04.692119+0000","last_peered":"2023-10-09T04:35:04.692119+0000","last_clean":"2023-10-09T04:34:44.059138+0000","last_became_active":"2023-10-09T04:34:51.321322+0000","last_became_peered":"2023-10-09T04:34:51.321322+0000","last_unstale":"2023-10-09T04:35:04.692119+0000","last_undegraded":"2023-10-09T04:35:04.692119+0000","last_fullsized":"2023-10-09T04:34:51.314501+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:46.254327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:11.566443+0000","last_change":"2023-10-09T04:34:51.318670+0000","last_active":"2023-10-09T04:35:11.566443+0000","last_peered":"2023-10-09T04:35:11.566443+0000","last_clean":"2023-10-09T04:34:44.058794+0000","last_became_active":"2023-10-09T04:34:51.318670+0000","last_became_peered":"2023-10-09T04:34:51.318670+0000","last_unstale":"2023-10-09T04:35:11.566443+0000","last_undegraded":"2023-10-09T04:35:11.566443+0000","last_fullsized":"2023-10-09T04:34:51.314163+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:43.814115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":69,"reported_epoch":204,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:04.709553+0000","last_change":"2023-10-09T04:34:51.325437+0000","last_active":"2023-10-09T04:35:04.709553+0000","last_peered":"2023-10-09T04:35:04.709553+0000","last_clean":"2023-10-09T04:34:44.071843+0000","last_became_active":"2023-10-09T04:34:51.325437+0000","last_became_peered":"2023-10-09T04:34:51.325437+0000","last_unstale":"2023-10-09T04:35:04.709553+0000","last_undegraded":"2023-10-09T04:34:51.324026+0000","last_fullsized":"2023-10-09T04:34:51.323882+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:24.344413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":182,"reported_epoch":204,"state":"active+undersized","last_fresh":"2023-10-09T04:35:04.711607+0000","last_change":"2023-10-09T04:34:51.326267+0000","last_active":"2023-10-09T04:35:04.711607+0000","last_peered":"2023-10-09T04:35:04.711607+0000","last_clean":"2023-10-09T04:34:44.060673+0000","last_became_active":"2023-10-09T04:34:51.326267+0000","last_became_peered":"2023-10-09T04:34:51.326267+0000","last_unstale":"2023-10-09T04:35:04.711607+0000","last_undegraded":"2023-10-09T04:35:04.711607+0000","last_fullsized":"2023-10-09T04:34:51.321105+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:44.409648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6161301504,"num_objects":94014,"num_object_clones":0,"num_object_copies":282042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":94014,"num_write_kb":6016896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18001952768,"data_stored":18001952768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94014,"ondisk_log_size":94014,"up":42,"acting":42,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":88,"acting":88,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234340,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2406496,"kb_used_data":1503704,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":91338656,"statfs":{"total":95995035648,"available":93530783744,"internally_reserved":0,"allocated":1539792896,"data_stored":1539267502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:21 2023","interfaces":[{"interface":"back","average":{"1min":7.202,"5min":7.2599999999999998,"15min":5.5759999999999996},"min":{"1min":0.42299999999999999,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":111.72199999999999,"5min":186.446,"15min":629.22500000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":7.1500000000000004,"5min":7.4850000000000003,"15min":5.4939999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":111.76600000000001,"5min":186.429,"15min":629.43100000000004},"last":0.34499999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:34:21 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":8.3019999999999996,"15min":4.1429999999999998},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.7010000000000001,"5min":224.13499999999999,"15min":224.13499999999999},"last":1.506},{"interface":"front","average":{"1min":0.751,"5min":8.3490000000000002,"15min":4.202},"min":{"1min":0.32600000000000001,"5min":0.27200000000000002,"15min":0.25700000000000001},"max":{"1min":2.681,"5min":224.203,"15min":224.203},"last":0.90600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:34:42 2023","interfaces":[{"interface":"back","average":{"1min":16.478000000000002,"5min":5.5570000000000004,"15min":2.2050000000000001},"min":{"1min":0.28499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":177.262,"5min":177.262,"15min":177.262},"last":2.2999999999999998},{"interface":"front","average":{"1min":16.478000000000002,"5min":5.5830000000000002,"15min":2.2229999999999999},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":177.24299999999999,"5min":177.24299999999999,"15min":177.24299999999999},"last":2.5419999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8580000000000001,"5min":11.728,"15min":5.3109999999999999},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":53.482999999999997,"5min":249.989,"15min":249.989},"last":0.84599999999999997},{"interface":"front","average":{"1min":3.6379999999999999,"5min":12.002000000000001,"15min":5.431},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":53.162999999999997,"5min":250.21700000000001,"15min":250.21700000000001},"last":0.58699999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:34:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.3560000000000001,"15min":1.216},"min":{"1min":0.29499999999999998,"5min":0.27200000000000002,"15min":0.23799999999999999},"max":{"1min":2.7469999999999999,"5min":10.869,"15min":10.869},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.0169999999999999,"15min":0.95299999999999996},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":3.5649999999999999,"5min":10.613,"15min":10.613},"last":0.55900000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:34:36 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.91200000000000003,"15min":0.95099999999999996},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":1.6120000000000001,"5min":5.2599999999999998,"15min":8.8610000000000007},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.3440000000000001,"15min":1.161},"min":{"1min":0.38100000000000001,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":2.0880000000000001,"5min":10.895,"15min":10.895},"last":0.434}]},{"osd":6,"last update":"Mon Oct 9 04:34:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.1619999999999999,"15min":1.165},"min":{"1min":0.35899999999999999,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":2.722,"5min":5.7270000000000003,"15min":12.007},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.331,"15min":1.2110000000000001},"min":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.31900000000000001},"max":{"1min":2.778,"5min":10.917,"15min":10.917},"last":0.497}]}]},{"osd":6,"up_from":46,"seq":197568495974,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3200336,"kb_used_data":2614664,"kb_used_omap":0,"kb_used_meta":585664,"kb_avail":90544816,"statfs":{"total":95995035648,"available":92717891584,"internally_reserved":0,"allocated":2677415936,"data_stored":2676906457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599719936},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,0,3,1,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:24 2023","interfaces":[{"interface":"back","average":{"1min":4.6529999999999996,"5min":16.414000000000001,"15min":6.6399999999999997},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":71.640000000000001,"5min":1284.521,"15min":1284.521},"last":0.41299999999999998},{"interface":"front","average":{"1min":5.1479999999999997,"5min":16.439,"15min":6.9370000000000003},"min":{"1min":0.34599999999999997,"5min":0.253,"15min":0.253},"max":{"1min":72.019000000000005,"5min":1284.4690000000001,"15min":1284.4690000000001},"last":3.508}]},{"osd":1,"last update":"Mon Oct 9 04:34:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2599999999999998,"5min":6.3719999999999999,"15min":3.4790000000000001},"min":{"1min":0.32100000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":81.126000000000005,"5min":185.74000000000001,"15min":185.74000000000001},"last":3.9950000000000001},{"interface":"front","average":{"1min":4.7770000000000001,"5min":6.2770000000000001,"15min":3.597},"min":{"1min":0.318,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":76.927999999999997,"5min":189.03999999999999,"15min":189.03999999999999},"last":0.38900000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.7110000000000001,"15min":0.95099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.6890000000000001,"5min":7.6890000000000001,"15min":7.6890000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.236,"15min":1.02},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.976,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":11.254}]},{"osd":3,"last update":"Mon Oct 9 04:35:13 2023","interfaces":[{"interface":"back","average":{"1min":11.717000000000001,"5min":5.0419999999999998,"15min":2.121},"min":{"1min":0.35999999999999999,"5min":0.314,"15min":0.314},"max":{"1min":144.99799999999999,"5min":144.99799999999999,"15min":144.99799999999999},"last":0.44},{"interface":"front","average":{"1min":11.445,"5min":5.1820000000000004,"15min":2.117},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.313},"max":{"1min":145.023,"5min":145.023,"15min":145.023},"last":3.4740000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:34:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.23,"15min":1.149},"min":{"1min":0.27300000000000002,"5min":0.26600000000000001,"15min":0.246},"max":{"1min":3.5310000000000001,"5min":8.0950000000000006,"15min":12.743},"last":4.0270000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.82699999999999996,"15min":1.0780000000000001},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.26000000000000001},"max":{"1min":1.242,"5min":8.9920000000000009,"15min":12.028},"last":0.35799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:34:24 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":2.468,"15min":1.5629999999999999},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":3.661,"5min":81.087000000000003,"15min":81.087000000000003},"last":4.0529999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.2320000000000002,"15min":1.6060000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.27400000000000002},"max":{"1min":1.0429999999999999,"5min":81.706999999999994,"15min":81.706999999999994},"last":2.3690000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":2.1320000000000001,"15min":1.492},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":11.226000000000001,"5min":81.674999999999997,"15min":81.674999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.246,"15min":1.5369999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":9.9009999999999998,"5min":81.055000000000007,"15min":81.055000000000007},"last":5.1870000000000003}]}]},{"osd":1,"up_from":118,"seq":506806141064,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1536216,"kb_used_data":1140880,"kb_used_omap":4,"kb_used_meta":395323,"kb_avail":92208936,"statfs":{"total":95995035648,"available":94421950464,"internally_reserved":0,"allocated":1168261120,"data_stored":1167450859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":404811662},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":4.8150000000000004,"15min":3.363},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":33.607999999999997,"5min":97.308000000000007,"15min":97.308000000000007},"last":0.89700000000000002},{"interface":"front","average":{"1min":2.6419999999999999,"5min":5.4180000000000001,"15min":3.669},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":33.697000000000003,"5min":97.055999999999997,"15min":97.055999999999997},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:34:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1530000000000005,"5min":3.544,"15min":1.417},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":143.11199999999999,"5min":143.11199999999999,"15min":143.11199999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":7.6529999999999996,"5min":2.8410000000000002,"15min":1.175},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.218},"max":{"1min":143.22,"5min":143.22,"15min":143.22},"last":1.1819999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:35:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6680000000000001,"5min":5.415,"15min":2.1059999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":58.673000000000002,"5min":195.035,"15min":195.035},"last":0.94799999999999995},{"interface":"front","average":{"1min":4.1550000000000002,"5min":6.0709999999999997,"15min":2.335},"min":{"1min":0.246,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":58.125999999999998,"5min":195.465,"15min":195.465},"last":0.79900000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.7949999999999999,"15min":2.6579999999999999},"min":{"1min":0.33800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":7.8799999999999999,"5min":12.537000000000001,"15min":107.795},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.306,"15min":1.6399999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5409999999999999,"5min":6.1369999999999996,"15min":109.167},"last":0.92600000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":2.1560000000000001,"15min":2.331},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.33700000000000002},"max":{"1min":3.6440000000000001,"5min":10.186999999999999,"15min":12.414999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.3280000000000001,"5min":2.0880000000000001,"15min":2.5219999999999998},"min":{"1min":0.44,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.944,"5min":12.605,"15min":13.933999999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":2.0899999999999999,"15min":2.2109999999999999},"min":{"1min":0.42699999999999999,"5min":0.39900000000000002,"15min":0.36599999999999999},"max":{"1min":5.5199999999999996,"5min":8.3770000000000007,"15min":13.388},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.831,"5min":2.0950000000000002,"15min":2.601},"min":{"1min":0.504,"5min":0.317,"15min":0.317},"max":{"1min":12.596,"5min":12.632999999999999,"15min":13.882},"last":0.48499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5820000000000001,"5min":2.077,"15min":2.6539999999999999},"min":{"1min":0.45500000000000002,"5min":0.374,"15min":0.374},"max":{"1min":8.4160000000000004,"5min":12.656000000000001,"15min":13.907},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.212,"5min":1.5529999999999999,"15min":1.573},"min":{"1min":0.441,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.2050000000000001,"5min":6.1799999999999997,"15min":7.8239999999999998},"last":1.2130000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024121,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4327364,"kb_used_data":3850748,"kb_used_omap":5,"kb_used_meta":476602,"kb_avail":89417788,"statfs":{"total":95995035648,"available":91563814912,"internally_reserved":0,"allocated":3943165952,"data_stored":3942373551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":488041082},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:34:19 2023","interfaces":[{"interface":"back","average":{"1min":6.0629999999999997,"5min":7.1719999999999997,"15min":4.3650000000000002},"min":{"1min":0.20499999999999999,"5min":0.192,"15min":0.192},"max":{"1min":83.260000000000005,"5min":167.92699999999999,"15min":167.92699999999999},"last":6.0179999999999998},{"interface":"front","average":{"1min":5.6349999999999998,"5min":7.3339999999999996,"15min":4.149},"min":{"1min":0.20799999999999999,"5min":0.20000000000000001,"15min":0.19600000000000001},"max":{"1min":82.870000000000005,"5min":169.78999999999999,"15min":169.78999999999999},"last":2.399}]},{"osd":2,"last update":"Mon Oct 9 04:34:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.024,"15min":1.26},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.3769999999999998,"5min":107.548,"15min":107.548},"last":1.548},{"interface":"front","average":{"1min":1.329,"5min":2.7210000000000001,"15min":1.1459999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.4989999999999997,"5min":107.35899999999999,"15min":107.35899999999999},"last":2.2469999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:35:11 2023","interfaces":[{"interface":"back","average":{"1min":5.8300000000000001,"5min":5.9370000000000003,"15min":2.3149999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":46.625,"5min":213.28399999999999,"15min":213.28399999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":6.0019999999999998,"5min":6.1059999999999999,"15min":2.3900000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":47.076000000000001,"5min":212.27000000000001,"15min":212.27000000000001},"last":2.4689999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.857,"15min":1.6839999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":3.734,"5min":10.717000000000001,"15min":10.73},"last":6.0549999999999997},{"interface":"front","average":{"1min":0.89800000000000002,"5min":2.0870000000000002,"15min":1.778},"min":{"1min":0.38400000000000001,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":3.5619999999999998,"5min":15.941000000000001,"15min":15.941000000000001},"last":2.5169999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":2.1930000000000001,"15min":1.901},"min":{"1min":0.34000000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.4580000000000002,"5min":12.807,"15min":12.807},"last":2.4329999999999998},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.6359999999999999,"15min":1.704},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":2.6509999999999998,"5min":8.4589999999999996,"15min":14.144},"last":1.4239999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:34:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":2.2250000000000001,"15min":2.0299999999999998},"min":{"1min":0.40100000000000002,"5min":0.374,"15min":0.35499999999999998},"max":{"1min":3.5089999999999999,"5min":16.369,"15min":16.369},"last":4.7130000000000001},{"interface":"front","average":{"1min":0.86899999999999999,"5min":1.601,"15min":1.8500000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.0209999999999999,"5min":8.4930000000000003,"15min":14.090999999999999},"last":5.024}]},{"osd":7,"last update":"Mon Oct 9 04:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.6879999999999999,"15min":1.833},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":3.0619999999999998,"5min":8.5229999999999997,"15min":11.890000000000001},"last":5.0679999999999996},{"interface":"front","average":{"1min":0.88400000000000001,"5min":2.0489999999999999,"15min":2.0070000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":3.7599999999999998,"5min":9.7110000000000003,"15min":11.176},"last":6.0819999999999999}]}]},{"osd":2,"up_from":135,"seq":579820585050,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1928196,"kb_used_data":1669308,"kb_used_omap":7,"kb_used_meta":258872,"kb_avail":91816956,"statfs":{"total":95995035648,"available":94020562944,"internally_reserved":0,"allocated":1709371392,"data_stored":1708661063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":265085009},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":9.0289999999999999,"5min":6.2489999999999997,"15min":2.3050000000000002},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":136.28100000000001,"5min":136.28100000000001,"15min":136.28100000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":15.180999999999999,"5min":7.3689999999999998,"15min":2.7370000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":136.239,"5min":136.239,"15min":136.239},"last":0.28299999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":7.6420000000000003,"5min":5.4779999999999998,"15min":2.081},"min":{"1min":0.246,"5min":0.193,"15min":0.193},"max":{"1min":119.748,"5min":119.748,"15min":119.748},"last":0.41499999999999998},{"interface":"front","average":{"1min":7.9009999999999998,"5min":5.8760000000000003,"15min":2.1899999999999999},"min":{"1min":0.23799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":118.024,"5min":118.024,"15min":118.024},"last":0.59499999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:34:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":6.093,"15min":2.2810000000000001},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4320000000000004,"5min":147.655,"15min":147.655},"last":0.438},{"interface":"front","average":{"1min":1.425,"5min":6.1150000000000002,"15min":2.3290000000000002},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.7850000000000001,"5min":147.607,"15min":147.607},"last":0.47799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":2.2669999999999999,"15min":1.1559999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.9699999999999998,"5min":12.765000000000001,"15min":12.765000000000001},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.389,"15min":0.872},"min":{"1min":0.373,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8399999999999999,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":7.5369999999999999,"5min":3.2389999999999999,"15min":1.5029999999999999},"min":{"1min":0.36299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":118.55800000000001,"5min":118.55800000000001,"15min":118.55800000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.339,"5min":2.7120000000000002,"15min":1.302},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.4640000000000004,"5min":21.716000000000001,"15min":21.716000000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.4819999999999993,"5min":2.972,"15min":1.4159999999999999},"min":{"1min":0.46300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":118.239,"5min":118.239,"15min":118.239},"last":0.79400000000000004},{"interface":"front","average":{"1min":8.298,"5min":3.149,"15min":1.51},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":118.498,"5min":118.498,"15min":118.498},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.0359999999999996,"5min":3.4750000000000001,"15min":1.5740000000000001},"min":{"1min":0.49099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":118.18899999999999,"5min":118.18899999999999,"15min":118.18899999999999},"last":0.621},{"interface":"front","average":{"1min":7.8730000000000002,"5min":3.391,"15min":1.5840000000000001},"min":{"1min":0.48299999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":118.629,"5min":118.629,"15min":118.629},"last":0.52900000000000003}]}]},{"osd":3,"up_from":142,"seq":609885356122,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2162404,"kb_used_data":1915932,"kb_used_omap":2,"kb_used_meta":246461,"kb_avail":91582748,"statfs":{"total":95995035648,"available":93780733952,"internally_reserved":0,"allocated":1961914368,"data_stored":1961206937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":252376117},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":2.7869999999999999,"15min":1.3740000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.8570000000000002,"5min":91.634,"15min":91.634},"last":3.2999999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":3.0339999999999998,"15min":1.296},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.7910000000000004,"5min":91.646000000000001,"15min":91.646000000000001},"last":3.3199999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":6.7699999999999996,"5min":6.593,"15min":3.1459999999999999},"min":{"1min":0.23400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":145.47499999999999,"5min":180.02500000000001,"15min":180.02500000000001},"last":1.1899999999999999},{"interface":"front","average":{"1min":7.0389999999999997,"5min":7.3730000000000002,"15min":2.972},"min":{"1min":0.25900000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":144.83799999999999,"5min":184.00899999999999,"15min":184.00899999999999},"last":0.247}]},{"osd":2,"last update":"Mon Oct 9 04:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":3.3260000000000001,"15min":1.3819999999999999},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.222},"max":{"1min":9.6660000000000004,"5min":118.238,"15min":118.238},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":3.0129999999999999,"15min":2.0129999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.7830000000000004,"5min":117.64400000000001,"15min":117.64400000000001},"last":0.28399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":5.3899999999999997,"5min":2.298,"15min":1.907},"min":{"1min":0.34599999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":103.208,"5min":103.208,"15min":103.208},"last":4.1689999999999996},{"interface":"front","average":{"1min":5.5979999999999999,"5min":2.5910000000000002,"15min":1.5249999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":103.218,"5min":103.218,"15min":103.218},"last":0.31}]},{"osd":5,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":2.742,"15min":1.554},"min":{"1min":0.38100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.7220000000000004,"5min":57.475000000000001,"15min":57.475000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.5229999999999999,"5min":3.0710000000000002,"15min":1.496},"min":{"1min":0.38100000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.7380000000000004,"5min":57.651000000000003,"15min":57.651000000000003},"last":0.438}]},{"osd":6,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":2.915,"15min":1.4470000000000001},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":104.422,"5min":104.422,"15min":104.422},"last":8.7469999999999999},{"interface":"front","average":{"1min":5.4960000000000004,"5min":2.5979999999999999,"15min":2.0299999999999998},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":104.50700000000001,"5min":104.50700000000001,"15min":104.50700000000001},"last":3.161}]},{"osd":7,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":2.7440000000000002,"15min":2.0489999999999999},"min":{"1min":0.34999999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.1800000000000002,"5min":57.399000000000001,"15min":57.399000000000001},"last":4.5700000000000003},{"interface":"front","average":{"1min":1.169,"5min":2.927,"15min":1.6519999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":5.7610000000000001,"5min":57.447000000000003,"15min":57.447000000000003},"last":0.57499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019248,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3864280,"kb_used_data":2972240,"kb_used_omap":0,"kb_used_meta":892032,"kb_avail":89880872,"statfs":{"total":95995035648,"available":92038012928,"internally_reserved":0,"allocated":3043573760,"data_stored":3043055944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913440768},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":5.7949999999999999,"15min":3.1179999999999999},"min":{"1min":0.24099999999999999,"5min":0.23000000000000001,"15min":0.20499999999999999},"max":{"1min":4.7679999999999998,"5min":219.577,"15min":219.577},"last":12.656000000000001},{"interface":"front","average":{"1min":0.98499999999999999,"5min":5.0510000000000002,"15min":2.9729999999999999},"min":{"1min":0.41899999999999998,"5min":0.27100000000000002,"15min":0.247},"max":{"1min":3.5139999999999998,"5min":218.96000000000001,"15min":218.96000000000001},"last":2.883}]},{"osd":1,"last update":"Mon Oct 9 04:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":9.2360000000000007,"15min":4.3220000000000001},"min":{"1min":0.27800000000000002,"5min":0.246,"15min":0.23400000000000001},"max":{"1min":10.599,"5min":142.74799999999999,"15min":142.74799999999999},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":9.4320000000000004,"15min":4.5449999999999999},"min":{"1min":0.39100000000000001,"5min":0.27800000000000002,"15min":0.26700000000000002},"max":{"1min":4.226,"5min":139.09700000000001,"15min":139.09700000000001},"last":12.725}]},{"osd":2,"last update":"Mon Oct 9 04:34:46 2023","interfaces":[{"interface":"back","average":{"1min":15.507,"5min":5.7279999999999998,"15min":2.2770000000000001},"min":{"1min":0.32000000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":197.31999999999999,"5min":197.31999999999999,"15min":197.31999999999999},"last":1.48},{"interface":"front","average":{"1min":15.843,"5min":4.9740000000000002,"15min":2.0449999999999999},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":197.536,"5min":197.536,"15min":197.536},"last":0.97399999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":9.7479999999999993,"15min":4.2350000000000003},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":61.372,"5min":262.524,"15min":262.524},"last":3.3119999999999998},{"interface":"front","average":{"1min":5.1210000000000004,"5min":10.284000000000001,"15min":4.4560000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":60.085000000000001,"5min":262.55099999999999,"15min":262.55099999999999},"last":2.1920000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":6.5149999999999997,"5min":3.2080000000000002,"15min":1.7210000000000001},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":107.476,"5min":107.476,"15min":107.476},"last":2.7250000000000001},{"interface":"front","average":{"1min":6.6040000000000001,"5min":3.6659999999999999,"15min":1.847},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":107.414,"5min":107.414,"15min":107.414},"last":5.9859999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":2.0089999999999999,"15min":1.3480000000000001},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":3.887,"5min":52.801000000000002,"15min":52.801000000000002},"last":7.3650000000000002},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.9850000000000001,"15min":1.3360000000000001},"min":{"1min":0.34599999999999997,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.855,"5min":51.064999999999998,"15min":51.064999999999998},"last":12.698}]},{"osd":7,"last update":"Mon Oct 9 04:34:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":2.2799999999999998,"15min":1.482},"min":{"1min":0.316,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":5.6909999999999998,"5min":76.283000000000001,"15min":76.283000000000001},"last":2.5190000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":2.2040000000000002,"15min":1.474},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.29199999999999998},"max":{"1min":2.6190000000000002,"5min":76.259,"15min":76.259},"last":1.966}]}]},{"osd":5,"up_from":38,"seq":163208757612,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2746888,"kb_used_data":1860672,"kb_used_omap":0,"kb_used_meta":886208,"kb_avail":90998264,"statfs":{"total":95995035648,"available":93182222336,"internally_reserved":0,"allocated":1905328128,"data_stored":1904822440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907476992},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":5.3360000000000003,"15min":2.9399999999999999},"min":{"1min":0.22700000000000001,"5min":0.22,"15min":0.22},"max":{"1min":5.2199999999999998,"5min":127.108,"15min":127.108},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":4.8470000000000004,"15min":2.5790000000000002},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.9380000000000002,"5min":128.036,"15min":128.036},"last":1.8660000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:34:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":5.327,"15min":2.9180000000000001},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":4.9930000000000003,"5min":109.95699999999999,"15min":109.95699999999999},"last":3.899},{"interface":"front","average":{"1min":1.1140000000000001,"5min":6.0730000000000004,"15min":3.7349999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.1699999999999999,"5min":110.542,"15min":116.04000000000001},"last":0.35399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:34:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":7.6879999999999997,"15min":2.9460000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.716000000000001,"5min":324.55000000000001,"15min":324.55000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.2639999999999998,"5min":7.2969999999999997,"15min":2.8170000000000002},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":21.704000000000001,"5min":323.94499999999999,"15min":323.94499999999999},"last":0.85099999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:14 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":8.4100000000000001,"15min":3.5099999999999998},"min":{"1min":0.41399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":18.940000000000001,"5min":259.053,"15min":259.053},"last":1.8999999999999999},{"interface":"front","average":{"1min":2.7429999999999999,"5min":8.1310000000000002,"15min":3.46},"min":{"1min":0.379,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":18.803999999999998,"5min":259.108,"15min":259.108},"last":4.0449999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":0.98499999999999999,"15min":0.99399999999999999},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":5.9340000000000002,"5min":7.2699999999999996,"15min":9.641},"last":4.0839999999999996},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.548,"15min":1.1659999999999999},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":4.8639999999999999,"5min":10.645,"15min":10.645},"last":0.49099999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:34:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.244,"15min":1.125},"min":{"1min":0.32300000000000001,"5min":0.316,"15min":0.26000000000000001},"max":{"1min":6.6909999999999998,"5min":11.192,"15min":11.192},"last":4.0119999999999996},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.306,"15min":1.1850000000000001},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.8280000000000003,"5min":6.8280000000000003,"15min":14.334},"last":3.3969999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:34:58 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.649,"15min":1.292},"min":{"1min":0.34799999999999998,"5min":0.317,"15min":0.27400000000000002},"max":{"1min":5.1150000000000002,"5min":11.093,"15min":11.093},"last":1.8300000000000001},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.5509999999999999,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.35999999999999999,"15min":0.33000000000000002},"max":{"1min":5.9640000000000004,"5min":9.8550000000000004,"15min":11.315},"last":3.9660000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3942252544,"data_stored":3942252544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1167458304,"data_stored":1167458304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1725693952,"data_stored":1725693952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1963851776,"data_stored":1963851776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3063218176,"data_stored":3063218176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1915944960,"data_stored":1915944960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2678194176,"data_stored":2678194176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1545338880,"data_stored":1545338880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:35:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:20.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:20.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-10-09T04:35:20.687 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:20.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:20.808 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:20 smithi029 ceph-mon[216903]: pgmap v766: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s; 35563/282114 objects degraded (12.606%) 2023-10-09T04:35:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:20 smithi029 ceph-mon[216903]: from='client.45611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:35:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35563/282114 objects degraded (12.606%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:20 smithi029 ceph-mon[223858]: pgmap v766: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s; 35563/282114 objects degraded (12.606%) 2023-10-09T04:35:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:20 smithi029 ceph-mon[223858]: from='client.45611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:35:20.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35563/282114 objects degraded (12.606%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:20.970 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:20.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:20.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:21.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:21.082 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 98984 98968 79.2822 57.25 0.00947911 0.0126081 2023-10-09T04:35:21.124 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:20 smithi177 ceph-mon[195797]: pgmap v766: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s; 35563/282114 objects degraded (12.606%) 2023-10-09T04:35:21.124 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:20 smithi177 ceph-mon[195797]: from='client.45611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:35:21.124 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35563/282114 objects degraded (12.606%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:21.152 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:21.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:21.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:21.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:21.513 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:21.719 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:21.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:21.830 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:21.913 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:21.926 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:21.926 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:22.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 99896 99880 79.0002 57 0.0128469 0.0126538 2023-10-09T04:35:22.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:22.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:22.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:22.359 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:22.451 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:22.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:22.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:22.524 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:22.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:22.581 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:22.622 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:22.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:22.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:22.789 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:22.826 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:35:22.880 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:22.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:22.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:23.027 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:23.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:35:23.083216+0000 min lat: 0.00100934 max lat: 1.04904 avg lat: 0.0127222 2023-10-09T04:35:23.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:35:23.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 100618 100602 78.5767 45.125 0.00947969 0.0127222 2023-10-09T04:35:23.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:22 smithi177 ceph-mon[195797]: pgmap v767: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 36817/292932 objects degraded (12.568%) 2023-10-09T04:35:23.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2792842091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:23.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:35:23.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:22 smithi029 ceph-mon[216903]: pgmap v767: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 36817/292932 objects degraded (12.568%) 2023-10-09T04:35:23.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2792842091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:23.154 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:35:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:35:22] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T04:35:23.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:22 smithi029 ceph-mon[223858]: pgmap v767: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 36817/292932 objects degraded (12.568%) 2023-10-09T04:35:23.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2792842091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:23.662 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:24.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 101437 101421 78.2384 51.1875 0.014277 0.0127768 2023-10-09T04:35:24.142 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:24.170 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[223858]: failed to encode map e206 with expected crc 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2792842091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[223858]: osdmap e206: 8 total, 7 up, 8 in 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[223858]: failed to encode map e206 with expected crc 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[223858]: pgmap v769: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 37054/294318 objects degraded (12.590%) 2023-10-09T04:35:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[216903]: failed to encode map e206 with expected crc 2023-10-09T04:35:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2792842091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[216903]: osdmap e206: 8 total, 7 up, 8 in 2023-10-09T04:35:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[216903]: failed to encode map e206 with expected crc 2023-10-09T04:35:24.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:23 smithi029 ceph-mon[216903]: pgmap v769: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 37054/294318 objects degraded (12.590%) 2023-10-09T04:35:24.204 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:24.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:23 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:24.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:23 smithi177 ceph-mon[195797]: failed to encode map e206 with expected crc 2023-10-09T04:35:24.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2792842091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:24.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:23 smithi177 ceph-mon[195797]: osdmap e206: 8 total, 7 up, 8 in 2023-10-09T04:35:24.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:23 smithi177 ceph-mon[195797]: failed to encode map e206 with expected crc 2023-10-09T04:35:24.205 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:23 smithi177 ceph-mon[195797]: pgmap v769: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 37054/294318 objects degraded (12.590%) 2023-10-09T04:35:24.234 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:24.280 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:24.323 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:24.325 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:24.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:24.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:24.365 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:24.365 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.365 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:24.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:35:24.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:24.503 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:24.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:24.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:24.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:24.739 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:25.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 102124 102108 77.8079 42.9375 0.0186192 0.0128475 2023-10-09T04:35:25.111 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2299258588' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:25.111 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2299258588' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:25.139 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:25.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:25.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2299258588' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:25.316 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:25.458 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:25.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:25.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:25.579 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:25.677 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:25.848 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:25.849 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:25.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:25.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:25.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:26.006 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:25 smithi029 ceph-mon[216903]: pgmap v770: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 37581/301404 objects degraded (12.469%) 2023-10-09T04:35:26.006 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37054/294318 objects degraded (12.590%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:26.006 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:25 smithi029 ceph-mon[223858]: pgmap v770: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 37581/301404 objects degraded (12.469%) 2023-10-09T04:35:26.007 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37054/294318 objects degraded (12.590%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:26.011 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:25 smithi177 ceph-mon[195797]: pgmap v770: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 37581/301404 objects degraded (12.469%) 2023-10-09T04:35:26.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37054/294318 objects degraded (12.590%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:26.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 103043 103027 77.5625 57.4375 0.00978381 0.0128884 2023-10-09T04:35:26.108 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:26.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:47618 10.f 10:fb59a7f5:::benchmark_data_smithi029_344401_object47617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:34:29.239229+0000", 2023-10-09T04:35:26.130 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 56.885218778000002, 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35756269800000001, 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 47618 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.239229+0000", 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.239229+0000", 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.239229+0000", 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.239274+0000", 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5129000000000001e-05 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.239288+0000", 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3987e-05 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.239293+0000", 2023-10-09T04:35:26.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0640000000000001e-06 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.241448+0000", 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021549339999999998 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.241558+0000", 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010994500000000001 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.241695+0000", 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013679999999999999 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.245259+0000", 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003564865 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.248908+0000", 2023-10-09T04:35:26.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036483489999999999 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596702+0000", 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34779427299999999 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596749+0000", 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7258000000000001e-05 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596792+0000", 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2639999999999998e-05 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:47636 10.f 10:f06ee121:::benchmark_data_smithi029_344401_object47635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:34:29.249318+0000", 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 56.875130079000002, 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34760200499999999, 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.137 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 47636 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249318+0000", 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249318+0000", 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249317+0000", 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249350+0000", 2023-10-09T04:35:26.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3016999999999999e-05 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249362+0000", 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2132e-05 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249367+0000", 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2480000000000004e-06 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249423+0000", 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5884999999999998e-05 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249576+0000", 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000153122 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.249734+0000", 2023-10-09T04:35:26.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015823500000000001 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.251559+0000", 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001825045 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253127+0000", 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001567794 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596841+0000", 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34371376599999998 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596894+0000", 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3279000000000003e-05 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596920+0000", 2023-10-09T04:35:26.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5653999999999999e-05 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:47648 10.f 10:f308e045:::benchmark_data_smithi029_344401_object47647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:34:29.253510+0000", 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 56.870938307000003, 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34350956500000002, 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.144 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 47648 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253510+0000", 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253510+0000", 2023-10-09T04:35:26.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253509+0000", 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253592+0000", 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3165999999999996e-05 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253602+0000", 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9979999999999998e-06 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.253608+0000", 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8139999999999997e-06 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.255567+0000", 2023-10-09T04:35:26.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019590760000000001 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.255681+0000", 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011406800000000001 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.255806+0000", 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012510200000000001 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.259130+0000", 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033231710000000002 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.259173+0000", 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3429e-05 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596956+0000", 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33778297200000001 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596986+0000", 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0088999999999999e-05 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.597019+0000", 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3157000000000001e-05 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.150 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:47669 10.f 10:f793525e:::benchmark_data_smithi029_344401_object47668:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:34:29.267796+0000", 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 56.856652337, 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32929472199999998, 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.151 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 47669 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267796+0000", 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267796+0000", 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267795+0000", 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267855+0000", 2023-10-09T04:35:26.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9808000000000001e-05 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267865+0000", 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0411000000000001e-05 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267869+0000", 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4379999999999997e-06 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.267905+0000", 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5165e-05 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.268005+0000", 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100455 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.268144+0000", 2023-10-09T04:35:26.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000139412 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.270112+0000", 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019678870000000002 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.271181+0000", 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010686580000000001 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.597045+0000", 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32586394899999999 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.597072+0000", 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6687999999999999e-05 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.597090+0000", 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8749999999999998e-05 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88626 10.f 10:fa1d3f17:::benchmark_data_smithi029_344401_object88625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.948176+0000", 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.176272066999999, 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29605953699999998, 2023-10-09T04:35:26.159 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88626 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948176+0000", 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948176+0000", 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948175+0000", 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948236+0000", 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0687999999999999e-05 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948245+0000", 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4010000000000003e-06 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948251+0000", 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2839999999999997e-06 2023-10-09T04:35:26.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948306+0000", 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5248999999999999e-05 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948408+0000", 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102197 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948536+0000", 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012803700000000001 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.954617+0000", 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060803560000000003 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244089+0000", 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28947209200000001 2023-10-09T04:35:26.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244134+0000", 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5096000000000003e-05 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244235+0000", 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101714 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88637 10.b 10:d616fc5d:::benchmark_data_smithi029_344401_object88636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.952729+0000", 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.171719247999999, 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28032315499999999, 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88637 2023-10-09T04:35:26.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952729+0000", 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952729+0000", 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952728+0000", 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952786+0000", 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8084999999999997e-05 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952794+0000", 2023-10-09T04:35:26.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3900000000000004e-06 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952798+0000", 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4730000000000002e-06 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.956034+0000", 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032358180000000001 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.956166+0000", 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131712 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.956306+0000", 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014045799999999999 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.959489+0000", 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031833289999999999 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977317+0000", 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017827945000000001 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.232959+0000", 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25564146999999998 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.233012+0000", 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3032999999999999e-05 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.233052+0000", 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0127999999999999e-05 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88639 10.0 10:0d9f3cb8:::benchmark_data_smithi029_344401_object88638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.957039+0000", 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.167409268, 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.278908182, 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88639 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.173 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957039+0000", 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957039+0000", 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957037+0000", 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957101+0000", 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3168000000000002e-05 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957113+0000", 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2758000000000001e-05 2023-10-09T04:35:26.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957120+0000", 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7229999999999998e-06 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957157+0000", 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7293999999999999e-05 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957242+0000", 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4962000000000004e-05 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.957325+0000", 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2299000000000006e-05 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.959869+0000", 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025444669999999999 2023-10-09T04:35:26.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.961065+0000", 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011964479999999999 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.235860+0000", 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27479405699999998 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.235909+0000", 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9716999999999998e-05 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.235947+0000", 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7669999999999997e-05 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88648 10.7 10:eb07d429:::benchmark_data_smithi029_344401_object88647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.977378+0000", 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.147069863999999, 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26689151999999999, 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88648 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977378+0000", 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977378+0000", 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977377+0000", 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977453+0000", 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6203000000000002e-05 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977466+0000", 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.241e-05 2023-10-09T04:35:26.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977472+0000", 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1e-06 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977587+0000", 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011529 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977711+0000", 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012325500000000001 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977794+0000", 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2996000000000003e-05 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.981122+0000", 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033287260000000002 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982412+0000", 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00128947 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244194+0000", 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26178187800000002 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244237+0000", 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3220999999999998e-05 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244270+0000", 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2799000000000002e-05 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88649 10.f 10:fa69f024:::benchmark_data_smithi029_344401_object88648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.977486+0000", 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.146962404, 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26706585599999999, 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88649 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.187 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977486+0000", 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977486+0000", 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977485+0000", 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977518+0000", 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3257999999999999e-05 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977526+0000", 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9759999999999995e-06 2023-10-09T04:35:26.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.977531+0000", 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2070000000000003e-06 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979682+0000", 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021505920000000002 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979817+0000", 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013515099999999999 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.979975+0000", 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015777400000000001 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982556+0000", 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025805989999999998 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244422+0000", 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26186611399999998 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244460+0000", 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7895000000000002e-05 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244551+0000", 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1866000000000004e-05 2023-10-09T04:35:26.192 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88653 10.7 10:e552285b:::benchmark_data_smithi029_344401_object88652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.982438+0000", 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.142010075999998, 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26194933199999998, 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.193 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88653 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982438+0000", 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982438+0000", 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982437+0000", 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982467+0000", 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9927999999999999e-05 2023-10-09T04:35:26.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982475+0000", 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9670000000000001e-06 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982478+0000", 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9419999999999997e-06 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982515+0000", 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6043e-05 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982606+0000", 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1862999999999998e-05 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982746+0000", 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000139667 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.988327+0000", 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055809090000000002 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.988331+0000", 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.794e-06 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244308+0000", 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25597715999999998 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244352+0000", 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4289000000000002e-05 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244387+0000", 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5017e-05 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.199 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88654 10.7 10:efc9f954:::benchmark_data_smithi029_344401_object88653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.982673+0000", 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.141775378999998, 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26199517300000003, 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.200 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88654 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982673+0000", 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982673+0000", 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982672+0000", 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982716+0000", 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3865000000000001e-05 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982726+0000", 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9469999999999998e-06 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.982731+0000", 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.985430+0000", 2023-10-09T04:35:26.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026989280000000002 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.985551+0000", 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120522 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.985694+0000", 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014317400000000001 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.988461+0000", 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027672640000000002 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.988666+0000", 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020432299999999999 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244585+0000", 2023-10-09T04:35:26.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25591915300000001 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244622+0000", 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7669999999999997e-05 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.244668+0000", 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5401000000000001e-05 2023-10-09T04:35:26.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88655 10.0 10:0225922a:::benchmark_data_smithi029_344401_object88654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.983634+0000", 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 17.140814055, 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25473818500000001, 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88655 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983634+0000", 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983634+0000", 2023-10-09T04:35:26.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983627+0000", 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999933 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983688+0000", 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0412999999999999e-05 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983699+0000", 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.182e-05 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983705+0000", 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2020000000000003e-06 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983759+0000", 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4126000000000003e-05 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.983893+0000", 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000134272 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.984026+0000", 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013330199999999999 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.987095+0000", 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030691709999999999 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.987920+0000", 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082428099999999995 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.238288+0000", 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25036813899999999 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.238335+0000", 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7624999999999999e-05 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.238372+0000", 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6659000000000003e-05 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.213 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:89561 10.0 10:0557da26:::benchmark_data_smithi029_344401_object89560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:10.152433+0000", 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 15.972015405, 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25902939200000002, 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.214 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89561 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152433+0000", 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152433+0000", 2023-10-09T04:35:26.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152431+0000", 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152519+0000", 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7733999999999997e-05 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152530+0000", 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1511999999999999e-05 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152535+0000", 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1039999999999998e-06 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152597+0000", 2023-10-09T04:35:26.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1815000000000004e-05 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152746+0000", 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000148956 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.152863+0000", 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116387 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.158857+0000", 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059941009999999999 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.158882+0000", 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5238000000000001e-05 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411378+0000", 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25249597499999998 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411425+0000", 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7024000000000002e-05 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411462+0000", 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7128000000000001e-05 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.220 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:89566 10.0 10:046569a1:::benchmark_data_smithi029_344401_object89565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:10.156277+0000", 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 15.968171422999999, 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25528092099999999, 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.221 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89566 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156277+0000", 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156277+0000", 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156275+0000", 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156347+0000", 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1864000000000002e-05 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156360+0000", 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3106e-05 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156368+0000", 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2119999999999997e-06 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156416+0000", 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7874000000000003e-05 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156549+0000", 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013357000000000001 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.156661+0000", 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000112038 2023-10-09T04:35:26.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.158965+0000", 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023037629999999999 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.162857+0000", 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003892427 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411494+0000", 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.24863684 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411529+0000", 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4929999999999999e-05 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411557+0000", 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8362000000000001e-05 2023-10-09T04:35:26.227 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:89574 10.d 10:bf7ffb1a:::benchmark_data_smithi029_344401_object89573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:10.160114+0000", 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 15.964334423, 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.245211288, 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.228 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89574 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160114+0000", 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160114+0000", 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160113+0000", 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160174+0000", 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0993999999999999e-05 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160181+0000", 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2139999999999999e-06 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160184+0000", 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1370000000000002e-06 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160228+0000", 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3337999999999997e-05 2023-10-09T04:35:26.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160354+0000", 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126618 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.160467+0000", 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000112716 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.162855+0000", 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023876320000000002 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163306+0000", 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00045172000000000002 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.405240+0000", 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.241933655 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.405288+0000", 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7958999999999999e-05 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.405325+0000", 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6772e-05 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:89575 10.0 10:02364b6d:::benchmark_data_smithi029_344401_object89574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:10.163600+0000", 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 15.960848402, 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.255499531, 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89575 2023-10-09T04:35:26.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163600+0000", 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163600+0000", 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163598+0000", 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163664+0000", 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6092000000000004e-05 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163672+0000", 2023-10-09T04:35:26.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4760000000000001e-06 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163676+0000", 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.907e-06 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163712+0000", 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5892000000000001e-05 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.163837+0000", 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012532099999999999 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.405194+0000", 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.24135727300000001 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.411593+0000", 2023-10-09T04:35:26.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063990760000000001 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.418861+0000", 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0072680269999999998 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.419032+0000", 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000170359 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.419076+0000", 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4629999999999998e-05 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:10.419099+0000", 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2759999999999999e-05 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:101954 10.7 10:ebe4e8ae:::benchmark_data_smithi029_344401_object101953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:24.687257+0000", 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 1.437191364, 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26351040399999998, 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101954 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687257+0000", 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687257+0000", 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687256+0000", 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687298+0000", 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2135000000000002e-05 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687310+0000", 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2274e-05 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687317+0000", 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7710000000000001e-06 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687748+0000", 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00043159999999999997 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687841+0000", 2023-10-09T04:35:26.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2267000000000003e-05 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687968+0000", 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127599 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.691514+0000", 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035454430000000001 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.700147+0000", 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008633646 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.950676+0000", 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25052875000000002 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.950729+0000", 2023-10-09T04:35:26.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2395999999999998e-05 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.950767+0000", 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8455999999999999e-05 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:101955 10.7 10:ed162a92:::benchmark_data_smithi029_344401_object101954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:24.687329+0000", 2023-10-09T04:35:26.248 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 1.437119367, 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27470255399999999, 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101955 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687329+0000", 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687329+0000", 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687328+0000", 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687359+0000", 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1310999999999999e-05 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687368+0000", 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4489999999999999e-06 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687373+0000", 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.327e-06 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.697246+0000", 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0098733429999999997 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.697355+0000", 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108076 2023-10-09T04:35:26.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.697463+0000", 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108806 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.700236+0000", 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027730010000000002 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.701015+0000", 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00077902699999999995 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.961968+0000", 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26095299799999999 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.962008+0000", 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9570999999999998e-05 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.962031+0000", 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3263000000000001e-05 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:101958 10.a 10:53c34928:::benchmark_data_smithi029_344401_object101957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:24.687871+0000", 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 1.4365773019999999, 2023-10-09T04:35:26.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.263096887, 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101958 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687871+0000", 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687871+0000", 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687870+0000", 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687939+0000", 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8997000000000004e-05 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687948+0000", 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8510000000000005e-06 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.687954+0000", 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9800000000000003e-06 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.688004+0000", 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0785999999999998e-05 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.688114+0000", 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109274 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.688230+0000", 2023-10-09T04:35:26.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116322 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.691708+0000", 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003478035 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.699028+0000", 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073198309999999997 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.950881+0000", 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25185303999999997 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.950928+0000", 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6931999999999997e-05 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.950968+0000", 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9657000000000003e-05 2023-10-09T04:35:26.261 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:101964 10.a 10:596ddd9d:::benchmark_data_smithi029_344401_object101963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:24.695361+0000", 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 1.4290873209999999, 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26357689000000001, 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:26.262 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101964 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695361+0000", 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695361+0000", 2023-10-09T04:35:26.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695359+0000", 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695412+0000", 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2871000000000003e-05 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695427+0000", 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5021000000000001e-05 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695435+0000", 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4760000000000001e-06 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695486+0000", 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1838000000000001e-05 2023-10-09T04:35:26.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695612+0000", 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012546599999999999 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.695764+0000", 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015195599999999999 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.699146+0000", 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033820450000000002 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.700058+0000", 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00091209699999999995 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.958838+0000", 2023-10-09T04:35:26.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.258779704 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.958898+0000", 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0229000000000002e-05 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:24.958938+0000", 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9614999999999999e-05 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.268 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.269 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:26.269 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:26.269 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:35:26.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:26.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:35:26.333 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:26.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:26.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:26.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:26.635 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:26.661 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:26.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:26.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:26.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:26.732 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:26.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:26.834 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:26.860 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pgp_num to 1 2023-10-09T04:35:27.083 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 103572 103556 77.0328 33.0625 0.0187999 0.012977 2023-10-09T04:35:27.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4233264984' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:35:27.109 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:35:27.114 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4233264984' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:35:27.114 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:35:27.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4233264984' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:35:27.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:35:27.342 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:35:27.807 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:27.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:35:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:35:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[216903]: osdmap e207: 8 total, 7 up, 8 in 2023-10-09T04:35:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[216903]: pgmap v772: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 38558/306576 objects degraded (12.577%) 2023-10-09T04:35:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[216903]: failed to encode map e207 with expected crc 2023-10-09T04:35:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:35:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[223858]: osdmap e207: 8 total, 7 up, 8 in 2023-10-09T04:35:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[223858]: pgmap v772: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 38558/306576 objects degraded (12.577%) 2023-10-09T04:35:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:27 smithi029 ceph-mon[223858]: failed to encode map e207 with expected crc 2023-10-09T04:35:28.084 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 104316 104300 76.6725 46.5 0.0260149 0.0130376 2023-10-09T04:35:28.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:35:28.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:27 smithi177 ceph-mon[195797]: osdmap e207: 8 total, 7 up, 8 in 2023-10-09T04:35:28.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:27 smithi177 ceph-mon[195797]: pgmap v772: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 38558/306576 objects degraded (12.577%) 2023-10-09T04:35:28.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:27 smithi177 ceph-mon[195797]: failed to encode map e207 with expected crc 2023-10-09T04:35:28.252 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:28.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:28.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:28.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:28.424 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:28.655 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:28.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:28.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:28.808 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:28.851 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:28.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:28.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:28.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:28.992 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:29.084 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 105112 105096 76.3594 49.75 0.0118601 0.013091 2023-10-09T04:35:29.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:28 smithi029 ceph-mon[216903]: failed to encode map e207 with expected crc 2023-10-09T04:35:29.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:28 smithi029 ceph-mon[216903]: failed to encode map e207 with expected crc 2023-10-09T04:35:29.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:28 smithi029 ceph-mon[223858]: failed to encode map e207 with expected crc 2023-10-09T04:35:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:28 smithi029 ceph-mon[223858]: failed to encode map e207 with expected crc 2023-10-09T04:35:29.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:28 smithi177 ceph-mon[195797]: failed to encode map e207 with expected crc 2023-10-09T04:35:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:28 smithi177 ceph-mon[195797]: failed to encode map e207 with expected crc 2023-10-09T04:35:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:29.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:29.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:29.313 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:29.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:29.449 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:29.704 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:29.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:29.810 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:29.952 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:30.000 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:29 smithi029 ceph-mon[216903]: pgmap v773: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 38558/306576 objects degraded (12.577%) 2023-10-09T04:35:30.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:29 smithi029 ceph-mon[223858]: pgmap v773: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 38558/306576 objects degraded (12.577%) 2023-10-09T04:35:30.085 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 106039 106023 76.1477 57.9375 0.0138188 0.0131272 2023-10-09T04:35:30.147 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:35:30.147 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:35:30.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.147 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:106083 10.8 10:1671d6a0:::benchmark_data_smithi029_344401_object106082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2023-10-09T04:35:30.147 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:30.130696+0000", 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.011244033, 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011281586, 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 106083 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130696+0000", 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130696+0000", 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130694+0000", 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:30.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130812+0000", 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011746 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130830+0000", 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8067000000000002e-05 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130837+0000", 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2520000000000002e-06 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.130893+0000", 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6379999999999999e-05 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.131034+0000", 2023-10-09T04:35:30.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014082400000000001 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.131185+0000", 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015131899999999999 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.135814+0000", 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046281669999999999 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.152 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45542.0:106073 10.1 e207/190 10:8bd19b64:::benchmark_data_smithi029_344401_object106072:head v 207'6580, mlcod=207'6577)", 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:30.137593+0000", 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0043470260000000004, 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044792490000000003, 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:30.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137593+0000", 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137593+0000", 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137592+0000", 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137616+0000", 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3881999999999999e-05 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137646+0000", 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0255000000000001e-05 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:30.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137653+0000", 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8360000000000003e-06 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137697+0000", 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3460000000000001e-05 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137717+0000", 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0758e-05 2023-10-09T04:35:30.156 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45542.0:106077 10.1 e207/190 10:82ff24d7:::benchmark_data_smithi029_344401_object106076:head v 207'6581, mlcod=207'6577)", 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:30.137967+0000", 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.003973308, 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041725169999999997, 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:30.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137967+0000", 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137967+0000", 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:30.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.137966+0000", 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.138002+0000", 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6090999999999997e-05 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.138016+0000", 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3818000000000001e-05 2023-10-09T04:35:30.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.138022+0000", 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0820000000000004e-06 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.141151+0000", 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031287709999999998 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.141172+0000", 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0516000000000001e-05 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:106087 10.8 10:11d88c0b:::benchmark_data_smithi029_344401_object106086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:30.139257+0000", 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.002682842, 2023-10-09T04:35:30.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029542280000000001, 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 106087 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:35:30.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139257+0000", 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139257+0000", 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139256+0000", 2023-10-09T04:35:30.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139309+0000", 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3041000000000003e-05 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139326+0000", 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6932999999999999e-05 2023-10-09T04:35:30.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139332+0000", 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5830000000000003e-06 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139386+0000", 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4128000000000001e-05 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:35:30.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139524+0000", 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000137591 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0", 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:30.139673+0000", 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014874800000000001 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:35:30.166 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 4 2023-10-09T04:35:30.167 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:35:30.167 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:30.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:30.176 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:29 smithi177 ceph-mon[195797]: pgmap v773: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 38558/306576 objects degraded (12.577%) 2023-10-09T04:35:30.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:30.348 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:30.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:30.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:30.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:30.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:30.581 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:30.658 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:30.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:30.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:30.790 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:30.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:30.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:30.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:30.913 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:35:30.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:30.973 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:31.085 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 106700 106684 75.7518 41.3125 0.0163529 0.0131967 2023-10-09T04:35:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3391165543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38558/306576 objects degraded (12.577%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3391165543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38558/306576 objects degraded (12.577%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:31.174 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:31.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:31.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:31.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3391165543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:31.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:35:31.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38558/306576 objects degraded (12.577%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:31.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:35:31.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:31.327 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:31.711 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:31.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:31.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:31.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:31.876 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:31.968 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:31.984 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:31.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:32.090 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 107620 107604 75.5449 57.5 0.00746137 0.0132328 2023-10-09T04:35:32.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:32.165 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: osdmap e208: 8 total, 7 up, 8 in 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:31 smithi177 ceph-mon[195797]: pgmap v775: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 39760/316743 objects degraded (12.553%) 2023-10-09T04:35:32.167 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:32.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: osdmap e208: 8 total, 7 up, 8 in 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[223858]: pgmap v775: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 39760/316743 objects degraded (12.553%) 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:35:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: osdmap e208: 8 total, 7 up, 8 in 2023-10-09T04:35:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: failed to encode map e208 with expected crc 2023-10-09T04:35:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:31 smithi029 ceph-mon[216903]: pgmap v775: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s; 39760/316743 objects degraded (12.553%) 2023-10-09T04:35:32.191 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:32.219 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:32.251 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:32.289 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:32.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:32.335 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:32.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.368 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.369 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:35:32.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:32.498 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:32.573 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:32.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:32.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:32.725 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:32.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:32.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:32.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:32.937 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:33.069 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:33.071 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1319306735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:33.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:33.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:33.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0088 2023-10-09T04:35:33.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 108368 2023-10-09T04:35:33.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:35:33.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:35:33.087 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 75.2482 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 35.8758 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 231.125 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 15.8125 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1203 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 574.013 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3698 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 253 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0132856 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.027043 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 1.04904 2023-10-09T04:35:33.088 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00100934 2023-10-09T04:35:33.092 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:35:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1319306735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1319306735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:33.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:35:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:35:32] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-10-09T04:35:33.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:33.220 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:33.300 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:33.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:33.358 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:33.359 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:33.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:33.453 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:33.460 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T04:35:33.460 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:35:33.461 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T04:35:33.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:35:33.524 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:33.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:33.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:33.691 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:33.779 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:33.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:33.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:33.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:35:33.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:33.951 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:34.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:33 smithi177 ceph-mon[195797]: pgmap v776: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 39982/318126 objects degraded (12.568%) 2023-10-09T04:35:34.047 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:34.049 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:34.054 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:33 smithi029 ceph-mon[216903]: pgmap v776: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 39982/318126 objects degraded (12.568%) 2023-10-09T04:35:34.054 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:33 smithi029 ceph-mon[223858]: pgmap v776: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 39982/318126 objects degraded (12.568%) 2023-10-09T04:35:34.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:34.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:34.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:34.197 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:34.336 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:34.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:34.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:34.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:34.448 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:34.448 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:34.479 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:34.628 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:34.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:34.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:34.788 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:34.889 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:34.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:34.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:34.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:35.022 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:35.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:35.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:35.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:35.229 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:35.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:35:35.570 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:35.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:35:35.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:35.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:35.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:35.753 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:36.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:36 smithi029 ceph-mon[223858]: pgmap v777: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 126 B/s rd, 41 MiB/s wr, 776 op/s; 40076/319662 objects degraded (12.537%) 2023-10-09T04:35:36.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40076/319662 objects degraded (12.537%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:36.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:36 smithi029 ceph-mon[216903]: pgmap v777: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 126 B/s rd, 41 MiB/s wr, 776 op/s; 40076/319662 objects degraded (12.537%) 2023-10-09T04:35:36.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40076/319662 objects degraded (12.537%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:36.269 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:35:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:36 smithi177 ceph-mon[195797]: pgmap v777: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 126 B/s rd, 41 MiB/s wr, 776 op/s; 40076/319662 objects degraded (12.537%) 2023-10-09T04:35:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40076/319662 objects degraded (12.537%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:36.342 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:36.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:36.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:36.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:36.485 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:36.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:36.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:36.613 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:35:36.615 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:35:36.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:36.766 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:37.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:37.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:37 smithi029 ceph-mon[216903]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:35:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:37 smithi029 ceph-mon[223858]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:35:37.177 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:37.238 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:37.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:37 smithi177 ceph-mon[195797]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:35:37.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:37.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:37.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:37.425 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:37.897 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:37.914 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:37.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:37.926 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":777,"stamp":"2023-10-09T04:35:34.955173+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6982080440,"num_objects":106554,"num_object_clones":0,"num_object_copies":319662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40076,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106554,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":108760,"num_write_kb":6882062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":3083510,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108676,"ondisk_log_size":108676,"up":242,"acting":242,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24326272,"kb_used_data":19600768,"kb_used_omap":20,"kb_used_meta":4725099,"kb_avail":725634944,"statfs":{"total":767960285184,"available":743050182656,"internally_reserved":0,"allocated":20071186432,"data_stored":20066078236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4838502030},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":285802520,"num_objects":4362,"num_object_clones":0,"num_object_copies":13086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1518,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4362,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6292,"num_write_kb":340865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.102829"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":194,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.938929+0000","last_change":"2023-10-09T04:34:51.322569+0000","last_active":"2023-10-09T04:35:26.938929+0000","last_peered":"2023-10-09T04:35:26.938929+0000","last_clean":"2023-10-09T04:34:44.061742+0000","last_became_active":"2023-10-09T04:34:51.322569+0000","last_became_peered":"2023-10-09T04:34:51.322569+0000","last_unstale":"2023-10-09T04:35:26.938929+0000","last_undegraded":"2023-10-09T04:35:26.938929+0000","last_fullsized":"2023-10-09T04:34:51.319415+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:46.193676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.856060+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:35:26.856060+0000","last_peered":"2023-10-09T04:35:26.856060+0000","last_clean":"2023-10-09T04:35:26.856060+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:35:26.856060+0000","last_undegraded":"2023-10-09T04:35:26.856060+0000","last_fullsized":"2023-10-09T04:35:26.856060+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":172,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.853691+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:35:26.853691+0000","last_peered":"2023-10-09T04:35:26.853691+0000","last_clean":"2023-10-09T04:35:26.853691+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:35:26.853691+0000","last_undegraded":"2023-10-09T04:35:26.853691+0000","last_fullsized":"2023-10-09T04:35:26.853691+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":120,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.826566+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:35:22.826566+0000","last_peered":"2023-10-09T04:35:22.826566+0000","last_clean":"2023-10-09T04:35:22.826566+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:35:22.826566+0000","last_undegraded":"2023-10-09T04:35:22.826566+0000","last_fullsized":"2023-10-09T04:35:22.826566+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":194,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855586+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:35:26.855586+0000","last_peered":"2023-10-09T04:35:26.855586+0000","last_clean":"2023-10-09T04:35:26.855586+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:35:26.855586+0000","last_undegraded":"2023-10-09T04:35:26.855586+0000","last_fullsized":"2023-10-09T04:35:26.855586+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":165,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.944757+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:35:26.944757+0000","last_peered":"2023-10-09T04:35:26.944757+0000","last_clean":"2023-10-09T04:35:26.944757+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:35:26.944757+0000","last_undegraded":"2023-10-09T04:35:26.944757+0000","last_fullsized":"2023-10-09T04:35:26.944757+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.820482+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:35:22.820482+0000","last_peered":"2023-10-09T04:35:22.820482+0000","last_clean":"2023-10-09T04:35:22.820482+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:35:22.820482+0000","last_undegraded":"2023-10-09T04:35:22.820482+0000","last_fullsized":"2023-10-09T04:35:22.820482+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855659+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:35:26.855659+0000","last_peered":"2023-10-09T04:35:26.855659+0000","last_clean":"2023-10-09T04:35:26.855659+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:35:26.855659+0000","last_undegraded":"2023-10-09T04:35:26.855659+0000","last_fullsized":"2023-10-09T04:35:26.855659+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":167,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.820567+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:35:22.820567+0000","last_peered":"2023-10-09T04:35:22.820567+0000","last_clean":"2023-10-09T04:35:22.820567+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:35:22.820567+0000","last_undegraded":"2023-10-09T04:35:22.820567+0000","last_fullsized":"2023-10-09T04:35:22.820567+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":131,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855586+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:35:26.855586+0000","last_peered":"2023-10-09T04:35:26.855586+0000","last_clean":"2023-10-09T04:35:26.855586+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:35:26.855586+0000","last_undegraded":"2023-10-09T04:35:26.855586+0000","last_fullsized":"2023-10-09T04:35:26.855586+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":169,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.826452+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:35:22.826452+0000","last_peered":"2023-10-09T04:35:22.826452+0000","last_clean":"2023-10-09T04:35:22.826452+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:35:22.826452+0000","last_undegraded":"2023-10-09T04:35:22.826452+0000","last_fullsized":"2023-10-09T04:35:22.826452+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":178,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.854073+0000","last_change":"2023-10-09T04:34:51.322394+0000","last_active":"2023-10-09T04:35:26.854073+0000","last_peered":"2023-10-09T04:35:26.854073+0000","last_clean":"2023-10-09T04:34:44.053596+0000","last_became_active":"2023-10-09T04:34:51.322394+0000","last_became_peered":"2023-10-09T04:34:51.322394+0000","last_unstale":"2023-10-09T04:35:26.854073+0000","last_undegraded":"2023-10-09T04:35:26.854073+0000","last_fullsized":"2023-10-09T04:34:51.313456+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:55.868058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":174,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.942596+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:35:26.942596+0000","last_peered":"2023-10-09T04:35:26.942596+0000","last_clean":"2023-10-09T04:35:26.942596+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:35:26.942596+0000","last_undegraded":"2023-10-09T04:35:26.942596+0000","last_fullsized":"2023-10-09T04:35:26.942596+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":168,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939673+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:35:26.939673+0000","last_peered":"2023-10-09T04:35:26.939673+0000","last_clean":"2023-10-09T04:35:26.939673+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:35:26.939673+0000","last_undegraded":"2023-10-09T04:35:26.939673+0000","last_fullsized":"2023-10-09T04:35:26.939673+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":175,"reported_epoch":206,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:22.826109+0000","last_change":"2023-10-09T04:34:51.334626+0000","last_active":"2023-10-09T04:35:22.826109+0000","last_peered":"2023-10-09T04:35:22.826109+0000","last_clean":"2023-10-09T04:34:44.064651+0000","last_became_active":"2023-10-09T04:34:51.334626+0000","last_became_peered":"2023-10-09T04:34:51.334626+0000","last_unstale":"2023-10-09T04:35:22.826109+0000","last_undegraded":"2023-10-09T04:34:51.326465+0000","last_fullsized":"2023-10-09T04:34:51.326345+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:10.974139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855789+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:35:26.855789+0000","last_peered":"2023-10-09T04:35:26.855789+0000","last_clean":"2023-10-09T04:35:26.855789+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:35:26.855789+0000","last_undegraded":"2023-10-09T04:35:26.855789+0000","last_fullsized":"2023-10-09T04:35:26.855789+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":119,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.826058+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:35:22.826058+0000","last_peered":"2023-10-09T04:35:22.826058+0000","last_clean":"2023-10-09T04:35:22.826058+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:35:22.826058+0000","last_undegraded":"2023-10-09T04:35:22.826058+0000","last_fullsized":"2023-10-09T04:35:22.826058+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":166,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.815242+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:35:22.815242+0000","last_peered":"2023-10-09T04:35:22.815242+0000","last_clean":"2023-10-09T04:35:22.815242+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:35:22.815242+0000","last_undegraded":"2023-10-09T04:35:22.815242+0000","last_fullsized":"2023-10-09T04:35:22.815242+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":149,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.825823+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:35:22.825823+0000","last_peered":"2023-10-09T04:35:22.825823+0000","last_clean":"2023-10-09T04:35:22.825823+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:35:22.825823+0000","last_undegraded":"2023-10-09T04:35:22.825823+0000","last_fullsized":"2023-10-09T04:35:22.825823+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":119,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.856134+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:35:26.856134+0000","last_peered":"2023-10-09T04:35:26.856134+0000","last_clean":"2023-10-09T04:35:26.856134+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:35:26.856134+0000","last_undegraded":"2023-10-09T04:35:26.856134+0000","last_fullsized":"2023-10-09T04:35:26.856134+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":214,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.944489+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:35:26.944489+0000","last_peered":"2023-10-09T04:35:26.944489+0000","last_clean":"2023-10-09T04:35:26.944489+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:35:26.944489+0000","last_undegraded":"2023-10-09T04:35:26.944489+0000","last_fullsized":"2023-10-09T04:35:26.944489+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.856272+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:35:26.856272+0000","last_peered":"2023-10-09T04:35:26.856272+0000","last_clean":"2023-10-09T04:35:26.856272+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:35:26.856272+0000","last_undegraded":"2023-10-09T04:35:26.856272+0000","last_fullsized":"2023-10-09T04:35:26.856272+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"207'6463","reported_seq":6487,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:30.006156+0000","last_change":"2023-10-09T04:33:51.283670+0000","last_active":"2023-10-09T04:35:30.006156+0000","last_peered":"2023-10-09T04:35:30.006156+0000","last_clean":"2023-10-09T04:35:30.006156+0000","last_became_active":"2023-10-09T04:33:51.283338+0000","last_became_peered":"2023-10-09T04:33:51.283338+0000","last_unstale":"2023-10-09T04:35:30.006156+0000","last_undegraded":"2023-10-09T04:35:30.006156+0000","last_fullsized":"2023-10-09T04:35:30.006156+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6463,"log_dups_size":0,"ondisk_log_size":6463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:51:57.393546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423559168,"num_objects":6463,"num_object_clones":0,"num_object_copies":19389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6463,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":195,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.826581+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:35:22.826581+0000","last_peered":"2023-10-09T04:35:22.826581+0000","last_clean":"2023-10-09T04:35:22.826581+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:35:22.826581+0000","last_undegraded":"2023-10-09T04:35:22.826581+0000","last_fullsized":"2023-10-09T04:35:22.826581+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":149,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.814824+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:35:22.814824+0000","last_peered":"2023-10-09T04:35:22.814824+0000","last_clean":"2023-10-09T04:35:22.814824+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:35:22.814824+0000","last_undegraded":"2023-10-09T04:35:22.814824+0000","last_fullsized":"2023-10-09T04:35:22.814824+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"208'6795","reported_seq":6820,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.724732+0000","last_change":"2023-10-09T04:33:51.451657+0000","last_active":"2023-10-09T04:35:34.724732+0000","last_peered":"2023-10-09T04:35:34.724732+0000","last_clean":"2023-10-09T04:35:34.724732+0000","last_became_active":"2023-10-09T04:33:51.451300+0000","last_became_peered":"2023-10-09T04:33:51.451300+0000","last_unstale":"2023-10-09T04:35:34.724732+0000","last_undegraded":"2023-10-09T04:35:34.724732+0000","last_fullsized":"2023-10-09T04:35:34.724732+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6795,"log_dups_size":0,"ondisk_log_size":6795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:02.431444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433389568,"num_objects":6613,"num_object_clones":0,"num_object_copies":19839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6795,"num_write_kb":429056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":168,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.826497+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:35:22.826497+0000","last_peered":"2023-10-09T04:35:22.826497+0000","last_clean":"2023-10-09T04:35:22.826497+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:35:22.826497+0000","last_undegraded":"2023-10-09T04:35:22.826497+0000","last_fullsized":"2023-10-09T04:35:22.826497+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939258+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:35:26.939258+0000","last_peered":"2023-10-09T04:35:26.939258+0000","last_clean":"2023-10-09T04:35:26.939258+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:35:26.939258+0000","last_undegraded":"2023-10-09T04:35:26.939258+0000","last_fullsized":"2023-10-09T04:35:26.939258+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"208'7027","reported_seq":7066,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.834046+0000","last_change":"2023-10-09T04:34:22.281046+0000","last_active":"2023-10-09T04:35:34.834046+0000","last_peered":"2023-10-09T04:35:34.834046+0000","last_clean":"2023-10-09T04:35:34.834046+0000","last_became_active":"2023-10-09T04:34:22.265736+0000","last_became_peered":"2023-10-09T04:34:22.265736+0000","last_unstale":"2023-10-09T04:35:34.834046+0000","last_undegraded":"2023-10-09T04:35:34.834046+0000","last_fullsized":"2023-10-09T04:35:34.834046+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7027,"log_dups_size":0,"ondisk_log_size":7027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:23.231373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446889984,"num_objects":6819,"num_object_clones":0,"num_object_copies":20457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7027,"num_write_kb":443072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":150,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.825890+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:35:22.825890+0000","last_peered":"2023-10-09T04:35:22.825890+0000","last_clean":"2023-10-09T04:35:22.825890+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:35:22.825890+0000","last_undegraded":"2023-10-09T04:35:22.825890+0000","last_fullsized":"2023-10-09T04:35:22.825890+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.945099+0000","last_change":"2023-10-09T04:34:51.328996+0000","last_active":"2023-10-09T04:35:26.945099+0000","last_peered":"2023-10-09T04:35:26.945099+0000","last_clean":"2023-10-09T04:34:44.068074+0000","last_became_active":"2023-10-09T04:34:51.328996+0000","last_became_peered":"2023-10-09T04:34:51.328996+0000","last_unstale":"2023-10-09T04:35:26.945099+0000","last_undegraded":"2023-10-09T04:35:26.945099+0000","last_fullsized":"2023-10-09T04:34:51.321084+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:56.215110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"208'6706","reported_seq":6736,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:31.562586+0000","last_change":"2023-10-09T04:34:51.318540+0000","last_active":"2023-10-09T04:35:31.562586+0000","last_peered":"2023-10-09T04:35:31.562586+0000","last_clean":"2023-10-09T04:34:50.268593+0000","last_became_active":"2023-10-09T04:34:51.318540+0000","last_became_peered":"2023-10-09T04:34:51.318540+0000","last_unstale":"2023-10-09T04:35:31.562586+0000","last_undegraded":"2023-10-09T04:34:51.315308+0000","last_fullsized":"2023-10-09T04:34:51.315200+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:05.444866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439484416,"num_objects":6706,"num_object_clones":0,"num_object_copies":20118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6706,"num_write_kb":429184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":6706}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.815091+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:35:22.815091+0000","last_peered":"2023-10-09T04:35:22.815091+0000","last_clean":"2023-10-09T04:35:22.815091+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:35:22.815091+0000","last_undegraded":"2023-10-09T04:35:22.815091+0000","last_fullsized":"2023-10-09T04:35:22.815091+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.815748+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:35:22.815748+0000","last_peered":"2023-10-09T04:35:22.815748+0000","last_clean":"2023-10-09T04:35:22.815748+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:35:22.815748+0000","last_undegraded":"2023-10-09T04:35:22.815748+0000","last_fullsized":"2023-10-09T04:35:22.815748+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.2","version":"208'6786","reported_seq":6813,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:31.631942+0000","last_change":"2023-10-09T04:33:51.279607+0000","last_active":"2023-10-09T04:35:31.631942+0000","last_peered":"2023-10-09T04:35:31.631942+0000","last_clean":"2023-10-09T04:35:31.631942+0000","last_became_active":"2023-10-09T04:33:51.279269+0000","last_became_peered":"2023-10-09T04:33:51.279269+0000","last_unstale":"2023-10-09T04:35:31.631942+0000","last_undegraded":"2023-10-09T04:35:31.631942+0000","last_fullsized":"2023-10-09T04:35:31.631942+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6786,"log_dups_size":0,"ondisk_log_size":6786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:48:05.128040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444727296,"num_objects":6786,"num_object_clones":0,"num_object_copies":20358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6786,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855791+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:35:26.855791+0000","last_peered":"2023-10-09T04:35:26.855791+0000","last_clean":"2023-10-09T04:35:26.855791+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:35:26.855791+0000","last_undegraded":"2023-10-09T04:35:26.855791+0000","last_fullsized":"2023-10-09T04:35:26.855791+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":168,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.942514+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:35:26.942514+0000","last_peered":"2023-10-09T04:35:26.942514+0000","last_clean":"2023-10-09T04:35:26.942514+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:35:26.942514+0000","last_undegraded":"2023-10-09T04:35:26.942514+0000","last_fullsized":"2023-10-09T04:35:26.942514+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"207'6599","reported_seq":6630,"reported_epoch":207,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:30.012496+0000","last_change":"2023-10-09T04:34:51.328975+0000","last_active":"2023-10-09T04:35:30.012496+0000","last_peered":"2023-10-09T04:35:30.012496+0000","last_clean":"2023-10-09T04:34:50.301002+0000","last_became_active":"2023-10-09T04:34:51.328975+0000","last_became_peered":"2023-10-09T04:34:51.328975+0000","last_unstale":"2023-10-09T04:35:30.012496+0000","last_undegraded":"2023-10-09T04:34:51.324496+0000","last_fullsized":"2023-10-09T04:34:51.324385+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6599,"log_dups_size":0,"ondisk_log_size":6599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:18:53.048540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432472064,"num_objects":6599,"num_object_clones":0,"num_object_copies":19797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6599,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6599,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":6599}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":166,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.944911+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:35:26.944911+0000","last_peered":"2023-10-09T04:35:26.944911+0000","last_clean":"2023-10-09T04:35:26.944911+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:35:26.944911+0000","last_undegraded":"2023-10-09T04:35:26.944911+0000","last_fullsized":"2023-10-09T04:35:26.944911+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":192,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.939800+0000","last_change":"2023-10-09T04:34:51.322935+0000","last_active":"2023-10-09T04:35:26.939800+0000","last_peered":"2023-10-09T04:35:26.939800+0000","last_clean":"2023-10-09T04:34:44.060875+0000","last_became_active":"2023-10-09T04:34:51.322935+0000","last_became_peered":"2023-10-09T04:34:51.322935+0000","last_unstale":"2023-10-09T04:35:26.939800+0000","last_undegraded":"2023-10-09T04:35:26.939800+0000","last_fullsized":"2023-10-09T04:34:51.319826+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:49.238888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"208'6926","reported_seq":6948,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.774983+0000","last_change":"2023-10-09T04:33:51.280550+0000","last_active":"2023-10-09T04:35:34.774983+0000","last_peered":"2023-10-09T04:35:34.774983+0000","last_clean":"2023-10-09T04:35:34.774983+0000","last_became_active":"2023-10-09T04:33:51.280262+0000","last_became_peered":"2023-10-09T04:33:51.280262+0000","last_unstale":"2023-10-09T04:35:34.774983+0000","last_undegraded":"2023-10-09T04:35:34.774983+0000","last_fullsized":"2023-10-09T04:35:34.774983+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6926,"log_dups_size":0,"ondisk_log_size":6926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:29.056390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438697984,"num_objects":6694,"num_object_clones":0,"num_object_copies":20082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6926,"num_write_kb":435840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.939002+0000","last_change":"2023-10-09T04:34:51.333617+0000","last_active":"2023-10-09T04:35:26.939002+0000","last_peered":"2023-10-09T04:35:26.939002+0000","last_clean":"2023-10-09T04:34:44.060999+0000","last_became_active":"2023-10-09T04:34:51.333617+0000","last_became_peered":"2023-10-09T04:34:51.333617+0000","last_unstale":"2023-10-09T04:35:26.939002+0000","last_undegraded":"2023-10-09T04:35:26.939002+0000","last_fullsized":"2023-10-09T04:34:51.322409+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:09.264091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":168,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939391+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:35:26.939391+0000","last_peered":"2023-10-09T04:35:26.939391+0000","last_clean":"2023-10-09T04:35:26.939391+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:35:26.939391+0000","last_undegraded":"2023-10-09T04:35:26.939391+0000","last_fullsized":"2023-10-09T04:35:26.939391+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"208'6799","reported_seq":6825,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.837654+0000","last_change":"2023-10-09T04:33:51.281183+0000","last_active":"2023-10-09T04:35:34.837654+0000","last_peered":"2023-10-09T04:35:34.837654+0000","last_clean":"2023-10-09T04:35:34.837654+0000","last_became_active":"2023-10-09T04:33:51.280920+0000","last_became_peered":"2023-10-09T04:33:51.280920+0000","last_unstale":"2023-10-09T04:35:34.837654+0000","last_undegraded":"2023-10-09T04:35:34.837654+0000","last_fullsized":"2023-10-09T04:35:34.837654+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6799,"log_dups_size":0,"ondisk_log_size":6799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:15.815193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432209920,"num_objects":6595,"num_object_clones":0,"num_object_copies":19785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6799,"num_write_kb":428608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":159,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855477+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:35:26.855477+0000","last_peered":"2023-10-09T04:35:26.855477+0000","last_clean":"2023-10-09T04:35:26.855477+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:35:26.855477+0000","last_undegraded":"2023-10-09T04:35:26.855477+0000","last_fullsized":"2023-10-09T04:35:26.855477+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":173,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.853778+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:35:26.853778+0000","last_peered":"2023-10-09T04:35:26.853778+0000","last_clean":"2023-10-09T04:35:26.853778+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:35:26.853778+0000","last_undegraded":"2023-10-09T04:35:26.853778+0000","last_fullsized":"2023-10-09T04:35:26.853778+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"207'6565","reported_seq":6589,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:29.964878+0000","last_change":"2023-10-09T04:33:51.281811+0000","last_active":"2023-10-09T04:35:29.964878+0000","last_peered":"2023-10-09T04:35:29.964878+0000","last_clean":"2023-10-09T04:35:29.964878+0000","last_became_active":"2023-10-09T04:33:51.281495+0000","last_became_peered":"2023-10-09T04:33:51.281495+0000","last_unstale":"2023-10-09T04:35:29.964878+0000","last_undegraded":"2023-10-09T04:35:29.964878+0000","last_fullsized":"2023-10-09T04:35:29.964878+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:53.315124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430243840,"num_objects":6565,"num_object_clones":0,"num_object_copies":19695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6565,"num_write_kb":420160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.854268+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:35:26.854268+0000","last_peered":"2023-10-09T04:35:26.854268+0000","last_clean":"2023-10-09T04:35:26.854268+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:35:26.854268+0000","last_undegraded":"2023-10-09T04:35:26.854268+0000","last_fullsized":"2023-10-09T04:35:26.854268+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":191,"reported_epoch":206,"state":"active+undersized","last_fresh":"2023-10-09T04:35:22.815328+0000","last_change":"2023-10-09T04:34:51.340476+0000","last_active":"2023-10-09T04:35:22.815328+0000","last_peered":"2023-10-09T04:35:22.815328+0000","last_clean":"2023-10-09T04:34:44.071691+0000","last_became_active":"2023-10-09T04:34:51.340476+0000","last_became_peered":"2023-10-09T04:34:51.340476+0000","last_unstale":"2023-10-09T04:35:22.815328+0000","last_undegraded":"2023-10-09T04:35:22.815328+0000","last_fullsized":"2023-10-09T04:34:51.323270+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:07.997213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.854184+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:35:26.854184+0000","last_peered":"2023-10-09T04:35:26.854184+0000","last_clean":"2023-10-09T04:35:26.854184+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:35:26.854184+0000","last_undegraded":"2023-10-09T04:35:26.854184+0000","last_fullsized":"2023-10-09T04:35:26.854184+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"208'6928","reported_seq":6953,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.748322+0000","last_change":"2023-10-09T04:33:51.277983+0000","last_active":"2023-10-09T04:35:34.748322+0000","last_peered":"2023-10-09T04:35:34.748322+0000","last_clean":"2023-10-09T04:35:34.748322+0000","last_became_active":"2023-10-09T04:33:51.277555+0000","last_became_peered":"2023-10-09T04:33:51.277555+0000","last_unstale":"2023-10-09T04:35:34.748322+0000","last_undegraded":"2023-10-09T04:35:34.748322+0000","last_fullsized":"2023-10-09T04:35:34.748322+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6928,"log_dups_size":0,"ondisk_log_size":6928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:59.460010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439222272,"num_objects":6702,"num_object_clones":0,"num_object_copies":20106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6928,"num_write_kb":436160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":171,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.818139+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:35:22.818139+0000","last_peered":"2023-10-09T04:35:22.818139+0000","last_clean":"2023-10-09T04:35:22.818139+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:35:22.818139+0000","last_undegraded":"2023-10-09T04:35:22.818139+0000","last_fullsized":"2023-10-09T04:35:22.818139+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":189,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939600+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:35:26.939600+0000","last_peered":"2023-10-09T04:35:26.939600+0000","last_clean":"2023-10-09T04:35:26.939600+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:35:26.939600+0000","last_undegraded":"2023-10-09T04:35:26.939600+0000","last_fullsized":"2023-10-09T04:35:26.939600+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":194,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.854150+0000","last_change":"2023-10-09T04:34:51.316666+0000","last_active":"2023-10-09T04:35:26.854150+0000","last_peered":"2023-10-09T04:35:26.854150+0000","last_clean":"2023-10-09T04:34:44.053522+0000","last_became_active":"2023-10-09T04:34:51.316666+0000","last_became_peered":"2023-10-09T04:34:51.316666+0000","last_unstale":"2023-10-09T04:35:26.854150+0000","last_undegraded":"2023-10-09T04:35:26.854150+0000","last_fullsized":"2023-10-09T04:34:51.313759+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:10.205902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"207'6730","reported_seq":6761,"reported_epoch":207,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:30.010749+0000","last_change":"2023-10-09T04:34:51.330190+0000","last_active":"2023-10-09T04:35:30.010749+0000","last_peered":"2023-10-09T04:35:30.010749+0000","last_clean":"2023-10-09T04:34:50.296861+0000","last_became_active":"2023-10-09T04:34:51.330190+0000","last_became_peered":"2023-10-09T04:34:51.330190+0000","last_unstale":"2023-10-09T04:35:30.010749+0000","last_undegraded":"2023-10-09T04:34:51.323489+0000","last_fullsized":"2023-10-09T04:34:51.323336+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:59.712251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441057280,"num_objects":6730,"num_object_clones":0,"num_object_copies":20190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6730,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6730,"num_write_kb":430720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":6730}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":126,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.855726+0000","last_change":"2023-10-09T04:34:51.318540+0000","last_active":"2023-10-09T04:35:26.855726+0000","last_peered":"2023-10-09T04:35:26.855726+0000","last_clean":"2023-10-09T04:29:34.560919+0000","last_became_active":"2023-10-09T04:34:51.318540+0000","last_became_peered":"2023-10-09T04:34:51.318540+0000","last_unstale":"2023-10-09T04:35:26.855726+0000","last_undegraded":"2023-10-09T04:35:26.855726+0000","last_fullsized":"2023-10-09T04:34:51.314590+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:41.112095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":179,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.853712+0000","last_change":"2023-10-09T04:34:51.321100+0000","last_active":"2023-10-09T04:35:26.853712+0000","last_peered":"2023-10-09T04:35:26.853712+0000","last_clean":"2023-10-09T04:34:44.053288+0000","last_became_active":"2023-10-09T04:34:51.321100+0000","last_became_peered":"2023-10-09T04:34:51.321100+0000","last_unstale":"2023-10-09T04:35:26.853712+0000","last_undegraded":"2023-10-09T04:35:26.853712+0000","last_fullsized":"2023-10-09T04:34:51.313128+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:11.901708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855031+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:35:26.855031+0000","last_peered":"2023-10-09T04:35:26.855031+0000","last_clean":"2023-10-09T04:35:26.855031+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:35:26.855031+0000","last_undegraded":"2023-10-09T04:35:26.855031+0000","last_fullsized":"2023-10-09T04:35:26.855031+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"208'6728","reported_seq":6753,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.708587+0000","last_change":"2023-10-09T04:33:51.277098+0000","last_active":"2023-10-09T04:35:34.708587+0000","last_peered":"2023-10-09T04:35:34.708587+0000","last_clean":"2023-10-09T04:35:34.708587+0000","last_became_active":"2023-10-09T04:33:51.276788+0000","last_became_peered":"2023-10-09T04:33:51.276788+0000","last_unstale":"2023-10-09T04:35:34.708587+0000","last_undegraded":"2023-10-09T04:35:34.708587+0000","last_fullsized":"2023-10-09T04:35:34.708587+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:20.524012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429916160,"num_objects":6560,"num_object_clones":0,"num_object_copies":19680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6728,"num_write_kb":425216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":178,"reported_epoch":206,"state":"active+undersized","last_fresh":"2023-10-09T04:35:22.815835+0000","last_change":"2023-10-09T04:34:51.334572+0000","last_active":"2023-10-09T04:35:22.815835+0000","last_peered":"2023-10-09T04:35:22.815835+0000","last_clean":"2023-10-09T04:34:44.071648+0000","last_became_active":"2023-10-09T04:34:51.334572+0000","last_became_peered":"2023-10-09T04:34:51.334572+0000","last_unstale":"2023-10-09T04:35:22.815835+0000","last_undegraded":"2023-10-09T04:35:22.815835+0000","last_fullsized":"2023-10-09T04:34:51.324871+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:03.116911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":599,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.824895+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:35:22.824895+0000","last_peered":"2023-10-09T04:35:22.824895+0000","last_clean":"2023-10-09T04:35:22.824895+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:35:22.824895+0000","last_undegraded":"2023-10-09T04:35:22.824895+0000","last_fullsized":"2023-10-09T04:35:22.824895+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":130,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855613+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:35:26.855613+0000","last_peered":"2023-10-09T04:35:26.855613+0000","last_clean":"2023-10-09T04:35:26.855613+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:35:26.855613+0000","last_undegraded":"2023-10-09T04:35:26.855613+0000","last_fullsized":"2023-10-09T04:35:26.855613+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":148,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.945251+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:35:26.945251+0000","last_peered":"2023-10-09T04:35:26.945251+0000","last_clean":"2023-10-09T04:35:26.945251+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:35:26.945251+0000","last_undegraded":"2023-10-09T04:35:26.945251+0000","last_fullsized":"2023-10-09T04:35:26.945251+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"207'6707","reported_seq":6748,"reported_epoch":207,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:29.882705+0000","last_change":"2023-10-09T04:34:51.327114+0000","last_active":"2023-10-09T04:35:29.882705+0000","last_peered":"2023-10-09T04:35:29.882705+0000","last_clean":"2023-10-09T04:34:50.268727+0000","last_became_active":"2023-10-09T04:34:51.327114+0000","last_became_peered":"2023-10-09T04:34:51.327114+0000","last_unstale":"2023-10-09T04:35:29.882705+0000","last_undegraded":"2023-10-09T04:34:51.325672+0000","last_fullsized":"2023-10-09T04:34:51.325531+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6707,"log_dups_size":0,"ondisk_log_size":6707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:20:06.884579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439549952,"num_objects":6707,"num_object_clones":0,"num_object_copies":20121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6707,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6707,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":6707}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":207,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.824845+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:35:22.824845+0000","last_peered":"2023-10-09T04:35:22.824845+0000","last_clean":"2023-10-09T04:35:22.824845+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:35:22.824845+0000","last_undegraded":"2023-10-09T04:35:22.824845+0000","last_fullsized":"2023-10-09T04:35:22.824845+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939408+0000","last_change":"2023-10-09T04:34:22.266723+0000","last_active":"2023-10-09T04:35:26.939408+0000","last_peered":"2023-10-09T04:35:26.939408+0000","last_clean":"2023-10-09T04:35:26.939408+0000","last_became_active":"2023-10-09T04:34:22.266261+0000","last_became_peered":"2023-10-09T04:34:22.266261+0000","last_unstale":"2023-10-09T04:35:26.939408+0000","last_undegraded":"2023-10-09T04:35:26.939408+0000","last_fullsized":"2023-10-09T04:35:26.939408+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:54.792194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.939989+0000","last_change":"2023-10-09T04:34:51.318703+0000","last_active":"2023-10-09T04:35:26.939989+0000","last_peered":"2023-10-09T04:35:26.939989+0000","last_clean":"2023-10-09T04:34:44.061887+0000","last_became_active":"2023-10-09T04:34:51.318703+0000","last_became_peered":"2023-10-09T04:34:51.318703+0000","last_unstale":"2023-10-09T04:35:26.939989+0000","last_undegraded":"2023-10-09T04:35:26.939989+0000","last_fullsized":"2023-10-09T04:34:51.317343+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:36.687606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"208'6905","reported_seq":6931,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-09T04:35:34.726747+0000","last_change":"2023-10-09T04:33:51.282958+0000","last_active":"2023-10-09T04:35:34.726747+0000","last_peered":"2023-10-09T04:35:34.726747+0000","last_clean":"2023-10-09T04:35:34.726747+0000","last_became_active":"2023-10-09T04:33:51.282685+0000","last_became_peered":"2023-10-09T04:33:51.282685+0000","last_unstale":"2023-10-09T04:35:34.726747+0000","last_undegraded":"2023-10-09T04:35:34.726747+0000","last_fullsized":"2023-10-09T04:35:34.726747+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:49.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436469784,"num_objects":6661,"num_object_clones":0,"num_object_copies":19983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6661,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6905,"num_write_kb":434049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":190,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.826382+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:35:22.826382+0000","last_peered":"2023-10-09T04:35:22.826382+0000","last_clean":"2023-10-09T04:35:22.826382+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:35:22.826382+0000","last_undegraded":"2023-10-09T04:35:22.826382+0000","last_fullsized":"2023-10-09T04:35:22.826382+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":179,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.854020+0000","last_change":"2023-10-09T04:34:51.325423+0000","last_active":"2023-10-09T04:35:26.854020+0000","last_peered":"2023-10-09T04:35:26.854020+0000","last_clean":"2023-10-09T04:34:44.053445+0000","last_became_active":"2023-10-09T04:34:51.325423+0000","last_became_peered":"2023-10-09T04:34:51.325423+0000","last_unstale":"2023-10-09T04:35:26.854020+0000","last_undegraded":"2023-10-09T04:35:26.854020+0000","last_fullsized":"2023-10-09T04:34:51.313428+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.954931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":51,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.815945+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:35:22.815945+0000","last_peered":"2023-10-09T04:35:22.815945+0000","last_clean":"2023-10-09T04:35:22.815945+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:35:22.815945+0000","last_undegraded":"2023-10-09T04:35:22.815945+0000","last_fullsized":"2023-10-09T04:35:22.815945+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"208'6782","reported_seq":6814,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:34.857018+0000","last_change":"2023-10-09T04:34:51.328209+0000","last_active":"2023-10-09T04:35:34.857018+0000","last_peered":"2023-10-09T04:35:34.857018+0000","last_clean":"2023-10-09T04:34:50.302298+0000","last_became_active":"2023-10-09T04:34:51.328209+0000","last_became_peered":"2023-10-09T04:34:51.328209+0000","last_unstale":"2023-10-09T04:35:34.857018+0000","last_undegraded":"2023-10-09T04:34:51.321394+0000","last_fullsized":"2023-10-09T04:34:51.321276+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6782,"log_dups_size":0,"ondisk_log_size":6782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:47.025638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428867584,"num_objects":6544,"num_object_clones":0,"num_object_copies":19632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6782,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":6544}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.856247+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:35:26.856247+0000","last_peered":"2023-10-09T04:35:26.856247+0000","last_clean":"2023-10-09T04:35:26.856247+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:35:26.856247+0000","last_undegraded":"2023-10-09T04:35:26.856247+0000","last_fullsized":"2023-10-09T04:35:26.856247+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.854649+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:35:26.854649+0000","last_peered":"2023-10-09T04:35:26.854649+0000","last_clean":"2023-10-09T04:35:26.854649+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:35:26.854649+0000","last_undegraded":"2023-10-09T04:35:26.854649+0000","last_fullsized":"2023-10-09T04:35:26.854649+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.855480+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:35:26.855480+0000","last_peered":"2023-10-09T04:35:26.855480+0000","last_clean":"2023-10-09T04:35:26.855480+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:35:26.855480+0000","last_undegraded":"2023-10-09T04:35:26.855480+0000","last_fullsized":"2023-10-09T04:35:26.855480+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"208'7014","reported_seq":7045,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:34.736527+0000","last_change":"2023-10-09T04:34:51.325469+0000","last_active":"2023-10-09T04:35:34.736527+0000","last_peered":"2023-10-09T04:35:34.736527+0000","last_clean":"2023-10-09T04:34:50.287068+0000","last_became_active":"2023-10-09T04:34:51.325469+0000","last_became_peered":"2023-10-09T04:34:51.325469+0000","last_unstale":"2023-10-09T04:35:34.736527+0000","last_undegraded":"2023-10-09T04:34:51.321417+0000","last_fullsized":"2023-10-09T04:34:51.321273+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:06.267427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444727296,"num_objects":6786,"num_object_clones":0,"num_object_copies":20358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6786,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7014,"num_write_kb":441600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":6786}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":177,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.944570+0000","last_change":"2023-10-09T04:34:51.331849+0000","last_active":"2023-10-09T04:35:26.944570+0000","last_peered":"2023-10-09T04:35:26.944570+0000","last_clean":"2023-10-09T04:34:44.065555+0000","last_became_active":"2023-10-09T04:34:51.331849+0000","last_became_peered":"2023-10-09T04:34:51.331849+0000","last_unstale":"2023-10-09T04:35:26.944570+0000","last_undegraded":"2023-10-09T04:35:26.944570+0000","last_fullsized":"2023-10-09T04:34:51.321695+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:09.071914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":61,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.854786+0000","last_change":"2023-10-09T04:34:22.266652+0000","last_active":"2023-10-09T04:35:26.854786+0000","last_peered":"2023-10-09T04:35:26.854786+0000","last_clean":"2023-10-09T04:35:26.854786+0000","last_became_active":"2023-10-09T04:34:22.266267+0000","last_became_peered":"2023-10-09T04:34:22.266267+0000","last_unstale":"2023-10-09T04:35:26.854786+0000","last_undegraded":"2023-10-09T04:35:26.854786+0000","last_fullsized":"2023-10-09T04:35:26.854786+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:50.646309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":151,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939122+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:35:26.939122+0000","last_peered":"2023-10-09T04:35:26.939122+0000","last_clean":"2023-10-09T04:35:26.939122+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:35:26.939122+0000","last_undegraded":"2023-10-09T04:35:26.939122+0000","last_fullsized":"2023-10-09T04:35:26.939122+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":48,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939530+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:35:26.939530+0000","last_peered":"2023-10-09T04:35:26.939530+0000","last_clean":"2023-10-09T04:35:26.939530+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:35:26.939530+0000","last_undegraded":"2023-10-09T04:35:26.939530+0000","last_fullsized":"2023-10-09T04:35:26.939530+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":119,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.853876+0000","last_change":"2023-10-09T04:34:51.316709+0000","last_active":"2023-10-09T04:35:26.853876+0000","last_peered":"2023-10-09T04:35:26.853876+0000","last_clean":"2023-10-09T04:27:36.073787+0000","last_became_active":"2023-10-09T04:34:51.316709+0000","last_became_peered":"2023-10-09T04:34:51.316709+0000","last_unstale":"2023-10-09T04:35:26.853876+0000","last_undegraded":"2023-10-09T04:35:26.853876+0000","last_fullsized":"2023-10-09T04:34:51.313259+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:43.330782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":169,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.814947+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:35:22.814947+0000","last_peered":"2023-10-09T04:35:22.814947+0000","last_clean":"2023-10-09T04:35:22.814947+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:35:22.814947+0000","last_undegraded":"2023-10-09T04:35:22.814947+0000","last_fullsized":"2023-10-09T04:35:22.814947+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939801+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:35:26.939801+0000","last_peered":"2023-10-09T04:35:26.939801+0000","last_clean":"2023-10-09T04:35:26.939801+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:35:26.939801+0000","last_undegraded":"2023-10-09T04:35:26.939801+0000","last_fullsized":"2023-10-09T04:35:26.939801+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":152,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-09T04:35:22.815460+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:35:22.815460+0000","last_peered":"2023-10-09T04:35:22.815460+0000","last_clean":"2023-10-09T04:35:22.815460+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:35:22.815460+0000","last_undegraded":"2023-10-09T04:35:22.815460+0000","last_fullsized":"2023-10-09T04:35:22.815460+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-09T04:35:26.939088+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:35:26.939088+0000","last_peered":"2023-10-09T04:35:26.939088+0000","last_clean":"2023-10-09T04:35:26.939088+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:35:26.939088+0000","last_undegraded":"2023-10-09T04:35:26.939088+0000","last_fullsized":"2023-10-09T04:35:26.939088+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":183,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.855967+0000","last_change":"2023-10-09T04:34:51.321322+0000","last_active":"2023-10-09T04:35:26.855967+0000","last_peered":"2023-10-09T04:35:26.855967+0000","last_clean":"2023-10-09T04:34:44.059138+0000","last_became_active":"2023-10-09T04:34:51.321322+0000","last_became_peered":"2023-10-09T04:34:51.321322+0000","last_unstale":"2023-10-09T04:35:26.855967+0000","last_undegraded":"2023-10-09T04:35:26.855967+0000","last_fullsized":"2023-10-09T04:34:51.314501+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:46.254327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":216,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.855557+0000","last_change":"2023-10-09T04:34:51.318670+0000","last_active":"2023-10-09T04:35:26.855557+0000","last_peered":"2023-10-09T04:35:26.855557+0000","last_clean":"2023-10-09T04:34:44.058794+0000","last_became_active":"2023-10-09T04:34:51.318670+0000","last_became_peered":"2023-10-09T04:34:51.318670+0000","last_unstale":"2023-10-09T04:35:26.855557+0000","last_undegraded":"2023-10-09T04:35:26.855557+0000","last_fullsized":"2023-10-09T04:34:51.314163+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:43.814115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":71,"reported_epoch":206,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:22.820687+0000","last_change":"2023-10-09T04:34:51.325437+0000","last_active":"2023-10-09T04:35:22.820687+0000","last_peered":"2023-10-09T04:35:22.820687+0000","last_clean":"2023-10-09T04:34:44.071843+0000","last_became_active":"2023-10-09T04:34:51.325437+0000","last_became_peered":"2023-10-09T04:34:51.325437+0000","last_unstale":"2023-10-09T04:35:22.820687+0000","last_undegraded":"2023-10-09T04:34:51.324026+0000","last_fullsized":"2023-10-09T04:34:51.323882+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:24.344413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+undersized","last_fresh":"2023-10-09T04:35:26.938810+0000","last_change":"2023-10-09T04:34:51.326267+0000","last_active":"2023-10-09T04:35:26.938810+0000","last_peered":"2023-10-09T04:35:26.938810+0000","last_clean":"2023-10-09T04:34:44.060673+0000","last_became_active":"2023-10-09T04:34:51.326267+0000","last_became_peered":"2023-10-09T04:34:51.326267+0000","last_unstale":"2023-10-09T04:35:26.938810+0000","last_undegraded":"2023-10-09T04:35:26.938810+0000","last_fullsized":"2023-10-09T04:34:51.321105+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:44.409648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6981484568,"num_objects":106530,"num_object_clones":0,"num_object_copies":319590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":40072,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106530,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108460,"num_write_kb":6879617,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20113203200,"data_stored":20113195056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108460,"ondisk_log_size":108460,"up":42,"acting":42,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":88,"acting":88,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234343,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2607088,"kb_used_data":1696104,"kb_used_omap":0,"kb_used_meta":910976,"kb_avail":91138064,"statfs":{"total":95995035648,"available":93325377536,"internally_reserved":0,"allocated":1736810496,"data_stored":1736284750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932839424},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5810000000000004,"5min":8.4450000000000003,"15min":5.8789999999999996},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.22900000000000001},"max":{"1min":68.495999999999995,"5min":186.446,"15min":629.22500000000002},"last":7.6500000000000004},{"interface":"front","average":{"1min":6.9870000000000001,"5min":8.6980000000000004,"15min":5.8559999999999999},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":68.540000000000006,"5min":186.429,"15min":629.43100000000004},"last":0.35199999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":7.6040000000000001,"15min":4.2119999999999997},"min":{"1min":0.28799999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":28.795999999999999,"5min":224.13499999999999,"15min":224.13499999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":3.0449999999999999,"5min":7.7329999999999997,"15min":4.2869999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":28.818999999999999,"5min":224.203,"15min":224.203},"last":0.436}]},{"osd":2,"last update":"Mon Oct 9 04:34:42 2023","interfaces":[{"interface":"back","average":{"1min":16.478000000000002,"5min":5.5570000000000004,"15min":2.2050000000000001},"min":{"1min":0.28499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":177.262,"5min":177.262,"15min":177.262},"last":2.2999999999999998},{"interface":"front","average":{"1min":16.478000000000002,"5min":5.5830000000000002,"15min":2.2229999999999999},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":177.24299999999999,"5min":177.24299999999999,"15min":177.24299999999999},"last":2.5419999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8580000000000001,"5min":11.728,"15min":5.3109999999999999},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":53.482999999999997,"5min":249.989,"15min":249.989},"last":7.6989999999999998},{"interface":"front","average":{"1min":3.6379999999999999,"5min":12.002000000000001,"15min":5.431},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":53.162999999999997,"5min":250.21700000000001,"15min":250.21700000000001},"last":2.6459999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:34:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.3560000000000001,"15min":1.216},"min":{"1min":0.29499999999999998,"5min":0.27200000000000002,"15min":0.23799999999999999},"max":{"1min":2.7469999999999999,"5min":10.869,"15min":10.869},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.0169999999999999,"15min":0.95299999999999996},"min":{"1min":0.32100000000000001,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":3.5649999999999999,"5min":10.613,"15min":10.613},"last":1.8819999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:34:36 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.91200000000000003,"15min":0.95099999999999996},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.26400000000000001},"max":{"1min":1.6120000000000001,"5min":5.2599999999999998,"15min":8.8610000000000007},"last":1.7609999999999999},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.3440000000000001,"15min":1.161},"min":{"1min":0.38100000000000001,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":2.0880000000000001,"5min":10.895,"15min":10.895},"last":3.1099999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:34:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.1619999999999999,"15min":1.165},"min":{"1min":0.35899999999999999,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":2.722,"5min":5.7270000000000003,"15min":12.007},"last":7.7279999999999998},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.331,"15min":1.2110000000000001},"min":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.31900000000000001},"max":{"1min":2.778,"5min":10.917,"15min":10.917},"last":4.024}]}]},{"osd":6,"up_from":46,"seq":197568495978,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3574316,"kb_used_data":2984356,"kb_used_omap":0,"kb_used_meta":589760,"kb_avail":90170836,"statfs":{"total":95995035648,"available":92334936064,"internally_reserved":0,"allocated":3055980544,"data_stored":3055466465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603914240},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":3.544,"5min":15.909000000000001,"15min":6.7400000000000002},"min":{"1min":0.26200000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":29.931999999999999,"5min":1284.521,"15min":1284.521},"last":1.105},{"interface":"front","average":{"1min":3.2919999999999998,"5min":15.868,"15min":6.9610000000000003},"min":{"1min":0.26600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":27.896000000000001,"5min":1284.4690000000001,"15min":1284.4690000000001},"last":0.70799999999999996}]},{"osd":1,"last update":"Mon Oct 9 04:35:22 2023","interfaces":[{"interface":"back","average":{"1min":6.4080000000000004,"5min":7.4939999999999998,"15min":3.7490000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":94.287000000000006,"5min":185.74000000000001,"15min":185.74000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":6.7709999999999999,"5min":7.4119999999999999,"15min":3.8839999999999999},"min":{"1min":0.34799999999999998,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":92.116,"5min":189.03999999999999,"15min":189.03999999999999},"last":1.1359999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.7110000000000001,"15min":0.95099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.6890000000000001,"5min":7.6890000000000001,"15min":7.6890000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.236,"15min":1.02},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.976,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":11.254}]},{"osd":3,"last update":"Mon Oct 9 04:35:13 2023","interfaces":[{"interface":"back","average":{"1min":11.717000000000001,"5min":5.0419999999999998,"15min":2.121},"min":{"1min":0.35999999999999999,"5min":0.314,"15min":0.314},"max":{"1min":144.99799999999999,"5min":144.99799999999999,"15min":144.99799999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":11.445,"5min":5.1820000000000004,"15min":2.117},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.313},"max":{"1min":145.023,"5min":145.023,"15min":145.023},"last":0.73799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":1.5720000000000001,"15min":1.268},"min":{"1min":0.28999999999999998,"5min":0.27300000000000002,"15min":0.246},"max":{"1min":9.8360000000000003,"5min":9.8360000000000003,"15min":12.743},"last":1.173},{"interface":"front","average":{"1min":1.738,"5min":1.028,"15min":1.1539999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26000000000000001},"max":{"1min":8.0129999999999999,"5min":8.9920000000000009,"15min":12.028},"last":1.01}]},{"osd":5,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.9239999999999999,"15min":1.7190000000000001},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":10.779,"5min":81.087000000000003,"15min":81.087000000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.8239999999999998,"5min":2.5750000000000002,"15min":1.7370000000000001},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":11.279999999999999,"5min":81.706999999999994,"15min":81.706999999999994},"last":5.7839999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":2.1320000000000001,"15min":1.492},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":11.226000000000001,"5min":81.674999999999997,"15min":81.674999999999997},"last":1.075},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.246,"15min":1.5369999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":9.9009999999999998,"5min":81.055000000000007,"15min":81.055000000000007},"last":0.57199999999999995}]}]},{"osd":1,"up_from":118,"seq":506806141067,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1681072,"kb_used_data":1277544,"kb_used_omap":4,"kb_used_meta":403515,"kb_avail":92064080,"statfs":{"total":95995035648,"available":94273617920,"internally_reserved":0,"allocated":1308205056,"data_stored":1307387283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":413200270},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":5.6849999999999996,"5min":4.8849999999999998,"15min":3.5350000000000001},"min":{"1min":0.189,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":64.037000000000006,"5min":97.308000000000007,"15min":97.308000000000007},"last":0.22500000000000001},{"interface":"front","average":{"1min":5.9980000000000002,"5min":5.4649999999999999,"15min":3.8540000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":66.388000000000005,"5min":97.055999999999997,"15min":97.055999999999997},"last":0.21299999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1530000000000005,"5min":3.544,"15min":1.417},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":143.11199999999999,"5min":143.11199999999999,"15min":143.11199999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":7.6529999999999996,"5min":2.8410000000000002,"15min":1.175},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.218},"max":{"1min":143.22,"5min":143.22,"15min":143.22},"last":1.1819999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:35:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6680000000000001,"5min":5.415,"15min":2.1059999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":58.673000000000002,"5min":195.035,"15min":195.035},"last":0.26900000000000002},{"interface":"front","average":{"1min":4.1550000000000002,"5min":6.0709999999999997,"15min":2.335},"min":{"1min":0.246,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":58.125999999999998,"5min":195.465,"15min":195.465},"last":0.29799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.804,"15min":2.597},"min":{"1min":0.32900000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.7919999999999998,"5min":8.0150000000000006,"15min":107.795},"last":0.311},{"interface":"front","average":{"1min":1.325,"5min":1.391,"15min":1.6539999999999999},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.1420000000000003,"5min":6.1420000000000003,"15min":109.167},"last":5.6639999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":2.105,"15min":2.214},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.186999999999999,"15min":12.414999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.7350000000000001,"5min":2.0680000000000001,"15min":2.431},"min":{"1min":0.42999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.734,"5min":10.061999999999999,"15min":13.933999999999999},"last":0.34699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":2.2549999999999999,"15min":2.173},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.8410000000000002,"5min":8.3770000000000007,"15min":13.388},"last":0.81200000000000006},{"interface":"front","average":{"1min":2.363,"5min":2.2050000000000001,"15min":2.5459999999999998},"min":{"1min":0.48499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":8.6549999999999994,"5min":12.596,"15min":13.882},"last":0.81899999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":2.1339999999999999,"15min":2.5920000000000001},"min":{"1min":0.438,"5min":0.374,"15min":0.374},"max":{"1min":7.6299999999999999,"5min":8.4160000000000004,"15min":13.907},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.391,"5min":1.607,"15min":1.5620000000000001},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9260000000000002,"5min":6.1799999999999997,"15min":7.8239999999999998},"last":0.95399999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024124,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4781280,"kb_used_data":4294808,"kb_used_omap":5,"kb_used_meta":486458,"kb_avail":88963872,"statfs":{"total":95995035648,"available":91099004928,"internally_reserved":0,"allocated":4397883392,"data_stored":4397086698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":498133626},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":7.1180000000000003,"15min":4.1980000000000004},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":6.0179999999999998,"5min":167.92699999999999,"15min":167.92699999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.484,"5min":7.3250000000000002,"15min":4.0510000000000002},"min":{"1min":0.28100000000000003,"5min":0.20799999999999999,"15min":0.19600000000000001},"max":{"1min":6.4960000000000004,"5min":169.78999999999999,"15min":169.78999999999999},"last":0.27200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:34:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.024,"15min":1.26},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.3769999999999998,"5min":107.548,"15min":107.548},"last":1.548},{"interface":"front","average":{"1min":1.329,"5min":2.7210000000000001,"15min":1.1459999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.4989999999999997,"5min":107.35899999999999,"15min":107.35899999999999},"last":2.2469999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:35:11 2023","interfaces":[{"interface":"back","average":{"1min":5.8300000000000001,"5min":5.9370000000000003,"15min":2.3149999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":46.625,"5min":213.28399999999999,"15min":213.28399999999999},"last":1.2809999999999999},{"interface":"front","average":{"1min":6.0019999999999998,"5min":6.1059999999999999,"15min":2.3900000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":47.076000000000001,"5min":212.27000000000001,"15min":212.27000000000001},"last":0.311}]},{"osd":4,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":3.1070000000000002,"5min":2.113,"15min":1.8069999999999999},"min":{"1min":0.38500000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":22.588000000000001,"5min":22.588000000000001,"15min":22.588000000000001},"last":5.5289999999999999},{"interface":"front","average":{"1min":3.6339999999999999,"5min":2.6030000000000002,"15min":1.9059999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":22.605,"5min":22.605,"15min":22.605},"last":5.5430000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":2.4350000000000001,"15min":1.948},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.27900000000000003},"max":{"1min":8.0510000000000002,"5min":12.807,"15min":12.807},"last":5.3099999999999996},{"interface":"front","average":{"1min":2.145,"5min":1.7789999999999999,"15min":1.7150000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.5129999999999999,"5min":8.4589999999999996,"15min":14.144},"last":5.1909999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":2.4390000000000001,"15min":2.0649999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.0749999999999993,"5min":16.369,"15min":16.369},"last":3.911},{"interface":"front","average":{"1min":3.0139999999999998,"5min":1.919,"15min":1.8899999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.4930000000000003,"5min":8.4930000000000003,"15min":14.090999999999999},"last":5.149}]},{"osd":7,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4529999999999998,"5min":1.905,"15min":1.8600000000000001},"min":{"1min":0.433,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.9770000000000003,"5min":8.5229999999999997,"15min":11.890000000000001},"last":6.9770000000000003},{"interface":"front","average":{"1min":2.2109999999999999,"5min":2.1629999999999998,"15min":2.0379999999999998},"min":{"1min":0.42399999999999999,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":6.9640000000000004,"5min":9.7110000000000003,"15min":11.176},"last":6.9640000000000004}]}]},{"osd":2,"up_from":135,"seq":579820585050,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1928196,"kb_used_data":1669308,"kb_used_omap":7,"kb_used_meta":258872,"kb_avail":91816956,"statfs":{"total":95995035648,"available":94020562944,"internally_reserved":0,"allocated":1709371392,"data_stored":1708661063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":265085009},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":9.0289999999999999,"5min":6.2489999999999997,"15min":2.3050000000000002},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":136.28100000000001,"5min":136.28100000000001,"15min":136.28100000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":15.180999999999999,"5min":7.3689999999999998,"15min":2.7370000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":136.239,"5min":136.239,"15min":136.239},"last":0.28299999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":7.6420000000000003,"5min":5.4779999999999998,"15min":2.081},"min":{"1min":0.246,"5min":0.193,"15min":0.193},"max":{"1min":119.748,"5min":119.748,"15min":119.748},"last":0.41499999999999998},{"interface":"front","average":{"1min":7.9009999999999998,"5min":5.8760000000000003,"15min":2.1899999999999999},"min":{"1min":0.23799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":118.024,"5min":118.024,"15min":118.024},"last":0.59499999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:34:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":6.093,"15min":2.2810000000000001},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4320000000000004,"5min":147.655,"15min":147.655},"last":0.438},{"interface":"front","average":{"1min":1.425,"5min":6.1150000000000002,"15min":2.3290000000000002},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.7850000000000001,"5min":147.607,"15min":147.607},"last":0.47799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":2.2669999999999999,"15min":1.1559999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.9699999999999998,"5min":12.765000000000001,"15min":12.765000000000001},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.389,"15min":0.872},"min":{"1min":0.373,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8399999999999999,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":7.5369999999999999,"5min":3.2389999999999999,"15min":1.5029999999999999},"min":{"1min":0.36299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":118.55800000000001,"5min":118.55800000000001,"15min":118.55800000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.339,"5min":2.7120000000000002,"15min":1.302},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.4640000000000004,"5min":21.716000000000001,"15min":21.716000000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.4819999999999993,"5min":2.972,"15min":1.4159999999999999},"min":{"1min":0.46300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":118.239,"5min":118.239,"15min":118.239},"last":0.79400000000000004},{"interface":"front","average":{"1min":8.298,"5min":3.149,"15min":1.51},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":118.498,"5min":118.498,"15min":118.498},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.0359999999999996,"5min":3.4750000000000001,"15min":1.5740000000000001},"min":{"1min":0.49099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":118.18899999999999,"5min":118.18899999999999,"15min":118.18899999999999},"last":0.621},{"interface":"front","average":{"1min":7.8730000000000002,"5min":3.391,"15min":1.5840000000000001},"min":{"1min":0.48299999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":118.629,"5min":118.629,"15min":118.629},"last":0.52900000000000003}]}]},{"osd":3,"up_from":142,"seq":609885356125,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2426436,"kb_used_data":2149628,"kb_used_omap":2,"kb_used_meta":276797,"kb_avail":91318716,"statfs":{"total":95995035648,"available":93510365184,"internally_reserved":0,"allocated":2201219072,"data_stored":2200503953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":283440181},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":2.7869999999999999,"15min":1.3740000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.8570000000000002,"5min":91.634,"15min":91.634},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":3.0339999999999998,"15min":1.296},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.7910000000000004,"5min":91.646000000000001,"15min":91.646000000000001},"last":1.768}]},{"osd":1,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":6.7699999999999996,"5min":6.593,"15min":3.1459999999999999},"min":{"1min":0.23400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":145.47499999999999,"5min":180.02500000000001,"15min":180.02500000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":7.0389999999999997,"5min":7.3730000000000002,"15min":2.972},"min":{"1min":0.25900000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":144.83799999999999,"5min":184.00899999999999,"15min":184.00899999999999},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":3.3260000000000001,"15min":1.3819999999999999},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.222},"max":{"1min":9.6660000000000004,"5min":118.238,"15min":118.238},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":3.0129999999999999,"15min":2.0129999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.7830000000000004,"5min":117.64400000000001,"15min":117.64400000000001},"last":0.28399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":5.3899999999999997,"5min":2.298,"15min":1.907},"min":{"1min":0.34599999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":103.208,"5min":103.208,"15min":103.208},"last":0.433},{"interface":"front","average":{"1min":5.5979999999999999,"5min":2.5910000000000002,"15min":1.5249999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":103.218,"5min":103.218,"15min":103.218},"last":0.48099999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":2.742,"15min":1.554},"min":{"1min":0.38100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.7220000000000004,"5min":57.475000000000001,"15min":57.475000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.5229999999999999,"5min":3.0710000000000002,"15min":1.496},"min":{"1min":0.38100000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.7380000000000004,"5min":57.651000000000003,"15min":57.651000000000003},"last":1.6950000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":2.915,"15min":1.4470000000000001},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":104.422,"5min":104.422,"15min":104.422},"last":1.7390000000000001},{"interface":"front","average":{"1min":5.4960000000000004,"5min":2.5979999999999999,"15min":2.0299999999999998},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":104.50700000000001,"5min":104.50700000000001,"15min":104.50700000000001},"last":0.68200000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":2.7440000000000002,"15min":2.0489999999999999},"min":{"1min":0.34999999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.1800000000000002,"5min":57.399000000000001,"15min":57.399000000000001},"last":1.6040000000000001},{"interface":"front","average":{"1min":1.169,"5min":2.927,"15min":1.6519999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":5.7610000000000001,"5min":57.447000000000003,"15min":57.447000000000003},"last":1.524}]}]},{"osd":4,"up_from":30,"seq":128849019252,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4326068,"kb_used_data":3425708,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":89419084,"statfs":{"total":95995035648,"available":91565142016,"internally_reserved":0,"allocated":3507924992,"data_stored":3507402576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":7.4189999999999996,"5min":6.2460000000000004,"15min":3.4900000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.20499999999999999},"max":{"1min":54.991999999999997,"5min":219.577,"15min":219.577},"last":5.0069999999999997},{"interface":"front","average":{"1min":7.2530000000000001,"5min":6.1029999999999998,"15min":3.286},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.247},"max":{"1min":54.841000000000001,"5min":218.96000000000001,"15min":218.96000000000001},"last":0.27100000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:35:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3129999999999997,"5min":7.4299999999999997,"15min":4.4790000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":48.633000000000003,"5min":142.74799999999999,"15min":142.74799999999999},"last":0.38},{"interface":"front","average":{"1min":4.593,"5min":7.6139999999999999,"15min":4.6929999999999996},"min":{"1min":0.28599999999999998,"5min":0.27800000000000002,"15min":0.26700000000000002},"max":{"1min":48.680999999999997,"5min":139.09700000000001,"15min":139.09700000000001},"last":4.9690000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:34:46 2023","interfaces":[{"interface":"back","average":{"1min":15.507,"5min":5.7279999999999998,"15min":2.2770000000000001},"min":{"1min":0.32000000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":197.31999999999999,"5min":197.31999999999999,"15min":197.31999999999999},"last":1.48},{"interface":"front","average":{"1min":15.843,"5min":4.9740000000000002,"15min":2.0449999999999999},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":197.536,"5min":197.536,"15min":197.536},"last":0.97399999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":9.7479999999999993,"15min":4.2350000000000003},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":61.372,"5min":262.524,"15min":262.524},"last":0.46999999999999997},{"interface":"front","average":{"1min":5.1210000000000004,"5min":10.284000000000001,"15min":4.4560000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":60.085000000000001,"5min":262.55099999999999,"15min":262.55099999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":6.5149999999999997,"5min":3.2080000000000002,"15min":1.7210000000000001},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":107.476,"5min":107.476,"15min":107.476},"last":0.34100000000000003},{"interface":"front","average":{"1min":6.6040000000000001,"5min":3.6659999999999999,"15min":1.847},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":107.414,"5min":107.414,"15min":107.414},"last":0.29999999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":7.173,"5min":2.6619999999999999,"15min":1.7609999999999999},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":65.75,"5min":65.75,"15min":65.75},"last":2.5030000000000001},{"interface":"front","average":{"1min":6.5510000000000002,"5min":2.5409999999999999,"15min":1.708},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":65.849999999999994,"5min":65.849999999999994,"15min":65.849999999999994},"last":9.2729999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:34:54 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":2.2799999999999998,"15min":1.482},"min":{"1min":0.316,"5min":0.309,"15min":0.30099999999999999},"max":{"1min":5.6909999999999998,"5min":76.283000000000001,"15min":76.283000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":2.2040000000000002,"15min":1.474},"min":{"1min":0.35999999999999999,"5min":0.33100000000000002,"15min":0.29199999999999998},"max":{"1min":2.6190000000000002,"5min":76.259,"15min":76.259},"last":0.55000000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757615,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3001816,"kb_used_data":2103312,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":90743336,"statfs":{"total":95995035648,"available":92921176064,"internally_reserved":0,"allocated":2153791488,"data_stored":2153285448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:26 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":5.4379999999999997,"15min":3.1709999999999998},"min":{"1min":0.26900000000000002,"5min":0.22,"15min":0.22},"max":{"1min":37.759,"5min":127.108,"15min":127.108},"last":4.3719999999999999},{"interface":"front","average":{"1min":5.6840000000000002,"5min":5.0659999999999998,"15min":2.8740000000000001},"min":{"1min":0.249,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":37.820999999999998,"5min":128.036,"15min":128.036},"last":0.249}]},{"osd":1,"last update":"Mon Oct 9 04:35:26 2023","interfaces":[{"interface":"back","average":{"1min":3.4300000000000002,"5min":4.7350000000000003,"15min":3.0299999999999998},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":38.015999999999998,"5min":109.95699999999999,"15min":109.95699999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":4.673,"5min":5.569,"15min":3.9129999999999998},"min":{"1min":0.35399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":38.045999999999999,"5min":110.542,"15min":116.04000000000001},"last":4.407}]},{"osd":2,"last update":"Mon Oct 9 04:34:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":7.6879999999999997,"15min":2.9460000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.716000000000001,"5min":324.55000000000001,"15min":324.55000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.2639999999999998,"5min":7.2969999999999997,"15min":2.8170000000000002},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":21.704000000000001,"5min":323.94499999999999,"15min":323.94499999999999},"last":0.85099999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:14 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":8.4100000000000001,"15min":3.5099999999999998},"min":{"1min":0.41399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":18.940000000000001,"5min":259.053,"15min":259.053},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.7429999999999999,"5min":8.1310000000000002,"15min":3.46},"min":{"1min":0.379,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":18.803999999999998,"5min":259.108,"15min":259.108},"last":2.7010000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":0.98499999999999999,"15min":0.99399999999999999},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":5.9340000000000002,"5min":7.2699999999999996,"15min":9.641},"last":361.09399999999999},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.548,"15min":1.1659999999999999},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":4.8639999999999999,"5min":10.645,"15min":10.645},"last":361.71300000000002}]},{"osd":6,"last update":"Mon Oct 9 04:34:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.244,"15min":1.125},"min":{"1min":0.32300000000000001,"5min":0.316,"15min":0.26000000000000001},"max":{"1min":6.6909999999999998,"5min":11.192,"15min":11.192},"last":3.0569999999999999},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.306,"15min":1.1850000000000001},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.8280000000000003,"5min":6.8280000000000003,"15min":14.334},"last":4.2409999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:34:58 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.649,"15min":1.292},"min":{"1min":0.34799999999999998,"5min":0.317,"15min":0.27400000000000002},"max":{"1min":5.1150000000000002,"5min":11.093,"15min":11.093},"last":4.1989999999999998},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.5509999999999999,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.35999999999999999,"15min":0.33000000000000002},"max":{"1min":5.9640000000000004,"5min":9.8550000000000004,"15min":11.315},"last":0.47599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4426104832,"data_stored":4426104832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1316814848,"data_stored":1316814848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1725693952,"data_stored":1725693952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2205220864,"data_stored":2205220864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3483176960,"data_stored":3483172888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2163146752,"data_stored":2163146752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3051425792,"data_stored":3051421720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1741619200,"data_stored":1741619200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:35:37.933 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:35:37.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-10-09T04:35:38.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:38.087 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:38 smithi029 ceph-mon[216903]: pgmap v778: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 802 op/s; 40129/319689 objects degraded (12.553%) 2023-10-09T04:35:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:38 smithi029 ceph-mon[223858]: pgmap v778: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 802 op/s; 40129/319689 objects degraded (12.553%) 2023-10-09T04:35:38.299 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:38.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:38.386 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:38 smithi177 ceph-mon[195797]: pgmap v778: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 802 op/s; 40129/319689 objects degraded (12.553%) 2023-10-09T04:35:38.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:38.471 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:38.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:38.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:38.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:38.684 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:38.684 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:38.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:38.841 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:38.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:38.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:38.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:39.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:39.044 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:35:39.048 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:39.103 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:39.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:39.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:39.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:39.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/702533515' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:39.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:39 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:39.232 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:39.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:39.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:39.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/702533515' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:39 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/702533515' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:39 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:39.424 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:39.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: pgmap v779: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 802 op/s; 40129/319689 objects degraded (12.553%) 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: osdmap e209: 8 total, 7 up, 8 in 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[223858]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.101 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:40.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:40.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:40.262 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:40.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: pgmap v779: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 802 op/s; 40129/319689 objects degraded (12.553%) 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: osdmap e209: 8 total, 7 up, 8 in 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:40 smithi177 ceph-mon[195797]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:40.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: pgmap v779: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 802 op/s; 40129/319689 objects degraded (12.553%) 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: osdmap e209: 8 total, 7 up, 8 in 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:40 smithi029 ceph-mon[216903]: failed to encode map e209 with expected crc 2023-10-09T04:35:40.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:40.546 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:40.616 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:40.662 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:40.698 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:40.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:40.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:40.736 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:40.775 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:40.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:40.810 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:40.846 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:40.864 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:40.884 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:40.922 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:40.947 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:40.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:40.964 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:40.964 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:35:40.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:35:41.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:41.142 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:41.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40129/319689 objects degraded (12.553%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:41.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3613293405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:41.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40129/319689 objects degraded (12.553%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:41.244 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3613293405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:41.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:41.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:41.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:41.426 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40129/319689 objects degraded (12.553%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:41.426 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3613293405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:41.429 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:41.530 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:41.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:41.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:41.671 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:41.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:41.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:41.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:41.876 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:41.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:41.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:42.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:42.142 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:42.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:35:42.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:42.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:42.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:42.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:42.443 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:42.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:42.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:42.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:42 smithi177 ceph-mon[195797]: pgmap v781: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 18 MiB/s wr, 789 op/s; 38918/309936 objects degraded (12.557%) 2023-10-09T04:35:42.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:42 smithi029 ceph-mon[216903]: pgmap v781: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 18 MiB/s wr, 789 op/s; 38918/309936 objects degraded (12.557%) 2023-10-09T04:35:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:42 smithi029 ceph-mon[223858]: pgmap v781: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 18 MiB/s wr, 789 op/s; 38918/309936 objects degraded (12.557%) 2023-10-09T04:35:42.674 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:42.756 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:42.760 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:42.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:42.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:42.952 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:43.001 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:35:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:35:42] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.43.0" 2023-10-09T04:35:43.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:43.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:43.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:43.212 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:43.495 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:43.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:43.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:43.673 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:43.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:43.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:43.823 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:43.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:43.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:43.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:44.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:44.087 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:44.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:44.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:44.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:44.381 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:44.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:44.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:44.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:44.598 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:44.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:44 smithi029 ceph-mon[216903]: pgmap v782: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 790 op/s; 38676/308499 objects degraded (12.537%) 2023-10-09T04:35:44.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:44.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:44.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:44 smithi029 ceph-mon[223858]: pgmap v782: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 790 op/s; 38676/308499 objects degraded (12.537%) 2023-10-09T04:35:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:44.674 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:44.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:35:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:44 smithi177 ceph-mon[195797]: pgmap v782: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 15 MiB/s wr, 790 op/s; 38676/308499 objects degraded (12.537%) 2023-10-09T04:35:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:44.942 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:44.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:44.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:45.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:45.113 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:45.195 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:45.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:45.271 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118323 10.4 e209/197 10:2746c7cb:::benchmark_data_smithi029_344401_object9952:head v 209'7570, mlcod=209'7567)", 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.240820+0000", 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.023868591000000002, 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023901170999999999, 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.240820+0000", 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.240820+0000", 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.240820+0000", 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.240827+0000", 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4510000000000003e-06 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.240839+0000", 2023-10-09T04:35:45.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1419e-05 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.240845+0000", 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8589999999999999e-06 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.244406+0000", 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035608990000000002 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.244420+0000", 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3516000000000001e-05 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118324 10.8 e209/201 10:1660a8dd:::benchmark_data_smithi029_344401_object9953:head v 209'7490, mlcod=209'7485)", 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.245781+0000", 2023-10-09T04:35:45.276 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.018907705, 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018999077, 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245781+0000", 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245781+0000", 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245781+0000", 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245785+0000", 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6210000000000001e-06 2023-10-09T04:35:45.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245790+0000", 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0069999999999997e-06 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245794+0000", 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9419999999999997e-06 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252343+0000", 2023-10-09T04:35:45.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065493110000000004 2023-10-09T04:35:45.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252367+0000", 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4562e-05 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118327 10.c e209/190 10:34c975b7:::benchmark_data_smithi029_344401_object9956:head v 209'7288, mlcod=209'7286)", 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.245797+0000", 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.018891654000000001, 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019041934, 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.282 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245797+0000", 2023-10-09T04:35:45.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245797+0000", 2023-10-09T04:35:45.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245797+0000", 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245803+0000", 2023-10-09T04:35:45.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2230000000000004e-06 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245808+0000", 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5650000000000003e-06 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.245811+0000", 2023-10-09T04:35:45.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5549999999999998e-06 2023-10-09T04:35:45.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.247974+0000", 2023-10-09T04:35:45.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021625479999999998 2023-10-09T04:35:45.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.247994+0000", 2023-10-09T04:35:45.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9528999999999999e-05 2023-10-09T04:35:45.289 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.289 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.289 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118334 10.1 e209/190 10:8440f909:::benchmark_data_smithi029_344401_object9963:head v 209'7339, mlcod=209'7337)", 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.252664+0000", 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.012024856, 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012234649, 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252664+0000", 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252664+0000", 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252663+0000", 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252674+0000", 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0879e-05 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252688+0000", 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4109e-05 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252698+0000", 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7909999999999999e-06 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252751+0000", 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3044000000000002e-05 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.252774+0000", 2023-10-09T04:35:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2761000000000001e-05 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118330 10.b e209/190 10:d22fcd72:::benchmark_data_smithi029_344401_object9959:head v 209'7217, mlcod=209'7214)", 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.255781+0000", 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0089080489999999995, 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091786009999999998, 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.294 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255781+0000", 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255781+0000", 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255780+0000", 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255789+0000", 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1730000000000001e-06 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255809+0000", 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9573999999999999e-05 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255818+0000", 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5130000000000002e-06 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256327+0000", 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00050903299999999999 2023-10-09T04:35:45.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256346+0000", 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8947999999999999e-05 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118336 10.a e209/190 10:5ca737e3:::benchmark_data_smithi029_344401_object9965:head v 209'7455, mlcod=209'7453)", 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.255834+0000", 2023-10-09T04:35:45.298 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0088551740000000004, 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091974759999999996, 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255834+0000", 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255834+0000", 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255833+0000", 2023-10-09T04:35:45.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255838+0000", 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7339999999999999e-06 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255845+0000", 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6259999999999997e-06 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255848+0000", 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6480000000000001e-06 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255886+0000", 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7147999999999997e-05 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255912+0000", 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6506999999999999e-05 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118332 10.b e209/190 10:d2adc2e8:::benchmark_data_smithi029_344401_object9961:head v 209'7218, mlcod=209'7214)", 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.255858+0000", 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00883106, 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092425860000000006, 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.303 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255858+0000", 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255858+0000", 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255858+0000", 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255865+0000", 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5610000000000001e-06 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255871+0000", 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0220000000000001e-06 2023-10-09T04:35:45.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.255877+0000", 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2340000000000002e-06 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.259859+0000", 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039824250000000004 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.259879+0000", 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9592000000000001e-05 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118335 10.b e209/190 10:db8c7be9:::benchmark_data_smithi029_344401_object9964:head v 209'7219, mlcod=209'7214)", 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.256766+0000", 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0079230480000000002, 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084086790000000005, 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.307 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256766+0000", 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256766+0000", 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256765+0000", 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256772+0000", 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.229e-06 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256782+0000", 2023-10-09T04:35:45.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0427e-05 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.256789+0000", 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6109999999999999e-06 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.263448+0000", 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006658967 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.263466+0000", 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8607999999999999e-05 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118339 10.4 e209/197 10:295c284d:::benchmark_data_smithi029_344401_object9968:head v 209'7571, mlcod=209'7567)", 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.260609+0000", 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0040804120000000003, 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046367159999999999, 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.311 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260609+0000", 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260609+0000", 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260608+0000", 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260617+0000", 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2660000000000007e-06 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260635+0000", 2023-10-09T04:35:45.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8084000000000002e-05 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260644+0000", 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1949999999999992e-06 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.265138+0000", 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044934529999999997 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.265158+0000", 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0718999999999999e-05 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118340 10.1 e209/190 10:8b1fee24:::benchmark_data_smithi029_344401_object9969:head v 209'7340, mlcod=209'7337)", 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.260657+0000", 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0040324289999999997, 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049136149999999996, 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.315 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260657+0000", 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260657+0000", 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260656+0000", 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260661+0000", 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0939999999999998e-06 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260668+0000", 2023-10-09T04:35:45.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0149999999999998e-06 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260674+0000", 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5650000000000001e-06 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260733+0000", 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9312999999999999e-05 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.260751+0000", 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7898999999999999e-05 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118338 10.8 e209/201 10:1b485906:::benchmark_data_smithi029_344401_object9967:head v 209'7491, mlcod=209'7488)", 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.261205+0000", 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.003484132, 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004435858, 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.319 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261205+0000", 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261205+0000", 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261204+0000", 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261212+0000", 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6159999999999996e-06 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261223+0000", 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1015e-05 2023-10-09T04:35:45.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261228+0000", 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4389999999999999e-06 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261281+0000", 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2516000000000003e-05 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.261296+0000", 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5532999999999998e-05 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118341 10.4 e209/197)", 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.264698+0000", 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4294967295.9999905, 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010069059999999999, 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.323 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.264698+0000", 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.264698+0000", 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.264697+0000", 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.264707+0000", 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5219999999999996e-06 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.264720+0000", 2023-10-09T04:35:45.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3094e-05 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.264729+0000", 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4090000000000003e-06 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:118319 10.4 e209/197 10:22aa1b38:::benchmark_data_smithi029_344401_object9948:head v 209'7569, mlcod=209'7567)", 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:45.232680+0000", 2023-10-09T04:35:45.326 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.032008883000000002, 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033082526000000001, 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232680+0000", 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:35:45.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232680+0000", 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232679+0000", 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232688+0000", 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232704+0000", 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6385e-05 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232715+0000", 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0747000000000001e-05 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232774+0000", 2023-10-09T04:35:45.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9697000000000001e-05 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:45.232796+0000", 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1291e-05 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 13 2023-10-09T04:35:45.330 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:35:45.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:45.378 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:45.511 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2378161227' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:35:45.511 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:45 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:35:45.512 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38676/308499 objects degraded (12.537%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:45.528 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 10 pg_num to 26 2023-10-09T04:35:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2378161227' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:35:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:45 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:35:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38676/308499 objects degraded (12.537%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2378161227' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:35:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:45 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:35:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38676/308499 objects degraded (12.537%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:46.611 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:46.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:46.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:46.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:46 smithi177 ceph-mon[195797]: pgmap v783: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 5.5 MiB/s wr, 784 op/s; 38147/301341 objects degraded (12.659%) 2023-10-09T04:35:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:46 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:35:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:46 smithi177 ceph-mon[195797]: osdmap e210: 8 total, 7 up, 8 in 2023-10-09T04:35:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:46 smithi177 ceph-mon[195797]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:46 smithi177 ceph-mon[195797]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:46 smithi177 ceph-mon[195797]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[216903]: pgmap v783: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 5.5 MiB/s wr, 784 op/s; 38147/301341 objects degraded (12.659%) 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[216903]: osdmap e210: 8 total, 7 up, 8 in 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[216903]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[216903]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[216903]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[223858]: pgmap v783: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 5.5 MiB/s wr, 784 op/s; 38147/301341 objects degraded (12.659%) 2023-10-09T04:35:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:35:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[223858]: osdmap e210: 8 total, 7 up, 8 in 2023-10-09T04:35:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[223858]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[223858]: failed to encode map e210 with expected crc 2023-10-09T04:35:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:46 smithi029 ceph-mon[223858]: failed to encode map e210 with expected crc 2023-10-09T04:35:47.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:35:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:35:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:35:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:35:48.056 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:48.085 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:48.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:48.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:48.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:48.509 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[216903]: pgmap v785: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 37033/295713 objects degraded (12.523%) 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[216903]: failed to encode map e211 with expected crc 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[216903]: osdmap e211: 8 total, 7 up, 8 in 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[216903]: failed to encode map e211 with expected crc 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[223858]: pgmap v785: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 37033/295713 objects degraded (12.523%) 2023-10-09T04:35:48.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[223858]: failed to encode map e211 with expected crc 2023-10-09T04:35:48.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:35:48.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[223858]: osdmap e211: 8 total, 7 up, 8 in 2023-10-09T04:35:48.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:48 smithi029 ceph-mon[223858]: failed to encode map e211 with expected crc 2023-10-09T04:35:48.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:48 smithi177 ceph-mon[195797]: pgmap v785: 89 pgs: 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 998 op/s; 37033/295713 objects degraded (12.523%) 2023-10-09T04:35:48.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:48 smithi177 ceph-mon[195797]: failed to encode map e211 with expected crc 2023-10-09T04:35:48.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:35:48.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:48 smithi177 ceph-mon[195797]: osdmap e211: 8 total, 7 up, 8 in 2023-10-09T04:35:48.814 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:48 smithi177 ceph-mon[195797]: failed to encode map e211 with expected crc 2023-10-09T04:35:48.830 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:49.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:49.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:49.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:49.133 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:49.174 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:49.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:49.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:49.308 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:49.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:49.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:49.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:49.526 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:49.555 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:35:49.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:49.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:49.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:49.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: osdmap e212: 8 total, 7 up, 8 in 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/635714607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:49.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:49 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:49.814 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:49.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: osdmap e212: 8 total, 7 up, 8 in 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/635714607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:49.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: osdmap e212: 8 total, 7 up, 8 in 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: failed to encode map e212 with expected crc 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/635714607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:49.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:49 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:35:49.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:49.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:50.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:50.095 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:50.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:50.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:50.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:50.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:50.400 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: pgmap v788: 99 pgs: 10 unknown, 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 37033/295713 objects degraded (12.523%) 2023-10-09T04:35:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: 10.6 scrub starts 2023-10-09T04:35:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:50.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: osdmap e213: 8 total, 7 up, 8 in 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37033/295713 objects degraded (12.523%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: pgmap v788: 99 pgs: 10 unknown, 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 37033/295713 objects degraded (12.523%) 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: 10.6 scrub starts 2023-10-09T04:35:50.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: osdmap e213: 8 total, 7 up, 8 in 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: failed to encode map e213 with expected crc 2023-10-09T04:35:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37033/295713 objects degraded (12.523%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:50.881 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:50.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:50.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:51.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: pgmap v788: 99 pgs: 10 unknown, 8 active+undersized+degraded, 17 active+undersized, 64 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 37033/295713 objects degraded (12.523%) 2023-10-09T04:35:51.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: 10.6 scrub starts 2023-10-09T04:35:51.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: failed to encode map e213 with expected crc 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: osdmap e213: 8 total, 7 up, 8 in 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: failed to encode map e213 with expected crc 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: failed to encode map e213 with expected crc 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: failed to encode map e213 with expected crc 2023-10-09T04:35:51.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37033/295713 objects degraded (12.523%), 8 pgs degraded (PG_DEGRADED) 2023-10-09T04:35:51.040 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:51.121 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:51.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:51.225 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:51.254 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:51.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:51.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:51.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:51.477 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:51.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:51.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:51.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:51.731 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:52.037 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:52.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:52.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:52.142 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.209 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:52.223 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:35:52.246 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:35:52.275 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:35:52.310 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:35:52.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:52.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:52.358 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:35:52.391 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:35:52.426 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:35:52.429 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.429 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.430 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:35:52.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:35:52.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:52.487 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:52.611 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:35:52.612 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:35:52.699 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:35:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:52.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:35:52.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:52.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:52 smithi029 ceph-mon[216903]: pgmap v790: 99 pgs: 2 unknown, 11 active+undersized+degraded, 17 active+undersized, 69 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 36015/287082 objects degraded (12.545%) 2023-10-09T04:35:52.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1670654235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:52.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:52 smithi029 ceph-mon[223858]: pgmap v790: 99 pgs: 2 unknown, 11 active+undersized+degraded, 17 active+undersized, 69 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 36015/287082 objects degraded (12.545%) 2023-10-09T04:35:52.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1670654235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:52.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:35:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:52 smithi177 ceph-mon[195797]: pgmap v790: 99 pgs: 2 unknown, 11 active+undersized+degraded, 17 active+undersized, 69 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 36015/287082 objects degraded (12.545%) 2023-10-09T04:35:52.862 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1670654235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:35:52.866 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:52.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:52.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:53.029 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T04:35:53.030 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:35:53.030 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-10-09T04:35:53.030 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:35:53.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:53.097 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:53.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:35:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:35:52] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-10-09T04:35:53.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:53.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:53.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:53.317 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:54.485 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:35:54.486 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:35:54.634 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:54.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:54.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:35:54.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:54 smithi029 ceph-mon[223858]: pgmap v791: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 35779/285774 objects degraded (12.520%) 2023-10-09T04:35:54.710 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:54 smithi029 ceph-mon[216903]: pgmap v791: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 35779/285774 objects degraded (12.520%) 2023-10-09T04:35:54.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:35:54.832 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:54.853 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:54.957 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:35:54.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:54.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:35:54.970 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:54 smithi177 ceph-mon[195797]: pgmap v791: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 35779/285774 objects degraded (12.520%) 2023-10-09T04:35:55.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:55.142 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:55.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:55.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:55.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:55.433 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:55.529 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:55.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:55.637 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:35:55.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35779/285774 objects degraded (12.520%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:55.707 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:55.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:35:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:55.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:35:55.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:35:55.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35779/285774 objects degraded (12.520%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35779/285774 objects degraded (12.520%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:35:55.935 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:56.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:35:56.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:56 smithi029 ceph-mon[216903]: pgmap v792: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 35076/276465 objects degraded (12.687%) 2023-10-09T04:35:56.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:56 smithi029 ceph-mon[223858]: pgmap v792: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 35076/276465 objects degraded (12.687%) 2023-10-09T04:35:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:56 smithi177 ceph-mon[195797]: pgmap v792: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 35076/276465 objects degraded (12.687%) 2023-10-09T04:35:57.242 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:35:57.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:57.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:35:57.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:57.396 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:57.494 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:57.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:57.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:57.544 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:57.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:35:57.655 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:57.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:35:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:57.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:35:57.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:35:57.927 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:58.033 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:35:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:58.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:35:58.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:35:58.183 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:35:58.224 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:58.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:35:58.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:35:58.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:35:58.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:35:58.503 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[216903]: pgmap v793: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 33657/269088 objects degraded (12.508%) 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2633145058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[223858]: pgmap v793: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 33657/269088 objects degraded (12.508%) 2023-10-09T04:35:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2633145058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:58 smithi177 ceph-mon[195797]: pgmap v793: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 33657/269088 objects degraded (12.508%) 2023-10-09T04:35:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:35:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:35:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:35:59.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2633145058' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:59.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:35:59.327 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:35:59.329 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:35:59.520 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:35:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:59 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:59 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:35:59 smithi029 ceph-mon[223858]: osdmap e214: 8 total, 7 up, 8 in 2023-10-09T04:35:59.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:59 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:35:59.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:59 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:35:59.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:35:59 smithi029 ceph-mon[216903]: osdmap e214: 8 total, 7 up, 8 in 2023-10-09T04:36:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:59 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:59 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:35:59 smithi177 ceph-mon[195797]: osdmap e214: 8 total, 7 up, 8 in 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: pgmap v794: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 33657/269088 objects degraded (12.508%) 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 33657/269088 objects degraded (12.508%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: pgmap v794: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 33657/269088 objects degraded (12.508%) 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: failed to encode map e214 with expected crc 2023-10-09T04:36:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 33657/269088 objects degraded (12.508%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:01.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: pgmap v794: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 33657/269088 objects degraded (12.508%) 2023-10-09T04:36:01.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:36:01.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: failed to encode map e214 with expected crc 2023-10-09T04:36:01.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: failed to encode map e214 with expected crc 2023-10-09T04:36:01.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: failed to encode map e214 with expected crc 2023-10-09T04:36:01.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: failed to encode map e214 with expected crc 2023-10-09T04:36:01.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: failed to encode map e214 with expected crc 2023-10-09T04:36:01.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 33657/269088 objects degraded (12.508%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:01.162 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":794,"stamp":"2023-10-09T04:35:58.966251+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5877274552,"num_objects":89696,"num_object_clones":0,"num_object_copies":269088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89696,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":127366,"num_write_kb":6937998,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":3083510,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200495,"ondisk_log_size":200495,"up":268,"acting":268,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":293,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21490864,"kb_used_data":16668848,"kb_used_omap":20,"kb_used_meta":4821355,"kb_avail":728470352,"statfs":{"total":767960285184,"available":745953640448,"internally_reserved":0,"allocated":17068900352,"data_stored":17063726821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":4937068174},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-581632000,"num_objects":-8875,"num_object_clones":0,"num_object_copies":-26625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8875,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003462"},"pg_stats":[{"pgid":"10.13","version":"213'7719","reported_seq":7763,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:54.900803+0000","last_change":"2023-10-09T04:35:48.580383+0000","last_active":"2023-10-09T04:35:54.900803+0000","last_peered":"2023-10-09T04:35:54.900803+0000","last_clean":"2023-10-09T04:34:50.301002+0000","last_became_active":"2023-10-09T04:35:48.580383+0000","last_became_peered":"2023-10-09T04:35:48.580383+0000","last_unstale":"2023-10-09T04:35:54.900803+0000","last_undegraded":"2023-10-09T04:35:48.573617+0000","last_fullsized":"2023-10-09T04:35:48.549238+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7719,"log_dups_size":0,"ondisk_log_size":7719,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184025088,"num_objects":2808,"num_object_clones":0,"num_object_copies":8424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2808,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3959,"num_write_kb":216544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2808}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":199,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.543805+0000","last_change":"2023-10-09T04:34:51.322569+0000","last_active":"2023-10-09T04:35:48.543805+0000","last_peered":"2023-10-09T04:35:48.543805+0000","last_clean":"2023-10-09T04:34:44.061742+0000","last_became_active":"2023-10-09T04:34:51.322569+0000","last_became_peered":"2023-10-09T04:34:51.322569+0000","last_unstale":"2023-10-09T04:35:48.543805+0000","last_undegraded":"2023-10-09T04:35:48.543805+0000","last_fullsized":"2023-10-09T04:34:51.319415+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:46.193676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542782+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:35:48.542782+0000","last_peered":"2023-10-09T04:35:48.542782+0000","last_clean":"2023-10-09T04:35:48.542782+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:35:48.542782+0000","last_undegraded":"2023-10-09T04:35:48.542782+0000","last_fullsized":"2023-10-09T04:35:48.542782+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"213'7731","reported_seq":7766,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.740706+0000","last_change":"2023-10-09T04:35:48.609418+0000","last_active":"2023-10-09T04:35:54.740706+0000","last_peered":"2023-10-09T04:35:54.740706+0000","last_clean":"2023-10-09T04:35:54.740706+0000","last_became_active":"2023-10-09T04:35:48.600846+0000","last_became_peered":"2023-10-09T04:35:48.600846+0000","last_unstale":"2023-10-09T04:35:54.740706+0000","last_undegraded":"2023-10-09T04:35:54.740706+0000","last_fullsized":"2023-10-09T04:35:54.740706+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186548224,"num_objects":2847,"num_object_clones":0,"num_object_copies":8541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3964,"num_write_kb":217920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.537181+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:35:48.537181+0000","last_peered":"2023-10-09T04:35:48.537181+0000","last_clean":"2023-10-09T04:35:48.537181+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:35:48.537181+0000","last_undegraded":"2023-10-09T04:35:48.537181+0000","last_fullsized":"2023-10-09T04:35:48.537181+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":126,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.547990+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:35:48.547990+0000","last_peered":"2023-10-09T04:35:48.547990+0000","last_clean":"2023-10-09T04:35:48.547990+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:35:48.547990+0000","last_undegraded":"2023-10-09T04:35:48.547990+0000","last_fullsized":"2023-10-09T04:35:48.547990+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.11","version":"213'7653","reported_seq":7694,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.855403+0000","last_change":"2023-10-09T04:35:48.598481+0000","last_active":"2023-10-09T04:35:54.855403+0000","last_peered":"2023-10-09T04:35:54.855403+0000","last_clean":"2023-10-09T04:35:54.855403+0000","last_became_active":"2023-10-09T04:35:48.591513+0000","last_became_peered":"2023-10-09T04:35:48.591513+0000","last_unstale":"2023-10-09T04:35:54.855403+0000","last_undegraded":"2023-10-09T04:35:54.855403+0000","last_fullsized":"2023-10-09T04:35:54.855403+0000","mapping_epoch":211,"log_start":"195'100","ondisk_log_start":"195'100","created":211,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181501952,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3928,"num_write_kb":214304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":199,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.541813+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:35:48.541813+0000","last_peered":"2023-10-09T04:35:48.541813+0000","last_clean":"2023-10-09T04:35:48.541813+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:35:48.541813+0000","last_undegraded":"2023-10-09T04:35:48.541813+0000","last_fullsized":"2023-10-09T04:35:48.541813+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":170,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542651+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:35:48.542651+0000","last_peered":"2023-10-09T04:35:48.542651+0000","last_clean":"2023-10-09T04:35:48.542651+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:35:48.542651+0000","last_undegraded":"2023-10-09T04:35:48.542651+0000","last_fullsized":"2023-10-09T04:35:48.542651+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":125,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.550899+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:35:48.550899+0000","last_peered":"2023-10-09T04:35:48.550899+0000","last_clean":"2023-10-09T04:35:48.550899+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:35:48.550899+0000","last_undegraded":"2023-10-09T04:35:48.550899+0000","last_fullsized":"2023-10-09T04:35:48.550899+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.541894+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:35:48.541894+0000","last_peered":"2023-10-09T04:35:48.541894+0000","last_clean":"2023-10-09T04:35:48.541894+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:35:48.541894+0000","last_undegraded":"2023-10-09T04:35:48.541894+0000","last_fullsized":"2023-10-09T04:35:48.541894+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.551268+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:35:48.551268+0000","last_peered":"2023-10-09T04:35:48.551268+0000","last_clean":"2023-10-09T04:35:48.551268+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:35:48.551268+0000","last_undegraded":"2023-10-09T04:35:48.551268+0000","last_fullsized":"2023-10-09T04:35:48.551268+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":136,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.541895+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:35:48.541895+0000","last_peered":"2023-10-09T04:35:48.541895+0000","last_clean":"2023-10-09T04:35:48.541895+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:35:48.541895+0000","last_undegraded":"2023-10-09T04:35:48.541895+0000","last_fullsized":"2023-10-09T04:35:48.541895+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.547744+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:35:48.547744+0000","last_peered":"2023-10-09T04:35:48.547744+0000","last_clean":"2023-10-09T04:35:48.547744+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:35:48.547744+0000","last_undegraded":"2023-10-09T04:35:48.547744+0000","last_fullsized":"2023-10-09T04:35:48.547744+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":184,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.538150+0000","last_change":"2023-10-09T04:34:51.322394+0000","last_active":"2023-10-09T04:35:48.538150+0000","last_peered":"2023-10-09T04:35:48.538150+0000","last_clean":"2023-10-09T04:34:44.053596+0000","last_became_active":"2023-10-09T04:34:51.322394+0000","last_became_peered":"2023-10-09T04:34:51.322394+0000","last_unstale":"2023-10-09T04:35:48.538150+0000","last_undegraded":"2023-10-09T04:35:48.538150+0000","last_fullsized":"2023-10-09T04:34:51.313456+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:25:55.868058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":179,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.546563+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:35:48.546563+0000","last_peered":"2023-10-09T04:35:48.546563+0000","last_clean":"2023-10-09T04:35:48.546563+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:35:48.546563+0000","last_undegraded":"2023-10-09T04:35:48.546563+0000","last_fullsized":"2023-10-09T04:35:48.546563+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542764+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:35:48.542764+0000","last_peered":"2023-10-09T04:35:48.542764+0000","last_clean":"2023-10-09T04:35:48.542764+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:35:48.542764+0000","last_undegraded":"2023-10-09T04:35:48.542764+0000","last_fullsized":"2023-10-09T04:35:48.542764+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":181,"reported_epoch":212,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:48.546179+0000","last_change":"2023-10-09T04:34:51.334626+0000","last_active":"2023-10-09T04:35:48.546179+0000","last_peered":"2023-10-09T04:35:48.546179+0000","last_clean":"2023-10-09T04:34:44.064651+0000","last_became_active":"2023-10-09T04:34:51.334626+0000","last_became_peered":"2023-10-09T04:34:51.334626+0000","last_unstale":"2023-10-09T04:35:48.546179+0000","last_undegraded":"2023-10-09T04:34:51.326465+0000","last_fullsized":"2023-10-09T04:34:51.326345+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:10.974139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542307+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:35:48.542307+0000","last_peered":"2023-10-09T04:35:48.542307+0000","last_clean":"2023-10-09T04:35:48.542307+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:35:48.542307+0000","last_undegraded":"2023-10-09T04:35:48.542307+0000","last_fullsized":"2023-10-09T04:35:48.542307+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":125,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.546344+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:35:48.546344+0000","last_peered":"2023-10-09T04:35:48.546344+0000","last_clean":"2023-10-09T04:35:48.546344+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:35:48.546344+0000","last_undegraded":"2023-10-09T04:35:48.546344+0000","last_fullsized":"2023-10-09T04:35:48.546344+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":172,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.550649+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:35:48.550649+0000","last_peered":"2023-10-09T04:35:48.550649+0000","last_clean":"2023-10-09T04:35:48.550649+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:35:48.550649+0000","last_undegraded":"2023-10-09T04:35:48.550649+0000","last_fullsized":"2023-10-09T04:35:48.550649+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"213'7821","reported_seq":7875,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:54.972245+0000","last_change":"2023-10-09T04:35:48.569809+0000","last_active":"2023-10-09T04:35:54.972245+0000","last_peered":"2023-10-09T04:35:54.972245+0000","last_clean":"2023-10-09T04:34:50.268727+0000","last_became_active":"2023-10-09T04:35:48.569809+0000","last_became_peered":"2023-10-09T04:35:48.569809+0000","last_unstale":"2023-10-09T04:35:54.972245+0000","last_undegraded":"2023-10-09T04:35:48.555406+0000","last_fullsized":"2023-10-09T04:35:48.544651+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7821,"log_dups_size":0,"ondisk_log_size":7821,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186351616,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2844,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4019,"num_write_kb":219584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2844}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":155,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.545840+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:35:48.545840+0000","last_peered":"2023-10-09T04:35:48.545840+0000","last_clean":"2023-10-09T04:35:48.545840+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:35:48.545840+0000","last_undegraded":"2023-10-09T04:35:48.545840+0000","last_fullsized":"2023-10-09T04:35:48.545840+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":124,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542965+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:35:48.542965+0000","last_peered":"2023-10-09T04:35:48.542965+0000","last_clean":"2023-10-09T04:35:48.542965+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:35:48.542965+0000","last_undegraded":"2023-10-09T04:35:48.542965+0000","last_fullsized":"2023-10-09T04:35:48.542965+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.19","version":"213'7859","reported_seq":7903,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:54.955338+0000","last_change":"2023-10-09T04:35:48.577454+0000","last_active":"2023-10-09T04:35:54.955338+0000","last_peered":"2023-10-09T04:35:54.955338+0000","last_clean":"2023-10-09T04:34:50.296861+0000","last_became_active":"2023-10-09T04:35:48.577454+0000","last_became_peered":"2023-10-09T04:35:48.577454+0000","last_unstale":"2023-10-09T04:35:54.955338+0000","last_undegraded":"2023-10-09T04:35:48.562766+0000","last_fullsized":"2023-10-09T04:35:48.548824+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7859,"log_dups_size":0,"ondisk_log_size":7859,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185958400,"num_objects":2838,"num_object_clones":0,"num_object_copies":8514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2838,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4035,"num_write_kb":219904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":2838}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":219,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.543804+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:35:48.543804+0000","last_peered":"2023-10-09T04:35:48.543804+0000","last_clean":"2023-10-09T04:35:48.543804+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:35:48.543804+0000","last_undegraded":"2023-10-09T04:35:48.543804+0000","last_fullsized":"2023-10-09T04:35:48.543804+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.543421+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:35:48.543421+0000","last_peered":"2023-10-09T04:35:48.543421+0000","last_clean":"2023-10-09T04:35:48.543421+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:35:48.543421+0000","last_undegraded":"2023-10-09T04:35:48.543421+0000","last_fullsized":"2023-10-09T04:35:48.543421+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"213'7515","reported_seq":7571,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:55.037903+0000","last_change":"2023-10-09T04:35:49.560942+0000","last_active":"2023-10-09T04:35:55.037903+0000","last_peered":"2023-10-09T04:35:55.037903+0000","last_clean":"2023-10-09T04:35:55.037903+0000","last_became_active":"2023-10-09T04:35:48.580345+0000","last_became_peered":"2023-10-09T04:35:48.580345+0000","last_unstale":"2023-10-09T04:35:55.037903+0000","last_undegraded":"2023-10-09T04:35:55.037903+0000","last_fullsized":"2023-10-09T04:35:55.037903+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":199,"log_size":7515,"log_dups_size":0,"ondisk_log_size":7515,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179437568,"num_objects":2738,"num_object_clones":0,"num_object_copies":8214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3861,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":201,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.548279+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:35:48.548279+0000","last_peered":"2023-10-09T04:35:48.548279+0000","last_clean":"2023-10-09T04:35:48.548279+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:35:48.548279+0000","last_undegraded":"2023-10-09T04:35:48.548279+0000","last_fullsized":"2023-10-09T04:35:48.548279+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.549797+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:35:48.549797+0000","last_peered":"2023-10-09T04:35:48.549797+0000","last_clean":"2023-10-09T04:35:48.549797+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:35:48.549797+0000","last_undegraded":"2023-10-09T04:35:48.549797+0000","last_fullsized":"2023-10-09T04:35:48.549797+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"213'7619","reported_seq":7660,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.760171+0000","last_change":"2023-10-09T04:35:48.589209+0000","last_active":"2023-10-09T04:35:54.760171+0000","last_peered":"2023-10-09T04:35:54.760171+0000","last_clean":"2023-10-09T04:35:54.760171+0000","last_became_active":"2023-10-09T04:35:48.571347+0000","last_became_peered":"2023-10-09T04:35:48.571347+0000","last_unstale":"2023-10-09T04:35:54.760171+0000","last_undegraded":"2023-10-09T04:35:54.760171+0000","last_fullsized":"2023-10-09T04:35:54.760171+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7619,"log_dups_size":0,"ondisk_log_size":7619,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183140352,"num_objects":2794,"num_object_clones":0,"num_object_copies":8382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3909,"num_write_kb":214528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":174,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.546081+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:35:48.546081+0000","last_peered":"2023-10-09T04:35:48.546081+0000","last_clean":"2023-10-09T04:35:48.546081+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:35:48.546081+0000","last_undegraded":"2023-10-09T04:35:48.546081+0000","last_fullsized":"2023-10-09T04:35:48.546081+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.544198+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:35:48.544198+0000","last_peered":"2023-10-09T04:35:48.544198+0000","last_clean":"2023-10-09T04:35:48.544198+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:35:48.544198+0000","last_undegraded":"2023-10-09T04:35:48.544198+0000","last_fullsized":"2023-10-09T04:35:48.544198+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"213'7874","reported_seq":7929,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.826947+0000","last_change":"2023-10-09T04:35:48.588540+0000","last_active":"2023-10-09T04:35:54.826947+0000","last_peered":"2023-10-09T04:35:54.826947+0000","last_clean":"2023-10-09T04:35:54.826947+0000","last_became_active":"2023-10-09T04:35:48.583314+0000","last_became_peered":"2023-10-09T04:35:48.583314+0000","last_unstale":"2023-10-09T04:35:54.826947+0000","last_undegraded":"2023-10-09T04:35:54.826947+0000","last_fullsized":"2023-10-09T04:35:54.826947+0000","mapping_epoch":197,"log_start":"195'300","ondisk_log_start":"195'300","created":190,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189300736,"num_objects":2888,"num_object_clones":0,"num_object_copies":8664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4034,"num_write_kb":221536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":156,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.547507+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:35:48.547507+0000","last_peered":"2023-10-09T04:35:48.547507+0000","last_clean":"2023-10-09T04:35:48.547507+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:35:48.547507+0000","last_undegraded":"2023-10-09T04:35:48.547507+0000","last_fullsized":"2023-10-09T04:35:48.547507+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.542039+0000","last_change":"2023-10-09T04:34:51.328996+0000","last_active":"2023-10-09T04:35:48.542039+0000","last_peered":"2023-10-09T04:35:48.542039+0000","last_clean":"2023-10-09T04:34:44.068074+0000","last_became_active":"2023-10-09T04:34:51.328996+0000","last_became_peered":"2023-10-09T04:34:51.328996+0000","last_unstale":"2023-10-09T04:35:48.542039+0000","last_undegraded":"2023-10-09T04:35:48.542039+0000","last_fullsized":"2023-10-09T04:34:51.321084+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:38:56.215110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"213'7801","reported_seq":7842,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:56.634818+0000","last_change":"2023-10-09T04:35:48.566701+0000","last_active":"2023-10-09T04:35:56.634818+0000","last_peered":"2023-10-09T04:35:56.634818+0000","last_clean":"2023-10-09T04:34:50.268593+0000","last_became_active":"2023-10-09T04:35:48.566701+0000","last_became_peered":"2023-10-09T04:35:48.566701+0000","last_unstale":"2023-10-09T04:35:56.634818+0000","last_undegraded":"2023-10-09T04:35:48.541777+0000","last_fullsized":"2023-10-09T04:35:48.541651+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7801,"log_dups_size":0,"ondisk_log_size":7801,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178978816,"num_objects":2731,"num_object_clones":0,"num_object_copies":8193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2731,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4044,"num_write_kb":216800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":2731}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.549884+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:35:48.549884+0000","last_peered":"2023-10-09T04:35:48.549884+0000","last_clean":"2023-10-09T04:35:48.549884+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:35:48.549884+0000","last_undegraded":"2023-10-09T04:35:48.549884+0000","last_fullsized":"2023-10-09T04:35:48.549884+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":70,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.550136+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:35:48.550136+0000","last_peered":"2023-10-09T04:35:48.550136+0000","last_clean":"2023-10-09T04:35:48.550136+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:35:48.550136+0000","last_undegraded":"2023-10-09T04:35:48.550136+0000","last_fullsized":"2023-10-09T04:35:48.550136+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.2","version":"213'7832","reported_seq":7874,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:56.657278+0000","last_change":"2023-10-09T04:35:48.572568+0000","last_active":"2023-10-09T04:35:56.657278+0000","last_peered":"2023-10-09T04:35:56.657278+0000","last_clean":"2023-10-09T04:35:56.657278+0000","last_became_active":"2023-10-09T04:35:48.565449+0000","last_became_peered":"2023-10-09T04:35:48.565449+0000","last_unstale":"2023-10-09T04:35:56.657278+0000","last_undegraded":"2023-10-09T04:35:56.657278+0000","last_fullsized":"2023-10-09T04:35:56.657278+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7832,"log_dups_size":0,"ondisk_log_size":7832,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183468032,"num_objects":2799,"num_object_clones":0,"num_object_copies":8397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4048,"num_write_kb":219136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":207,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542624+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:35:48.542624+0000","last_peered":"2023-10-09T04:35:48.542624+0000","last_clean":"2023-10-09T04:35:48.542624+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:35:48.542624+0000","last_undegraded":"2023-10-09T04:35:48.542624+0000","last_fullsized":"2023-10-09T04:35:48.542624+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.546533+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:35:48.546533+0000","last_peered":"2023-10-09T04:35:48.546533+0000","last_clean":"2023-10-09T04:35:48.546533+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:35:48.546533+0000","last_undegraded":"2023-10-09T04:35:48.546533+0000","last_fullsized":"2023-10-09T04:35:48.546533+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"213'7718","reported_seq":7763,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:55.016681+0000","last_change":"2023-10-09T04:35:48.555365+0000","last_active":"2023-10-09T04:35:55.016681+0000","last_peered":"2023-10-09T04:35:55.016681+0000","last_clean":"2023-10-09T04:34:50.301002+0000","last_became_active":"2023-10-09T04:35:48.555365+0000","last_became_peered":"2023-10-09T04:35:48.555365+0000","last_unstale":"2023-10-09T04:35:55.016681+0000","last_undegraded":"2023-10-09T04:35:48.548651+0000","last_fullsized":"2023-10-09T04:35:48.548498+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184090624,"num_objects":2809,"num_object_clones":0,"num_object_copies":8427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3958,"num_write_kb":216544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2809}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.541937+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:35:48.541937+0000","last_peered":"2023-10-09T04:35:48.541937+0000","last_clean":"2023-10-09T04:35:48.541937+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:35:48.541937+0000","last_undegraded":"2023-10-09T04:35:48.541937+0000","last_fullsized":"2023-10-09T04:35:48.541937+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":197,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.542074+0000","last_change":"2023-10-09T04:34:51.322935+0000","last_active":"2023-10-09T04:35:48.542074+0000","last_peered":"2023-10-09T04:35:48.542074+0000","last_clean":"2023-10-09T04:34:44.060875+0000","last_became_active":"2023-10-09T04:34:51.322935+0000","last_became_peered":"2023-10-09T04:34:51.322935+0000","last_unstale":"2023-10-09T04:35:48.542074+0000","last_undegraded":"2023-10-09T04:35:48.542074+0000","last_fullsized":"2023-10-09T04:34:51.319826+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:49.238888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"213'7712","reported_seq":7748,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.746523+0000","last_change":"2023-10-09T04:35:48.588082+0000","last_active":"2023-10-09T04:35:54.746523+0000","last_peered":"2023-10-09T04:35:54.746523+0000","last_clean":"2023-10-09T04:35:54.746523+0000","last_became_active":"2023-10-09T04:35:48.577627+0000","last_became_peered":"2023-10-09T04:35:48.577627+0000","last_unstale":"2023-10-09T04:35:54.746523+0000","last_undegraded":"2023-10-09T04:35:54.746523+0000","last_fullsized":"2023-10-09T04:35:54.746523+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7712,"log_dups_size":0,"ondisk_log_size":7712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187793408,"num_objects":2865,"num_object_clones":0,"num_object_copies":8595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3944,"num_write_kb":217920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":181,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.544042+0000","last_change":"2023-10-09T04:34:51.333617+0000","last_active":"2023-10-09T04:35:48.544042+0000","last_peered":"2023-10-09T04:35:48.544042+0000","last_clean":"2023-10-09T04:34:44.060999+0000","last_became_active":"2023-10-09T04:34:51.333617+0000","last_became_peered":"2023-10-09T04:34:51.333617+0000","last_unstale":"2023-10-09T04:35:48.544042+0000","last_undegraded":"2023-10-09T04:35:48.544042+0000","last_fullsized":"2023-10-09T04:34:51.322409+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:09.264091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.543157+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:35:48.543157+0000","last_peered":"2023-10-09T04:35:48.543157+0000","last_clean":"2023-10-09T04:35:48.543157+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:35:48.543157+0000","last_undegraded":"2023-10-09T04:35:48.543157+0000","last_fullsized":"2023-10-09T04:35:48.543157+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"213'7656","reported_seq":7698,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.844096+0000","last_change":"2023-10-09T04:35:48.556467+0000","last_active":"2023-10-09T04:35:54.844096+0000","last_peered":"2023-10-09T04:35:54.844096+0000","last_clean":"2023-10-09T04:35:54.844096+0000","last_became_active":"2023-10-09T04:35:48.548442+0000","last_became_peered":"2023-10-09T04:35:48.548442+0000","last_unstale":"2023-10-09T04:35:54.844096+0000","last_undegraded":"2023-10-09T04:35:54.844096+0000","last_fullsized":"2023-10-09T04:35:54.844096+0000","mapping_epoch":190,"log_start":"195'100","ondisk_log_start":"195'100","created":190,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181305344,"num_objects":2766,"num_object_clones":0,"num_object_copies":8298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3930,"num_write_kb":214304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":164,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.541713+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:35:48.541713+0000","last_peered":"2023-10-09T04:35:48.541713+0000","last_clean":"2023-10-09T04:35:48.541713+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:35:48.541713+0000","last_undegraded":"2023-10-09T04:35:48.541713+0000","last_fullsized":"2023-10-09T04:35:48.541713+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":178,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.538326+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:35:48.538326+0000","last_peered":"2023-10-09T04:35:48.538326+0000","last_clean":"2023-10-09T04:35:48.538326+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:35:48.538326+0000","last_undegraded":"2023-10-09T04:35:48.538326+0000","last_fullsized":"2023-10-09T04:35:48.538326+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"213'7838","reported_seq":7869,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.983437+0000","last_change":"2023-10-09T04:33:51.281811+0000","last_active":"2023-10-09T04:35:54.983437+0000","last_peered":"2023-10-09T04:35:54.983437+0000","last_clean":"2023-10-09T04:35:54.983437+0000","last_became_active":"2023-10-09T04:33:51.281495+0000","last_became_peered":"2023-10-09T04:33:51.281495+0000","last_unstale":"2023-10-09T04:35:54.983437+0000","last_undegraded":"2023-10-09T04:35:54.983437+0000","last_fullsized":"2023-10-09T04:35:54.983437+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7838,"log_dups_size":0,"ondisk_log_size":7838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:29:53.315124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365428736,"num_objects":5576,"num_object_clones":0,"num_object_copies":16728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7838,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":187,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.537762+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:35:48.537762+0000","last_peered":"2023-10-09T04:35:48.537762+0000","last_clean":"2023-10-09T04:35:48.537762+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:35:48.537762+0000","last_undegraded":"2023-10-09T04:35:48.537762+0000","last_fullsized":"2023-10-09T04:35:48.537762+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":197,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.550100+0000","last_change":"2023-10-09T04:34:51.340476+0000","last_active":"2023-10-09T04:35:48.550100+0000","last_peered":"2023-10-09T04:35:48.550100+0000","last_clean":"2023-10-09T04:34:44.071691+0000","last_became_active":"2023-10-09T04:34:51.340476+0000","last_became_peered":"2023-10-09T04:34:51.340476+0000","last_unstale":"2023-10-09T04:35:48.550100+0000","last_undegraded":"2023-10-09T04:35:48.550100+0000","last_fullsized":"2023-10-09T04:34:51.323270+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:07.997213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.538275+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:35:48.538275+0000","last_peered":"2023-10-09T04:35:48.538275+0000","last_clean":"2023-10-09T04:35:48.538275+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:35:48.538275+0000","last_undegraded":"2023-10-09T04:35:48.538275+0000","last_fullsized":"2023-10-09T04:35:48.538275+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"213'7993","reported_seq":8023,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.761387+0000","last_change":"2023-10-09T04:33:51.277983+0000","last_active":"2023-10-09T04:35:54.761387+0000","last_peered":"2023-10-09T04:35:54.761387+0000","last_clean":"2023-10-09T04:35:54.761387+0000","last_became_active":"2023-10-09T04:33:51.277555+0000","last_became_peered":"2023-10-09T04:33:51.277555+0000","last_unstale":"2023-10-09T04:35:54.761387+0000","last_undegraded":"2023-10-09T04:35:54.761387+0000","last_fullsized":"2023-10-09T04:35:54.761387+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7993,"log_dups_size":0,"ondisk_log_size":7993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:59.460010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369426432,"num_objects":5637,"num_object_clones":0,"num_object_copies":16911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7993,"num_write_kb":436160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":177,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.550151+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:35:48.550151+0000","last_peered":"2023-10-09T04:35:48.550151+0000","last_clean":"2023-10-09T04:35:48.550151+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:35:48.550151+0000","last_undegraded":"2023-10-09T04:35:48.550151+0000","last_fullsized":"2023-10-09T04:35:48.550151+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":194,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542675+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:35:48.542675+0000","last_peered":"2023-10-09T04:35:48.542675+0000","last_clean":"2023-10-09T04:35:48.542675+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:35:48.542675+0000","last_undegraded":"2023-10-09T04:35:48.542675+0000","last_fullsized":"2023-10-09T04:35:48.542675+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":199,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.537709+0000","last_change":"2023-10-09T04:34:51.316666+0000","last_active":"2023-10-09T04:35:48.537709+0000","last_peered":"2023-10-09T04:35:48.537709+0000","last_clean":"2023-10-09T04:34:44.053522+0000","last_became_active":"2023-10-09T04:34:51.316666+0000","last_became_peered":"2023-10-09T04:34:51.316666+0000","last_unstale":"2023-10-09T04:35:48.537709+0000","last_undegraded":"2023-10-09T04:35:48.537709+0000","last_fullsized":"2023-10-09T04:34:51.313759+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:10.205902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"213'7885","reported_seq":7930,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:55.024767+0000","last_change":"2023-10-09T04:35:48.558400+0000","last_active":"2023-10-09T04:35:55.024767+0000","last_peered":"2023-10-09T04:35:55.024767+0000","last_clean":"2023-10-09T04:34:50.296861+0000","last_became_active":"2023-10-09T04:35:48.558400+0000","last_became_peered":"2023-10-09T04:35:48.558400+0000","last_unstale":"2023-10-09T04:35:55.024767+0000","last_undegraded":"2023-10-09T04:35:48.548591+0000","last_fullsized":"2023-10-09T04:35:48.548471+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7885,"log_dups_size":0,"ondisk_log_size":7885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184254464,"num_objects":2811,"num_object_clones":0,"num_object_copies":8433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2811,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4060,"num_write_kb":219904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":2811}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":131,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.542059+0000","last_change":"2023-10-09T04:34:51.318540+0000","last_active":"2023-10-09T04:35:48.542059+0000","last_peered":"2023-10-09T04:35:48.542059+0000","last_clean":"2023-10-09T04:29:34.560919+0000","last_became_active":"2023-10-09T04:34:51.318540+0000","last_became_peered":"2023-10-09T04:34:51.318540+0000","last_unstale":"2023-10-09T04:35:48.542059+0000","last_undegraded":"2023-10-09T04:35:48.542059+0000","last_fullsized":"2023-10-09T04:34:51.314590+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:41.112095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":184,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.537598+0000","last_change":"2023-10-09T04:34:51.321100+0000","last_active":"2023-10-09T04:35:48.537598+0000","last_peered":"2023-10-09T04:35:48.537598+0000","last_clean":"2023-10-09T04:34:44.053288+0000","last_became_active":"2023-10-09T04:34:51.321100+0000","last_became_peered":"2023-10-09T04:34:51.321100+0000","last_unstale":"2023-10-09T04:35:48.537598+0000","last_undegraded":"2023-10-09T04:35:48.537598+0000","last_fullsized":"2023-10-09T04:34:51.313128+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:18:11.901708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":182,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.538235+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:35:48.538235+0000","last_peered":"2023-10-09T04:35:48.538235+0000","last_clean":"2023-10-09T04:35:48.538235+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:35:48.538235+0000","last_undegraded":"2023-10-09T04:35:48.538235+0000","last_fullsized":"2023-10-09T04:35:48.538235+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"213'7758","reported_seq":7786,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.790086+0000","last_change":"2023-10-09T04:33:51.277098+0000","last_active":"2023-10-09T04:35:54.790086+0000","last_peered":"2023-10-09T04:35:54.790086+0000","last_clean":"2023-10-09T04:35:54.790086+0000","last_became_active":"2023-10-09T04:33:51.276788+0000","last_became_peered":"2023-10-09T04:33:51.276788+0000","last_unstale":"2023-10-09T04:35:54.790086+0000","last_undegraded":"2023-10-09T04:35:54.790086+0000","last_fullsized":"2023-10-09T04:35:54.790086+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7758,"log_dups_size":0,"ondisk_log_size":7758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:20.524012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362414080,"num_objects":5530,"num_object_clones":0,"num_object_copies":16590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7758,"num_write_kb":425216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":184,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.550228+0000","last_change":"2023-10-09T04:34:51.334572+0000","last_active":"2023-10-09T04:35:48.550228+0000","last_peered":"2023-10-09T04:35:48.550228+0000","last_clean":"2023-10-09T04:34:44.071648+0000","last_became_active":"2023-10-09T04:34:51.334572+0000","last_became_peered":"2023-10-09T04:34:51.334572+0000","last_unstale":"2023-10-09T04:35:48.550228+0000","last_undegraded":"2023-10-09T04:35:48.550228+0000","last_fullsized":"2023-10-09T04:34:51.324871+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:03.116911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":605,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.545262+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:35:48.545262+0000","last_peered":"2023-10-09T04:35:48.545262+0000","last_clean":"2023-10-09T04:35:48.545262+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:35:48.545262+0000","last_undegraded":"2023-10-09T04:35:48.545262+0000","last_fullsized":"2023-10-09T04:35:48.545262+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.541994+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:35:48.541994+0000","last_peered":"2023-10-09T04:35:48.541994+0000","last_clean":"2023-10-09T04:35:48.541994+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:35:48.541994+0000","last_undegraded":"2023-10-09T04:35:48.541994+0000","last_fullsized":"2023-10-09T04:35:48.541994+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":153,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542634+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:35:48.542634+0000","last_peered":"2023-10-09T04:35:48.542634+0000","last_clean":"2023-10-09T04:35:48.542634+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:35:48.542634+0000","last_undegraded":"2023-10-09T04:35:48.542634+0000","last_fullsized":"2023-10-09T04:35:48.542634+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"213'7804","reported_seq":7859,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:54.929991+0000","last_change":"2023-10-09T04:35:48.563659+0000","last_active":"2023-10-09T04:35:54.929991+0000","last_peered":"2023-10-09T04:35:54.929991+0000","last_clean":"2023-10-09T04:34:50.268727+0000","last_became_active":"2023-10-09T04:35:48.563659+0000","last_became_peered":"2023-10-09T04:35:48.563659+0000","last_unstale":"2023-10-09T04:35:54.929991+0000","last_undegraded":"2023-10-09T04:35:48.544493+0000","last_fullsized":"2023-10-09T04:35:48.544356+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7804,"log_dups_size":0,"ondisk_log_size":7804,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187465728,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2860,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4001,"num_write_kb":219584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2860}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":213,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.544974+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:35:48.544974+0000","last_peered":"2023-10-09T04:35:48.544974+0000","last_clean":"2023-10-09T04:35:48.544974+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:35:48.544974+0000","last_undegraded":"2023-10-09T04:35:48.544974+0000","last_fullsized":"2023-10-09T04:35:48.544974+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.544265+0000","last_change":"2023-10-09T04:34:22.266723+0000","last_active":"2023-10-09T04:35:48.544265+0000","last_peered":"2023-10-09T04:35:48.544265+0000","last_clean":"2023-10-09T04:35:48.544265+0000","last_became_active":"2023-10-09T04:34:22.266261+0000","last_became_peered":"2023-10-09T04:34:22.266261+0000","last_unstale":"2023-10-09T04:35:48.544265+0000","last_undegraded":"2023-10-09T04:35:48.544265+0000","last_fullsized":"2023-10-09T04:35:48.544265+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:54.792194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":77,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.540453+0000","last_change":"2023-10-09T04:34:51.318703+0000","last_active":"2023-10-09T04:35:48.540453+0000","last_peered":"2023-10-09T04:35:48.540453+0000","last_clean":"2023-10-09T04:34:44.061887+0000","last_became_active":"2023-10-09T04:34:51.318703+0000","last_became_peered":"2023-10-09T04:34:51.318703+0000","last_unstale":"2023-10-09T04:35:48.540453+0000","last_undegraded":"2023-10-09T04:35:48.540453+0000","last_fullsized":"2023-10-09T04:34:51.317343+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:36.687606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"213'7951","reported_seq":7982,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.757702+0000","last_change":"2023-10-09T04:33:51.282958+0000","last_active":"2023-10-09T04:35:54.757702+0000","last_peered":"2023-10-09T04:35:54.757702+0000","last_clean":"2023-10-09T04:35:54.757702+0000","last_became_active":"2023-10-09T04:33:51.282685+0000","last_became_peered":"2023-10-09T04:33:51.282685+0000","last_unstale":"2023-10-09T04:35:54.757702+0000","last_undegraded":"2023-10-09T04:35:54.757702+0000","last_fullsized":"2023-10-09T04:35:54.757702+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:49.468338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367919128,"num_objects":5615,"num_object_clones":0,"num_object_copies":16845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5615,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7951,"num_write_kb":434049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":196,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.547653+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:35:48.547653+0000","last_peered":"2023-10-09T04:35:48.547653+0000","last_clean":"2023-10-09T04:35:48.547653+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:35:48.547653+0000","last_undegraded":"2023-10-09T04:35:48.547653+0000","last_fullsized":"2023-10-09T04:35:48.547653+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":184,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.537612+0000","last_change":"2023-10-09T04:34:51.325423+0000","last_active":"2023-10-09T04:35:48.537612+0000","last_peered":"2023-10-09T04:35:48.537612+0000","last_clean":"2023-10-09T04:34:44.053445+0000","last_became_active":"2023-10-09T04:34:51.325423+0000","last_became_peered":"2023-10-09T04:34:51.325423+0000","last_unstale":"2023-10-09T04:35:48.537612+0000","last_undegraded":"2023-10-09T04:35:48.537612+0000","last_fullsized":"2023-10-09T04:34:51.313428+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:46:29.954931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":57,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.550367+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:35:48.550367+0000","last_peered":"2023-10-09T04:35:48.550367+0000","last_clean":"2023-10-09T04:35:48.550367+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:35:48.550367+0000","last_undegraded":"2023-10-09T04:35:48.550367+0000","last_fullsized":"2023-10-09T04:35:48.550367+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"213'7782","reported_seq":7819,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:54.864671+0000","last_change":"2023-10-09T04:34:51.328209+0000","last_active":"2023-10-09T04:35:54.864671+0000","last_peered":"2023-10-09T04:35:54.864671+0000","last_clean":"2023-10-09T04:34:50.302298+0000","last_became_active":"2023-10-09T04:34:51.328209+0000","last_became_peered":"2023-10-09T04:34:51.328209+0000","last_unstale":"2023-10-09T04:35:54.864671+0000","last_undegraded":"2023-10-09T04:34:51.321394+0000","last_fullsized":"2023-10-09T04:34:51.321276+0000","mapping_epoch":201,"log_start":"195'1234","ondisk_log_start":"195'1234","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:35:47.025638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363331584,"num_objects":5544,"num_object_clones":0,"num_object_copies":16632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7782,"num_write_kb":426432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":5544}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":207,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.543354+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:35:48.543354+0000","last_peered":"2023-10-09T04:35:48.543354+0000","last_clean":"2023-10-09T04:35:48.543354+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:35:48.543354+0000","last_undegraded":"2023-10-09T04:35:48.543354+0000","last_fullsized":"2023-10-09T04:35:48.543354+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.537947+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:35:48.537947+0000","last_peered":"2023-10-09T04:35:48.537947+0000","last_clean":"2023-10-09T04:35:48.537947+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:35:48.537947+0000","last_undegraded":"2023-10-09T04:35:48.537947+0000","last_fullsized":"2023-10-09T04:35:48.537947+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:49.556404+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:35:49.556404+0000","last_peered":"2023-10-09T04:35:49.556404+0000","last_clean":"2023-10-09T04:35:49.556404+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:35:49.556404+0000","last_undegraded":"2023-10-09T04:35:49.556404+0000","last_fullsized":"2023-10-09T04:35:49.556404+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"213'8121","reported_seq":8157,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:54.759975+0000","last_change":"2023-10-09T04:34:51.325469+0000","last_active":"2023-10-09T04:35:54.759975+0000","last_peered":"2023-10-09T04:35:54.759975+0000","last_clean":"2023-10-09T04:34:50.287068+0000","last_became_active":"2023-10-09T04:34:51.325469+0000","last_became_peered":"2023-10-09T04:34:51.325469+0000","last_unstale":"2023-10-09T04:35:54.759975+0000","last_undegraded":"2023-10-09T04:34:51.321417+0000","last_fullsized":"2023-10-09T04:34:51.321273+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":190,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":8121,"log_dups_size":0,"ondisk_log_size":8121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:06.267427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372178944,"num_objects":5679,"num_object_clones":0,"num_object_copies":17037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5679,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8121,"num_write_kb":441600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":5679}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":182,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.542516+0000","last_change":"2023-10-09T04:34:51.331849+0000","last_active":"2023-10-09T04:35:48.542516+0000","last_peered":"2023-10-09T04:35:48.542516+0000","last_clean":"2023-10-09T04:34:44.065555+0000","last_became_active":"2023-10-09T04:34:51.331849+0000","last_became_peered":"2023-10-09T04:34:51.331849+0000","last_unstale":"2023-10-09T04:35:48.542516+0000","last_undegraded":"2023-10-09T04:35:48.542516+0000","last_fullsized":"2023-10-09T04:34:51.321695+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:09.071914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":66,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.538022+0000","last_change":"2023-10-09T04:34:22.266652+0000","last_active":"2023-10-09T04:35:48.538022+0000","last_peered":"2023-10-09T04:35:48.538022+0000","last_clean":"2023-10-09T04:35:48.538022+0000","last_became_active":"2023-10-09T04:34:22.266267+0000","last_became_peered":"2023-10-09T04:34:22.266267+0000","last_unstale":"2023-10-09T04:35:48.538022+0000","last_undegraded":"2023-10-09T04:35:48.538022+0000","last_fullsized":"2023-10-09T04:35:48.538022+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:50.646309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":156,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.544028+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:35:48.544028+0000","last_peered":"2023-10-09T04:35:48.544028+0000","last_clean":"2023-10-09T04:35:48.544028+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:35:48.544028+0000","last_undegraded":"2023-10-09T04:35:48.544028+0000","last_fullsized":"2023-10-09T04:35:48.544028+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"213'7823","reported_seq":7864,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:56.572246+0000","last_change":"2023-10-09T04:35:48.580879+0000","last_active":"2023-10-09T04:35:56.572246+0000","last_peered":"2023-10-09T04:35:56.572246+0000","last_clean":"2023-10-09T04:35:56.572246+0000","last_became_active":"2023-10-09T04:35:48.572680+0000","last_became_peered":"2023-10-09T04:35:48.572680+0000","last_unstale":"2023-10-09T04:35:56.572246+0000","last_undegraded":"2023-10-09T04:35:56.572246+0000","last_fullsized":"2023-10-09T04:35:56.572246+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7823,"log_dups_size":0,"ondisk_log_size":7823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184057856,"num_objects":2809,"num_object_clones":0,"num_object_copies":8427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4040,"num_write_kb":219136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":53,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542558+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:35:48.542558+0000","last_peered":"2023-10-09T04:35:48.542558+0000","last_clean":"2023-10-09T04:35:48.542558+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:35:48.542558+0000","last_undegraded":"2023-10-09T04:35:48.542558+0000","last_fullsized":"2023-10-09T04:35:48.542558+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.537518+0000","last_change":"2023-10-09T04:34:51.316709+0000","last_active":"2023-10-09T04:35:48.537518+0000","last_peered":"2023-10-09T04:35:48.537518+0000","last_clean":"2023-10-09T04:27:36.073787+0000","last_became_active":"2023-10-09T04:34:51.316709+0000","last_became_peered":"2023-10-09T04:34:51.316709+0000","last_unstale":"2023-10-09T04:35:48.537518+0000","last_undegraded":"2023-10-09T04:35:48.537518+0000","last_fullsized":"2023-10-09T04:34:51.313259+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:43.330782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.15","version":"213'7803","reported_seq":7844,"reported_epoch":213,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:56.638339+0000","last_change":"2023-10-09T04:35:48.567263+0000","last_active":"2023-10-09T04:35:56.638339+0000","last_peered":"2023-10-09T04:35:56.638339+0000","last_clean":"2023-10-09T04:34:50.268593+0000","last_became_active":"2023-10-09T04:35:48.567263+0000","last_became_peered":"2023-10-09T04:35:48.567263+0000","last_unstale":"2023-10-09T04:35:56.638339+0000","last_undegraded":"2023-10-09T04:35:48.548125+0000","last_fullsized":"2023-10-09T04:35:48.541956+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7803,"log_dups_size":0,"ondisk_log_size":7803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178847744,"num_objects":2729,"num_object_clones":0,"num_object_copies":8187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4046,"num_write_kb":216800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":2729}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.549724+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:35:48.549724+0000","last_peered":"2023-10-09T04:35:48.549724+0000","last_clean":"2023-10-09T04:35:48.549724+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:35:48.549724+0000","last_undegraded":"2023-10-09T04:35:48.549724+0000","last_fullsized":"2023-10-09T04:35:48.549724+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.542919+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:35:48.542919+0000","last_peered":"2023-10-09T04:35:48.542919+0000","last_clean":"2023-10-09T04:35:48.542919+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:35:48.542919+0000","last_undegraded":"2023-10-09T04:35:48.542919+0000","last_fullsized":"2023-10-09T04:35:48.542919+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"213'7905","reported_seq":7959,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.774297+0000","last_change":"2023-10-09T04:35:48.599126+0000","last_active":"2023-10-09T04:35:54.774297+0000","last_peered":"2023-10-09T04:35:54.774297+0000","last_clean":"2023-10-09T04:35:54.774297+0000","last_became_active":"2023-10-09T04:35:48.592913+0000","last_became_peered":"2023-10-09T04:35:48.592913+0000","last_unstale":"2023-10-09T04:35:54.774297+0000","last_undegraded":"2023-10-09T04:35:54.774297+0000","last_fullsized":"2023-10-09T04:35:54.774297+0000","mapping_epoch":211,"log_start":"195'300","ondisk_log_start":"195'300","created":211,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7605,"log_dups_size":0,"ondisk_log_size":7605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187269120,"num_objects":2858,"num_object_clones":0,"num_object_copies":8574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4066,"num_write_kb":221536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":158,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.550242+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:35:48.550242+0000","last_peered":"2023-10-09T04:35:48.550242+0000","last_clean":"2023-10-09T04:35:48.550242+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:35:48.550242+0000","last_undegraded":"2023-10-09T04:35:48.550242+0000","last_fullsized":"2023-10-09T04:35:48.550242+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":210,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-10-09T04:35:48.544011+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:35:48.544011+0000","last_peered":"2023-10-09T04:35:48.544011+0000","last_clean":"2023-10-09T04:35:48.544011+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:35:48.544011+0000","last_undegraded":"2023-10-09T04:35:48.544011+0000","last_fullsized":"2023-10-09T04:35:48.544011+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"213'7623","reported_seq":7663,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.727612+0000","last_change":"2023-10-09T04:35:48.603203+0000","last_active":"2023-10-09T04:35:54.727612+0000","last_peered":"2023-10-09T04:35:54.727612+0000","last_clean":"2023-10-09T04:35:54.727612+0000","last_became_active":"2023-10-09T04:35:48.596432+0000","last_became_peered":"2023-10-09T04:35:48.596432+0000","last_unstale":"2023-10-09T04:35:54.727612+0000","last_undegraded":"2023-10-09T04:35:54.727612+0000","last_fullsized":"2023-10-09T04:35:54.727612+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7623,"log_dups_size":0,"ondisk_log_size":7623,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182878208,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3914,"num_write_kb":214528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":189,"reported_epoch":213,"state":"active+undersized","last_fresh":"2023-10-09T04:35:49.558076+0000","last_change":"2023-10-09T04:34:51.321322+0000","last_active":"2023-10-09T04:35:49.558076+0000","last_peered":"2023-10-09T04:35:49.558076+0000","last_clean":"2023-10-09T04:34:44.059138+0000","last_became_active":"2023-10-09T04:34:51.321322+0000","last_became_peered":"2023-10-09T04:34:51.321322+0000","last_unstale":"2023-10-09T04:35:49.558076+0000","last_undegraded":"2023-10-09T04:35:49.558076+0000","last_fullsized":"2023-10-09T04:34:51.314501+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:36:46.254327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":221,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.541848+0000","last_change":"2023-10-09T04:34:51.318670+0000","last_active":"2023-10-09T04:35:48.541848+0000","last_peered":"2023-10-09T04:35:48.541848+0000","last_clean":"2023-10-09T04:34:44.058794+0000","last_became_active":"2023-10-09T04:34:51.318670+0000","last_became_peered":"2023-10-09T04:34:51.318670+0000","last_unstale":"2023-10-09T04:35:48.541848+0000","last_undegraded":"2023-10-09T04:35:48.541848+0000","last_fullsized":"2023-10-09T04:34:51.314163+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:43.814115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"213'7517","reported_seq":7558,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-10-09T04:35:54.981495+0000","last_change":"2023-10-09T04:35:48.598346+0000","last_active":"2023-10-09T04:35:54.981495+0000","last_peered":"2023-10-09T04:35:54.981495+0000","last_clean":"2023-10-09T04:35:54.981495+0000","last_became_active":"2023-10-09T04:35:48.593355+0000","last_became_peered":"2023-10-09T04:35:48.593355+0000","last_unstale":"2023-10-09T04:35:54.981495+0000","last_undegraded":"2023-10-09T04:35:54.981495+0000","last_fullsized":"2023-10-09T04:35:54.981495+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":211,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:33:50.264488+0000","last_clean_scrub_stamp":"2023-10-09T04:33:50.264488+0000","objects_scrubbed":0,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179306496,"num_objects":2736,"num_object_clones":0,"num_object_copies":8208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3863,"num_write_kb":211168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":77,"reported_epoch":212,"state":"active+undersized+degraded","last_fresh":"2023-10-09T04:35:48.551103+0000","last_change":"2023-10-09T04:34:51.325437+0000","last_active":"2023-10-09T04:35:48.551103+0000","last_peered":"2023-10-09T04:35:48.551103+0000","last_clean":"2023-10-09T04:34:44.071843+0000","last_became_active":"2023-10-09T04:34:51.325437+0000","last_became_peered":"2023-10-09T04:34:51.325437+0000","last_unstale":"2023-10-09T04:35:48.551103+0000","last_undegraded":"2023-10-09T04:34:51.324026+0000","last_fullsized":"2023-10-09T04:34:51.323882+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:24.344413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":190,"reported_epoch":212,"state":"active+undersized","last_fresh":"2023-10-09T04:35:48.542180+0000","last_change":"2023-10-09T04:34:51.326267+0000","last_active":"2023-10-09T04:35:48.542180+0000","last_peered":"2023-10-09T04:35:48.542180+0000","last_clean":"2023-10-09T04:34:44.060673+0000","last_became_active":"2023-10-09T04:34:51.326267+0000","last_became_peered":"2023-10-09T04:34:51.326267+0000","last_unstale":"2023-10-09T04:35:48.542180+0000","last_undegraded":"2023-10-09T04:35:48.542180+0000","last_fullsized":"2023-10-09T04:34:51.321105+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:44.409648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":88,"acting":88,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":5876678680,"num_objects":89672,"num_object_clones":0,"num_object_copies":269016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33653,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89672,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127066,"num_write_kb":6935553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16991531008,"data_stored":16991518792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":200279,"ondisk_log_size":200279,"up":68,"acting":68,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":88,"acting":88,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234348,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2346912,"kb_used_data":1448216,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":91398240,"statfs":{"total":95995035648,"available":93591797760,"internally_reserved":0,"allocated":1482973184,"data_stored":1482446390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":6.5810000000000004,"5min":8.4450000000000003,"15min":5.8789999999999996},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.22900000000000001},"max":{"1min":68.495999999999995,"5min":186.446,"15min":629.22500000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":6.9870000000000001,"5min":8.6980000000000004,"15min":5.8559999999999999},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":68.540000000000006,"5min":186.429,"15min":629.43100000000004},"last":1.6080000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":7.6040000000000001,"15min":4.2119999999999997},"min":{"1min":0.28799999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":28.795999999999999,"5min":224.13499999999999,"15min":224.13499999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":3.0449999999999999,"5min":7.7329999999999997,"15min":4.2869999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":28.818999999999999,"5min":224.203,"15min":224.203},"last":0.42899999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:42 2023","interfaces":[{"interface":"back","average":{"1min":16.478000000000002,"5min":5.5570000000000004,"15min":2.2050000000000001},"min":{"1min":0.28499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":177.262,"5min":177.262,"15min":177.262},"last":2.2999999999999998},{"interface":"front","average":{"1min":16.478000000000002,"5min":5.5830000000000002,"15min":2.2229999999999999},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":177.24299999999999,"5min":177.24299999999999,"15min":177.24299999999999},"last":2.5419999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8580000000000001,"5min":11.728,"15min":5.3109999999999999},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":53.482999999999997,"5min":249.989,"15min":249.989},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.6379999999999999,"5min":12.002000000000001,"15min":5.431},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":53.162999999999997,"5min":250.21700000000001,"15min":250.21700000000001},"last":1.639}]},{"osd":4,"last update":"Mon Oct 9 04:35:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2349999999999999,"5min":1.619,"15min":1.3089999999999999},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.253},"max":{"1min":9.4290000000000003,"5min":10.869,"15min":10.869},"last":1.6870000000000001},{"interface":"front","average":{"1min":2.1699999999999999,"5min":1.2949999999999999,"15min":1.0389999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":7.4119999999999999,"5min":10.613,"15min":10.613},"last":2.7109999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:35:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.111,"15min":1.016},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.0410000000000004,"5min":6.0410000000000004,"15min":8.8610000000000007},"last":0.372},{"interface":"front","average":{"1min":2.448,"5min":1.623,"15min":1.2589999999999999},"min":{"1min":0.35799999999999998,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":10.326000000000001,"5min":10.895,"15min":10.895},"last":1.6619999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:35:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0150000000000001,"5min":1.427,"15min":1.226},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.7279999999999998,"5min":7.7279999999999998,"15min":12.007},"last":1.8120000000000001},{"interface":"front","average":{"1min":3.0990000000000002,"5min":1.7609999999999999,"15min":1.351},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":9.3819999999999997,"5min":10.917,"15min":10.917},"last":6.3449999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495982,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3120212,"kb_used_data":2521996,"kb_used_omap":0,"kb_used_meta":597952,"kb_avail":90624940,"statfs":{"total":95995035648,"available":92799938560,"internally_reserved":0,"allocated":2582523904,"data_stored":2582008953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612302848},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":3.544,"5min":15.909000000000001,"15min":6.7400000000000002},"min":{"1min":0.26200000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":29.931999999999999,"5min":1284.521,"15min":1284.521},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.2919999999999998,"5min":15.868,"15min":6.9610000000000003},"min":{"1min":0.26600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":27.896000000000001,"5min":1284.4690000000001,"15min":1284.4690000000001},"last":0.28899999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:35:22 2023","interfaces":[{"interface":"back","average":{"1min":6.4080000000000004,"5min":7.4939999999999998,"15min":3.7490000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":94.287000000000006,"5min":185.74000000000001,"15min":185.74000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":6.7709999999999999,"5min":7.4119999999999999,"15min":3.8839999999999999},"min":{"1min":0.34799999999999998,"5min":0.26300000000000001,"15min":0.24299999999999999},"max":{"1min":92.116,"5min":189.03999999999999,"15min":189.03999999999999},"last":0.39700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":1.7110000000000001,"15min":0.95099999999999996},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.6890000000000001,"5min":7.6890000000000001,"15min":7.6890000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.236,"15min":1.02},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.976,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":11.254}]},{"osd":3,"last update":"Mon Oct 9 04:35:13 2023","interfaces":[{"interface":"back","average":{"1min":11.717000000000001,"5min":5.0419999999999998,"15min":2.121},"min":{"1min":0.35999999999999999,"5min":0.314,"15min":0.314},"max":{"1min":144.99799999999999,"5min":144.99799999999999,"15min":144.99799999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":11.445,"5min":5.1820000000000004,"15min":2.117},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.313},"max":{"1min":145.023,"5min":145.023,"15min":145.023},"last":0.33900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":1.5720000000000001,"15min":1.268},"min":{"1min":0.28999999999999998,"5min":0.27300000000000002,"15min":0.246},"max":{"1min":9.8360000000000003,"5min":9.8360000000000003,"15min":12.743},"last":1.788},{"interface":"front","average":{"1min":1.738,"5min":1.028,"15min":1.1539999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26000000000000001},"max":{"1min":8.0129999999999999,"5min":8.9920000000000009,"15min":12.028},"last":1.0900000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:35:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":2.9239999999999999,"15min":1.7190000000000001},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28100000000000003},"max":{"1min":10.779,"5min":81.087000000000003,"15min":81.087000000000003},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.8239999999999998,"5min":2.5750000000000002,"15min":1.7370000000000001},"min":{"1min":0.35099999999999998,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":11.279999999999999,"5min":81.706999999999994,"15min":81.706999999999994},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:34:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":2.1320000000000001,"15min":1.492},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":11.226000000000001,"5min":81.674999999999997,"15min":81.674999999999997},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.246,"15min":1.5369999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.32300000000000001},"max":{"1min":9.9009999999999998,"5min":81.055000000000007,"15min":81.055000000000007},"last":0.38600000000000001}]}]},{"osd":1,"up_from":118,"seq":506806141072,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487224,"kb_used_data":1065648,"kb_used_omap":4,"kb_used_meta":421563,"kb_avail":92257928,"statfs":{"total":95995035648,"available":94472118272,"internally_reserved":0,"allocated":1091223552,"data_stored":1090382869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":431681422},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":5.6849999999999996,"5min":4.8849999999999998,"15min":3.5350000000000001},"min":{"1min":0.189,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":64.037000000000006,"5min":97.308000000000007,"15min":97.308000000000007},"last":0.23599999999999999},{"interface":"front","average":{"1min":5.9980000000000002,"5min":5.4649999999999999,"15min":3.8540000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":66.388000000000005,"5min":97.055999999999997,"15min":97.055999999999997},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:34:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1530000000000005,"5min":3.544,"15min":1.417},"min":{"1min":0.26600000000000001,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":143.11199999999999,"5min":143.11199999999999,"15min":143.11199999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":7.6529999999999996,"5min":2.8410000000000002,"15min":1.175},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.218},"max":{"1min":143.22,"5min":143.22,"15min":143.22},"last":1.1819999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:35:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6680000000000001,"5min":5.415,"15min":2.1059999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":58.673000000000002,"5min":195.035,"15min":195.035},"last":0.27200000000000002},{"interface":"front","average":{"1min":4.1550000000000002,"5min":6.0709999999999997,"15min":2.335},"min":{"1min":0.246,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":58.125999999999998,"5min":195.465,"15min":195.465},"last":0.28199999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.804,"15min":2.597},"min":{"1min":0.32900000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.7919999999999998,"5min":8.0150000000000006,"15min":107.795},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.325,"5min":1.391,"15min":1.6539999999999999},"min":{"1min":0.35699999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.1420000000000003,"5min":6.1420000000000003,"15min":109.167},"last":0.46700000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":2.105,"15min":2.214},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.8810000000000002,"5min":10.186999999999999,"15min":12.414999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.7350000000000001,"5min":2.0680000000000001,"15min":2.431},"min":{"1min":0.42999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.734,"5min":10.061999999999999,"15min":13.933999999999999},"last":0.72199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":2.2549999999999999,"15min":2.173},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.8410000000000002,"5min":8.3770000000000007,"15min":13.388},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.363,"5min":2.2050000000000001,"15min":2.5459999999999998},"min":{"1min":0.48499999999999999,"5min":0.317,"15min":0.317},"max":{"1min":8.6549999999999994,"5min":12.596,"15min":13.882},"last":0.65600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:35:27 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":2.1339999999999999,"15min":2.5920000000000001},"min":{"1min":0.438,"5min":0.374,"15min":0.374},"max":{"1min":7.6299999999999999,"5min":8.4160000000000004,"15min":13.907},"last":2.5600000000000001},{"interface":"front","average":{"1min":1.391,"5min":1.607,"15min":1.5620000000000001},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9260000000000002,"5min":6.1799999999999997,"15min":7.8239999999999998},"last":2.593}]}]},{"osd":0,"up_from":91,"seq":390842024129,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4015260,"kb_used_data":3488596,"kb_used_omap":5,"kb_used_meta":526650,"kb_avail":89729892,"statfs":{"total":95995035648,"available":91883409408,"internally_reserved":0,"allocated":3572322304,"data_stored":3571505994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":539290234},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:35:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":7.1180000000000003,"15min":4.1980000000000004},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":6.0179999999999998,"5min":167.92699999999999,"15min":167.92699999999999},"last":0.315},{"interface":"front","average":{"1min":1.484,"5min":7.3250000000000002,"15min":4.0510000000000002},"min":{"1min":0.28100000000000003,"5min":0.20799999999999999,"15min":0.19600000000000001},"max":{"1min":6.4960000000000004,"5min":169.78999999999999,"15min":169.78999999999999},"last":2.4790000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:34:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":3.024,"15min":1.26},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.3769999999999998,"5min":107.548,"15min":107.548},"last":1.548},{"interface":"front","average":{"1min":1.329,"5min":2.7210000000000001,"15min":1.1459999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.4989999999999997,"5min":107.35899999999999,"15min":107.35899999999999},"last":2.2469999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:35:11 2023","interfaces":[{"interface":"back","average":{"1min":5.8300000000000001,"5min":5.9370000000000003,"15min":2.3149999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":46.625,"5min":213.28399999999999,"15min":213.28399999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":6.0019999999999998,"5min":6.1059999999999999,"15min":2.3900000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":47.076000000000001,"5min":212.27000000000001,"15min":212.27000000000001},"last":2.4279999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":3.1070000000000002,"5min":2.113,"15min":1.8069999999999999},"min":{"1min":0.38500000000000001,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":22.588000000000001,"5min":22.588000000000001,"15min":22.588000000000001},"last":2.5390000000000001},{"interface":"front","average":{"1min":3.6339999999999999,"5min":2.6030000000000002,"15min":1.9059999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.30499999999999999},"max":{"1min":22.605,"5min":22.605,"15min":22.605},"last":3.4849999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4289999999999998,"5min":2.4350000000000001,"15min":1.948},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.27900000000000003},"max":{"1min":8.0510000000000002,"5min":12.807,"15min":12.807},"last":2.528},{"interface":"front","average":{"1min":2.145,"5min":1.7789999999999999,"15min":1.7150000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.5129999999999999,"5min":8.4589999999999996,"15min":14.144},"last":2.9279999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3820000000000001,"5min":2.4390000000000001,"15min":2.0649999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.0749999999999993,"5min":16.369,"15min":16.369},"last":3.9849999999999999},{"interface":"front","average":{"1min":3.0139999999999998,"5min":1.919,"15min":1.8899999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.4930000000000003,"5min":8.4930000000000003,"15min":14.090999999999999},"last":1.1899999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4529999999999998,"5min":1.905,"15min":1.8600000000000001},"min":{"1min":0.433,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.9770000000000003,"5min":8.5229999999999997,"15min":11.890000000000001},"last":2.903},{"interface":"front","average":{"1min":2.2109999999999999,"5min":2.1629999999999998,"15min":2.0379999999999998},"min":{"1min":0.42399999999999999,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":6.9640000000000004,"5min":9.7110000000000003,"15min":11.176},"last":2.5880000000000001}]}]},{"osd":2,"up_from":135,"seq":579820585050,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1928196,"kb_used_data":1669308,"kb_used_omap":7,"kb_used_meta":258872,"kb_avail":91816956,"statfs":{"total":95995035648,"available":94020562944,"internally_reserved":0,"allocated":1709371392,"data_stored":1708661063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":265085009},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":9.0289999999999999,"5min":6.2489999999999997,"15min":2.3050000000000002},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":136.28100000000001,"5min":136.28100000000001,"15min":136.28100000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":15.180999999999999,"5min":7.3689999999999998,"15min":2.7370000000000001},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":136.239,"5min":136.239,"15min":136.239},"last":0.28299999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":7.6420000000000003,"5min":5.4779999999999998,"15min":2.081},"min":{"1min":0.246,"5min":0.193,"15min":0.193},"max":{"1min":119.748,"5min":119.748,"15min":119.748},"last":0.41499999999999998},{"interface":"front","average":{"1min":7.9009999999999998,"5min":5.8760000000000003,"15min":2.1899999999999999},"min":{"1min":0.23799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":118.024,"5min":118.024,"15min":118.024},"last":0.59499999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:34:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":6.093,"15min":2.2810000000000001},"min":{"1min":0.28100000000000003,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.4320000000000004,"5min":147.655,"15min":147.655},"last":0.438},{"interface":"front","average":{"1min":1.425,"5min":6.1150000000000002,"15min":2.3290000000000002},"min":{"1min":0.312,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.7850000000000001,"5min":147.607,"15min":147.607},"last":0.47799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":2.2669999999999999,"15min":1.1559999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.9699999999999998,"5min":12.765000000000001,"15min":12.765000000000001},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.389,"15min":0.872},"min":{"1min":0.373,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.8399999999999999,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":7.5369999999999999,"5min":3.2389999999999999,"15min":1.5029999999999999},"min":{"1min":0.36299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":118.55800000000001,"5min":118.55800000000001,"15min":118.55800000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.339,"5min":2.7120000000000002,"15min":1.302},"min":{"1min":0.34599999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.4640000000000004,"5min":21.716000000000001,"15min":21.716000000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.4819999999999993,"5min":2.972,"15min":1.4159999999999999},"min":{"1min":0.46300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":118.239,"5min":118.239,"15min":118.239},"last":0.79400000000000004},{"interface":"front","average":{"1min":8.298,"5min":3.149,"15min":1.51},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":118.498,"5min":118.498,"15min":118.498},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:34:47 2023","interfaces":[{"interface":"back","average":{"1min":8.0359999999999996,"5min":3.4750000000000001,"15min":1.5740000000000001},"min":{"1min":0.49099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":118.18899999999999,"5min":118.18899999999999,"15min":118.18899999999999},"last":0.621},{"interface":"front","average":{"1min":7.8730000000000002,"5min":3.391,"15min":1.5840000000000001},"min":{"1min":0.48299999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":118.629,"5min":118.629,"15min":118.629},"last":0.52900000000000003}]}]},{"osd":3,"up_from":142,"seq":609885356130,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2107392,"kb_used_data":1776184,"kb_used_omap":2,"kb_used_meta":331197,"kb_avail":91637760,"statfs":{"total":95995035648,"available":93837066240,"internally_reserved":0,"allocated":1818812416,"data_stored":1818078328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":339145781},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":2.7869999999999999,"15min":1.3740000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.8570000000000002,"5min":91.634,"15min":91.634},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":3.0339999999999998,"15min":1.296},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.7910000000000004,"5min":91.646000000000001,"15min":91.646000000000001},"last":0.307}]},{"osd":1,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":6.7699999999999996,"5min":6.593,"15min":3.1459999999999999},"min":{"1min":0.23400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":145.47499999999999,"5min":180.02500000000001,"15min":180.02500000000001},"last":0.223},{"interface":"front","average":{"1min":7.0389999999999997,"5min":7.3730000000000002,"15min":2.972},"min":{"1min":0.25900000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":144.83799999999999,"5min":184.00899999999999,"15min":184.00899999999999},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":3.3260000000000001,"15min":1.3819999999999999},"min":{"1min":0.26200000000000001,"5min":0.222,"15min":0.222},"max":{"1min":9.6660000000000004,"5min":118.238,"15min":118.238},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":3.0129999999999999,"15min":2.0129999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":4.7830000000000004,"5min":117.64400000000001,"15min":117.64400000000001},"last":0.28399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":5.3899999999999997,"5min":2.298,"15min":1.907},"min":{"1min":0.34599999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":103.208,"5min":103.208,"15min":103.208},"last":0.28699999999999998},{"interface":"front","average":{"1min":5.5979999999999999,"5min":2.5910000000000002,"15min":1.5249999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.313},"max":{"1min":103.218,"5min":103.218,"15min":103.218},"last":0.40699999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":2.742,"15min":1.554},"min":{"1min":0.38100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.7220000000000004,"5min":57.475000000000001,"15min":57.475000000000001},"last":0.436},{"interface":"front","average":{"1min":1.5229999999999999,"5min":3.0710000000000002,"15min":1.496},"min":{"1min":0.38100000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.7380000000000004,"5min":57.651000000000003,"15min":57.651000000000003},"last":0.35099999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":5.5579999999999998,"5min":2.915,"15min":1.4470000000000001},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":104.422,"5min":104.422,"15min":104.422},"last":0.39400000000000002},{"interface":"front","average":{"1min":5.4960000000000004,"5min":2.5979999999999999,"15min":2.0299999999999998},"min":{"1min":0.39400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":104.50700000000001,"5min":104.50700000000001,"15min":104.50700000000001},"last":0.48599999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:35:05 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":2.7440000000000002,"15min":2.0489999999999999},"min":{"1min":0.34999999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.1800000000000002,"5min":57.399000000000001,"15min":57.399000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.169,"5min":2.927,"15min":1.6519999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":5.7610000000000001,"5min":57.447000000000003,"15min":57.447000000000003},"last":0.45800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019256,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3790364,"kb_used_data":2894164,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":89954788,"statfs":{"total":95995035648,"available":92113702912,"internally_reserved":0,"allocated":2963623936,"data_stored":2963100648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":7.4189999999999996,"5min":6.2460000000000004,"15min":3.4900000000000002},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.20499999999999999},"max":{"1min":54.991999999999997,"5min":219.577,"15min":219.577},"last":1.02},{"interface":"front","average":{"1min":7.2530000000000001,"5min":6.1029999999999998,"15min":3.286},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.247},"max":{"1min":54.841000000000001,"5min":218.96000000000001,"15min":218.96000000000001},"last":1.034}]},{"osd":1,"last update":"Mon Oct 9 04:35:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3129999999999997,"5min":7.4299999999999997,"15min":4.4790000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":48.633000000000003,"5min":142.74799999999999,"15min":142.74799999999999},"last":3.681},{"interface":"front","average":{"1min":4.593,"5min":7.6139999999999999,"15min":4.6929999999999996},"min":{"1min":0.28599999999999998,"5min":0.27800000000000002,"15min":0.26700000000000002},"max":{"1min":48.680999999999997,"5min":139.09700000000001,"15min":139.09700000000001},"last":1.0549999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:46 2023","interfaces":[{"interface":"back","average":{"1min":15.507,"5min":5.7279999999999998,"15min":2.2770000000000001},"min":{"1min":0.32000000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":197.31999999999999,"5min":197.31999999999999,"15min":197.31999999999999},"last":1.48},{"interface":"front","average":{"1min":15.843,"5min":4.9740000000000002,"15min":2.0449999999999999},"min":{"1min":0.377,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":197.536,"5min":197.536,"15min":197.536},"last":0.97399999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":9.7479999999999993,"15min":4.2350000000000003},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":61.372,"5min":262.524,"15min":262.524},"last":1.071},{"interface":"front","average":{"1min":5.1210000000000004,"5min":10.284000000000001,"15min":4.4560000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":60.085000000000001,"5min":262.55099999999999,"15min":262.55099999999999},"last":3.6419999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":6.5149999999999997,"5min":3.2080000000000002,"15min":1.7210000000000001},"min":{"1min":0.27100000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":107.476,"5min":107.476,"15min":107.476},"last":3.706},{"interface":"front","average":{"1min":6.6040000000000001,"5min":3.6659999999999999,"15min":1.847},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":107.414,"5min":107.414,"15min":107.414},"last":1.9750000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:35:32 2023","interfaces":[{"interface":"back","average":{"1min":7.173,"5min":2.6619999999999999,"15min":1.7609999999999999},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":65.75,"5min":65.75,"15min":65.75},"last":3.7309999999999999},{"interface":"front","average":{"1min":6.5510000000000002,"5min":2.5409999999999999,"15min":1.708},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":65.849999999999994,"5min":65.849999999999994,"15min":65.849999999999994},"last":1.6559999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:35:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4750000000000001,"5min":2.5640000000000001,"15min":1.573},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.8179999999999996,"5min":76.283000000000001,"15min":76.283000000000001},"last":2.1829999999999998},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.492,"15min":1.5860000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.061,"5min":76.259,"15min":76.259},"last":4.2320000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757620,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2695304,"kb_used_data":1804736,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":91049848,"statfs":{"total":95995035648,"available":93235044352,"internally_reserved":0,"allocated":1848049664,"data_stored":1847542576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:35:26 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":5.4379999999999997,"15min":3.1709999999999998},"min":{"1min":0.26900000000000002,"5min":0.22,"15min":0.22},"max":{"1min":37.759,"5min":127.108,"15min":127.108},"last":0.435},{"interface":"front","average":{"1min":5.6840000000000002,"5min":5.0659999999999998,"15min":2.8740000000000001},"min":{"1min":0.249,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":37.820999999999998,"5min":128.036,"15min":128.036},"last":0.99199999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:35:26 2023","interfaces":[{"interface":"back","average":{"1min":3.4300000000000002,"5min":4.7350000000000003,"15min":3.0299999999999998},"min":{"1min":0.32300000000000001,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":38.015999999999998,"5min":109.95699999999999,"15min":109.95699999999999},"last":1.7230000000000001},{"interface":"front","average":{"1min":4.673,"5min":5.569,"15min":3.9129999999999998},"min":{"1min":0.35399999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":38.045999999999999,"5min":110.542,"15min":116.04000000000001},"last":0.49099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:34:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":7.6879999999999997,"15min":2.9460000000000002},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":21.716000000000001,"5min":324.55000000000001,"15min":324.55000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.2639999999999998,"5min":7.2969999999999997,"15min":2.8170000000000002},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":21.704000000000001,"5min":323.94499999999999,"15min":323.94499999999999},"last":0.85099999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:35:14 2023","interfaces":[{"interface":"back","average":{"1min":3.472,"5min":8.4100000000000001,"15min":3.5099999999999998},"min":{"1min":0.41399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":18.940000000000001,"5min":259.053,"15min":259.053},"last":1.546},{"interface":"front","average":{"1min":2.7429999999999999,"5min":8.1310000000000002,"15min":3.46},"min":{"1min":0.379,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":18.803999999999998,"5min":259.108,"15min":259.108},"last":0.92000000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:35:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":0.98499999999999999,"15min":0.99399999999999999},"min":{"1min":0.28999999999999998,"5min":0.248,"15min":0.248},"max":{"1min":5.9340000000000002,"5min":7.2699999999999996,"15min":9.641},"last":1.6919999999999999},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.548,"15min":1.1659999999999999},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.245},"max":{"1min":4.8639999999999999,"5min":10.645,"15min":10.645},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:35:31 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":1.6060000000000001,"15min":1.2190000000000001},"min":{"1min":0.38800000000000001,"5min":0.317,"15min":0.29599999999999999},"max":{"1min":10.166,"5min":11.192,"15min":11.192},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.6759999999999999,"15min":1.2909999999999999},"min":{"1min":0.34799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.974,"5min":11.974,"15min":14.334},"last":0.36099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:34:58 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.649,"15min":1.292},"min":{"1min":0.34799999999999998,"5min":0.317,"15min":0.27400000000000002},"max":{"1min":5.1150000000000002,"5min":11.093,"15min":11.093},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.8939999999999999,"5min":1.5509999999999999,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.35999999999999999,"15min":0.33000000000000002},"max":{"1min":5.9640000000000004,"5min":9.8550000000000004,"15min":11.315},"last":15.257999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3548516352,"data_stored":3548512280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1087766528,"data_stored":1087766528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1725693952,"data_stored":1725693952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1808203776,"data_stored":1808203776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2944012288,"data_stored":2944008216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1830158336,"data_stored":1830158336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2575634432,"data_stored":2575630360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1471545344,"data_stored":1471545344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:36:01.169 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:36:01.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 36 2023-10-09T04:36:01.222 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:01.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:01.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:01.383 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:01.813 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:36:01.813 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:36:01.884 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:01.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:01.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:02.039 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:02 smithi029 ceph-mon[216903]: pgmap v796: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 31924/254892 objects degraded (12.525%) 2023-10-09T04:36:02.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:36:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:36:02] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T04:36:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:02 smithi029 ceph-mon[223858]: pgmap v796: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 31924/254892 objects degraded (12.525%) 2023-10-09T04:36:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:02 smithi177 ceph-mon[195797]: pgmap v796: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 31924/254892 objects degraded (12.525%) 2023-10-09T04:36:03.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:36:03.362 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:03.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:03.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:03.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:36:03.545 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:03.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:03.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:03.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:03.868 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:03.991 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:04.043 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:04.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:04.081 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:36:04.114 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:36:04.146 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:36:04.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:04.189 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:36:04.202 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:04.206 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:04.232 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:36:04.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:04.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:04.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:04.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:36:04.319 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:36:04.320 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.320 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.320 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:36:04.320 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.321 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.321 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.321 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.321 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:36:04.359 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:04.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:04.441 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:04.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:04 smithi177 ceph-mon[195797]: pgmap v797: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 31607/253041 objects degraded (12.491%) 2023-10-09T04:36:04.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:04.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:04.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1912396497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:04.549 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[216903]: pgmap v797: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 31607/253041 objects degraded (12.491%) 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1912396497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[223858]: pgmap v797: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 31607/253041 objects degraded (12.491%) 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:04.550 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1912396497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:04.563 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:04.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:04.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:04.708 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:05.524 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:05.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:05.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:05.717 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 31607/253041 objects degraded (12.491%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:05.718 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 31607/253041 objects degraded (12.491%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 31607/253041 objects degraded (12.491%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:05.780 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:05.869 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:36:05.870 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:36:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:06 smithi029 ceph-mon[216903]: pgmap v798: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 30980/244599 objects degraded (12.666%) 2023-10-09T04:36:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:06 smithi029 ceph-mon[223858]: pgmap v798: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 30980/244599 objects degraded (12.666%) 2023-10-09T04:36:06.785 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:06 smithi177 ceph-mon[195797]: pgmap v798: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 30980/244599 objects degraded (12.666%) 2023-10-09T04:36:07.151 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:07.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:07.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:07.313 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:07.396 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:07.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:07.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:07.551 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:07.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:36:08.074 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:08.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:08.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:08.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:08.234 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:08.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:08 smithi029 ceph-mon[216903]: pgmap v799: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 29705/237993 objects degraded (12.481%) 2023-10-09T04:36:08.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:08 smithi029 ceph-mon[223858]: pgmap v799: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 29705/237993 objects degraded (12.481%) 2023-10-09T04:36:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:08 smithi177 ceph-mon[195797]: pgmap v799: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 29705/237993 objects degraded (12.481%) 2023-10-09T04:36:09.328 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:09.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:09.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:09.515 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:09.670 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:09.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:09.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:09.850 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:09.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:09.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:10.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:10.146 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:10.221 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:10.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:10.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:10.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:10 smithi029 ceph-mon[216903]: pgmap v800: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 29705/237993 objects degraded (12.481%) 2023-10-09T04:36:10.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/109189146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T04:36:10.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 29705/237993 objects degraded (12.481%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:10.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:10.367 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:36:10.367 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:36:10.367 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:36:10.367 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:36:10.374 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 10 pg_num to 36 2023-10-09T04:36:10.438 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:10 smithi177 ceph-mon[195797]: pgmap v800: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 29705/237993 objects degraded (12.481%) 2023-10-09T04:36:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/109189146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T04:36:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 29705/237993 objects degraded (12.481%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:10 smithi029 ceph-mon[223858]: pgmap v800: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 29705/237993 objects degraded (12.481%) 2023-10-09T04:36:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/109189146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T04:36:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 29705/237993 objects degraded (12.481%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:10.743 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:10.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:10.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:10.876 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:11.357 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/109189146' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[223858]: osdmap e215: 8 total, 7 up, 8 in 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[223858]: failed to encode map e215 with expected crc 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/888577608' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/109189146' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[216903]: osdmap e215: 8 total, 7 up, 8 in 2023-10-09T04:36:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[216903]: failed to encode map e215 with expected crc 2023-10-09T04:36:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/888577608' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:36:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:36:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T04:36:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/109189146' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2023-10-09T04:36:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:11 smithi177 ceph-mon[195797]: osdmap e215: 8 total, 7 up, 8 in 2023-10-09T04:36:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:11 smithi177 ceph-mon[195797]: failed to encode map e215 with expected crc 2023-10-09T04:36:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/888577608' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:36:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:36:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T04:36:12.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: pgmap v802: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 28061/224412 objects degraded (12.504%) 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: osdmap e216: 8 total, 7 up, 8 in 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.606 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[223858]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: pgmap v802: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 28061/224412 objects degraded (12.504%) 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: osdmap e216: 8 total, 7 up, 8 in 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.607 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:12 smithi029 ceph-mon[216903]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: pgmap v802: 99 pgs: 12 active+undersized+degraded, 17 active+undersized, 70 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 28061/224412 objects degraded (12.504%) 2023-10-09T04:36:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: osdmap e216: 8 total, 7 up, 8 in 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:12 smithi177 ceph-mon[195797]: failed to encode map e216 with expected crc 2023-10-09T04:36:12.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:36:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:36:12] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-10-09T04:36:13.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:36:13.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: osdmap e217: 8 total, 7 up, 8 in 2023-10-09T04:36:13.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: 10.12 scrub starts 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: 10.16 scrub starts 2023-10-09T04:36:13.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[216903]: 10.12 scrub ok 2023-10-09T04:36:13.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: osdmap e217: 8 total, 7 up, 8 in 2023-10-09T04:36:13.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: 10.12 scrub starts 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: 10.16 scrub starts 2023-10-09T04:36:13.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:13 smithi029 ceph-mon[223858]: 10.12 scrub ok 2023-10-09T04:36:13.605 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:13.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:13.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:13.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:13.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: osdmap e217: 8 total, 7 up, 8 in 2023-10-09T04:36:13.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: failed to encode map e217 with expected crc 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: 10.12 scrub starts 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: 10.16 scrub starts 2023-10-09T04:36:13.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:13 smithi177 ceph-mon[195797]: 10.12 scrub ok 2023-10-09T04:36:13.870 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:14.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:14.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:14.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:14.240 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:14.306 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:14.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:14.415 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:14.441 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:14.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:14.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:14 smithi177 ceph-mon[195797]: pgmap v805: 109 pgs: 10 unknown, 1 peering, 12 active+undersized+degraded, 17 active+undersized, 69 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 27736/222447 objects degraded (12.469%) 2023-10-09T04:36:14.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:14.517 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:14 smithi177 ceph-mon[195797]: 10.6 scrub starts 2023-10-09T04:36:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:14.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:14.619 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:14.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:14 smithi029 ceph-mon[223858]: pgmap v805: 109 pgs: 10 unknown, 1 peering, 12 active+undersized+degraded, 17 active+undersized, 69 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 27736/222447 objects degraded (12.469%) 2023-10-09T04:36:14.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:14.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:14 smithi029 ceph-mon[223858]: 10.6 scrub starts 2023-10-09T04:36:14.646 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:14 smithi029 ceph-mon[216903]: pgmap v805: 109 pgs: 10 unknown, 1 peering, 12 active+undersized+degraded, 17 active+undersized, 69 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 27736/222447 objects degraded (12.469%) 2023-10-09T04:36:14.646 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:14.646 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:14 smithi029 ceph-mon[216903]: 10.6 scrub starts 2023-10-09T04:36:14.649 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:14.807 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:14.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:14.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:14.979 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:15.087 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:15.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:15.198 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:15.276 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:15.341 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:15.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:15.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:15.446 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:15.495 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:15 smithi177 ceph-mon[195797]: 10.2 scrub starts 2023-10-09T04:36:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:15 smithi177 ceph-mon[195797]: 10.c scrub starts 2023-10-09T04:36:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:15 smithi177 ceph-mon[195797]: 10.6 scrub ok 2023-10-09T04:36:15.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 27736/222447 objects degraded (12.469%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:15.641 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[216903]: 10.2 scrub starts 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[216903]: 10.c scrub starts 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[216903]: 10.6 scrub ok 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 27736/222447 objects degraded (12.469%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[223858]: 10.2 scrub starts 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[223858]: 10.c scrub starts 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[223858]: 10.6 scrub ok 2023-10-09T04:36:15.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 27736/222447 objects degraded (12.469%), 12 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:15.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:36:15.668 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:15.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:15.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:15.724 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:15.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:15.850 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:15.988 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:36:16.030 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:36:16.062 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:36:16.077 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:16.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:16.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:16.095 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:36:16.127 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:36:16.164 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:36:16.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:16.201 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.202 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:36:16.242 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:16.329 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:16.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:16.442 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:16.490 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:16.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:16 smithi177 ceph-mon[195797]: pgmap v806: 109 pgs: 3 unknown, 1 peering, 14 active+undersized+degraded, 17 active+undersized, 74 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s; 27199/215154 objects degraded (12.642%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:16.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:16 smithi177 ceph-mon[195797]: 10.16 scrub starts 2023-10-09T04:36:16.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:16 smithi177 ceph-mon[195797]: 10.22 scrub starts 2023-10-09T04:36:16.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:16 smithi177 ceph-mon[195797]: 10.16 scrub ok 2023-10-09T04:36:16.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/62594177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[223858]: pgmap v806: 109 pgs: 3 unknown, 1 peering, 14 active+undersized+degraded, 17 active+undersized, 74 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s; 27199/215154 objects degraded (12.642%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[223858]: 10.16 scrub starts 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[223858]: 10.22 scrub starts 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[223858]: 10.16 scrub ok 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/62594177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[216903]: pgmap v806: 109 pgs: 3 unknown, 1 peering, 14 active+undersized+degraded, 17 active+undersized, 74 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s; 27199/215154 objects degraded (12.642%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[216903]: 10.16 scrub starts 2023-10-09T04:36:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[216903]: 10.22 scrub starts 2023-10-09T04:36:16.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[216903]: 10.16 scrub ok 2023-10-09T04:36:16.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/62594177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:17.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:17 smithi029 ceph-mon[216903]: 10.7 scrub starts 2023-10-09T04:36:17.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:17 smithi029 ceph-mon[223858]: 10.7 scrub starts 2023-10-09T04:36:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:17 smithi177 ceph-mon[195797]: 10.7 scrub starts 2023-10-09T04:36:17.803 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:17.807 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:36:17.807 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:36:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:17.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:17.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:17.960 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:18.079 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:18.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:18.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:18.224 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[216903]: pgmap v807: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25984/209100 objects degraded (12.427%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[216903]: 10.7 scrub ok 2023-10-09T04:36:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[216903]: 10.2 scrub starts 2023-10-09T04:36:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[216903]: 10.2 scrub ok 2023-10-09T04:36:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[223858]: pgmap v807: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25984/209100 objects degraded (12.427%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[223858]: 10.7 scrub ok 2023-10-09T04:36:18.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[223858]: 10.2 scrub starts 2023-10-09T04:36:18.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:18 smithi029 ceph-mon[223858]: 10.2 scrub ok 2023-10-09T04:36:18.684 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T04:36:18.684 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:36:18.684 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-09T04:36:18.685 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:36:18.685 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:36:18.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-09T04:36:18.709 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:18 smithi177 ceph-mon[195797]: pgmap v807: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 25984/209100 objects degraded (12.427%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:18.709 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:18 smithi177 ceph-mon[195797]: 10.7 scrub ok 2023-10-09T04:36:18.709 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:18 smithi177 ceph-mon[195797]: 10.2 scrub starts 2023-10-09T04:36:18.709 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:18 smithi177 ceph-mon[195797]: 10.2 scrub ok 2023-10-09T04:36:19.316 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:19.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:19.366 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:19.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:19.465 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:19.531 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:19.561 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:19.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:19.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:19 smithi029 ceph-mon[216903]: 10.10 scrub starts 2023-10-09T04:36:19.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:19 smithi029 ceph-mon[223858]: 10.10 scrub starts 2023-10-09T04:36:19.662 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:19 smithi177 ceph-mon[195797]: 10.10 scrub starts 2023-10-09T04:36:19.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:19.728 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:19.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:36:19.893 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:19.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:20.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:20.055 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:20.236 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:20.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:20.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:20.425 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: pgmap v808: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 25984/209100 objects degraded (12.427%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: 10.10 scrub ok 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: 10.4 scrub starts 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 25984/209100 objects degraded (12.427%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:20.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: pgmap v808: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 25984/209100 objects degraded (12.427%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: 10.10 scrub ok 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: 10.4 scrub starts 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 25984/209100 objects degraded (12.427%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:20.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:20.907 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:36:20.907 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:36:20.989 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:20.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: pgmap v808: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 25984/209100 objects degraded (12.427%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: 10.10 scrub ok 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: 10.4 scrub starts 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 25984/209100 objects degraded (12.427%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:36:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:21.031 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:21.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:21.144 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:21 smithi029 ceph-mon[223858]: 10.4 scrub ok 2023-10-09T04:36:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:21 smithi029 ceph-mon[223858]: 10.22 scrub starts 2023-10-09T04:36:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:21 smithi029 ceph-mon[223858]: 10.22 scrub ok 2023-10-09T04:36:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:21 smithi029 ceph-mon[216903]: 10.4 scrub ok 2023-10-09T04:36:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:21 smithi029 ceph-mon[216903]: 10.22 scrub starts 2023-10-09T04:36:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:21 smithi029 ceph-mon[216903]: 10.22 scrub ok 2023-10-09T04:36:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:21 smithi177 ceph-mon[195797]: 10.4 scrub ok 2023-10-09T04:36:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:21 smithi177 ceph-mon[195797]: 10.22 scrub starts 2023-10-09T04:36:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:21 smithi177 ceph-mon[195797]: 10.22 scrub ok 2023-10-09T04:36:22.861 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:22.863 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:36:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:36:22] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T04:36:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:22.864 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:22 smithi029 ceph-mon[216903]: pgmap v809: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 23925/191931 objects degraded (12.465%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:22.864 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:22 smithi029 ceph-mon[216903]: 10.1c scrub starts 2023-10-09T04:36:22.865 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:22 smithi029 ceph-mon[223858]: pgmap v809: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 23925/191931 objects degraded (12.465%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:22.865 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:22 smithi029 ceph-mon[223858]: 10.1c scrub starts 2023-10-09T04:36:22.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:22.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:23.023 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:36:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:22 smithi177 ceph-mon[195797]: pgmap v809: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 23925/191931 objects degraded (12.465%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:22 smithi177 ceph-mon[195797]: 10.1c scrub starts 2023-10-09T04:36:23.045 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:23.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:23.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:23.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:23.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:23.328 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:23.429 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:36:23.429 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:36:23.429 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:36:23.473 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:23.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:23.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:23.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:23.634 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:23.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:23 smithi177 ceph-mon[195797]: 10.1c scrub ok 2023-10-09T04:36:23.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:23 smithi177 ceph-mon[195797]: 10.17 scrub starts 2023-10-09T04:36:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:23 smithi029 ceph-mon[216903]: 10.1c scrub ok 2023-10-09T04:36:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:23 smithi029 ceph-mon[216903]: 10.17 scrub starts 2023-10-09T04:36:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:23 smithi029 ceph-mon[223858]: 10.1c scrub ok 2023-10-09T04:36:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:23 smithi029 ceph-mon[223858]: 10.17 scrub starts 2023-10-09T04:36:24.409 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:24.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:24.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:24.562 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:24.635 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:36:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:24 smithi029 ceph-mon[216903]: pgmap v810: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23528/189519 objects degraded (12.415%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:24 smithi029 ceph-mon[216903]: 10.17 scrub ok 2023-10-09T04:36:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1269665860' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:24.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:24 smithi029 ceph-mon[223858]: pgmap v810: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23528/189519 objects degraded (12.415%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:24.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:24 smithi029 ceph-mon[223858]: 10.17 scrub ok 2023-10-09T04:36:24.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1269665860' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:24.935 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:24.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:24.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:24 smithi177 ceph-mon[195797]: pgmap v810: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23528/189519 objects degraded (12.415%); 0 B/s, 0 objects/s recovering 2023-10-09T04:36:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:24 smithi177 ceph-mon[195797]: 10.17 scrub ok 2023-10-09T04:36:25.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1269665860' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:25.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:25.094 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:25.704 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:25.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:25.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:25.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:25.870 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: 10.c scrub starts 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1269665860' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: osdmap e218: 8 total, 7 up, 8 in 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23528/189519 objects degraded (12.415%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: 10.c scrub starts 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1269665860' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: osdmap e218: 8 total, 7 up, 8 in 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: failed to encode map e218 with expected crc 2023-10-09T04:36:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23528/189519 objects degraded (12.415%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:25.950 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:25.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:25.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: 10.c scrub starts 2023-10-09T04:36:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:36:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1269665860' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: osdmap e218: 8 total, 7 up, 8 in 2023-10-09T04:36:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: failed to encode map e218 with expected crc 2023-10-09T04:36:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: failed to encode map e218 with expected crc 2023-10-09T04:36:26.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: failed to encode map e218 with expected crc 2023-10-09T04:36:26.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: failed to encode map e218 with expected crc 2023-10-09T04:36:26.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: failed to encode map e218 with expected crc 2023-10-09T04:36:26.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23528/189519 objects degraded (12.415%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:26.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:26.127 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:26.140 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:26.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:26.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:26.283 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:36:26.315 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:36:26.326 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:26.348 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:36:26.382 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:26.382 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:36:26.429 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:36:26.464 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:36:26.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:26.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:36:26.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:26.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:26 smithi177 ceph-mon[195797]: pgmap v812: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 22730/178980 objects degraded (12.700%) 2023-10-09T04:36:26.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:26 smithi177 ceph-mon[195797]: 10.c scrub ok 2023-10-09T04:36:26.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:26 smithi177 ceph-mon[195797]: 10.0 scrub starts 2023-10-09T04:36:26.639 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:26 smithi177 ceph-mon[195797]: 10.0 scrub ok 2023-10-09T04:36:26.640 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2044154765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:26.640 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:26.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[216903]: pgmap v812: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 22730/178980 objects degraded (12.700%) 2023-10-09T04:36:26.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[216903]: 10.c scrub ok 2023-10-09T04:36:26.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[216903]: 10.0 scrub starts 2023-10-09T04:36:26.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[216903]: 10.0 scrub ok 2023-10-09T04:36:26.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2044154765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[223858]: pgmap v812: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 22730/178980 objects degraded (12.700%) 2023-10-09T04:36:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[223858]: 10.c scrub ok 2023-10-09T04:36:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[223858]: 10.0 scrub starts 2023-10-09T04:36:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[223858]: 10.0 scrub ok 2023-10-09T04:36:26.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2044154765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:27.319 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:27.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:27.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:27.469 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:27.540 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:27.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:27.644 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:27.670 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:27.711 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:27.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:27.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:27.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:27.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:36:27.887 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:27.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:28.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:28.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:28.153 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:28.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:28.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:28.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:28.339 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:28.738 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:28.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:28.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:28.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:36:28.876 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:36:28.898 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:28 smithi177 ceph-mon[195797]: pgmap v813: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21196/170877 objects degraded (12.404%) 2023-10-09T04:36:28.898 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:28.898 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:28.900 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:28 smithi029 ceph-mon[216903]: pgmap v813: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21196/170877 objects degraded (12.404%) 2023-10-09T04:36:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:28 smithi029 ceph-mon[223858]: pgmap v813: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 21196/170877 objects degraded (12.404%) 2023-10-09T04:36:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:28.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:28.984 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:29.006 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:29.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:29.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:29.164 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:29.253 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:29.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:29.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216847 8.a 8:5fc75d23:::benchmark_data_smithi029_252925_object100223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-10-09T04:36:29.350 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 563.33149242599995, 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.090995154, 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:36:29.351 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216847 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014630+0000", 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014633+0000", 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7219999999999999e-06 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014641+0000", 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3270000000000001e-06 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.014646+0000", 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9479999999999999e-06 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019056+0000", 2023-10-09T04:36:29.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044100279999999999 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019186+0000", 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130605 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.019311+0000", 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000124444 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.028477+0000", 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091663030000000006 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.031031+0000", 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025535000000000002 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.041136+0000", 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010105124 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.041146+0000", 2023-10-09T04:36:29.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0193e-05 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.105547+0000", 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0644010289999999 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.105625+0000", 2023-10-09T04:36:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8319999999999996e-05 2023-10-09T04:36:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.359 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.34629.0:216860 8.1 8:8d7bc4ee:::benchmark_data_smithi029_252925_object100236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-10-09T04:36:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:27:06.036509+0000", 2023-10-09T04:36:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 563.30961407799998, 2023-10-09T04:36:29.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0727470320000001, 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.34629", 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3609583692", 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 216860 2023-10-09T04:36:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036509+0000", 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036509+0000", 2023-10-09T04:36:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036507+0000", 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036515+0000", 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3799999999999994e-06 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036532+0000", 2023-10-09T04:36:29.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6906e-05 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.036540+0000", 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4490000000000002e-06 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.046000+0000", 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094603259999999998 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:06.046015+0000", 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4802e-05 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.083499+0000", 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0374842280000001 2023-10-09T04:36:29.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.083617+0000", 2023-10-09T04:36:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011813 2023-10-09T04:36:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0", 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.083749+0000", 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013148000000000001 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.102159+0000", 2023-10-09T04:36:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018410037000000001 2023-10-09T04:36:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.109121+0000", 2023-10-09T04:36:29.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069619340000000004 2023-10-09T04:36:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.109171+0000", 2023-10-09T04:36:29.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0247999999999997e-05 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:27:07.109256+0000", 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4887000000000002e-05 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.371 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38515 9.5 9:a1f72eec:::benchmark_data_smithi029_299184_object38514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020059+0000", 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.326063347, 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.199031669, 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.372 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38515 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020059+0000", 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020059+0000", 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020057+0000", 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020093+0000", 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5815000000000002e-05 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020111+0000", 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7827999999999999e-05 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020123+0000", 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1863e-05 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020175+0000", 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1601000000000003e-05 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020186+0000", 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1617000000000001e-05 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.206129+0000", 2023-10-09T04:36:29.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1859429779999999 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.207366+0000", 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012366600000000001 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.207499+0000", 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013340100000000001 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212549+0000", 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050496259999999998 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212591+0000", 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1692e-05 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212644+0000", 2023-10-09T04:36:29.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3046e-05 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217457+0000", 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048135089999999997 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217527+0000", 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9851000000000006e-05 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219091+0000", 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015640490000000001 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38517 9.5 9:a2f1fa15:::benchmark_data_smithi029_299184_object38516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020134+0000", 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.32598904000002, 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2006594580000001, 2023-10-09T04:36:29.380 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38517 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020134+0000", 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020134+0000", 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020133+0000", 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020153+0000", 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0139000000000001e-05 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020159+0000", 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6080000000000003e-06 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020165+0000", 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6210000000000001e-06 2023-10-09T04:36:29.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020214+0000", 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9135999999999998e-05 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020221+0000", 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1980000000000004e-06 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210594+0000", 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1903735600000001 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210757+0000", 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000162552 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.210947+0000", 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190216 2023-10-09T04:36:29.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212633+0000", 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001685913 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.212764+0000", 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013039099999999999 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.217187+0000", 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044233149999999997 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219145+0000", 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019578719999999998 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.219205+0000", 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9941000000000001e-05 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220793+0000", 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015884250000000001 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.388 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38536 9.5 9:ad9b3f04:::benchmark_data_smithi029_299184_object38535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.32595445700002, 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2106301479999999, 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38536 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020168+0000", 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020185+0000", 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7113999999999999e-05 2023-10-09T04:36:29.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020189+0000", 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.245e-06 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020193+0000", 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3909999999999998e-06 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020248+0000", 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5114000000000001e-05 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.020256+0000", 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4859999999999997e-06 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.214300+0000", 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1940435709999999 2023-10-09T04:36:29.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.214417+0000", 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011676900000000001 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.214586+0000", 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000169545 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216587+0000", 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020009870000000001 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.225062+0000", 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084753099999999998 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.225143+0000", 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0411e-05 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.229153+0000", 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040097960000000004 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.229213+0000", 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0350000000000003e-05 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230798+0000", 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015853530000000001 2023-10-09T04:36:29.397 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:138758 10.12 10:4b286a4b:::benchmark_data_smithi029_344401_object30387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:06.409892+0000", 2023-10-09T04:36:29.398 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 22.936230952999999, 2023-10-09T04:36:29.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.128170585, 2023-10-09T04:36:29.399 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.399 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.400 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138758 2023-10-09T04:36:29.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.400 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409892+0000", 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409892+0000", 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409890+0000", 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409896+0000", 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0630000000000002e-06 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409907+0000", 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0579e-05 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409913+0000", 2023-10-09T04:36:29.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.409961+0000", 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7797000000000003e-05 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.410051+0000", 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0414999999999994e-05 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.410117+0000", 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6395000000000004e-05 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.411823+0000", 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017059530000000001 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.412160+0000", 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033707600000000001 2023-10-09T04:36:29.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.537961+0000", 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12580068899999999 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538023+0000", 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2373999999999996e-05 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538062+0000", 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8896000000000002e-05 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:138769 10.2 10:45fcf1af:::benchmark_data_smithi029_344401_object30398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:06.538301+0000", 2023-10-09T04:36:29.407 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 22.807822028, 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.120471987, 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138769 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.408 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538301+0000", 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538301+0000", 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538300+0000", 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538305+0000", 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6450000000000002e-06 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538316+0000", 2023-10-09T04:36:29.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0380999999999999e-05 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.538322+0000", 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1630000000000001e-06 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.648739+0000", 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.110417663 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.648881+0000", 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014155799999999999 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.648971+0000", 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0288e-05 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.650989+0000", 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020180799999999998 2023-10-09T04:36:29.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.651307+0000", 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031764299999999999 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.658657+0000", 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073498440000000003 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.658710+0000", 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3046e-05 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:06.658773+0000", 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2811000000000006e-05 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:143832 10.b e215/190 10:d980bc3b:::benchmark_data_smithi029_344401_object35461:head v 215'8780, mlcod=215'8775)", 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:11.347546+0000", 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.998576296, 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22330248699999999, 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347546+0000", 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347546+0000", 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347545+0000", 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347553+0000", 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8199999999999997e-06 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347564+0000", 2023-10-09T04:36:29.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1439e-05 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347572+0000", 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8849999999999999e-06 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347613+0000", 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0722000000000001e-05 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347632+0000", 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8508e-05 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.570849+0000", 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22321730100000001 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:143840 10.12 10:4acbb7c5:::benchmark_data_smithi029_344401_object35469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:11.347554+0000", 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.998568348999999, 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.28889141400000001, 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 143840 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.421 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347554+0000", 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347554+0000", 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347553+0000", 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347559+0000", 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1319999999999999e-06 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347570+0000", 2023-10-09T04:36:29.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1141000000000001e-05 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347577+0000", 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347624+0000", 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7067000000000002e-05 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347715+0000", 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0844e-05 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.348048+0000", 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033330300000000002 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.349685+0000", 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016366729999999999 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417342+0000", 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067657439 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.636345+0000", 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21900275899999999 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.636407+0000", 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1891999999999996e-05 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.636446+0000", 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8884000000000002e-05 2023-10-09T04:36:29.427 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:143838 10.b e215/190)", 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:11.417112+0000", 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.929010315999999, 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15369728399999999, 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417112+0000", 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417112+0000", 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417111+0000", 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417123+0000", 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2103000000000001e-05 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417134+0000", 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1003e-05 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417139+0000", 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1560000000000003e-06 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.570793+0000", 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15365329699999999 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.570810+0000", 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7028e-05 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:143848 10.12 10:4e7cd2a0:::benchmark_data_smithi029_344401_object35477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:11.470338+0000", 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.875784236000001, 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.176258954, 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 143848 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470338+0000", 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470338+0000", 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470335+0000", 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999967 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470350+0000", 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4639e-05 2023-10-09T04:36:29.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470378+0000", 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7990000000000001e-05 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470394+0000", 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6660999999999999e-05 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470469+0000", 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4350000000000005e-05 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.470627+0000", 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015864199999999999 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.471089+0000", 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046105899999999999 2023-10-09T04:36:29.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.514814+0000", 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043725845999999999 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.635001+0000", 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.120186502 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.646529+0000", 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011527762 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.646567+0000", 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8566000000000003e-05 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.646597+0000", 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0219999999999999e-05 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.439 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:144335 10.12 10:4de8e1fe:::benchmark_data_smithi029_344401_object35964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:12.738967+0000", 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 16.607155685999999, 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22950683599999999, 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.440 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144335 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.738967+0000", 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.738967+0000", 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.738965+0000", 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.738973+0000", 2023-10-09T04:36:29.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5630000000000003e-06 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.738986+0000", 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3086e-05 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.738992+0000", 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8529999999999997e-06 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.746475+0000", 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074834369999999999 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.746604+0000", 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012833300000000001 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.746968+0000", 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036448200000000001 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.749324+0000", 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023554880000000002 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.756761+0000", 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074372969999999998 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968386+0000", 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.211625432 2023-10-09T04:36:29.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968428+0000", 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1958e-05 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968474+0000", 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5528999999999997e-05 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:144361 10.12 10:48968ae1:::benchmark_data_smithi029_344401_object35990:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:12.751149+0000", 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 16.594973752000001, 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21746884599999999, 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144361 2023-10-09T04:36:29.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751149+0000", 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751149+0000", 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751148+0000", 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751153+0000", 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.587e-06 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751167+0000", 2023-10-09T04:36:29.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3654e-05 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751173+0000", 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4350000000000002e-06 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751224+0000", 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1028e-05 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751372+0000", 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014745000000000001 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.751797+0000", 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00042515899999999999 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.753617+0000", 2023-10-09T04:36:29.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001819661 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.756801+0000", 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003183913 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968538+0000", 2023-10-09T04:36:29.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21173767399999999 2023-10-09T04:36:29.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968577+0000", 2023-10-09T04:36:29.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8702999999999999e-05 2023-10-09T04:36:29.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968618+0000", 2023-10-09T04:36:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0864e-05 2023-10-09T04:36:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:144379 10.12 10:4b60e6d8:::benchmark_data_smithi029_344401_object36008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:12.771394+0000", 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 16.574729140999999, 2023-10-09T04:36:29.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.197362077, 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144379 2023-10-09T04:36:29.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.458 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771394+0000", 2023-10-09T04:36:29.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771394+0000", 2023-10-09T04:36:29.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771392+0000", 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771399+0000", 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6610000000000003e-06 2023-10-09T04:36:29.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771413+0000", 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4803999999999999e-05 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771422+0000", 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8619999999999992e-06 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771476+0000", 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3965000000000003e-05 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771582+0000", 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105447 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.771900+0000", 2023-10-09T04:36:29.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031807999999999999 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.773310+0000", 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00140974 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.774758+0000", 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014482290000000001 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968670+0000", 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19391255800000001 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968718+0000", 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7296000000000003e-05 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:12.968756+0000", 2023-10-09T04:36:29.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7966999999999997e-05 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:146459 10.12 10:4bff3c90:::benchmark_data_smithi029_344401_object38088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:14.516443+0000", 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 14.829679357, 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.233737273, 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.465 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 146459 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516443+0000", 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516443+0000", 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516442+0000", 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516450+0000", 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1689999999999992e-06 2023-10-09T04:36:29.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516463+0000", 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2505e-05 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516470+0000", 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6090000000000003e-06 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516508+0000", 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7784999999999998e-05 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516584+0000", 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5694999999999999e-05 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.516814+0000", 2023-10-09T04:36:29.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000230335 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.518339+0000", 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015252250000000001 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.518514+0000", 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017436900000000001 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.750076+0000", 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23156246599999999 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.750131+0000", 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4622e-05 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.750181+0000", 2023-10-09T04:36:29.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9743000000000001e-05 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:146463 10.12 10:4b088091:::benchmark_data_smithi029_344401_object38092:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:14.519168+0000", 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 14.826954967000001, 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.236492067, 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.472 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 146463 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519168+0000", 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519168+0000", 2023-10-09T04:36:29.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519167+0000", 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519173+0000", 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0839999999999997e-06 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519197+0000", 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3818000000000001e-05 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519206+0000", 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9250000000000001e-06 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519315+0000", 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109255 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745605+0000", 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22629059400000001 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.746105+0000", 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049923199999999995 2023-10-09T04:36:29.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755525+0000", 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0094206340000000006 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755567+0000", 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1675000000000003e-05 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755594+0000", 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7214999999999999e-05 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755630+0000", 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6115000000000002e-05 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755660+0000", 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9492000000000002e-05 2023-10-09T04:36:29.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:146457 10.b e217/216 10:d7cb820f:::benchmark_data_smithi029_344401_object38086:head v 217'8883, mlcod=217'8881)", 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:14.519185+0000", 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 14.826937684000001, 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.226508456, 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519185+0000", 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519185+0000", 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519185+0000", 2023-10-09T04:36:29.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519188+0000", 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8380000000000004e-06 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519194+0000", 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0399999999999998e-06 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519199+0000", 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1420000000000003e-06 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519233+0000", 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4755000000000002e-05 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.519247+0000", 2023-10-09T04:36:29.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3417999999999999e-05 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745663+0000", 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22641597099999999 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745693+0000", 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0701000000000003e-05 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.483 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:146466 10.1b e217/216 10:d9ae40c9:::benchmark_data_smithi029_344401_object38095:head v 217'8869, mlcod=217'8867)", 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:14.521650+0000", 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 14.824472941, 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23553228000000001, 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.521650+0000", 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.521650+0000", 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.521649+0000", 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.521656+0000", 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4039999999999998e-06 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745572+0000", 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22391612899999999 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745581+0000", 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.242e-06 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745720+0000", 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139312 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745743+0000", 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3334000000000001e-05 2023-10-09T04:36:29.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.757148+0000", 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011405258999999999 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.757182+0000", 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3544000000000003e-05 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.488 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:146470 10.22 10:4462109b:::benchmark_data_smithi029_344401_object38099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:14.524468+0000", 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 14.821654864999999, 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23105023999999999, 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 146470 2023-10-09T04:36:29.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.524468+0000", 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.524468+0000", 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.524466+0000", 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.524473+0000", 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5420000000000002e-06 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745531+0000", 2023-10-09T04:36:29.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.221058383 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745550+0000", 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8749e-05 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745610+0000", 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9651999999999997e-05 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.745726+0000", 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011597 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.746000+0000", 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00027376800000000003 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.752966+0000", 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069665040000000001 2023-10-09T04:36:29.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755442+0000", 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002476055 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755449+0000", 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1069999999999999e-06 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755488+0000", 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8532000000000003e-05 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:14.755518+0000", 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0304e-05 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:162910 10.12 10:4c49ae6a:::benchmark_data_smithi029_344401_object54539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:27.529839+0000", 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.8162838990000001, 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19700272199999999, 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 162910 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:29.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529839+0000", 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529839+0000", 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529837+0000", 2023-10-09T04:36:29.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529845+0000", 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2609999999999993e-06 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529860+0000", 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4693e-05 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529868+0000", 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3650000000000004e-06 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.529914+0000", 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6010000000000002e-05 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.530006+0000", 2023-10-09T04:36:29.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1963e-05 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.530419+0000", 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041282200000000003 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.532269+0000", 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001849626 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.535981+0000", 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037123630000000002 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.726733+0000", 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19075219199999999 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:29.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.726795+0000", 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1557999999999996e-05 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:27.726842+0000", 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6548999999999998e-05 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:29.502 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:36:29.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:29.551 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:29.681 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:29.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:29.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:29.844 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:29.875 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-10-09T04:36:29.875 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-10-09T04:36:29.875 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-10-09T04:36:29.875 DEBUG:teuthology.orchestra.run.smithi029:osd.2> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-09T04:36:29.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl start ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T04:36:29.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:29.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:29.986 INFO:journalctl@ceph.osd.2.smithi029.stdout:-- Logs begin at Mon 2023-10-09 03:43:45 UTC. -- 2023-10-09T04:36:30.067 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:30.097 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:30.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:36:30.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:30 smithi029 ceph-mon[223858]: pgmap v814: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 21196/170877 objects degraded (12.404%) 2023-10-09T04:36:30.904 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 21196/170877 objects degraded (12.404%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:30.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:30 smithi029 ceph-mon[216903]: pgmap v814: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 21196/170877 objects degraded (12.404%) 2023-10-09T04:36:30.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 21196/170877 objects degraded (12.404%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:30.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:36:30.916 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:30.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:31.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:30 smithi177 ceph-mon[195797]: pgmap v814: 109 pgs: 1 active+clean+scrubbing, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 21196/170877 objects degraded (12.404%) 2023-10-09T04:36:31.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 21196/170877 objects degraded (12.404%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:31.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:31.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:31.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:31.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:31.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:31.375 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:31.840 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:31.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:31.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:31.945 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:32.012 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:32.152 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:32.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:32.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:32.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:32.304 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:32.405 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:32.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:32.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:32.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:32.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:32.538 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:32.580 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:32.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:32.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:32.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:32.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:32.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:32 smithi029 ceph-mon[216903]: pgmap v815: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 19296/154710 objects degraded (12.472%) 2023-10-09T04:36:32.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:32 smithi029 ceph-mon[216903]: 10.11 scrub starts 2023-10-09T04:36:32.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:32 smithi029 ceph-mon[216903]: 10.11 scrub ok 2023-10-09T04:36:32.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:32 smithi029 ceph-mon[223858]: pgmap v815: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 19296/154710 objects degraded (12.472%) 2023-10-09T04:36:32.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:32 smithi029 ceph-mon[223858]: 10.11 scrub starts 2023-10-09T04:36:32.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:32 smithi029 ceph-mon[223858]: 10.11 scrub ok 2023-10-09T04:36:32.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:32 smithi177 ceph-mon[195797]: pgmap v815: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 19296/154710 objects degraded (12.472%) 2023-10-09T04:36:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:32 smithi177 ceph-mon[195797]: 10.11 scrub starts 2023-10-09T04:36:32.854 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:32 smithi177 ceph-mon[195797]: 10.11 scrub ok 2023-10-09T04:36:32.856 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:32.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:32.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:33.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:33.111 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:33.150 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-09T04:36:33.151 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:32 smithi029 bash[372721]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-09T04:36:33.151 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:36:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:36:32] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T04:36:33.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:33.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:33.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:33.354 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:33.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:33.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:33.582 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:33.629 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:33.697 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:36:33.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3889217002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:33.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3889217002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:33.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3889217002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: pgmap v816: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 18904/152286 objects degraded (12.413%) 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: 10.14 deep-scrub starts 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3889217002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: osdmap e219: 8 total, 7 up, 8 in 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[216903]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: pgmap v816: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 18904/152286 objects degraded (12.413%) 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: 10.14 deep-scrub starts 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3889217002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: osdmap e219: 8 total, 7 up, 8 in 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: failed to encode map e219 with expected crc 2023-10-09T04:36:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:34 smithi029 ceph-mon[223858]: failed to encode map e219 with expected crc 2023-10-09T04:36:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: pgmap v816: 109 pgs: 15 active+undersized+degraded, 17 active+undersized, 77 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 18904/152286 objects degraded (12.413%) 2023-10-09T04:36:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: 10.14 deep-scrub starts 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3889217002' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: osdmap e219: 8 total, 7 up, 8 in 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: failed to encode map e219 with expected crc 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: failed to encode map e219 with expected crc 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: failed to encode map e219 with expected crc 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: failed to encode map e219 with expected crc 2023-10-09T04:36:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:34 smithi177 ceph-mon[195797]: failed to encode map e219 with expected crc 2023-10-09T04:36:35.479 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:35.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:35.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:35.669 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:35.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 18904/152286 objects degraded (12.413%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:35.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 18904/152286 objects degraded (12.413%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:35.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:35.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:35.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:35.926 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 18904/152286 objects degraded (12.413%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:36.601 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:36.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:36.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:36.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:36.739 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:36 smithi029 ceph-mon[216903]: pgmap v818: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 18131/141948 objects degraded (12.773%) 2023-10-09T04:36:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:36 smithi029 ceph-mon[223858]: pgmap v818: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 18131/141948 objects degraded (12.773%) 2023-10-09T04:36:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:36 smithi177 ceph-mon[195797]: pgmap v818: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 18131/141948 objects degraded (12.773%) 2023-10-09T04:36:37.251 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:37.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:37.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:37.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:37.425 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:37.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:37.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:37.603 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:37.661 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:37.732 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:37.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:37.806 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:36:37.806 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122457 10.14 e219/211 10:2c1d4078:::benchmark_data_smithi029_344401_object76225:head v 219'10201, mlcod=219'10184)", 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.787699+0000", 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.016578618, 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016612821999999999, 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787699+0000", 2023-10-09T04:36:37.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787699+0000", 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787699+0000", 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787702+0000", 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2890000000000002e-06 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787707+0000", 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.707e-06 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787710+0000", 2023-10-09T04:36:37.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.737e-06 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800695+0000", 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01298508 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800714+0000", 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8272000000000001e-05 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.810 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122458 10.14 e219/211 10:2c1f9484:::benchmark_data_smithi029_344401_object86550:head v 219'10202, mlcod=219'10184)", 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.788628+0000", 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.015649451000000002, 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015729894000000001, 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.788628+0000", 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.788628+0000", 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.788627+0000", 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:37.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.788632+0000", 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9359999999999998e-06 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.788651+0000", 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8315e-05 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.788658+0000", 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1199999999999996e-06 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802626+0000", 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013968292 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802646+0000", 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9539e-05 2023-10-09T04:36:37.814 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122461 10.14 e219/211)", 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.790715+0000", 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.013562092, 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013689700000000001, 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.790715+0000", 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.790715+0000", 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.790715+0000", 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.790721+0000", 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2960000000000004e-06 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.790735+0000", 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4611e-05 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.790742+0000", 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8990000000000004e-06 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122463 10.14 e219/211)", 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.793234+0000", 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.011043747, 2023-10-09T04:36:37.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011199369000000001, 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793234+0000", 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793234+0000", 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793233+0000", 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793238+0000", 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5090000000000001e-06 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793246+0000", 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7740000000000004e-06 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793251+0000", 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.113e-06 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122465 10.14 e219/211)", 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.796984+0000", 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0072930900000000003, 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074763720000000002, 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.796984+0000", 2023-10-09T04:36:37.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.796984+0000", 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.796983+0000", 2023-10-09T04:36:37.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.796989+0000", 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7620000000000001e-06 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797000+0000", 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0835e-05 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797006+0000", 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.212e-06 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122467 10.14 e219/211)", 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.797971+0000", 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0063064030000000004, 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065493699999999997, 2023-10-09T04:36:37.825 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797971+0000", 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797971+0000", 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797970+0000", 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797976+0000", 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8209999999999998e-06 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797989+0000", 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2858e-05 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.797996+0000", 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1729999999999998e-06 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.828 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122468 10.14 e219/211)", 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.800696+0000", 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.003581692, 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038824010000000002, 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800696+0000", 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.829 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800696+0000", 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800695+0000", 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800702+0000", 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.737e-06 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800713+0000", 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1474000000000001e-05 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.800721+0000", 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0949999999999996e-06 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.831 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122469 10.14 e219/211)", 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.802338+0000", 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.001938899, 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022810130000000001, 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802338+0000", 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802338+0000", 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802338+0000", 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:37.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802344+0000", 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2550000000000003e-06 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802358+0000", 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4209999999999999e-05 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.802368+0000", 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4769999999999992e-06 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:174570 10.17 e219/211 10:eff2b3f0:::benchmark_data_smithi029_344401_object66199:head v 219'9072, mlcod=219'9070)", 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.804504+0000", 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4294967295.999773, 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015444699999999999, 2023-10-09T04:36:37.835 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804504+0000", 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804504+0000", 2023-10-09T04:36:37.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804503+0000", 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804511+0000", 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0409999999999998e-06 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804527+0000", 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5410999999999999e-05 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804535+0000", 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8310000000000001e-06 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:37.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804583+0000", 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8687999999999999e-05 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.804605+0000", 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1818999999999999e-05 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122454 10.14 e219/211 10:2c172cbb:::benchmark_data_smithi029_344401_object97803:head v 219'10198, mlcod=219'10184)", 2023-10-09T04:36:37.839 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.787633+0000", 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.016643848999999999, 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017080234, 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787633+0000", 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787633+0000", 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787633+0000", 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787641+0000", 2023-10-09T04:36:37.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7709999999999995e-06 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787650+0000", 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5709999999999998e-06 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787658+0000", 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7919999999999993e-06 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793102+0000", 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054439220000000003 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.793130+0000", 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8067000000000001e-05 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.843 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122455 10.14 e219/211 10:2c196d02:::benchmark_data_smithi029_344401_object69465:head v 219'10199, mlcod=219'10184)", 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.787664+0000", 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.016613187000000001, 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017091948999999999, 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787664+0000", 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787664+0000", 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787664+0000", 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787667+0000", 2023-10-09T04:36:37.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4929999999999998e-06 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787672+0000", 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0350000000000001e-06 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787676+0000", 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9079999999999997e-06 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.795890+0000", 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082140719999999993 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.795913+0000", 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2560000000000001e-05 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.847 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(osd.4.0:122456 10.14 e219/211 10:2c1a1fe8:::benchmark_data_smithi029_344401_object77606:head v 219'10200, mlcod=219'10184)", 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:37.787680+0000", 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.016597499000000002, 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017114556, 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787680+0000", 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787680+0000", 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787680+0000", 2023-10-09T04:36:37.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787686+0000", 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.827e-06 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787691+0000", 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9649999999999999e-06 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.787695+0000", 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7900000000000001e-06 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.798507+0000", 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010811803 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:37.798530+0000", 2023-10-09T04:36:37.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2486000000000001e-05 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 12 2023-10-09T04:36:37.852 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:36:37.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:37.919 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:37.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:38.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:38.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:38.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:38.172 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:38.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:38.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:38.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:38.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:38.428 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:38.503 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:38.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:38.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:38.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:38.650 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:38.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:38.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:38.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:38.864 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:38.870 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:36:38.870 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:36:38.870 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:36:38.876 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:36:38.923 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:36:38.961 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:36:38.999 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:36:38.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:36:38.999 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:36:38.999 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-10-09T04:36:39.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:36:39.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:36:39.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:36:39.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:36:39.000 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:36:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:38 smithi029 ceph-mon[216903]: pgmap v819: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 16604/134025 objects degraded (12.389%) 2023-10-09T04:36:39.017 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/434457203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:39.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:38 smithi029 ceph-mon[223858]: pgmap v819: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 16604/134025 objects degraded (12.389%) 2023-10-09T04:36:39.017 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/434457203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:38 smithi177 ceph-mon[195797]: pgmap v819: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 16604/134025 objects degraded (12.389%) 2023-10-09T04:36:39.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/434457203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:39.347 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:39.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:39.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:39.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:39.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:36:39.513 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:39.592 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:39.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:39.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:39.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:39.728 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:39.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:39.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:39.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:39.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:39.902 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:39.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:39.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:40.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:40.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:36:40.124 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:40.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:40.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:40.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:40.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:40.390 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:40.461 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:36:40.462 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:36:40.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:40.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:40.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:36:40.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:40.620 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:40.706 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:40 smithi029 bash[373915]: 0979e032489c1c475f793a6a77b5e4861c0b48f6882252a7efb628bfeca4f05a 2023-10-09T04:36:40.706 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:40 smithi029 systemd[1]: Started Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:36:40.707 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:40 smithi177 ceph-mon[195797]: pgmap v820: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 16604/134025 objects degraded (12.389%) 2023-10-09T04:36:40.707 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 16604/134025 objects degraded (12.389%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:40.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:40.711 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:40.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:40.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:40.827 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:40.897 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:41.024 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:41.026 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:40 smithi029 ceph-mon[216903]: pgmap v820: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 16604/134025 objects degraded (12.389%) 2023-10-09T04:36:41.026 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 16604/134025 objects degraded (12.389%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:41.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:40 smithi029 ceph-mon[223858]: pgmap v820: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 16604/134025 objects degraded (12.389%) 2023-10-09T04:36:41.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 16604/134025 objects degraded (12.389%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:41.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:41.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:41.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:41.203 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:41.323 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:41 smithi029 conmon[374274]: 2023-10-09T04:36:41.119+0000 7faaebc07640 -1 Falling back to public interface 2023-10-09T04:36:41.335 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:41.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:41.442 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:41.516 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:41.707 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:41.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:41.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:41.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:41.874 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:41.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:42.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:42.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:42.106 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:42.149 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:42.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:36:42.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:42 smithi029 ceph-mon[216903]: pgmap v821: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 15178/122040 objects degraded (12.437%) 2023-10-09T04:36:42.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:42 smithi029 ceph-mon[216903]: 10.14 deep-scrub ok 2023-10-09T04:36:42.822 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:36:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:36:42] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T04:36:42.823 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:42 smithi029 ceph-mon[223858]: pgmap v821: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 15178/122040 objects degraded (12.437%) 2023-10-09T04:36:42.823 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:42 smithi029 ceph-mon[223858]: 10.14 deep-scrub ok 2023-10-09T04:36:43.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:42 smithi177 ceph-mon[195797]: pgmap v821: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 15178/122040 objects degraded (12.437%) 2023-10-09T04:36:43.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:42 smithi177 ceph-mon[195797]: 10.14 deep-scrub ok 2023-10-09T04:36:43.991 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:43.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:44.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:44.141 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:44.142 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:44.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:44.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:44.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:44.390 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:44.431 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:44.510 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:44 smithi177 ceph-mon[195797]: pgmap v822: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 14857/120063 objects degraded (12.374%) 2023-10-09T04:36:44.510 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:44.510 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:44 smithi029 ceph-mon[216903]: pgmap v822: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 14857/120063 objects degraded (12.374%) 2023-10-09T04:36:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:44 smithi029 ceph-mon[223858]: pgmap v822: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 14857/120063 objects degraded (12.374%) 2023-10-09T04:36:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:45.171 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:44 smithi029 conmon[374274]: 2023-10-09T04:36:44.527+0000 7faaebc07640 -1 osd.2 200 log_to_monitors true 2023-10-09T04:36:45.231 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:45.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:45.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:45.400 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:45.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:45.513 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:45 smithi029 ceph-mon[216903]: from='osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:45.513 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:45 smithi029 ceph-mon[216903]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:45.514 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 14857/120063 objects degraded (12.374%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:45.514 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:45 smithi029 ceph-mon[223858]: from='osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:45.514 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:45 smithi029 ceph-mon[223858]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:45.514 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 14857/120063 objects degraded (12.374%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:45.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:45.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:45.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:45.702 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:36:45.703 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:36:45.703 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:45 smithi177 ceph-mon[195797]: from='osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:45 smithi177 ceph-mon[195797]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-09T04:36:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 14857/120063 objects degraded (12.374%), 15 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:45.921 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 04:36:45 smithi029 conmon[374274]: 2023-10-09T04:36:45.546+0000 7faad906b700 -1 osd.2 200 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:36:46.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: pgmap v823: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 14540/114066 objects degraded (12.747%) 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: osdmap e220: 8 total, 7 up, 8 in 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: from='osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[216903]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: pgmap v823: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 14540/114066 objects degraded (12.747%) 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: osdmap e220: 8 total, 7 up, 8 in 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: from='osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: failed to encode map e220 with expected crc 2023-10-09T04:36:46.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:46 smithi029 ceph-mon[223858]: failed to encode map e220 with expected crc 2023-10-09T04:36:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: pgmap v823: 109 pgs: 1 active+clean+scrubbing+deep, 15 active+undersized+degraded, 17 active+undersized, 76 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 14540/114066 objects degraded (12.747%) 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: osdmap e220: 8 total, 7 up, 8 in 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: from='osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: failed to encode map e220 with expected crc 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: failed to encode map e220 with expected crc 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: failed to encode map e220 with expected crc 2023-10-09T04:36:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:46 smithi177 ceph-mon[195797]: failed to encode map e220 with expected crc 2023-10-09T04:36:47.753 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:47.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:47.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:36:47.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:36:47.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881] boot 2023-10-09T04:36:47.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T04:36:47.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:47.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[216903]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881] boot 2023-10-09T04:36:47.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:47 smithi029 ceph-mon[223858]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T04:36:47.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:47.886 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:36:47.886 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: osd.2 [v2:172.21.15.29:6818/2824881881,v1:172.21.15.29:6819/2824881881] boot 2023-10-09T04:36:47.886 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: osdmap e221: 8 total, 8 up, 8 in 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: failed to encode map e221 with expected crc 2023-10-09T04:36:47.887 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:47 smithi177 ceph-mon[195797]: osdmap e222: 8 total, 8 up, 8 in 2023-10-09T04:36:47.889 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:47.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:47.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:48.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:48.068 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:48.109 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:48.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:48.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:48.242 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:48.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:48.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:48.388 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:48.420 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:48.465 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:48.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:48.535 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:36:48.569 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:48.612 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: pgmap v826: 109 pgs: 6 peering, 15 active+undersized+degraded, 11 active+undersized, 77 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 13155/105144 objects degraded (12.511%) 2023-10-09T04:36:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1591196172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1591196172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:48.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[216903]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: pgmap v826: 109 pgs: 6 peering, 15 active+undersized+degraded, 11 active+undersized, 77 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 13155/105144 objects degraded (12.511%) 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: failed to encode map e222 with expected crc 2023-10-09T04:36:48.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1591196172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:48.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:48.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1591196172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:48.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:48 smithi029 ceph-mon[223858]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T04:36:48.676 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:48.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:48.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:48.837 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:49.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: pgmap v826: 109 pgs: 6 peering, 15 active+undersized+degraded, 11 active+undersized, 77 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 13155/105144 objects degraded (12.511%) 2023-10-09T04:36:49.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: failed to encode map e222 with expected crc 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1591196172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:36:49.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1591196172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:36:49.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:48 smithi177 ceph-mon[195797]: osdmap e223: 8 total, 8 up, 8 in 2023-10-09T04:36:49.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[216903]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[216903]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[216903]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[216903]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[223858]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[223858]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[223858]: failed to encode map e223 with expected crc 2023-10-09T04:36:49.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:49 smithi029 ceph-mon[223858]: failed to encode map e223 with expected crc 2023-10-09T04:36:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:49 smithi177 ceph-mon[195797]: failed to encode map e223 with expected crc 2023-10-09T04:36:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:49 smithi177 ceph-mon[195797]: failed to encode map e223 with expected crc 2023-10-09T04:36:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:49 smithi177 ceph-mon[195797]: failed to encode map e223 with expected crc 2023-10-09T04:36:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:49 smithi177 ceph-mon[195797]: failed to encode map e223 with expected crc 2023-10-09T04:36:50.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:50.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:50.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:50 smithi029 ceph-mon[216903]: pgmap v829: 109 pgs: 4 remapped+peering, 6 peering, 11 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 8630/103851 objects degraded (8.310%) 2023-10-09T04:36:50.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8630/103851 objects degraded (8.310%), 11 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:50.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:50 smithi029 ceph-mon[223858]: pgmap v829: 109 pgs: 4 remapped+peering, 6 peering, 11 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 8630/103851 objects degraded (8.310%) 2023-10-09T04:36:50.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8630/103851 objects degraded (8.310%), 11 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:50.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:50 smithi177 ceph-mon[195797]: pgmap v829: 109 pgs: 4 remapped+peering, 6 peering, 11 active+undersized+degraded, 10 active+undersized, 78 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 8630/103851 objects degraded (8.310%) 2023-10-09T04:36:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8630/103851 objects degraded (8.310%), 11 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T04:36:51.041 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:51.154 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:51.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:51.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:51.316 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:36:51.316 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:36:51.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.316 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:185850 10.e e223/222 10:75adcb40:::benchmark_data_smithi029_344401_object77479:head v 223'10157, mlcod=202'4700)", 2023-10-09T04:36:51.316 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:51.297101+0000", 2023-10-09T04:36:51.316 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0083476090000000006, 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083798080000000007, 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297101+0000", 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:51.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297101+0000", 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297100+0000", 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297109+0000", 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1530000000000003e-06 2023-10-09T04:36:51.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297124+0000", 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4796e-05 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297136+0000", 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1422e-05 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297188+0000", 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2371999999999999e-05 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.297212+0000", 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4358999999999998e-05 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:185849 10.5 e223/222 10:a050c28d:::benchmark_data_smithi029_344401_object77478:head v 223'9581, mlcod=202'4766)", 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:51.298418+0000", 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0070309830000000002, 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071354579999999999, 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298418+0000", 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:51.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298418+0000", 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298417+0000", 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:51.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298426+0000", 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0710000000000001e-06 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298438+0000", 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2016e-05 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298446+0000", 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1440000000000003e-06 2023-10-09T04:36:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298589+0000", 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014274900000000001 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.298619+0000", 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0249000000000001e-05 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:51.324 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:185852 10.13 e223/222 10:cdc74be0:::benchmark_data_smithi029_344401_object77481:head v 223'9606, mlcod=202'4687)", 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:51.305307+0000", 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.00014196200000000001, 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031114700000000001, 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305307+0000", 2023-10-09T04:36:51.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305307+0000", 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305306+0000", 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305314+0000", 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4120000000000004e-06 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305329+0000", 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5426000000000001e-05 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:51.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305338+0000", 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4849999999999992e-06 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305427+0000", 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9728999999999994e-05 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:51.305448+0000", 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1086000000000001e-05 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.328 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:51.329 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.329 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:51.329 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:36:51.329 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 3 2023-10-09T04:36:51.329 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:36:51.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:51.431 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:51.623 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:51.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:51.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:36:51.734 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:36:51.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:51.782 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:36:51.824 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:36:51.827 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:51.864 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:36:51.904 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:36:51.937 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:36:51.972 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:36:52.005 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:36:52.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:52 smithi029 ceph-mon[216903]: pgmap v830: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 6 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 20230/97167 objects degraded (20.820%); 468 KiB/s, 7 objects/s recovering 2023-10-09T04:36:52.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2585445694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:52.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:36:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:36:52] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2023-10-09T04:36:52.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:52 smithi029 ceph-mon[223858]: pgmap v830: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 6 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 20230/97167 objects degraded (20.820%); 468 KiB/s, 7 objects/s recovering 2023-10-09T04:36:52.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2585445694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:52.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:36:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:52 smithi177 ceph-mon[195797]: pgmap v830: 109 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 6 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 20230/97167 objects degraded (20.820%); 468 KiB/s, 7 objects/s recovering 2023-10-09T04:36:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2585445694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:36:53.035 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:53.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:53.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:53.204 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:54.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:36:54.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:54.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:54.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:54.380 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:54.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:54 smithi029 ceph-mon[216903]: pgmap v831: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 38425/94422 objects degraded (40.695%); 436 KiB/s, 6 objects/s recovering 2023-10-09T04:36:54.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:54 smithi029 ceph-mon[223858]: pgmap v831: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 38425/94422 objects degraded (40.695%); 436 KiB/s, 6 objects/s recovering 2023-10-09T04:36:54.719 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:54.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:54.826 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:54.899 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:55.008 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:55.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:54 smithi177 ceph-mon[195797]: pgmap v831: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 38425/94422 objects degraded (40.695%); 436 KiB/s, 6 objects/s recovering 2023-10-09T04:36:55.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:55.182 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:55.318 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:55.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:55.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:55.475 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:55.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38425/94422 objects degraded (40.695%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:36:55.828 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:55.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:55.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:55.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38425/94422 objects degraded (40.695%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:36:55.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:55.944 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:55.966 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38425/94422 objects degraded (40.695%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:36:55.968 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:56.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:56.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:56.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:56.141 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:36:56.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:36:56.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:36:56.224 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:56.288 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:36:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:56.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:36:56.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:56.485 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:56.742 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:36:56.742 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:36:56.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:36:56.742 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38520 9.8 9:108867f8:::benchmark_data_smithi029_299184_object38519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 426.70478231700002, 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1949972569999998, 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38520 2023-10-09T04:36:56.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033218+0000", 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033216+0000", 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033239+0000", 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2768999999999998e-05 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033248+0000", 2023-10-09T04:36:56.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6910000000000003e-06 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033254+0000", 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0399999999999998e-06 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033339+0000", 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5388000000000003e-05 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033348+0000", 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7469999999999994e-06 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022545+0000", 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98919705899999999 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022559+0000", 2023-10-09T04:36:56.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3551e-05 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216745+0000", 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.194186607 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216864+0000", 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011896300000000001 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.216980+0000", 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116215 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.222890+0000", 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059099820000000003 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.224040+0000", 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001149583 2023-10-09T04:36:56.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226281+0000", 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022405580000000001 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226344+0000", 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2959999999999994e-05 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.228215+0000", 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001871295 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:38523 9.8 9:186f9d6c:::benchmark_data_smithi029_299184_object38522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e153)", 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 426.70428893399998, 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1990103689999998, 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.751 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 38523 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033711+0000", 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033710+0000", 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033730+0000", 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0020000000000001e-05 2023-10-09T04:36:56.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033742+0000", 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2299e-05 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033748+0000", 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4079999999999997e-06 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033788+0000", 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0441000000000001e-05 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:50.033795+0000", 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1439999999999997e-06 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022741+0000", 2023-10-09T04:36:56.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98894592299999995 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:51.022751+0000", 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4960000000000002e-06 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.220959+0000", 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.198207902 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221125+0000", 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016651700000000001 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.221423+0000", 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000297767 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.224239+0000", 2023-10-09T04:36:56.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028161560000000002 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.226232+0000", 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019932280000000001 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230759+0000", 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045263370000000001 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.230821+0000", 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1879999999999997e-05 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:29:52.232721+0000", 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001900723 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112523 9.8 9:166f8be2:::benchmark_data_smithi029_299184_object14220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e166)", 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.62749506199998, 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.01694084, 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112523 2023-10-09T04:36:56.760 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110505+0000", 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110503+0000", 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110510+0000", 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4899999999999997e-06 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110525+0000", 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4864e-05 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110551+0000", 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5894e-05 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110637+0000", 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6575000000000004e-05 2023-10-09T04:36:56.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110776+0000", 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013886399999999999 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.110903+0000", 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012670300000000001 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.117848+0000", 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069447629999999996 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.117863+0000", 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5756999999999998e-05 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127354+0000", 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0094912250000001 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127446+0000", 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1144999999999996e-05 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112554 9.8 9:196c9f66:::benchmark_data_smithi029_299184_object14251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.58835637099997, 2023-10-09T04:36:56.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99418602, 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112554 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:36:56.767 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149643+0000", 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149642+0000", 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149649+0000", 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6830000000000008e-06 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149667+0000", 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7935999999999999e-05 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149676+0000", 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8149999999999995e-06 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149759+0000", 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3293000000000004e-05 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.149772+0000", 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.225e-05 2023-10-09T04:36:56.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127865+0000", 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.9780934 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.127992+0000", 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127248 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.771 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.128124+0000", 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013166 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137717+0000", 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095931829999999999 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137731+0000", 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3898e-05 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.772 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143735+0000", 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060035740000000002 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143787+0000", 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2207999999999999e-05 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.773 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143830+0000", 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2657000000000002e-05 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112566 9.8 9:1591ae10:::benchmark_data_smithi029_299184_object14263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.58229014, 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98828444299999996, 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.774 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112566 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.775 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155710+0000", 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155708+0000", 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155715+0000", 2023-10-09T04:36:56.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.737e-06 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155732+0000", 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6733000000000001e-05 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155740+0000", 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3159999999999997e-06 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155798+0000", 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8369000000000003e-05 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.155810+0000", 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1187e-05 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131524+0000", 2023-10-09T04:36:56.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97571455100000004 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131668+0000", 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000144115 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.131781+0000", 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011272 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137768+0000", 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059868439999999998 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137815+0000", 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7286000000000001e-05 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143926+0000", 2023-10-09T04:36:56.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061107280000000002 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143955+0000", 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9420999999999998e-05 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143994+0000", 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8894999999999999e-05 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.781 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112581 9.8 9:188eb5a8:::benchmark_data_smithi029_299184_object14278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.56516402, 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97130810899999998, 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.782 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112581 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172836+0000", 2023-10-09T04:36:56.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172834+0000", 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172841+0000", 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0890000000000002e-06 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172854+0000", 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2677000000000001e-05 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172863+0000", 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172924+0000", 2023-10-09T04:36:56.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1587999999999994e-05 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.172935+0000", 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0971e-05 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134673+0000", 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96173812199999997 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.786 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134756+0000", 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2958000000000001e-05 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.134831+0000", 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4935e-05 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137804+0000", 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029725469999999999 2023-10-09T04:36:56.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137823+0000", 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9485000000000001e-05 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144083+0000", 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062598230000000003 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144109+0000", 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5782000000000002e-05 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144144+0000", 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5181000000000001e-05 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.789 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112605 9.8 9:173f4bff:::benchmark_data_smithi029_299184_object14302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.53580050900001, 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95321568300000004, 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112605 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202199+0000", 2023-10-09T04:36:56.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202203+0000", 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0260000000000001e-06 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202215+0000", 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1698e-05 2023-10-09T04:36:56.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202222+0000", 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0870000000000001e-06 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202287+0000", 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5141999999999994e-05 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.202297+0000", 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0135e-05 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137839+0000", 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.935542184 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.137923+0000", 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.365e-05 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.138000+0000", 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6852000000000002e-05 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143877+0000", 2023-10-09T04:36:56.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058773489999999996 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.143886+0000", 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9919999999999996e-06 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155344+0000", 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011457606 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155380+0000", 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6705000000000002e-05 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155415+0000", 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4813e-05 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112609 9.8 9:1697f7cc:::benchmark_data_smithi029_299184_object14306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.53215068999998, 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94970479100000005, 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112609 2023-10-09T04:36:56.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205849+0000", 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205848+0000", 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205853+0000", 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2599999999999999e-06 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205861+0000", 2023-10-09T04:36:56.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5339999999999999e-06 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205867+0000", 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1129999999999997e-06 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205926+0000", 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8548000000000003e-05 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.205937+0000", 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1100999999999999e-05 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140841+0000", 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93490438799999998 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140924+0000", 2023-10-09T04:36:56.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2818000000000006e-05 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.140995+0000", 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0840000000000006e-05 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144049+0000", 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030538369999999998 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144062+0000", 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2706e-05 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155470+0000", 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011407907 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155514+0000", 2023-10-09T04:36:56.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4409999999999997e-05 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155554+0000", 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0036000000000001e-05 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112659 9.8 9:1ba9566d:::benchmark_data_smithi029_299184_object14356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:36:56.805 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.46682990099998, 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.89206917200000002, 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112659 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271170+0000", 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271169+0000", 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271174+0000", 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.592e-06 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271186+0000", 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.153e-05 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271194+0000", 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2179999999999999e-06 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271280+0000", 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6273999999999994e-05 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.271288+0000", 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7749999999999993e-06 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144186+0000", 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.87289803399999999 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144264+0000", 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8158000000000006e-05 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.144352+0000", 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7874000000000006e-05 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155299+0000", 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010946424999999999 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155630+0000", 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033138200000000002 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163143+0000", 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007512627 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163198+0000", 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5139999999999997e-05 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163239+0000", 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1260000000000001e-05 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112683 9.8 9:1e5d2014:::benchmark_data_smithi029_299184_object14380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.43565766199998, 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.868701897, 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.813 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112683 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302342+0000", 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302341+0000", 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302347+0000", 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5869999999999998e-06 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302361+0000", 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3838e-05 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302369+0000", 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2539999999999992e-06 2023-10-09T04:36:56.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302430+0000", 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0968999999999999e-05 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.302446+0000", 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5427e-05 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148056+0000", 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84561038799999999 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148181+0000", 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012532600000000001 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.148295+0000", 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113237 2023-10-09T04:36:56.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155432+0000", 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071375379999999997 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163283+0000", 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078503070000000008 2023-10-09T04:36:56.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170952+0000", 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076698299999999999 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170999+0000", 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6375000000000002e-05 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171044+0000", 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5340000000000003e-05 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112688 9.8 9:1bc999dd:::benchmark_data_smithi029_299184_object14385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.432860103, 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86601131600000003, 2023-10-09T04:36:56.821 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112688 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305140+0000", 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305139+0000", 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305143+0000", 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9339999999999999e-06 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305155+0000", 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1765e-05 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305161+0000", 2023-10-09T04:36:56.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3829999999999997e-06 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305229+0000", 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7902000000000001e-05 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.305244+0000", 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.464e-05 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151650+0000", 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84640602600000003 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151753+0000", 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103168 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.151867+0000", 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001144 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163313+0000", 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011445079 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163377+0000", 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4084000000000005e-05 2023-10-09T04:36:56.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171081+0000", 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077041899999999996 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171119+0000", 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8631000000000003e-05 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.171151+0000", 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1636e-05 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112697 9.8 9:12bbffcf:::benchmark_data_smithi029_299184_object14394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.41947883799997, 2023-10-09T04:36:56.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85940739799999999, 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112697 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:36:56.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318521+0000", 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318520+0000", 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318526+0000", 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7739999999999999e-06 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318544+0000", 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8156e-05 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318553+0000", 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8859999999999993e-06 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318616+0000", 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2434000000000006e-05 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.318627+0000", 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0893000000000001e-05 2023-10-09T04:36:56.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155677+0000", 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83705072999999997 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155793+0000", 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115912 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.155913+0000", 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012010000000000001 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163366+0000", 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074525160000000002 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163387+0000", 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0649000000000002e-05 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177849+0000", 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014462773999999999 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177892+0000", 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2336999999999998e-05 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177928+0000", 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6739000000000002e-05 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.838 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112698 9.8 9:1730e59e:::benchmark_data_smithi029_299184_object14395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.41880821400002, 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85884108599999998, 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112698 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319192+0000", 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319191+0000", 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319196+0000", 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8250000000000004e-06 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319210+0000", 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4318000000000001e-05 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319217+0000", 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0319999999999998e-06 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319272+0000", 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4486e-05 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.319283+0000", 2023-10-09T04:36:56.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1447000000000001e-05 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159355+0000", 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84007240500000002 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159465+0000", 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110021 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.159570+0000", 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000104728 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163437+0000", 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003866754 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.166866+0000", 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034293549999999998 2023-10-09T04:36:56.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177962+0000", 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011095997999999999 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.177995+0000", 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2712999999999997e-05 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178033+0000", 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7809999999999999e-05 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112742 9.8 9:1d544479:::benchmark_data_smithi029_299184_object14439:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.35489747899999, 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.806861312, 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.846 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112742 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383102+0000", 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383101+0000", 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383107+0000", 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6359999999999999e-06 2023-10-09T04:36:56.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383119+0000", 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2502e-05 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383126+0000", 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3629999999999999e-06 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383193+0000", 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7113999999999995e-05 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383204+0000", 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1276e-05 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163543+0000", 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78033866200000002 2023-10-09T04:36:56.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163658+0000", 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011532299999999999 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.163768+0000", 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110164 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.170921+0000", 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071524010000000001 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178095+0000", 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071740179999999999 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189883+0000", 2023-10-09T04:36:56.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011788378 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189927+0000", 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4149e-05 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189964+0000", 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6482000000000001e-05 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.853 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:112743 9.8 9:12e2e405:::benchmark_data_smithi029_299184_object14440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 333.354850554, 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.806888253, 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.854 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112743 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:36:56.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383149+0000", 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383152+0000", 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9610000000000001e-06 2023-10-09T04:36:56.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383159+0000", 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9990000000000002e-06 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383164+0000", 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8269999999999997e-06 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383278+0000", 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114821 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:23.383291+0000", 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2601e-05 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.166912+0000", 2023-10-09T04:36:56.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78362119900000005 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.167043+0000", 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013115800000000001 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.167159+0000", 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115868 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.178066+0000", 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010906701 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189514+0000", 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011448309 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.189971+0000", 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045658199999999997 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190001+0000", 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0080999999999999e-05 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:31:24.190038+0000", 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6575999999999997e-05 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166781 9.8 9:1ce571f8:::benchmark_data_smithi029_299184_object68478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:36:56.861 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 248.656667363, 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3412964679999999, 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166781 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081332+0000", 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:36:56.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081336+0000", 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5390000000000001e-06 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081346+0000", 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1819999999999995e-06 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081351+0000", 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7729999999999996e-06 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081392+0000", 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.121e-05 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081516+0000", 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012359900000000001 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081633+0000", 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011679399999999999 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.092967+0000", 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011334426 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106870+0000", 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013902978999999999 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422497+0000", 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3156270430000001 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422563+0000", 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5446999999999999e-05 2023-10-09T04:36:56.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422629+0000", 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6181999999999998e-05 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166784 9.8 9:1040dc07:::benchmark_data_smithi029_299184_object68481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:36:56.868 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.089057+0000", 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 248.64894251499999, 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.333752869, 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166784 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089057+0000", 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089057+0000", 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089056+0000", 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:56.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089062+0000", 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3899999999999998e-06 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089079+0000", 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6588000000000001e-05 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089087+0000", 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2099999999999993e-06 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089131+0000", 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3968999999999999e-05 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089249+0000", 2023-10-09T04:36:56.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011747300000000001 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089362+0000", 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113409 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093090+0000", 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037278599999999999 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106901+0000", 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013810835 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422722+0000", 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.315821562 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422766+0000", 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3812e-05 2023-10-09T04:36:56.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422810+0000", 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4121e-05 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166793 9.8 9:1d211ece:::benchmark_data_smithi029_299184_object68490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.102951+0000", 2023-10-09T04:36:56.875 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 248.635049313, 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.331321475, 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166793 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102951+0000", 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102951+0000", 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102949+0000", 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102957+0000", 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5089999999999996e-06 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102972+0000", 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5571000000000001e-05 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.102980+0000", 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7440000000000007e-06 2023-10-09T04:36:56.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.103061+0000", 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1113999999999996e-05 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.103210+0000", 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014861599999999999 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.103351+0000", 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000141543 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106931+0000", 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035802289999999999 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.106940+0000", 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3780000000000001e-06 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434173+0000", 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3272329950000001 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434229+0000", 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5723999999999997e-05 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434272+0000", 2023-10-09T04:36:56.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3520999999999999e-05 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166821 9.8 9:1a502545:::benchmark_data_smithi029_299184_object68518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.151353+0000", 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 248.586646976, 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2830994099999999, 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:36:56.882 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166821 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151353+0000", 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151353+0000", 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151352+0000", 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151357+0000", 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4159999999999997e-06 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151370+0000", 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.326e-05 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151378+0000", 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1380000000000007e-06 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151414+0000", 2023-10-09T04:36:56.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.557e-05 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151500+0000", 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6058999999999997e-05 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5", 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.151570+0000", 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0357000000000004e-05 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.154437+0000", 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028668280000000001 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.155093+0000", 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00065607000000000003 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434364+0000", 2023-10-09T04:36:56.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.279271429 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434411+0000", 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7063e-05 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.434452+0000", 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0942999999999997e-05 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.888 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:120135 10.5 10:a674fd63:::benchmark_data_smithi029_344401_object11764:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:35:47.592591+0000", 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 69.145409177000005, 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98066541699999998, 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:36:56.889 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 120135 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.592591+0000", 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.592591+0000", 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.592590+0000", 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.592596+0000", 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1929999999999998e-06 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.592608+0000", 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1952e-05 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.592616+0000", 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3569999999999998e-06 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.594583+0000", 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019662120000000002 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T04:36:56.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.594590+0000", 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.153e-06 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.912069+0000", 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.31747952000000002 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:47.912086+0000", 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6827999999999999e-05 2023-10-09T04:36:56.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:48.573193+0000", 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.66110734000000004 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:35:48.573256+0000", 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2747000000000001e-05 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:36:56.896 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:36:56.896 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:36:56.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:56 smithi029 ceph-mon[223858]: pgmap v832: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 38155/89583 objects degraded (42.592%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:36:56.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:56 smithi029 ceph-mon[216903]: pgmap v832: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 38155/89583 objects degraded (42.592%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:36:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:56.897 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:56.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:56.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:36:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:56 smithi177 ceph-mon[195797]: pgmap v832: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 38155/89583 objects degraded (42.592%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:36:57.043 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:57.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:36:57.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:57.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:36:57.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:57.329 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:57.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:57.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:57.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:57.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:36:57.589 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:57.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:36:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:57.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:36:57.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:36:57.812 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:58.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:58 smithi029 ceph-mon[216903]: pgmap v833: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 38155/83067 objects degraded (45.933%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:36:58.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:58.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:36:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:58 smithi029 ceph-mon[223858]: pgmap v833: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 38155/83067 objects degraded (45.933%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:36:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:58.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:36:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:58.704 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:36:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:58.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:36:58.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:58.869 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:58.980 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:58.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:58.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:58.988 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:58 smithi177 ceph-mon[195797]: pgmap v833: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 38155/83067 objects degraded (45.933%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:36:58.988 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:36:58.988 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:36:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:36:59.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:36:59.140 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:59.186 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:36:59.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:36:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:59.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:36:59.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:36:59.403 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:59.744 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:36:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:59.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:36:59.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:36:59.871 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:36:59.882 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:36:59.929 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:36:59.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:36:59.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:00.033 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:00.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:00.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:00.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:00.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:00.365 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:00.659 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:00.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:00.721 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:00.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:00.815 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:00.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:00.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:00 smithi177 ceph-mon[195797]: pgmap v834: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 38155/81426 objects degraded (46.858%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:37:00.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38155/81426 objects degraded (46.858%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:00.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:00.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:00 smithi029 ceph-mon[216903]: pgmap v834: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 38155/81426 objects degraded (46.858%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:37:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38155/81426 objects degraded (46.858%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:00.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:00 smithi029 ceph-mon[223858]: pgmap v834: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 38155/81426 objects degraded (46.858%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T04:37:00.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38155/81426 objects degraded (46.858%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:01.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:01.082 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:01.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:01.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:01.266 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:01.333 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:01.692 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:37:01.824 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:01.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:01.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2009705239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:01.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2009705239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:01.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:01.979 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2009705239' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:02.695 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:37:02.963 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: pgmap v835: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 37893/71955 objects degraded (52.662%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:37:02.963 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:02.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: failed to encode map e224 with expected crc 2023-10-09T04:37:02.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2009705239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:02.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T04:37:02.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: failed to encode map e224 with expected crc 2023-10-09T04:37:02.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1502354472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:37:02.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:37:03.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:37:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:37:02] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: pgmap v835: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 37893/71955 objects degraded (52.662%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: failed to encode map e224 with expected crc 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2009705239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: failed to encode map e224 with expected crc 2023-10-09T04:37:03.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1502354472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: pgmap v835: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 37893/71955 objects degraded (52.662%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: failed to encode map e224 with expected crc 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2009705239' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: osdmap e224: 8 total, 8 up, 8 in 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: failed to encode map e224 with expected crc 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1502354472' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:37:03.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:37:03.271 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:03.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:03.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:03.286 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:37:03.286 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:37:03.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.286 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:195709 10.e 10:737d9800:::benchmark_data_smithi029_344401_object87338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:03.274576+0000", 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0075008619999999996, 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0075333520000000001, 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 195709 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.287 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274576+0000", 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274576+0000", 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274575+0000", 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274580+0000", 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7489999999999998e-06 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274591+0000", 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1248e-05 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274596+0000", 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5349999999999998e-06 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:03.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274644+0000", 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8884000000000002e-05 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274752+0000", 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010717099999999999 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.275202+0000", 2023-10-09T04:37:03.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00045084900000000002 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277562+0000", 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002359398 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.281087+0000", 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00352469 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.281356+0000", 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026981599999999998 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.281430+0000", 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3757999999999994e-05 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:195711 10.e 10:72f4f9d7:::benchmark_data_smithi029_344401_object87340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:03.277647+0000", 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0044300980000000004, 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045838179999999999, 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 195711 2023-10-09T04:37:03.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277647+0000", 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277647+0000", 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277645+0000", 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277656+0000", 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0307e-05 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:03.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277673+0000", 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7581e-05 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277681+0000", 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5499999999999997e-06 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277815+0000", 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013455100000000001 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.277949+0000", 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000133895 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.278288+0000", 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033870200000000002 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:03.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.280553+0000", 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022649020000000001 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:195702 10.14 10:2f1f9c58:::benchmark_data_smithi029_344401_object87331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:03.261119+0000", 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.020957955, 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021193683000000001, 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:03.299 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 195702 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.261119+0000", 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.261119+0000", 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.261118+0000", 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.261121+0000", 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.227e-06 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.261126+0000", 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6540000000000004e-06 2023-10-09T04:37:03.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.261130+0000", 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8310000000000003e-06 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.271150+0000", 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010019742999999999 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.271276+0000", 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126451 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.271761+0000", 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00048457600000000002 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.274267+0000", 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025056179999999998 2023-10-09T04:37:03.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:03.275545+0000", 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001278324 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 3 2023-10-09T04:37:03.305 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:37:03.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:03.403 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:04.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:03 smithi177 ceph-mon[195797]: failed to encode map e225 with expected crc 2023-10-09T04:37:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:37:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:03 smithi177 ceph-mon[195797]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T04:37:04.128 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:04.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:03 smithi029 ceph-mon[216903]: failed to encode map e225 with expected crc 2023-10-09T04:37:04.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:37:04.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:03 smithi029 ceph-mon[216903]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T04:37:04.136 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:03 smithi029 ceph-mon[223858]: failed to encode map e225 with expected crc 2023-10-09T04:37:04.136 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:37:04.136 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:03 smithi029 ceph-mon[223858]: osdmap e225: 8 total, 8 up, 8 in 2023-10-09T04:37:04.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:04.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:04.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:04.304 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:04.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:37:04.934 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:04.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:04.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:04 smithi029 ceph-mon[216903]: pgmap v838: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 37893/69069 objects degraded (54.863%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:37:04.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:04 smithi029 ceph-mon[223858]: pgmap v838: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 37893/69069 objects degraded (54.863%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:37:04.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:04 smithi177 ceph-mon[195797]: pgmap v838: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 854 op/s; 37893/69069 objects degraded (54.863%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:37:05.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:05.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:37:05.128 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:05.348 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:05.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:05.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:05.504 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:05.507 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:05.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:05.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:05.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:37:05.659 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:37:05.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:05.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:37:05.751 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:37:05.756 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:05.801 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:37:05.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:37:05.880 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:05.880 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:05.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:05.897 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:37:05.945 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:37:05.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.954 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.955 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.955 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.955 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.955 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.955 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.955 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:37:05.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:06.055 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:06.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:06 smithi029 ceph-mon[216903]: pgmap v839: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 37522/65256 objects degraded (57.500%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T04:37:06.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:06 smithi029 ceph-mon[223858]: pgmap v839: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 37522/65256 objects degraded (57.500%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T04:37:06.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37893/69069 objects degraded (54.863%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:06.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1440130161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:06 smithi177 ceph-mon[195797]: pgmap v839: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 37522/65256 objects degraded (57.500%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T04:37:06.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37893/69069 objects degraded (54.863%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:06.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1440130161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:06.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37893/69069 objects degraded (54.863%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:06.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1440130161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:06.773 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:06.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:06.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:37:06.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:06.974 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:07.615 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:07.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:07.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:07.794 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:08.150 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:08.151 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:08.151 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:08.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:08.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:08.326 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:08 smithi029 ceph-mon[216903]: pgmap v840: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 37522/59997 objects degraded (62.540%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T04:37:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:08 smithi029 ceph-mon[223858]: pgmap v840: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 37522/59997 objects degraded (62.540%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T04:37:08.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:08.476 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:08 smithi177 ceph-mon[195797]: pgmap v840: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 37522/59997 objects degraded (62.540%); 4.9 MiB/s, 79 objects/s recovering 2023-10-09T04:37:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:08.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:08.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:08.621 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:08.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:08.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:08.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:08.911 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:09.056 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:09.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:09.095 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:09.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:09.230 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:09.424 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:09.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:09.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:09.580 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:09.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:37:09.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:37:09.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:09.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:09.830 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:09.913 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:10.002 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:10.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:10.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:10.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:10.191 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:10.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:10.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:10.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:10.309 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:10.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:10.429 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:10 smithi177 ceph-mon[195797]: pgmap v841: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 37522/58479 objects degraded (64.163%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:37:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37522/58479 objects degraded (64.163%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:10.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:10 smithi029 ceph-mon[216903]: pgmap v841: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 37522/58479 objects degraded (64.163%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:37:10.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37522/58479 objects degraded (64.163%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:10 smithi029 ceph-mon[223858]: pgmap v841: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 37522/58479 objects degraded (64.163%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:37:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37522/58479 objects degraded (64.163%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:11.110 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:11.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:11.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:11.270 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:11.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:11.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:11.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1366634310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:37:11.393 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:37:11.421 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:11.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1366634310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:37:11.558 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1366634310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:37:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[216903]: pgmap v842: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1022 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 37457/49107 objects degraded (76.276%); 2.9 MiB/s, 72 objects/s recovering 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1366634310' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[216903]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[216903]: failed to encode map e226 with expected crc 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[216903]: failed to encode map e226 with expected crc 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[223858]: pgmap v842: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1022 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 37457/49107 objects degraded (76.276%); 2.9 MiB/s, 72 objects/s recovering 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1366634310' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[223858]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[223858]: failed to encode map e226 with expected crc 2023-10-09T04:37:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:12 smithi029 ceph-mon[223858]: failed to encode map e226 with expected crc 2023-10-09T04:37:12.745 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:12 smithi177 ceph-mon[195797]: pgmap v842: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 1022 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 37457/49107 objects degraded (76.276%); 2.9 MiB/s, 72 objects/s recovering 2023-10-09T04:37:12.745 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1366634310' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:37:12.745 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:12 smithi177 ceph-mon[195797]: osdmap e226: 8 total, 8 up, 8 in 2023-10-09T04:37:12.745 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:12 smithi177 ceph-mon[195797]: failed to encode map e226 with expected crc 2023-10-09T04:37:12.745 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:12 smithi177 ceph-mon[195797]: failed to encode map e226 with expected crc 2023-10-09T04:37:13.018 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:13.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:13.080 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:13.080 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:13.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:13.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:37:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:37:12] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T04:37:13.200 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:13.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:37:13.336 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:13.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:13.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/188584802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:13.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:13 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:13.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:13.461 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/188584802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:13.462 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:13 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:13.463 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/188584802' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:13.464 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:13 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:13.476 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:37:13.561 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:13.846 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:13.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:13.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:13.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:13.980 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:14.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:14.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:14.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:14.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:14.201 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:14.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:14.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:14.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:14.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:37:14.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:14.502 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: pgmap v844: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 959 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 37457/46068 objects degraded (81.308%); 2.7 MiB/s, 67 objects/s recovering 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: failed to encode map e227 with expected crc 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: failed to encode map e227 with expected crc 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:14.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: pgmap v844: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 959 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 37457/46068 objects degraded (81.308%); 2.7 MiB/s, 67 objects/s recovering 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: failed to encode map e227 with expected crc 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: failed to encode map e227 with expected crc 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:14.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: pgmap v844: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 959 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 37457/46068 objects degraded (81.308%); 2.7 MiB/s, 67 objects/s recovering 2023-10-09T04:37:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:37:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: osdmap e227: 8 total, 8 up, 8 in 2023-10-09T04:37:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: failed to encode map e227 with expected crc 2023-10-09T04:37:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: failed to encode map e227 with expected crc 2023-10-09T04:37:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:14.907 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:15.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:15.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:15.414 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37457/46068 objects degraded (81.308%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:15.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:15.560 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37457/46068 objects degraded (81.308%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:15.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37457/46068 objects degraded (81.308%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:15.699 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:15.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:15.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:15.845 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:15.878 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:15.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:15.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:16.062 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:16.152 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:16.349 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:37:16.385 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:37:16.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:16 smithi029 ceph-mon[216903]: pgmap v846: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 875 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 37457/42027 objects degraded (89.126%); 520 KiB/s, 83 objects/s recovering 2023-10-09T04:37:16.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2627523642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:16.417 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:16 smithi029 ceph-mon[223858]: pgmap v846: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 875 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 37457/42027 objects degraded (89.126%); 520 KiB/s, 83 objects/s recovering 2023-10-09T04:37:16.417 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2627523642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:16.421 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:37:16.424 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:16.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:16.447 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:37:16.475 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:37:16.515 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:37:16.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:16.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:37:16.609 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:37:16.622 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:16.635 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.636 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.637 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:37:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:16 smithi177 ceph-mon[195797]: pgmap v846: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 875 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 37457/42027 objects degraded (89.126%); 520 KiB/s, 83 objects/s recovering 2023-10-09T04:37:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2627523642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:16.889 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:16.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:16.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:17.077 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:17.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:37:17.532 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:37:17.532 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:206585 10.6 10:64a0de0e:::benchmark_data_smithi029_344401_object98214:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:17.464702+0000", 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.067144480000000006, 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.067170137000000005, 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:17.533 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 206585 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464702+0000", 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464702+0000", 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464701+0000", 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464707+0000", 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6389999999999997e-06 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464719+0000", 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2840000000000001e-05 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464725+0000", 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6749999999999999e-06 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464765+0000", 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9487999999999999e-05 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464881+0000", 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011660299999999999 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.465442+0000", 2023-10-09T04:37:17.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00056049799999999999 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.468474+0000", 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030327029999999999 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.480122+0000", 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011647931 2023-10-09T04:37:17.538 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:17.539 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:17.539 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:17.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.539 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:206580 10.16 10:6fc2dbe9:::benchmark_data_smithi029_344401_object98209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e227)", 2023-10-09T04:37:17.539 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:17.460047+0000", 2023-10-09T04:37:17.540 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.071799663, 2023-10-09T04:37:17.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.071886241000000003, 2023-10-09T04:37:17.540 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:17.540 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:37:17.540 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:17.540 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:17.541 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:17.541 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 206580 2023-10-09T04:37:17.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.541 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:17.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:17.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460047+0000", 2023-10-09T04:37:17.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:17.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:17.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460047+0000", 2023-10-09T04:37:17.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:17.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:17.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460045+0000", 2023-10-09T04:37:17.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:17.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:17.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460052+0000", 2023-10-09T04:37:17.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2179999999999996e-06 2023-10-09T04:37:17.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:17.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460079+0000", 2023-10-09T04:37:17.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7518e-05 2023-10-09T04:37:17.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:17.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460098+0000", 2023-10-09T04:37:17.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.853e-05 2023-10-09T04:37:17.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:17.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460158+0000", 2023-10-09T04:37:17.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0587000000000001e-05 2023-10-09T04:37:17.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:17.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460299+0000", 2023-10-09T04:37:17.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014091799999999999 2023-10-09T04:37:17.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.460925+0000", 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062628499999999997 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.464544+0000", 2023-10-09T04:37:17.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036183119999999998 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:17.478832+0000", 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014287876 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:17.550 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:17.551 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:17.551 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:17.551 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:37:17.551 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 2 2023-10-09T04:37:17.551 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:37:18.428 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:18.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:18.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:18.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:18.598 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:18 smithi029 ceph-mon[216903]: pgmap v847: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 766 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 37457/36795 objects degraded (101.799%); 520 KiB/s, 83 objects/s recovering 2023-10-09T04:37:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:18 smithi029 ceph-mon[223858]: pgmap v847: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 766 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 37457/36795 objects degraded (101.799%); 520 KiB/s, 83 objects/s recovering 2023-10-09T04:37:18.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:18 smithi177 ceph-mon[195797]: pgmap v847: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 766 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 37457/36795 objects degraded (101.799%); 520 KiB/s, 83 objects/s recovering 2023-10-09T04:37:18.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:18.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:37:18.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:18.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:19.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:19.086 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:19.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:19.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:19.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:19.409 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:19.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:19.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:19.568 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:19.570 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:19.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:19.665 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:19.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:19.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:19.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:19.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:19.902 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:20.077 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:20.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:20.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:20.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:20.263 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:20.344 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:20.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:20.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:20.489 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:37:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:20 smithi177 ceph-mon[195797]: pgmap v848: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 738 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 37457/35475 objects degraded (105.587%); 0 B/s, 46 objects/s recovering 2023-10-09T04:37:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3285338922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:37:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:20 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:37:20.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37457/35475 objects degraded (105.587%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:20.535 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:20.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[216903]: pgmap v848: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 738 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 37457/35475 objects degraded (105.587%); 0 B/s, 46 objects/s recovering 2023-10-09T04:37:20.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3285338922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:37:20.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:37:20.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37457/35475 objects degraded (105.587%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:20.537 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[223858]: pgmap v848: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 738 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 37457/35475 objects degraded (105.587%); 0 B/s, 46 objects/s recovering 2023-10-09T04:37:20.537 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3285338922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:37:20.537 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:37:20.537 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37457/35475 objects degraded (105.587%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:20.638 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:20.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:20.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:20.789 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:20.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:20.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:20.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:20.963 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:20.994 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:21.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:37:21.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:21.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:21.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:21.645 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:21.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:21 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:37:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:37:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: osdmap e228: 8 total, 8 up, 8 in 2023-10-09T04:37:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: failed to encode map e228 with expected crc 2023-10-09T04:37:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:21 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:37:22.220 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:37:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:37:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:37:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:37:22.221 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:37:22.494 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:22 smithi177 ceph-mon[195797]: pgmap v850: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 550 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 37457/26439 objects degraded (141.673%); 0 B/s, 92 objects/s recovering 2023-10-09T04:37:22.797 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:22 smithi029 ceph-mon[223858]: pgmap v850: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 550 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 37457/26439 objects degraded (141.673%); 0 B/s, 92 objects/s recovering 2023-10-09T04:37:22.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:22 smithi029 ceph-mon[216903]: pgmap v850: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 550 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 37457/26439 objects degraded (141.673%); 0 B/s, 92 objects/s recovering 2023-10-09T04:37:22.855 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:22.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:22.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:22.893 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:37:22.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:23.089 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:37:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:37:22] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-10-09T04:37:23.090 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:23.162 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:23.343 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:23.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:23.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:23.534 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:23.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:23.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:23.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:23.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:37:23.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:23.988 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:24.096 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:24.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:24.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:24.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:24.265 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:24.286 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:24.529 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:37:24.665 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:24 smithi177 ceph-mon[195797]: pgmap v851: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 490 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 37457/23565 objects degraded (158.952%); 0 B/s, 77 objects/s recovering 2023-10-09T04:37:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/655942986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:37:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:24 smithi029 ceph-mon[216903]: pgmap v851: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 490 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 37457/23565 objects degraded (158.952%); 0 B/s, 77 objects/s recovering 2023-10-09T04:37:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/655942986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:37:24.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:24 smithi029 ceph-mon[223858]: pgmap v851: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 490 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 37457/23565 objects degraded (158.952%); 0 B/s, 77 objects/s recovering 2023-10-09T04:37:24.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/655942986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:37:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:25 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:25.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:25 smithi177 ceph-mon[195797]: failed to encode map e229 with expected crc 2023-10-09T04:37:25.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/655942986' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:37:25.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:25 smithi177 ceph-mon[195797]: osdmap e229: 8 total, 8 up, 8 in 2023-10-09T04:37:25.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:25 smithi177 ceph-mon[195797]: failed to encode map e229 with expected crc 2023-10-09T04:37:25.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37457/23565 objects degraded (158.952%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[223858]: failed to encode map e229 with expected crc 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/655942986' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[223858]: osdmap e229: 8 total, 8 up, 8 in 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[223858]: failed to encode map e229 with expected crc 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37457/23565 objects degraded (158.952%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[216903]: failed to encode map e229 with expected crc 2023-10-09T04:37:25.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/655942986' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:37:25.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[216903]: osdmap e229: 8 total, 8 up, 8 in 2023-10-09T04:37:25.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[216903]: failed to encode map e229 with expected crc 2023-10-09T04:37:25.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37457/23565 objects degraded (158.952%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:26.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:26.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:26.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:26.305 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:26.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:26.550 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:26 smithi177 ceph-mon[195797]: pgmap v853: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 403 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 37457/19404 objects degraded (193.038%); 0 B/s, 89 objects/s recovering 2023-10-09T04:37:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:26.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:26.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:26.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:26 smithi029 ceph-mon[216903]: pgmap v853: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 403 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 37457/19404 objects degraded (193.038%); 0 B/s, 89 objects/s recovering 2023-10-09T04:37:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:26 smithi029 ceph-mon[223858]: pgmap v853: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 403 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 37457/19404 objects degraded (193.038%); 0 B/s, 89 objects/s recovering 2023-10-09T04:37:27.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:27.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:27.225 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:37:27.225 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:37:27.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:37:27.225 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:37:27.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38499 9.9 9:9a8fb78d:::benchmark_data_smithi029_299184_object38498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19997639000002, 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.179240123, 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38499 2023-10-09T04:37:27.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020773+0000", 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020771+0000", 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020915+0000", 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000143788 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020932+0000", 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.698e-05 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020941+0000", 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2529999999999993e-06 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022592+0000", 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016505980000000001 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022601+0000", 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9099999999999994e-06 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.199150+0000", 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.17654937 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.200013+0000", 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000862749 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38501 9.9 9:91e7d993:::benchmark_data_smithi029_299184_object38500:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19978434799998, 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1804904860000001, 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38501 2023-10-09T04:37:27.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020965+0000", 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.020964+0000", 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021040+0000", 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5575999999999996e-05 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021060+0000", 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9754000000000001e-05 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021065+0000", 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4999999999999999e-06 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022627+0000", 2023-10-09T04:37:27.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015613140000000001 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022638+0000", 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1204e-05 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201357+0000", 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1787194400000001 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201456+0000", 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8263000000000004e-05 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38505 9.9 9:9fe8cf27:::benchmark_data_smithi029_299184_object38504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19967841099998, 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1806352849999999, 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.237 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38505 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021071+0000", 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021070+0000", 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021146+0000", 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5275e-05 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021154+0000", 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8720000000000002e-06 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021170+0000", 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6252999999999998e-05 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022657+0000", 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001487588 2023-10-09T04:37:27.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022662+0000", 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5549999999999996e-06 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201579+0000", 2023-10-09T04:37:27.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1789165530000001 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201706+0000", 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127687 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38509 9.9 9:938e198b:::benchmark_data_smithi029_299184_object38508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19957394300002, 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.180647043, 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.244 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38509 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021175+0000", 2023-10-09T04:37:27.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021220+0000", 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5445000000000003e-05 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021249+0000", 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8875000000000001e-05 2023-10-09T04:37:27.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021254+0000", 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1209999999999998e-06 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022682+0000", 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014274299999999999 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022687+0000", 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4149999999999998e-06 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201740+0000", 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1790529270000001 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201822+0000", 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2379000000000005e-05 2023-10-09T04:37:27.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38510 9.9 9:98f164af:::benchmark_data_smithi029_299184_object38509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19949010200003, 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.180751331, 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38510 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021259+0000", 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021326+0000", 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6715000000000001e-05 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021334+0000", 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7129999999999994e-06 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021340+0000", 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1510000000000001e-06 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022709+0000", 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013682690000000001 2023-10-09T04:37:27.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022715+0000", 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0990000000000004e-06 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.201936+0000", 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1792214670000001 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202011+0000", 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4392000000000002e-05 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38528 9.9 9:91ac7278:::benchmark_data_smithi029_299184_object38527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19938268599998, 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.203529793, 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.255 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38528 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021367+0000", 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021366+0000", 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021405+0000", 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8535999999999998e-05 2023-10-09T04:37:27.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021412+0000", 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8129999999999996e-06 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021418+0000", 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2499999999999997e-06 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022736+0000", 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001317948 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022741+0000", 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.096e-06 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202061+0000", 2023-10-09T04:37:27.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1793201959999999 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202400+0000", 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033863900000000001 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.202619+0000", 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000219619 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.207410+0000", 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047910080000000002 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209061+0000", 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001650257 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.222756+0000", 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013695182 2023-10-09T04:37:27.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.222822+0000", 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6346000000000005e-05 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.224896+0000", 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002074471 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38539 9.9 9:931725d0:::benchmark_data_smithi029_299184_object38538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19932680900001, 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2048092829999999, 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38539 2023-10-09T04:37:27.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021423+0000", 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021422+0000", 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021457+0000", 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5354000000000001e-05 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021465+0000", 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2339999999999997e-06 2023-10-09T04:37:27.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021469+0000", 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3619999999999999e-06 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022985+0000", 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015154610000000001 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.022992+0000", 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3250000000000002e-06 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204186+0000", 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.181193653 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204309+0000", 2023-10-09T04:37:27.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123767 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.204574+0000", 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026451699999999997 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209071+0000", 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044968439999999998 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209400+0000", 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032915499999999997 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.225011+0000", 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015611534 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.225082+0000", 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0273999999999998e-05 2023-10-09T04:37:27.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226232+0000", 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001150264 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:38547 9.9 9:9345b2c2:::benchmark_data_smithi029_299184_object38546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e154)", 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:37:27.270 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 456.19927552799999, 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.206218247, 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 38547 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:37:27.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021474+0000", 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021473+0000", 2023-10-09T04:37:27.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021502+0000", 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8175999999999999e-05 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021508+0000", 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.021513+0000", 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3660000000000002e-06 2023-10-09T04:37:27.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.023032+0000", 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015193610000000001 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:51.023040+0000", 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6899999999999992e-06 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.205732+0000", 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1826917210000001 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.205900+0000", 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000168913 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.206070+0000", 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016947199999999999 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209441+0000", 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033709899999999999 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.209549+0000", 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108433 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226306+0000", 2023-10-09T04:37:27.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016757141 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.226351+0000", 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4988999999999999e-05 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:29:52.227692+0000", 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001340651 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.278 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:47657 10.9 10:9a4d341d:::benchmark_data_smithi029_344401_object47656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:34:29.261744+0000", 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 177.95900568299999, 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33510193500000002, 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.279 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 47657 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261744+0000", 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261744+0000", 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261739+0000", 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999957 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261790+0000", 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0769000000000001e-05 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261807+0000", 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6840000000000001e-05 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261815+0000", 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0299999999999994e-06 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261877+0000", 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1622000000000001e-05 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.261981+0000", 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000104526 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.262071+0000", 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9870000000000005e-05 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.264281+0000", 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022103489999999999 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.265935+0000", 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001654097 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596749+0000", 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.330813093 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596804+0000", 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5828999999999997e-05 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:34:29.596846+0000", 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1291999999999997e-05 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88629 10.3 10:c811f6cb:::benchmark_data_smithi029_344401_object88628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.948379+0000", 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 138.27237015099999, 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27891803100000001, 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88629 2023-10-09T04:37:27.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948379+0000", 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948379+0000", 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948378+0000", 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948410+0000", 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2598000000000002e-05 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948421+0000", 2023-10-09T04:37:27.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0132000000000001e-05 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948426+0000", 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7479999999999999e-06 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948590+0000", 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016400299999999999 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948722+0000", 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013193100000000001 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.948924+0000", 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000202108 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952152+0000", 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032279940000000001 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.226908+0000", 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27475596800000002 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.227191+0000", 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028242099999999998 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.227297+0000", 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106526 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:88633 10.3 10:c172d3cd:::benchmark_data_smithi029_344401_object88632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:08.952469+0000", 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 138.26828013900001, 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28261339000000002, 2023-10-09T04:37:27.292 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 88633 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952469+0000", 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952469+0000", 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952468+0000", 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952583+0000", 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114634 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952599+0000", 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6863000000000001e-05 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952623+0000", 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3331000000000001e-05 2023-10-09T04:37:27.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952692+0000", 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9302999999999997e-05 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952822+0000", 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000130302 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:37:27.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.952961+0000", 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013819700000000001 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:08.956280+0000", 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033191800000000001 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.234943+0000", 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.27866359000000002 2023-10-09T04:37:27.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.234995+0000", 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2012999999999998e-05 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:09.235083+0000", 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7232999999999996e-05 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.298 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:103487 10.6 10:61d448ad:::benchmark_data_smithi029_344401_object103486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:26.573228+0000", 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 120.647521246, 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36493930200000002, 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.299 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 103487 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573228+0000", 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573228+0000", 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573227+0000", 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573263+0000", 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6353999999999998e-05 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573276+0000", 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.253e-05 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573281+0000", 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.164e-06 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573337+0000", 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5745000000000002e-05 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573468+0000", 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000131724 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573606+0000", 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013798099999999999 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.577261+0000", 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036541690000000001 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.581898+0000", 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004637492 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.938067+0000", 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35616884500000001 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.938127+0000", 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9925e-05 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.938167+0000", 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0593000000000003e-05 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120129 10.19 10:9dbedecc:::benchmark_data_smithi029_344401_object11758:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.593087+0000", 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 99.627662150000006, 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0070726539999999, 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120129 2023-10-09T04:37:27.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593087+0000", 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593087+0000", 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593085+0000", 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593094+0000", 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3999999999999992e-06 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593120+0000", 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5780999999999999e-05 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593131+0000", 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1008e-05 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.735165+0000", 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14203438700000001 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.735172+0000", 2023-10-09T04:37:27.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9399999999999996e-06 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.585926+0000", 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85075447699999995 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.590090+0000", 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041638459999999997 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 7", 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.590193+0000", 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102508 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.593257+0000", 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030638290000000001 2023-10-09T04:37:27.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600029+0000", 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067728179999999999 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600080+0000", 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0160000000000001e-05 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600160+0000", 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0267999999999998e-05 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.314 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143839 10.23 10:c6eaaf06:::benchmark_data_smithi029_344401_object35468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418089+0000", 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.802660850999999, 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96140360199999997, 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.317 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143839 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418089+0000", 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418089+0000", 2023-10-09T04:37:27.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418088+0000", 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418096+0000", 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3059999999999998e-06 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418110+0000", 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4487999999999999e-05 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418117+0000", 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6229999999999999e-06 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.531432+0000", 2023-10-09T04:37:27.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.11331483000000001 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.531439+0000", 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9630000000000001e-06 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.379415+0000", 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.84797630999999996 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.379492+0000", 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6917999999999998e-05 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:196420 10.16 10:6ebbf647:::benchmark_data_smithi029_344401_object88049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:04.328242+0000", 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 22.892507221999999, 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30011083100000002, 2023-10-09T04:37:27.322 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 196420 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328242+0000", 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328242+0000", 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328242+0000", 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328245+0000", 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9519999999999999e-06 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328251+0000", 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6449999999999999e-06 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328256+0000", 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3030000000000001e-06 2023-10-09T04:37:27.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328309+0000", 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3780999999999999e-05 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328423+0000", 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113826 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.328981+0000", 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055784400000000005 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.332197+0000", 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032156009999999998 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.335140+0000", 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002943378 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.628287+0000", 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.293146727 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.628323+0000", 2023-10-09T04:37:27.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6640999999999997e-05 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:04.628353+0000", 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9561999999999999e-05 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:209188 10.19 10:9d2f763b:::benchmark_data_smithi029_344401_object100817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:20.796311+0000", 2023-10-09T04:37:27.329 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.4244386359999996, 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.18956505100000001, 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 209188 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796311+0000", 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796311+0000", 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796309+0000", 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796317+0000", 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0809999999999994e-06 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796332+0000", 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5029e-05 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796340+0000", 2023-10-09T04:37:27.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8760000000000005e-06 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796398+0000", 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8346e-05 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.796556+0000", 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015749500000000001 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.797191+0000", 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00063463899999999999 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.800580+0000", 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00338967 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.827036+0000", 2023-10-09T04:37:27.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026456160999999999 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.985054+0000", 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.15801799599999999 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.985109+0000", 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5049000000000001e-05 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.985876+0000", 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00076631900000000001 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:209199 10.9 10:94638836:::benchmark_data_smithi029_344401_object100828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:20.803589+0000", 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.4171600309999999, 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.18390606900000001, 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.336 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 209199 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.803589+0000", 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.803589+0000", 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.803588+0000", 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.803593+0000", 2023-10-09T04:37:27.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5669999999999996e-06 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.803602+0000", 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9930000000000001e-06 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.803607+0000", 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8149999999999996e-06 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.804450+0000", 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00084380000000000002 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.804564+0000", 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113787 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.805223+0000", 2023-10-09T04:37:27.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00065888899999999996 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.810858+0000", 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056350580000000001 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.827075+0000", 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016217266000000001 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.986691+0000", 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.15961578700000001 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.986747+0000", 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5754000000000002e-05 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.987495+0000", 2023-10-09T04:37:27.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00074853400000000003 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:209205 10.9 10:91c5648f:::benchmark_data_smithi029_344401_object100834:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:20.813315+0000", 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.4074347700000001, 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.17489397300000001, 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 209205 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813315+0000", 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813315+0000", 2023-10-09T04:37:27.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813313+0000", 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813322+0000", 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9719999999999998e-06 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813337+0000", 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.519e-05 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813345+0000", 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7859999999999997e-06 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813400+0000", 2023-10-09T04:37:27.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4969999999999997e-05 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.813570+0000", 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000170326 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.814224+0000", 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00065414599999999998 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.817307+0000", 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030822739999999999 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.827108+0000", 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0098015830000000009 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.987589+0000", 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.16048027400000001 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.987645+0000", 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6328e-05 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.988209+0000", 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00056374799999999998 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.349 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:209212 10.6 10:66cbb4a3:::benchmark_data_smithi029_344401_object100841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:20.822432+0000", 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.3983169599999998, 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.17967174699999999, 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.350 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 209212 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.822432+0000", 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.822432+0000", 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.822431+0000", 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.822437+0000", 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.003e-06 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.822449+0000", 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1748e-05 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.822455+0000", 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4119999999999998e-06 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.825756+0000", 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003300538 2023-10-09T04:37:27.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.825887+0000", 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013093600000000001 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.826447+0000", 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055962500000000001 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.830568+0000", 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041215959999999999 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:20.830830+0000", 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026188999999999997 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:21.002013+0000", 2023-10-09T04:37:27.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.17118272800000001 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:21.002067+0000", 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4339999999999998e-05 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:21.002104+0000", 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7106e-05 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.356 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:211996 10.16 10:6cdeb975:::benchmark_data_smithi029_344401_object103625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:37:24.657795+0000", 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 2.5629546059999999, 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.18337389400000001, 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:27.357 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 211996 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657795+0000", 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657795+0000", 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:27.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657793+0000", 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657801+0000", 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9019999999999999e-06 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:27.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657820+0000", 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9202e-05 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657828+0000", 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7270000000000007e-06 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.657885+0000", 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6418000000000001e-05 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.658020+0000", 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013568900000000001 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.658622+0000", 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000601663 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:27.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.662673+0000", 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040507030000000001 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.664420+0000", 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001747583 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.841064+0000", 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.176644156 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.841124+0000", 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9697000000000001e-05 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:37:24.841169+0000", 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4659000000000001e-05 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.363 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:27.364 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:27.364 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:37:27.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:27.454 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:28.204 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:28.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:28.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:28.418 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:28.631 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:37:28.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:28 smithi029 ceph-mon[216903]: pgmap v854: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 291 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 37457/13992 objects degraded (267.703%); 0 B/s, 89 objects/s recovering 2023-10-09T04:37:28.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:28.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:28.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:28 smithi029 ceph-mon[223858]: pgmap v854: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 291 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 37457/13992 objects degraded (267.703%); 0 B/s, 89 objects/s recovering 2023-10-09T04:37:28.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:28.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:28.687 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:37:28.705 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:28.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:28.733 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:37:28.775 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:37:28.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:28 smithi177 ceph-mon[195797]: pgmap v854: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 291 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 37457/13992 objects degraded (267.703%); 0 B/s, 89 objects/s recovering 2023-10-09T04:37:28.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:28.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:28.801 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:37:28.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:28.844 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:37:28.889 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:28.891 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:37:28.941 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:37:28.959 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:37:28.960 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:37:29.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:37:29.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:29.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:29.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:29.167 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:29.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:29.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:29.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:29.442 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:29.508 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:29.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:37:29.737 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:29.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3320611746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:29.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3320611746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:29.957 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:29.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:30.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3320611746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:30.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:30.135 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:30.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:30.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:30.583 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:30 smithi177 ceph-mon[195797]: pgmap v855: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 263 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 37457/12654 objects degraded (296.009%); 0 B/s, 39 objects/s recovering 2023-10-09T04:37:30.583 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:30.583 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:30.584 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37457/12654 objects degraded (296.009%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:30.627 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:30.711 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:30.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[216903]: pgmap v855: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 263 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 37457/12654 objects degraded (296.009%); 0 B/s, 39 objects/s recovering 2023-10-09T04:37:30.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:30.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:30.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37457/12654 objects degraded (296.009%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:30.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[223858]: pgmap v855: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 263 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 37457/12654 objects degraded (296.009%); 0 B/s, 39 objects/s recovering 2023-10-09T04:37:30.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:30.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:30.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37457/12654 objects degraded (296.009%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:30.924 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:30.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:31.032 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:31.074 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:31.123 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:31.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:31.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:31.297 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:31.498 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:31.503 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:31.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:31.563 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 108368 objects 2023-10-09T04:37:31.563 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :118.471 2023-10-09T04:37:31.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:31.649 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:31.828 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:31.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:31.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:31.964 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:32.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:32.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:32.120 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:32.120 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:32.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:32.194 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:32.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:32.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:32.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:32.479 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:32.589 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:37:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:32 smithi029 ceph-mon[216903]: pgmap v856: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 100 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 37457/4863 objects degraded (770.245%); 0 B/s, 79 objects/s recovering 2023-10-09T04:37:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/721834889' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:37:32.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:32 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:37:32.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:32 smithi029 ceph-mon[223858]: pgmap v856: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 100 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 37457/4863 objects degraded (770.245%); 0 B/s, 79 objects/s recovering 2023-10-09T04:37:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/721834889' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:37:32.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:32 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:37:32.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:37:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:32 smithi177 ceph-mon[195797]: pgmap v856: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 100 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 37457/4863 objects degraded (770.245%); 0 B/s, 79 objects/s recovering 2023-10-09T04:37:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/721834889' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:37:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:32 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:37:33.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:37:33.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:37:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:37:32] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T04:37:33.454 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:33.666 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:33.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:33.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:33.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:33.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:33.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:33 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:37:33.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:33 smithi177 ceph-mon[195797]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T04:37:33.858 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:33 smithi177 ceph-mon[195797]: failed to encode map e230 with expected crc 2023-10-09T04:37:33.859 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:33 smithi177 ceph-mon[195797]: failed to encode map e230 with expected crc 2023-10-09T04:37:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:37:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[216903]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T04:37:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[216903]: failed to encode map e230 with expected crc 2023-10-09T04:37:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[216903]: failed to encode map e230 with expected crc 2023-10-09T04:37:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:37:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[223858]: osdmap e230: 8 total, 8 up, 8 in 2023-10-09T04:37:33.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[223858]: failed to encode map e230 with expected crc 2023-10-09T04:37:33.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:33 smithi029 ceph-mon[223858]: failed to encode map e230 with expected crc 2023-10-09T04:37:34.263 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:34.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:34.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:34.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:37:34.405 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:34.517 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:34.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:34.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:34.625 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:34.651 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:34.697 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:34.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:34.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:34.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[216903]: pgmap v858: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 74 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 37457/3618 objects degraded (1035.296%); 0 B/s, 93 objects/s recovering 2023-10-09T04:37:34.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[216903]: failed to encode map e230 with expected crc 2023-10-09T04:37:34.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[216903]: failed to encode map e230 with expected crc 2023-10-09T04:37:34.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[216903]: failed to encode map e231 with expected crc 2023-10-09T04:37:34.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[216903]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T04:37:34.829 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[216903]: failed to encode map e231 with expected crc 2023-10-09T04:37:34.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[223858]: pgmap v858: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 74 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 37457/3618 objects degraded (1035.296%); 0 B/s, 93 objects/s recovering 2023-10-09T04:37:34.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[223858]: failed to encode map e230 with expected crc 2023-10-09T04:37:34.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[223858]: failed to encode map e230 with expected crc 2023-10-09T04:37:34.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[223858]: failed to encode map e231 with expected crc 2023-10-09T04:37:34.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[223858]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T04:37:34.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:34 smithi029 ceph-mon[223858]: failed to encode map e231 with expected crc 2023-10-09T04:37:34.832 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:35.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:35.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:34 smithi177 ceph-mon[195797]: pgmap v858: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 74 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 37457/3618 objects degraded (1035.296%); 0 B/s, 93 objects/s recovering 2023-10-09T04:37:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:34 smithi177 ceph-mon[195797]: failed to encode map e230 with expected crc 2023-10-09T04:37:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:34 smithi177 ceph-mon[195797]: failed to encode map e230 with expected crc 2023-10-09T04:37:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:34 smithi177 ceph-mon[195797]: failed to encode map e231 with expected crc 2023-10-09T04:37:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:34 smithi177 ceph-mon[195797]: osdmap e231: 8 total, 8 up, 8 in 2023-10-09T04:37:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:34 smithi177 ceph-mon[195797]: failed to encode map e231 with expected crc 2023-10-09T04:37:35.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:35.134 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:35.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:35.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:35.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:35.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:35.310 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:35.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:35.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:35.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:35.401 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:35.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:35.496 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:35.636 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T04:37:35.636 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37457/3618 objects degraded (1035.296%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3975151069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37457/3618 objects degraded (1035.296%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3975151069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:35.700 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:35.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:35.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:35.834 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:35.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: osdmap e232: 8 total, 8 up, 8 in 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: failed to encode map e232 with expected crc 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37457/3618 objects degraded (1035.296%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:35.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3975151069' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:35.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:37:36.285 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:36.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:36.389 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:36.418 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:36.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:36.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:36.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:36.500 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:37:36.524 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:36.524 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:36.571 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:36.604 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:36.618 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-09T04:37:36.619 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-09T04:37:36.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:36.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:36.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:36.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:36.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[216903]: pgmap v861: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 36 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 37456/1758 objects degraded (2130.603%); 0 B/s, 175 objects/s recovering 2023-10-09T04:37:36.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[216903]: failed to encode map e232 with expected crc 2023-10-09T04:37:36.872 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[216903]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[223858]: pgmap v861: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 36 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 37456/1758 objects degraded (2130.603%); 0 B/s, 175 objects/s recovering 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[223858]: failed to encode map e232 with expected crc 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:36.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:36 smithi029 ceph-mon[223858]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T04:37:36.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:36 smithi177 ceph-mon[195797]: pgmap v861: 109 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 36 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 37456/1758 objects degraded (2130.603%); 0 B/s, 175 objects/s recovering 2023-10-09T04:37:36.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:36 smithi177 ceph-mon[195797]: failed to encode map e232 with expected crc 2023-10-09T04:37:36.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:36 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:36.879 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:36 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:37:36.879 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:36 smithi177 ceph-mon[195797]: osdmap e233: 8 total, 8 up, 8 in 2023-10-09T04:37:36.880 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:36.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:36.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:37.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:37.058 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:37.155 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:37:37.175 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:37:37.200 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:37:37.215 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:37:37.235 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:37:37.253 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:37:37.272 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:37:37.291 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:37:37.292 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.293 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:37:37.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:37 smithi029 ceph-mon[223858]: failed to encode map e233 with expected crc 2023-10-09T04:37:37.609 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/752491328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:37.609 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:37 smithi029 ceph-mon[223858]: failed to encode map e233 with expected crc 2023-10-09T04:37:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:37 smithi029 ceph-mon[216903]: failed to encode map e233 with expected crc 2023-10-09T04:37:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/752491328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:37 smithi029 ceph-mon[216903]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:37 smithi177 ceph-mon[195797]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/752491328' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:37 smithi177 ceph-mon[195797]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.367 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:38.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:38.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:37:38.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:38.531 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:38.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:38.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:38.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:38.713 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:38 smithi177 ceph-mon[195797]: pgmap v863: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 244 objects/s recovering 2023-10-09T04:37:38.714 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:38 smithi177 ceph-mon[195797]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.714 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:38 smithi177 ceph-mon[195797]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.716 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:38.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:38.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:38.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:38.840 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:38.871 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:38.900 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:38 smithi029 ceph-mon[216903]: pgmap v863: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 244 objects/s recovering 2023-10-09T04:37:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:38 smithi029 ceph-mon[216903]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:38 smithi029 ceph-mon[216903]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:38 smithi029 ceph-mon[223858]: pgmap v863: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 244 objects/s recovering 2023-10-09T04:37:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:38 smithi029 ceph-mon[223858]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:38 smithi029 ceph-mon[223858]: failed to encode map e233 with expected crc 2023-10-09T04:37:38.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:38.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:38.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:39.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:39.091 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:39.161 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:39.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:39.264 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:39.294 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:39.443 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:39.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:39.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:39.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:39.576 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:39.610 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:39.632 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:39.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:39.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:39.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:39.766 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:39.916 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:39.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:40.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:40.051 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:40.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:40.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:40.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:40.231 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:40.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:37:40.335 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:40.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:40.441 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:40.469 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:40.512 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:40.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:40.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:40.619 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:37:40.649 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:40 smithi177 ceph-mon[195797]: pgmap v864: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 190 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 180 objects/s recovering 2023-10-09T04:37:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1051321014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:37:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35006/732 objects degraded (4782.240%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:40 smithi029 ceph-mon[216903]: pgmap v864: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 190 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 180 objects/s recovering 2023-10-09T04:37:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1051321014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:37:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35006/732 objects degraded (4782.240%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:40 smithi029 ceph-mon[223858]: pgmap v864: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 190 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 180 objects/s recovering 2023-10-09T04:37:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1051321014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:37:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35006/732 objects degraded (4782.240%), 12 pgs degraded (PG_DEGRADED) 2023-10-09T04:37:41.036 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:41.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:41.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:41.169 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:41.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:41.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:37:41.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:41.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:41.348 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:37:41.348 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:37:41.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120082 10.10 10:085e1ab9:::benchmark_data_smithi029_344401_object11711:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596697+0000", 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.749093324, 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0098806279999999, 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.349 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120082 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596697+0000", 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596697+0000", 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596690+0000", 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999928 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596707+0000", 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7484000000000001e-05 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596729+0000", 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1865000000000002e-05 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596740+0000", 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0244000000000001e-05 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707518+0000", 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.11077846199999999 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707524+0000", 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.993e-06 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606328+0000", 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89880406199999996 2023-10-09T04:37:41.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606578+0000", 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024985699999999998 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120087 10.10 10:0811bfd4:::benchmark_data_smithi029_344401_object11716:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596752+0000", 2023-10-09T04:37:41.354 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.749038462, 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0097016990000001, 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120087 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596752+0000", 2023-10-09T04:37:41.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596752+0000", 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596752+0000", 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596754+0000", 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4549999999999998e-06 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596760+0000", 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1959999999999996e-06 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596765+0000", 2023-10-09T04:37:41.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3900000000000003e-06 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707540+0000", 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110774794 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707543+0000", 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8940000000000003e-06 2023-10-09T04:37:41.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606401+0000", 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89885776100000003 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606454+0000", 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2672e-05 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.359 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120113 10.0 10:002fa587:::benchmark_data_smithi029_344401_object11742:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596769+0000", 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.74902163900001, 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98701352200000003, 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.360 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120113 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596769+0000", 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596769+0000", 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596769+0000", 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596771+0000", 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7269999999999999e-06 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596777+0000", 2023-10-09T04:37:41.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4269999999999999e-06 2023-10-09T04:37:41.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596781+0000", 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9439999999999998e-06 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707560+0000", 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110779302 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707566+0000", 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5729999999999998e-06 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.583707+0000", 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.87614096299999999 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.583783+0000", 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5970000000000006e-05 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120117 10.0 10:010f71f3:::benchmark_data_smithi029_344401_object11746:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596785+0000", 2023-10-09T04:37:41.366 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.74900583900001, 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98706538600000004, 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120117 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.367 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596785+0000", 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596785+0000", 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596784+0000", 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596787+0000", 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.34e-06 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596792+0000", 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.293e-06 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596796+0000", 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4089999999999999e-06 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707582+0000", 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110786422 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707587+0000", 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2299999999999999e-06 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.583803+0000", 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.87621563199999997 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.583850+0000", 2023-10-09T04:37:41.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7394000000000001e-05 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120120 10.10 10:0ceb740a:::benchmark_data_smithi029_344401_object11749:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596800+0000", 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.74899050400001, 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0097226509999999, 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.372 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120120 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596800+0000", 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596800+0000", 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596800+0000", 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596802+0000", 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0279999999999999e-06 2023-10-09T04:37:41.374 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596808+0000", 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8119999999999996e-06 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596811+0000", 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6830000000000001e-06 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707602+0000", 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110791167 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707607+0000", 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1450000000000001e-06 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606473+0000", 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89886606599999996 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606523+0000", 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0089000000000001e-05 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.377 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120125 10.10 10:0c360ae0:::benchmark_data_smithi029_344401_object11754:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596815+0000", 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.748975556, 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0216638, 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120125 2023-10-09T04:37:41.378 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596815+0000", 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596815+0000", 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596815+0000", 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596817+0000", 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0080000000000001e-06 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596822+0000", 2023-10-09T04:37:41.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.48e-06 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596826+0000", 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2830000000000002e-06 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707621+0000", 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110795885 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707625+0000", 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0230000000000003e-06 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606542+0000", 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89891648700000004 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606623+0000", 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0628000000000001e-05 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606748+0000", 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012505 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.611352+0000", 2023-10-09T04:37:41.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046042649999999997 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.616436+0000", 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050840759999999999 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.618399+0000", 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019626050000000001 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.618444+0000", 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5563999999999999e-05 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.618479+0000", 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4774000000000003e-05 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120130 10.0 10:047fa9c1:::benchmark_data_smithi029_344401_object11759:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596829+0000", 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.748961337, 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.009645659, 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120130 2023-10-09T04:37:41.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596829+0000", 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596829+0000", 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596829+0000", 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596831+0000", 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1490000000000001e-06 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596836+0000", 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2290000000000002e-06 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596840+0000", 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6459999999999999e-06 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707650+0000", 2023-10-09T04:37:41.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110810406 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707656+0000", 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8089999999999998e-06 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.583869+0000", 2023-10-09T04:37:41.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.876212451 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.583960+0000", 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1689999999999998e-05 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.584078+0000", 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011743000000000001 2023-10-09T04:37:41.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.589932+0000", 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058539739999999996 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600530+0000", 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010598715999999999 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606395+0000", 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058643259999999996 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606438+0000", 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2762999999999997e-05 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606475+0000", 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7404000000000003e-05 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120133 10.0 10:045c11ac:::benchmark_data_smithi029_344401_object11762:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.596844+0000", 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.74894656399999, 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.009706759, 2023-10-09T04:37:41.394 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120133 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596844+0000", 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596844+0000", 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596844+0000", 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596846+0000", 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1849999999999998e-06 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596851+0000", 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3249999999999998e-06 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.596854+0000", 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1999999999999999e-06 2023-10-09T04:37:41.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707674+0000", 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110819818 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.707680+0000", 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0190000000000003e-06 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.585906+0000", 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.87822562500000001 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.586002+0000", 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6477999999999993e-05 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:37:41.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.586099+0000", 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6877000000000001e-05 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600502+0000", 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014402991 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600554+0000", 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1963999999999999e-05 2023-10-09T04:37:41.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606481+0000", 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059269809999999996 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606520+0000", 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8933000000000001e-05 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.606551+0000", 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0676999999999998e-05 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120136 10.7 10:e03e2ea4:::benchmark_data_smithi029_344401_object11765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.678876+0000", 2023-10-09T04:37:41.402 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 113.666914431, 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91744313099999997, 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120136 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.403 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.678876+0000", 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.678876+0000", 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.678874+0000", 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T04:37:41.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.678890+0000", 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6141e-05 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.678921+0000", 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1236999999999999e-05 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.678933+0000", 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1960999999999999e-05 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.730684+0000", 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.051750448999999997 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.730695+0000", 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1442e-05 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.577419+0000", 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.84672431599999998 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.577526+0000", 2023-10-09T04:37:41.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010693500000000001 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.577640+0000", 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113906 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.589278+0000", 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011637484999999999 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.589450+0000", 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017241700000000001 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.596228+0000", 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067775379999999996 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.596270+0000", 2023-10-09T04:37:41.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2642000000000003e-05 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.596319+0000", 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9144999999999998e-05 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.410 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143828 10.1b 10:d9775463:::benchmark_data_smithi029_344401_object35457:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418172+0000", 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927619007999994, 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0204485969999999, 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143828 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418172+0000", 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418172+0000", 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.412 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418170+0000", 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418177+0000", 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0369999999999997e-06 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418195+0000", 2023-10-09T04:37:41.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7895999999999999e-05 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418203+0000", 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7249999999999997e-06 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703680+0000", 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28547725099999999 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703687+0000", 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0940000000000002e-06 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438552+0000", 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73486447799999999 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438620+0000", 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8555999999999999e-05 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143830 10.1b 10:df9650ab:::benchmark_data_smithi029_344401_object35459:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418218+0000", 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927572413000007, 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0204952469999999, 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.416 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143830 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418218+0000", 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418218+0000", 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418218+0000", 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418221+0000", 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.224e-06 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418253+0000", 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2289000000000002e-05 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418259+0000", 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3720000000000004e-06 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703717+0000", 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28545869000000001 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703723+0000", 2023-10-09T04:37:41.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0809999999999999e-06 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438648+0000", 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73492474500000005 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438713+0000", 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5263000000000002e-05 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.421 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143832 10.1b 10:d980bc3b:::benchmark_data_smithi029_344401_object35461:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418266+0000", 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927524161999997, 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0205260599999999, 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.422 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143832 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418266+0000", 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418266+0000", 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418266+0000", 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418269+0000", 2023-10-09T04:37:41.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.055e-06 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418276+0000", 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7800000000000003e-06 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418280+0000", 2023-10-09T04:37:41.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9689999999999996e-06 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703754+0000", 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28547458199999998 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703760+0000", 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.553e-06 2023-10-09T04:37:41.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438737+0000", 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73497731300000002 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438793+0000", 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5253000000000001e-05 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.427 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143837 10.a 10:57f32b18:::benchmark_data_smithi029_344401_object35466:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418333+0000", 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927457446000005, 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.002577381, 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.428 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143837 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418333+0000", 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418333+0000", 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418333+0000", 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418336+0000", 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3900000000000002e-06 2023-10-09T04:37:41.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418344+0000", 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2700000000000004e-06 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418349+0000", 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9819999999999999e-06 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418406+0000", 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6329000000000002e-05 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418413+0000", 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9630000000000001e-06 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.733026+0000", 2023-10-09T04:37:41.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31461308599999999 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.733038+0000", 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2544999999999999e-05 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.420844+0000", 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68780584899999997 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.420911+0000", 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6489000000000007e-05 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143838 10.1b 10:dd47399a:::benchmark_data_smithi029_344401_object35467:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418369+0000", 2023-10-09T04:37:41.434 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927422066000005, 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0287571980000001, 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143838 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418369+0000", 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418369+0000", 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418368+0000", 2023-10-09T04:37:41.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418371+0000", 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7650000000000002e-06 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418378+0000", 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6959999999999999e-06 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418382+0000", 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1679999999999997e-06 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703793+0000", 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28541124800000001 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.703798+0000", 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438808+0000", 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73501022999999999 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:37:41.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438928+0000", 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119084 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.447056+0000", 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081285260000000005 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.447126+0000", 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9703999999999995e-05 2023-10-09T04:37:41.440 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143841 10.a 10:56506fa7:::benchmark_data_smithi029_344401_object35470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418389+0000", 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927401196000005, 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0172594669999999, 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.441 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143841 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418389+0000", 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418389+0000", 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418389+0000", 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418391+0000", 2023-10-09T04:37:41.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3190000000000002e-06 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418397+0000", 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0090000000000004e-06 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418401+0000", 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8130000000000002e-06 2023-10-09T04:37:41.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418476+0000", 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5244e-05 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418482+0000", 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4620000000000004e-06 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.733061+0000", 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31457919099999998 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.733068+0000", 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3509999999999996e-06 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.420935+0000", 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68786606500000003 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.421027+0000", 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2072999999999998e-05 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.435579+0000", 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014552726 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.435649+0000", 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9553000000000003e-05 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143842 10.a 10:56d6907c:::benchmark_data_smithi029_344401_object35471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418406+0000", 2023-10-09T04:37:41.448 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.927385045999998, 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.017524324, 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143842 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418406+0000", 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418406+0000", 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418405+0000", 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:37:41.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418408+0000", 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.542e-06 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418413+0000", 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8609999999999997e-06 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418417+0000", 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4070000000000003e-06 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418505+0000", 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7992999999999996e-05 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting_for_map not empty", 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418513+0000", 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4309999999999997e-06 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.733090+0000", 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31457737600000002 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.733097+0000", 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8360000000000003e-06 2023-10-09T04:37:41.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.421048+0000", 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68795107499999997 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.421134+0000", 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6440999999999995e-05 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.435882+0000", 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014747830999999999 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.435930+0000", 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7836000000000001e-05 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143851 10.1b 10:d9bb8905:::benchmark_data_smithi029_344401_object35480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.516746+0000", 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.829045072, 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93003210199999997, 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143851 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.456 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.516746+0000", 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.516746+0000", 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.516743+0000", 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.516757+0000", 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4048e-05 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.516782+0000", 2023-10-09T04:37:41.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4904000000000001e-05 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.516794+0000", 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1821000000000001e-05 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.704432+0000", 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.18763803100000001 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.704437+0000", 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4439999999999999e-06 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.438958+0000", 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73452035599999999 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.439055+0000", 2023-10-09T04:37:41.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7441000000000004e-05 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.439337+0000", 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028128400000000003 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.444623+0000", 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005286308 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.446607+0000", 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001983687 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.446648+0000", 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1403000000000003e-05 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.446690+0000", 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2292000000000001e-05 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.446778+0000", 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7451e-05 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143856 10.1b 10:de471c6d:::benchmark_data_smithi029_344401_object35485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.638564+0000", 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.707226892999998, 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.80872634300000001, 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143856 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.464 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.638564+0000", 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.638564+0000", 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.638561+0000", 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.638575+0000", 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3896999999999999e-05 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.638605+0000", 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0156999999999999e-05 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.638618+0000", 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2328e-05 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.704458+0000", 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.065840009000000005 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.704464+0000", 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7159999999999997e-06 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.440900+0000", 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73643547200000004 2023-10-09T04:37:41.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.441005+0000", 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105585 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.441236+0000", 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000230366 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.444664+0000", 2023-10-09T04:37:41.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034287530000000001 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.446611+0000", 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019468650000000001 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.447160+0000", 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00054863100000000005 2023-10-09T04:37:41.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.447200+0000", 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9669000000000003e-05 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.447290+0000", 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0465999999999997e-05 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.471 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143857 10.f 10:f559e582:::benchmark_data_smithi029_344401_object35486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.647663+0000", 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.698127806000002, 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.77862930299999999, 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.472 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143857 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.647663+0000", 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.647663+0000", 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.647656+0000", 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999928 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.647671+0000", 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5216e-05 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.647696+0000", 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5418e-05 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.647705+0000", 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7159999999999992e-06 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.735013+0000", 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.087308396999999996 2023-10-09T04:37:41.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.735019+0000", 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.626e-06 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.404927+0000", 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66990823799999999 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.405060+0000", 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013281400000000001 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.405516+0000", 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00045644799999999998 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.410904+0000", 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053878830000000004 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.426163+0000", 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015258412000000001 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.426208+0000", 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4805000000000002e-05 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.426292+0000", 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4555000000000006e-05 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143860 10.1f 10:fc43b2ac:::benchmark_data_smithi029_344401_object35489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.739338+0000", 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 89.606452539000003, 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.688230746, 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143860 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.480 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739338+0000", 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739338+0000", 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739337+0000", 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739349+0000", 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1953999999999999e-05 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739374+0000", 2023-10-09T04:37:41.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5272000000000001e-05 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739386+0000", 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.187e-05 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739448+0000", 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2224000000000006e-05 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.739460+0000", 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1946999999999999e-05 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.423762+0000", 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68430167200000003 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.423860+0000", 2023-10-09T04:37:41.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8688000000000001e-05 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3", 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.424203+0000", 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034273600000000001 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.426448+0000", 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022444629999999999 2023-10-09T04:37:41.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.427450+0000", 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001002045 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.427488+0000", 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8578000000000002e-05 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:37:41.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.427569+0000", 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0649999999999995e-05 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:37:41.487 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:37:41.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:41.515 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:41.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:41.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:41.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:41.694 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:41.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:41.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:41.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:41.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:37:41.842 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:41.869 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:41.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:41.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:41 smithi177 ceph-mon[195797]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1051321014' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:37:41.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:41 smithi177 ceph-mon[195797]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:37:41.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:41 smithi177 ceph-mon[195797]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:41 smithi177 ceph-mon[195797]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:41 smithi177 ceph-mon[195797]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:41.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[223858]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1051321014' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[223858]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[223858]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[223858]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[223858]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[216903]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1051321014' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:37:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[216903]: osdmap e234: 8 total, 8 up, 8 in 2023-10-09T04:37:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[216903]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[216903]: failed to encode map e234 with expected crc 2023-10-09T04:37:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:41 smithi029 ceph-mon[216903]: failed to encode map e234 with expected crc 2023-10-09T04:37:42.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:42.042 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:42.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:42.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:42.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:42.098 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:42.137 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:42.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:42.217 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:42.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:42.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:42.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:42.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:42.391 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:42.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:42.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:42.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:42.566 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:42.610 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:42.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:42.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:42.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:42 smithi029 ceph-mon[223858]: pgmap v866: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 303 objects/s recovering 2023-10-09T04:37:42.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1387194246' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:37:42.637 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_5' removed 2023-10-09T04:37:42.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:42.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:42.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:42 smithi029 ceph-mon[216903]: pgmap v866: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 303 objects/s recovering 2023-10-09T04:37:42.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1387194246' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:37:42.889 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:37:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:37:42] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-10-09T04:37:42.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:42.975 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:42 smithi177 ceph-mon[195797]: pgmap v866: 109 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 14 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 133 op/s; 35006/732 objects degraded (4782.240%); 0 B/s, 303 objects/s recovering 2023-10-09T04:37:42.975 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1387194246' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:37:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:42.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:43.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:37:43.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:43.108 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:43.407 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:43.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[216903]: failed to encode map e235 with expected crc 2023-10-09T04:37:43.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1387194246' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:37:43.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[216903]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:37:43.896 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:43.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[223858]: failed to encode map e235 with expected crc 2023-10-09T04:37:43.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1387194246' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:37:43.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[223858]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:37:43.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:43.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:43 smithi177 ceph-mon[195797]: failed to encode map e235 with expected crc 2023-10-09T04:37:43.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1387194246' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:37:43.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:43 smithi177 ceph-mon[195797]: osdmap e235: 8 total, 8 up, 8 in 2023-10-09T04:37:43.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:44.324 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:44.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:44.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:44.462 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:44.616 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:37:44.617 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:37:44.617 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-10-09T04:37:44.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-10-09T04:37:44.709 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:44.715 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:44.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:44.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:44.850 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:44 smithi177 ceph-mon[195797]: pgmap v868: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:44.861 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:44 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35006/732 objects degraded (4782.240%), 12 pgs degraded) 2023-10-09T04:37:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:44 smithi029 ceph-mon[216903]: pgmap v868: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:44 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35006/732 objects degraded (4782.240%), 12 pgs degraded) 2023-10-09T04:37:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:44 smithi029 ceph-mon[223858]: pgmap v868: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:44 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35006/732 objects degraded (4782.240%), 12 pgs degraded) 2023-10-09T04:37:44.935 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:37:44.954 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:37:44.973 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:37:44.993 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:37:45.004 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:45.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:45.015 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:37:45.034 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:37:45.053 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:37:45.072 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:37:45.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.074 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:37:45.085 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:45.086 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:45.108 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:45.138 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:45.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:45.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:45.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:45.517 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:45.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:45.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:45.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:37:45.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:45.840 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:45.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:45.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4085046691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4085046691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:45.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:46.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4085046691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:46.017 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:46.115 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:46.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:46.122 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:46.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:46.245 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:46.389 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:46.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:46.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:46.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:46.561 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:46.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:46 smithi029 ceph-mon[216903]: pgmap v869: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:46.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:46 smithi029 ceph-mon[223858]: pgmap v869: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:46 smithi177 ceph-mon[195797]: pgmap v869: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:47.064 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:47.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:47.098 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:47.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.196 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:47.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:47.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:47.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:47.376 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:47.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:47.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:47.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:47.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:47.603 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:47.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:47.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:47.654 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:37:47.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4230513481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:47.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:47.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4230513481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:47.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:47.698 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:47.699 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:47.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4230513481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:47.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:37:47.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:47.780 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:47.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:47.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:47.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:47.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:48.003 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:48.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:48.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:48.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:48.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:48.183 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:48.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:37:48.435 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:48.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:48.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:37:48.539 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:48.567 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:48.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:48.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:48.700 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:37:48.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:48.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: pgmap v870: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:48.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:48.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:48.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:37:48.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[216903]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: pgmap v870: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.746 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:48 smithi029 ceph-mon[223858]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.747 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:48.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: pgmap v870: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:48.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:37:48.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:37:48.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: osdmap e236: 8 total, 8 up, 8 in 2023-10-09T04:37:48.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.899 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.900 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: failed to encode map e236 with expected crc 2023-10-09T04:37:48.900 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:48 smithi177 ceph-mon[195797]: failed to encode map e236 with expected crc 2023-10-09T04:37:49.659 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:49.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:49.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:49.799 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:50.099 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:50.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:50.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:50.165 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:37:50.165 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:37:50.187 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:49 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:50.187 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:49 smithi177 ceph-mon[195797]: pgmap v872: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:50.203 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:50.224 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:50.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:49 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:50.232 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:49 smithi029 ceph-mon[216903]: pgmap v872: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:50.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:49 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:50.232 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:49 smithi029 ceph-mon[223858]: pgmap v872: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:50.237 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:50.295 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:50.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:50.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:50.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:50.428 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:50.477 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:50.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:50.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:50.608 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:50.859 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:50.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:50.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:37:50.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:50.990 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:51.011 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:37:51.013 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_6' created 2023-10-09T04:37:51.049 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:51.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:51.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1242842724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:51 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1242842724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:51 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:51.180 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1242842724' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:51.180 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:51 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:37:51.184 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:51.230 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:51.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:51.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:51.366 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:51.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:37:51.968 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:51.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:52.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:52.102 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:52.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:52.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:52.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/959671111' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T04:37:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: failed to encode map e237 with expected crc 2023-10-09T04:37:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: failed to encode map e237 with expected crc 2023-10-09T04:37:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:37:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/959671111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T04:37:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:37:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: pgmap v874: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:52 smithi177 ceph-mon[195797]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/959671111' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: failed to encode map e237 with expected crc 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: failed to encode map e237 with expected crc 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/959671111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: pgmap v874: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[216903]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/959671111' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-09T04:37:52.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: failed to encode map e237 with expected crc 2023-10-09T04:37:52.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: failed to encode map e237 with expected crc 2023-10-09T04:37:52.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:37:52.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/959671111' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-09T04:37:52.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: osdmap e237: 8 total, 8 up, 8 in 2023-10-09T04:37:52.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: pgmap v874: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:52.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:52 smithi029 ceph-mon[223858]: osdmap e238: 8 total, 8 up, 8 in 2023-10-09T04:37:52.282 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:52.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:37:52.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-10-09T04:37:52.494 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:52.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:52.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:52.629 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:52.686 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:52.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:52.793 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:52.822 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:52.823 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:52.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:52.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:52.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:37:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:37:52] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.43.0" 2023-10-09T04:37:52.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:53.007 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:53.213 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:53.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:53.269 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:53 smithi177 ceph-mon[195797]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:53 smithi177 ceph-mon[195797]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:53 smithi177 ceph-mon[195797]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:53 smithi177 ceph-mon[195797]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:53 smithi177 ceph-mon[195797]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:53 smithi177 ceph-mon[195797]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:37:53.315 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:37:53.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:53.337 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:37:53.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[216903]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[216903]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[216903]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[216903]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[216903]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[216903]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[223858]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[223858]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[223858]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[223858]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[223858]: failed to encode map e238 with expected crc 2023-10-09T04:37:53.343 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:53 smithi029 ceph-mon[223858]: osdmap e239: 8 total, 8 up, 8 in 2023-10-09T04:37:53.346 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:53.359 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:37:53.390 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:37:53.408 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:53.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:53.416 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:37:53.435 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:37:53.455 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:37:53.466 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:53.466 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:37:53.478 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:37:53.479 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.480 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:37:53.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:53.542 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:53.743 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:53.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:53.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:53.876 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:53.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:53.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:53.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:54.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:37:54.062 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:54.158 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:37:54.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:54.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:37:54.261 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:54 smithi177 ceph-mon[195797]: failed to encode map e239 with expected crc 2023-10-09T04:37:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:54 smithi177 ceph-mon[195797]: pgmap v877: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/466829861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:54.290 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:54.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:54.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:54.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:54 smithi029 ceph-mon[216903]: failed to encode map e239 with expected crc 2023-10-09T04:37:54.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:54 smithi029 ceph-mon[216903]: pgmap v877: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:54.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/466829861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:54 smithi029 ceph-mon[223858]: failed to encode map e239 with expected crc 2023-10-09T04:37:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:54 smithi029 ceph-mon[223858]: pgmap v877: 89 pgs: 6 creating+peering, 10 unknown, 73 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:54.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/466829861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:37:54.441 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:37:54.469 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:54.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:37:54.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:54.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:37:54.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:54.643 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:54.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:54.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:54.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:54.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:54.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:55.025 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:55.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:55.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:55.162 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:55.817 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:55.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:55.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:55.954 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:56.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:56 smithi029 ceph-mon[216903]: pgmap v878: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:56.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:56 smithi029 ceph-mon[223858]: pgmap v878: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:56.632 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:56.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:56.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:56.766 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:56 smithi177 ceph-mon[195797]: pgmap v878: 89 pgs: 6 creating+peering, 8 unknown, 75 active+clean; 582 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:56.769 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:56.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:56.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:56.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:56.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:56.955 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:57.153 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:57.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:57.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:57.289 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:57.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:57.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:57.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:57.445 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:37:57.475 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:57.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:37:57.847 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:37:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:57.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:37:57.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:57.983 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:58.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:58.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:58.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:58.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:58.168 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:58.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:58.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:37:58.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:37:58.343 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:58.355 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-10-09T04:37:58.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:58 smithi029 ceph-mon[216903]: pgmap v879: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:58.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1275471438' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:58.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:58.379 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:58 smithi029 ceph-mon[223858]: pgmap v879: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:58.379 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1275471438' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:58.379 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:58.385 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:37:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:58.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:37:58.418 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:37:58.418 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:37:58.418 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:37:58.419 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:37:58.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:58.527 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:58.691 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:58.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:58 smithi177 ceph-mon[195797]: pgmap v879: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:37:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1275471438' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:37:58.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:37:58.825 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:59.375 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:37:59.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:59.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:37:59.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:37:59.512 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:59.513 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:37:59.560 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:37:59.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:59.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:37:59.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:37:59.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1809628265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:37:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:37:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:37:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1809628265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:37:59.698 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:59.746 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:37:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:59.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:37:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:37:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: osdmap e240: 8 total, 8 up, 8 in 2023-10-09T04:37:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:37:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:37:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: failed to encode map e240 with expected crc 2023-10-09T04:37:59.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:37:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1809628265' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:37:59.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:37:59.878 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:37:59.923 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:37:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:37:59.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:00.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:00.054 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:00.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:00.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:00.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:00.229 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:00.271 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:00.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:00.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:00.402 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:00.445 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:00.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:00.522 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:38:00.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:00.584 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:00 smithi177 ceph-mon[195797]: pgmap v881: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1809628265' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:38:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:00 smithi177 ceph-mon[195797]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:38:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:00 smithi177 ceph-mon[195797]: failed to encode map e241 with expected crc 2023-10-09T04:38:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3711238325' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:00.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:00 smithi177 ceph-mon[195797]: failed to encode map e241 with expected crc 2023-10-09T04:38:00.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[216903]: pgmap v881: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:00.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1809628265' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:38:00.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[216903]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:38:00.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[216903]: failed to encode map e241 with expected crc 2023-10-09T04:38:00.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3711238325' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[216903]: failed to encode map e241 with expected crc 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[223858]: pgmap v881: 89 pgs: 89 active+clean; 582 KiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1809628265' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[223858]: osdmap e241: 8 total, 8 up, 8 in 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[223858]: failed to encode map e241 with expected crc 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3711238325' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:00.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:00 smithi029 ceph-mon[223858]: failed to encode map e241 with expected crc 2023-10-09T04:38:00.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:38:00.889 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:00.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:00.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:01.021 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:01.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:01.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:01.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:01.200 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:01.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:01.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:01.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:38:01.349 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:01.376 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:01.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:01.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:01.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:01.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:01.551 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:01.553 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:01.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:01.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3711238325' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[216903]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3711238325' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:01 smithi029 ceph-mon[223858]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:01.734 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:38:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:38:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3711238325' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: osdmap e242: 8 total, 8 up, 8 in 2023-10-09T04:38:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:01 smithi177 ceph-mon[195797]: failed to encode map e242 with expected crc 2023-10-09T04:38:01.883 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:01.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:01.903 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-10-09T04:38:01.929 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:38:01.929 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:38:01.976 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:38:01.976 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:38:01.976 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_392508 2023-10-09T04:38:01.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:38:01.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:38:01.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:02.051 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:02.257 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:02.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:02.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:02.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:02.388 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:02.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:02.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:02.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:02.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:02.577 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:02.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:02.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:02.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:02.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:02.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:02 smithi029 ceph-mon[216903]: pgmap v884: 89 pgs: 89 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:02.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:02 smithi029 ceph-mon[223858]: pgmap v884: 89 pgs: 89 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:02.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:02 smithi177 ceph-mon[195797]: pgmap v884: 89 pgs: 89 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:02.779 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:02.881 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:02.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:02.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 15 1885 1870 116.869 116.875 0.00238565 0.00853667 2023-10-09T04:38:02.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:03.037 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:03.124 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:03.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:03.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:38:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:38:02] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.43.0" 2023-10-09T04:38:03.222 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:03.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:03.296 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:03.317 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:03.536 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:03.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:03.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:38:03.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:03.729 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:03.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:03.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:03.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:03.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 4767 4751 148.453 180.062 0.00563193 0.00671394 2023-10-09T04:38:03.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:04.046 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:04.297 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:04.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:04.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:04.458 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:04.516 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:38:04.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:04.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:04.559 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:38:04.594 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:38:04.642 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:38:04.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:04.683 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:38:04.716 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:04.723 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:38:04.765 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:38:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:04 smithi177 ceph-mon[195797]: pgmap v885: 89 pgs: 89 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3496015167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:04.803 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:38:04.810 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.810 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:38:04.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:04 smithi029 ceph-mon[216903]: pgmap v885: 89 pgs: 89 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:04.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:04.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3496015167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:04.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:04 smithi029 ceph-mon[223858]: pgmap v885: 89 pgs: 89 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-09T04:38:04.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:04.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3496015167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:04.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 7494 7478 155.772 170.438 0.00383453 0.00640793 2023-10-09T04:38:05.977 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 10610 10594 165.503 194.75 0.00610332 0.00603359 2023-10-09T04:38:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:06 smithi029 ceph-mon[216903]: pgmap v886: 89 pgs: 89 active+clean; 55 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 8.2 MiB/s wr, 130 op/s 2023-10-09T04:38:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:06 smithi029 ceph-mon[223858]: pgmap v886: 89 pgs: 89 active+clean; 55 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 8.2 MiB/s wr, 130 op/s 2023-10-09T04:38:06.978 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 15 13533 13518 168.943 182.75 0.00513523 0.00591196 2023-10-09T04:38:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:06 smithi177 ceph-mon[195797]: pgmap v886: 89 pgs: 89 active+clean; 55 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 8.2 MiB/s wr, 130 op/s 2023-10-09T04:38:07.978 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 15811 15795 164.503 142.312 0.00814731 0.00607347 2023-10-09T04:38:08.216 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:08.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:08.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:08.323 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:08.377 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:08.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:08.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:08.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:08.623 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:08 smithi177 ceph-mon[195797]: pgmap v887: 89 pgs: 89 active+clean; 621 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T04:38:08.629 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:08.695 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:08.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:08.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:08.853 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:08 smithi029 ceph-mon[216903]: pgmap v887: 89 pgs: 89 active+clean; 621 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T04:38:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:08 smithi029 ceph-mon[223858]: pgmap v887: 89 pgs: 89 active+clean; 621 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-09T04:38:08.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:08.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:08.978 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 18470 18454 164.742 166.188 0.00312971 0.00606364 2023-10-09T04:38:09.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:09.083 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:09.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:09.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:09.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:09.317 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:09.395 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:09.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:09.503 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:09.551 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:09.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:09.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:09.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:09.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:09.792 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:09.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:09.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:09.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:09.978 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 20807 20791 162.405 146.062 0.00460346 0.00615191 2023-10-09T04:38:09.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:10.051 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:10.251 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:10.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:10.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:10.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:10.421 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:10.596 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:10.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:10.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:10 smithi029 ceph-mon[223858]: pgmap v888: 89 pgs: 89 active+clean; 621 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:38:10.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:10 smithi029 ceph-mon[216903]: pgmap v888: 89 pgs: 89 active+clean; 621 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:38:10.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:10.757 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:10.855 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:10.860 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:10 smithi177 ceph-mon[195797]: pgmap v888: 89 pgs: 89 active+clean; 621 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-09T04:38:10.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:10.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:10.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:10.978 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 23161 23145 160.706 147.125 0.00551276 0.0062166 2023-10-09T04:38:11.009 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:11.084 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:11.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:11.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:11.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:11.231 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:11.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:11.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:11.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:11.483 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:11.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:11.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:11.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:11.723 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:11.970 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:11.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:11.979 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 25458 25442 158.99 143.562 0.00705061 0.00628403 2023-10-09T04:38:12.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:12.131 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:12.221 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:12.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:12.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:12.351 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:12.442 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:12.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:12.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:12.588 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:12.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:12.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:12 smithi029 ceph-mon[223858]: pgmap v889: 89 pgs: 89 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:38:12.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:12 smithi029 ceph-mon[216903]: pgmap v889: 89 pgs: 89 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:38:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:12.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:12.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:12.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:12 smithi177 ceph-mon[195797]: pgmap v889: 89 pgs: 89 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-09T04:38:12.814 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:12.893 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:12.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:12.978 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 27319 27303 155.11 116.312 0.00466561 0.00644109 2023-10-09T04:38:13.003 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:13.058 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:38:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:38:12] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.43.0" 2023-10-09T04:38:13.189 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:13.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:13.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:13.339 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:13.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:13.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:13.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:13.577 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:13.762 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:13.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:13.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:13.905 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:13.949 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:13.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:13.979 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 29136 29120 151.64 113.562 0.0046041 0.00658934 2023-10-09T04:38:14.053 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:14.114 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:14.254 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:14.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:14.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:14.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:14.415 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:14.512 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:14 smithi029 ceph-mon[216903]: pgmap v890: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:38:14.513 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:14.514 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:14.516 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:14.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:14.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:14.658 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:14.658 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:14 smithi177 ceph-mon[195797]: pgmap v890: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:38:14.658 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:14.659 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:14.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:14.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:14.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:14.850 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:14.896 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:14 smithi029 ceph-mon[223858]: pgmap v890: 89 pgs: 89 active+clean; 1.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-09T04:38:14.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:14.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:14.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:14.980 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 31200 31184 149.885 129 0.00732546 0.00666664 2023-10-09T04:38:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:14.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:15.087 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:15.137 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:15.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:15.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:15.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:15.328 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:15.365 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:15.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:15.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:15.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:15.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:15.716 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:15.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:15.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:15.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:15.919 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:15.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:15.980 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 32548 32532 145.197 84.25 0.00390685 0.00688257 2023-10-09T04:38:16.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:16.089 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:16.410 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:16.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:16.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:16.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:16.585 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:16 smithi029 ceph-mon[216903]: pgmap v891: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:38:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:16 smithi029 ceph-mon[223858]: pgmap v891: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:38:16.716 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:16.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:16.773 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:16 smithi177 ceph-mon[195797]: pgmap v891: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-09T04:38:16.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:16.872 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:16.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:16.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:16.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:16.980 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 34473 34457 143.537 120.312 0.0050707 0.00696159 2023-10-09T04:38:17.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:17.117 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:17.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:17.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:17.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:17.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:17.293 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:17.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:17.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:17.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:17.466 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:17.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:17.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:17.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:17.665 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:17.707 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:17.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:17.813 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:17.838 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:17.880 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:17.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:17.980 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 35158 35142 137.243 42.8125 0.00512013 0.00728113 2023-10-09T04:38:17.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:18.019 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:18.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:18.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:18.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:18.263 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:18.527 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:18 smithi029 ceph-mon[223858]: pgmap v892: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2023-10-09T04:38:18.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:18 smithi177 ceph-mon[195797]: pgmap v892: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2023-10-09T04:38:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:18 smithi029 ceph-mon[216903]: pgmap v892: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 164 MiB/s wr, 2.63k op/s 2023-10-09T04:38:18.981 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 36417 36401 133.794 78.6875 0.00588014 0.00744364 2023-10-09T04:38:19.179 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:19.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:19.286 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:19.333 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:19.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:19.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:19.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:19.604 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:19.729 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:19.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:19.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:19.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:19.914 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:19.939 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:19.986 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 15 37735 37720 130.903 82.4375 0.0133748 0.00763318 2023-10-09T04:38:20.117 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:20.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:20.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:20.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:20.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:20.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:20.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:20.480 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:20.545 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:20 smithi029 ceph-mon[223858]: pgmap v893: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:38:20.562 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:20.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:20.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:20.714 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:20.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:20.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:20.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:20 smithi029 ceph-mon[216903]: pgmap v893: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:38:20.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:20.917 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:20 smithi177 ceph-mon[195797]: pgmap v893: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s 2023-10-09T04:38:20.922 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:20.986 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 39378 39362 129.415 102.625 0.0108913 0.00772266 2023-10-09T04:38:20.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:21.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:21.102 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:21.139 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:38:21.139 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:38:21.142 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:21.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:21.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:21.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:21.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:21.352 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:21.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:21.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:21.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:21.574 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:38:21.579 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:21.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:21.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:21.738 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:38:21.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:21.859 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:21.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1442193866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:38:21.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:38:21.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1442193866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:38:21.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:38:21.986 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:38:21.986976+0000 min lat: 0.00104865 max lat: 0.325301 avg lat: 0.00781366 2023-10-09T04:38:21.986 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:38:21.987 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 40728 40712 127.163 84.375 0.00264461 0.00781366 2023-10-09T04:38:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1442193866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:38:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:38:22.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: pgmap v894: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-09T04:38:22.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:38:22.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:38:22.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[216903]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: pgmap v894: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:22 smithi029 ceph-mon[223858]: failed to encode map e243 with expected crc 2023-10-09T04:38:22.987 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 42266 42250 125.684 96.125 0.00932922 0.00795153 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: pgmap v894: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: osdmap e243: 8 total, 8 up, 8 in 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: failed to encode map e243 with expected crc 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: failed to encode map e243 with expected crc 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: failed to encode map e243 with expected crc 2023-10-09T04:38:23.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: failed to encode map e243 with expected crc 2023-10-09T04:38:23.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: failed to encode map e243 with expected crc 2023-10-09T04:38:23.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:22 smithi177 ceph-mon[195797]: failed to encode map e243 with expected crc 2023-10-09T04:38:23.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:38:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:38:22] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.43.0" 2023-10-09T04:38:23.987 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 43722 43706 124.108 91 0.0121595 0.0080528 2023-10-09T04:38:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:24 smithi029 ceph-mon[216903]: pgmap v896: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:38:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:24 smithi029 ceph-mon[223858]: pgmap v896: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:38:24.989 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 45093 45077 122.427 85.6875 0.0140561 0.00816284 2023-10-09T04:38:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:24 smithi177 ceph-mon[195797]: pgmap v896: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-09T04:38:25.924 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:25.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:25.989 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 45994 45978 119.674 56.3125 0.0076511 0.0083501 2023-10-09T04:38:26.030 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:26.071 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:26.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:26.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:26.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:26.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:26.282 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:26.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:26.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:26.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:26.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:38:26.456 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:26.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:38:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:26.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:38:26.608 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:26.634 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:26.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:26.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:26.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:26.839 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:26 smithi177 ceph-mon[195797]: pgmap v897: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:38:26.851 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:26 smithi029 ceph-mon[216903]: pgmap v897: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:38:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:26 smithi029 ceph-mon[223858]: pgmap v897: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-09T04:38:26.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:26.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:26.989 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 47017 47001 117.445 63.9375 0.0097943 0.00850853 2023-10-09T04:38:27.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:27.108 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:27.330 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:38:27.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:38:27.336 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:27.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:27.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:38:27.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:27.481 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:27.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:27.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:27.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:27.582 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:27.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:27.758 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:27.989 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 48247 48231 115.885 76.875 0.00979037 0.00862155 2023-10-09T04:38:28.175 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:28.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:28.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:28.346 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:28.540 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:28.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:28.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:28.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:28.712 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:28.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:28 smithi029 ceph-mon[216903]: pgmap v898: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:38:28.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:28.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:28 smithi029 ceph-mon[223858]: pgmap v898: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:38:28.773 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:28.972 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:28.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:28.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:28.990 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 49477 49461 114.435 76.875 0.0073707 0.0087334 2023-10-09T04:38:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:28 smithi177 ceph-mon[195797]: pgmap v898: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:38:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:29.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:29.138 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:29.245 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:29.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:29.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:29.402 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:29.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:29.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:29.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:29.615 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:38:29.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3989230515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:29.649 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:29 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:29.650 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:29.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:29.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:29.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:29.870 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:29.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:29.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3989230515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:29.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:29 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3989230515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:29 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:38:29.994 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 50870 50854 113.45 87.0625 0.00715957 0.00880937 2023-10-09T04:38:30.018 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:30.087 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: pgmap v899: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[223858]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: pgmap v899: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:38:30.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:30.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:38:30.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:38:30.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:30 smithi029 ceph-mon[216903]: failed to encode map e244 with expected crc 2023-10-09T04:38:30.993 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 52173 52157 112.342 81.4375 0.0085985 0.0088966 2023-10-09T04:38:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: pgmap v899: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:38:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: osdmap e244: 8 total, 8 up, 8 in 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: failed to encode map e244 with expected crc 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: failed to encode map e244 with expected crc 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: failed to encode map e244 with expected crc 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: failed to encode map e244 with expected crc 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: failed to encode map e244 with expected crc 2023-10-09T04:38:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:30 smithi177 ceph-mon[195797]: failed to encode map e244 with expected crc 2023-10-09T04:38:31.835 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:31.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:31.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:31.972 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:31.993 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 53268 53252 110.879 68.4375 0.00229086 0.00900028 2023-10-09T04:38:32.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:32.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:32.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:32.122 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:32.150 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:32.687 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:32.694 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:32.697 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:32.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:32.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:32.847 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:32.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:32 smithi029 ceph-mon[223858]: pgmap v901: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:38:32.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:32 smithi029 ceph-mon[216903]: pgmap v901: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:38:32.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:38:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:38:32] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2023-10-09T04:38:32.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:32 smithi177 ceph-mon[195797]: pgmap v901: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-09T04:38:32.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:32.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:32.994 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 54422 54406 109.63 72.125 0.0125527 0.00911659 2023-10-09T04:38:33.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:33.090 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:33.338 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:33.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:33.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:33.488 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:33.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:33.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:33.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:33.715 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:33.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:33.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:33.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:33.937 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:33.998 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 55722 55706 108.737 81.25 0.00972997 0.0091917 2023-10-09T04:38:34.312 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:34.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:34.416 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:38:34.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:34.478 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:34.495 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:38:34.528 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:38:34.624 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:34 smithi029 ceph-mon[216903]: pgmap v902: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:38:34.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:34 smithi029 ceph-mon[223858]: pgmap v902: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:38:34.685 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:38:34.724 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:38:34.763 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:38:34.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:38:34.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:38:34.839 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:38:34.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2893461590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:34.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2893461590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:34.942 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:34.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:34.995 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 56771 56755 107.429 65.5625 0.011227 0.00929299 2023-10-09T04:38:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:34 smithi177 ceph-mon[195797]: pgmap v902: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-09T04:38:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2893461590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:35.049 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:35.111 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:35.996 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 57756 57740 106.081 61.5625 0.0146078 0.00942226 2023-10-09T04:38:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:36 smithi177 ceph-mon[195797]: pgmap v903: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:38:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:36 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:36 smithi029 ceph-mon[216903]: pgmap v903: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:38:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:36 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:36 smithi029 ceph-mon[223858]: pgmap v903: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:38:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:36 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:38:36.996 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 58452 58436 104.293 43.5 0.00317825 0.00956047 2023-10-09T04:38:37.975 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:37.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:37.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:37.996 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 59459 59443 103.143 62.9375 0.0164688 0.00968989 2023-10-09T04:38:38.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:38.143 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:38.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:38.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:38.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:38.466 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:38.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:38 smithi029 ceph-mon[216903]: pgmap v904: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:38:38.894 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:38 smithi029 ceph-mon[223858]: pgmap v904: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:38:38.985 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:38.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:38.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:39.002 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 60555 60539 102.193 68.5 0.0101403 0.00978075 2023-10-09T04:38:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:38 smithi177 ceph-mon[195797]: pgmap v904: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:38:39.044 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:38:39.044 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:38:39.044 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183705 10.13 10.550def73 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570147+0000", 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.47170020999999, 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96301429199999999, 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.045 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183705 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570147+0000", 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570147+0000", 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570146+0000", 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570164+0000", 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7838000000000001e-05 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570176+0000", 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2649999999999999e-05 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570186+0000", 2023-10-09T04:38:39.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.026e-05 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627369+0000", 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057182589999999998 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627380+0000", 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.079e-05 2023-10-09T04:38:39.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533151+0000", 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90577155499999995 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533162+0000", 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0236e-05 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.050 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183707 10.23 10.be4d5fe3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570198+0000", 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.471650036, 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96303084900000002, 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.051 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183707 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570198+0000", 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570198+0000", 2023-10-09T04:38:39.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570197+0000", 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570200+0000", 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5969999999999999e-06 2023-10-09T04:38:39.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570205+0000", 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.84e-06 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570211+0000", 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0610000000000003e-06 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627385+0000", 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057174306000000001 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627394+0000", 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3950000000000007e-06 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533219+0000", 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90582470800000003 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533228+0000", 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4620000000000002e-06 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183713 10.13 10.8cd385d3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.47163306900001, 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96298166600000001, 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183713 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.057 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570216+0000", 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9259999999999999e-06 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570222+0000", 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1240000000000002e-06 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570226+0000", 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1760000000000003e-06 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627405+0000", 2023-10-09T04:38:39.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057178527999999999 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627412+0000", 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5000000000000002e-06 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533186+0000", 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90577387600000003 2023-10-09T04:38:39.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.062 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.062 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.062 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533196+0000", 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8530000000000004e-06 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.063 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183714 10.15 10.f5e10575 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.471617341, 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96215133799999997, 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183714 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570232+0000", 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0679999999999999e-06 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570237+0000", 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7389999999999999e-06 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570242+0000", 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0509999999999996e-06 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607094+0000", 2023-10-09T04:38:39.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036852426000000001 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607105+0000", 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0959e-05 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532372+0000", 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92526699999999995 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532382+0000", 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4830000000000005e-06 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183718 10.8 10.ac006e88 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570247+0000", 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.471600823, 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96047792499999995, 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.070 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183718 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570247+0000", 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.071 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570247+0000", 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570246+0000", 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570254+0000", 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6129999999999998e-06 2023-10-09T04:38:39.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570259+0000", 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3469999999999998e-06 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570264+0000", 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1960000000000001e-06 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593203+0000", 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022939365 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593219+0000", 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5820000000000001e-05 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530714+0000", 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93749524500000003 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530725+0000", 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0659000000000001e-05 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183731 10.15 10.37a67415 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.583550+0000", 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.458297692, 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94886407800000006, 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183731 2023-10-09T04:38:39.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583550+0000", 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583550+0000", 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583549+0000", 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583554+0000", 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9749999999999998e-06 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583562+0000", 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0220000000000004e-06 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583568+0000", 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7660000000000001e-06 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607153+0000", 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023584963 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607159+0000", 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.117e-06 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532404+0000", 2023-10-09T04:38:39.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92524475299999998 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532414+0000", 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7100000000000002e-06 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.081 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183735 10.5 10.e68c7665 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.593341+0000", 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.448506464, 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93897671800000004, 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183735 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593341+0000", 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593341+0000", 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593339+0000", 2023-10-09T04:38:39.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593350+0000", 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0436e-05 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593363+0000", 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3105000000000001e-05 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593375+0000", 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2364999999999999e-05 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607125+0000", 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013750149 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607132+0000", 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.224e-06 2023-10-09T04:38:39.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532305+0000", 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92517361099999995 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532318+0000", 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2707999999999999e-05 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.086 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183739 10.15 10.23ae88b5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.600199+0000", 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.441648552, 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93224512900000001, 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183739 2023-10-09T04:38:39.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600199+0000", 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600199+0000", 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600198+0000", 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600205+0000", 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2069999999999998e-06 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600220+0000", 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5398999999999999e-05 2023-10-09T04:38:39.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600230+0000", 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0006e-05 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607186+0000", 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006955653 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607193+0000", 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.973e-06 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532435+0000", 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.925241804 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532444+0000", 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5999999999999996e-06 2023-10-09T04:38:39.091 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183746 10.5 10.14db3525 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.611980+0000", 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.429867757, 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92037076200000001, 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.092 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183746 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611980+0000", 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611980+0000", 2023-10-09T04:38:39.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611978+0000", 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611986+0000", 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2899999999999997e-06 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611998+0000", 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2118e-05 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.612012+0000", 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4317e-05 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.612243+0000", 2023-10-09T04:38:39.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023038900000000001 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.612250+0000", 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6009999999999997e-06 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532341+0000", 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92009069700000001 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532351+0000", 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6870000000000006e-06 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183750 10.18 10.1e4f0c38 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.624114+0000", 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.417733704, 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90657211299999996, 2023-10-09T04:38:39.097 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183750 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624114+0000", 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624114+0000", 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624112+0000", 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624121+0000", 2023-10-09T04:38:39.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3180000000000007e-06 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624140+0000", 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9738e-05 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624149+0000", 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3850000000000002e-06 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624201+0000", 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2445999999999999e-05 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624211+0000", 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3680000000000008e-06 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530672+0000", 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90646186799999995 2023-10-09T04:38:39.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530686+0000", 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3596e-05 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(10.1f 225/222 [PushOp(10:f9cae83a:::benchmark_data_smithi029_344401_object105776:head, version: 207'6728, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:f9cae83a:::benchmark_data_smithi029_344401_object105776:head@207'6728, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:04.389215+0000", 2023-10-09T04:38:39.102 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 94.652633064, 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23008295200000001, 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.389215+0000", 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.389215+0000", 2023-10-09T04:38:39.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.389214+0000", 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.389241+0000", 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6667999999999999e-05 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.389251+0000", 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0845999999999999e-05 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.389260+0000", 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9020000000000005e-06 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.617689+0000", 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.228428825 2023-10-09T04:38:39.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.617716+0000", 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7061000000000001e-05 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.619297+0000", 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001581184 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.106 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:212397 10.8 e229/222 10:1673175d:::benchmark_data_smithi029_344401_object104026:head v 229'10551, mlcod=202'4776)", 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:25.297489+0000", 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 73.744358996000003, 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.174648681, 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.297489+0000", 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.297489+0000", 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.297488+0000", 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.297495+0000", 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8800000000000002e-06 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.297504+0000", 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7139999999999999e-06 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.297515+0000", 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1252e-05 2023-10-09T04:38:39.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.300148+0000", 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002633182 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.300175+0000", 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7025999999999998e-05 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.472087+0000", 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17191187199999999 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.472137+0000", 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0327000000000001e-05 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.111 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:212399 10.19 e229/222 10:986d908f:::benchmark_data_smithi029_344401_object104028:head v 229'10584, mlcod=202'4868)", 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:25.301094+0000", 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 73.740753209000005, 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.166425502, 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.301094+0000", 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.301094+0000", 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.301093+0000", 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.301101+0000", 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7949999999999996e-06 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.453569+0000", 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15246794 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.453582+0000", 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3049e-05 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.453652+0000", 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9645000000000002e-05 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.454709+0000", 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001057332 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.467473+0000", 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012763762999999999 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:25.467520+0000", 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6709000000000003e-05 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:1003 11.6 11:6527dac1:::benchmark_data_smithi029_392508_object1002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:02.470734+0000", 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 36.571113357000002, 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27092683899999997, 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 1003 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470734+0000", 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470734+0000", 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470734+0000", 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470755+0000", 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0954999999999998e-05 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470762+0000", 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2980000000000003e-06 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470768+0000", 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2439999999999998e-06 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470840+0000", 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1790000000000002e-05 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470952+0000", 2023-10-09T04:38:39.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112515 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.471106+0000", 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015394200000000001 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.472788+0000", 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001681245 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.472831+0000", 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3460000000000001e-05 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.741565+0000", 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26873381800000001 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.741622+0000", 2023-10-09T04:38:39.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6712999999999997e-05 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.741661+0000", 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9431999999999998e-05 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:1005 11.6 11:6d482bd1:::benchmark_data_smithi029_392508_object1004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:02.470826+0000", 2023-10-09T04:38:39.123 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 36.571021096999999, 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27136526799999999, 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 1005 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470826+0000", 2023-10-09T04:38:39.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470826+0000", 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470826+0000", 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470850+0000", 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4368000000000001e-05 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470858+0000", 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8650000000000001e-06 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470864+0000", 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8200000000000002e-06 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.471391+0000", 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052661699999999995 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.471499+0000", 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108795 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.471634+0000", 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013486999999999999 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.472798+0000", 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011634309999999999 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.472861+0000", 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3534000000000005e-05 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.742106+0000", 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26924466699999999 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.742156+0000", 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0306999999999998e-05 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.742192+0000", 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5568999999999998e-05 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:1028 11.6 11:6e793e2f:::benchmark_data_smithi029_392508_object1027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:02.477918+0000", 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 36.563929465000001, 2023-10-09T04:38:39.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26438680399999998, 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 1028 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.477918+0000", 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.477918+0000", 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.477917+0000", 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.477938+0000", 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.084e-05 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.477950+0000", 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2577e-05 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.477957+0000", 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0559999999999999e-06 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.478021+0000", 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3888999999999998e-05 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.478130+0000", 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108743 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:38:39.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.478286+0000", 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015551900000000001 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.478848+0000", 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00056227600000000001 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.479129+0000", 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028123299999999998 2023-10-09T04:38:39.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.742232+0000", 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26310242900000003 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.742271+0000", 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9708e-05 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.742305+0000", 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3658000000000002e-05 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:57935 11.6 11:667539a3:::benchmark_data_smithi029_392508_object57934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:36.122647+0000", 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.9192004909999998, 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23627731900000001, 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.137 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 57935 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122647+0000", 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122647+0000", 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122646+0000", 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122676+0000", 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9964999999999999e-05 2023-10-09T04:38:39.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122692+0000", 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6690000000000001e-05 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122703+0000", 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0282000000000001e-05 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122750+0000", 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7170999999999999e-05 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.122876+0000", 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000126298 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.123033+0000", 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015716300000000001 2023-10-09T04:38:39.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.126075+0000", 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030415759999999998 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.126771+0000", 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00069630099999999997 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.358830+0000", 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23205877 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.358883+0000", 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2781000000000002e-05 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.358924+0000", 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1601999999999999e-05 2023-10-09T04:38:39.143 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:57945 11.6 11:63f26f5d:::benchmark_data_smithi029_392508_object57944:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:36.131679+0000", 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.9101680820000002, 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23274052100000001, 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.144 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 57945 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131679+0000", 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131679+0000", 2023-10-09T04:38:39.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131678+0000", 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131702+0000", 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3546000000000001e-05 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131715+0000", 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3536000000000001e-05 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131722+0000", 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.708e-06 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131769+0000", 2023-10-09T04:38:39.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6795000000000002e-05 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131875+0000", 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000106053 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.131981+0000", 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010653399999999999 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.133995+0000", 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020134760000000002 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.134611+0000", 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000616027 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.364331+0000", 2023-10-09T04:38:39.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22971981899999999 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.364382+0000", 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1363000000000003e-05 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:36.364420+0000", 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8015000000000001e-05 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.150 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:59007 11.2 11:4cf1493b:::benchmark_data_smithi029_392508_object59006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:37.477467+0000", 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.564380358, 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20285362100000001, 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 59007 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477467+0000", 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477467+0000", 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477466+0000", 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477489+0000", 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3583000000000001e-05 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477500+0000", 2023-10-09T04:38:39.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0521e-05 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477507+0000", 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2309999999999999e-06 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477548+0000", 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0611000000000001e-05 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477648+0000", 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100751 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.477789+0000", 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140211 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.482164+0000", 2023-10-09T04:38:39.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004375744 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.482627+0000", 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046225699999999999 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.680230+0000", 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.197603055 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.680280+0000", 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0565000000000001e-05 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.680321+0000", 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0679000000000001e-05 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:59010 11.2 11:43ffb887:::benchmark_data_smithi029_392508_object59009:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:37.479629+0000", 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.5622181230000001, 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.201324856, 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 59010 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.158 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479629+0000", 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479629+0000", 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:38:39.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479628+0000", 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479659+0000", 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1083e-05 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479674+0000", 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4620999999999999e-05 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479683+0000", 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9050000000000003e-06 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479832+0000", 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014935799999999999 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.479986+0000", 2023-10-09T04:38:39.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015332200000000001 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.480149+0000", 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000163005 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.482321+0000", 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021726800000000002 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.482778+0000", 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045713200000000001 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.680864+0000", 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19808606300000001 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.680914+0000", 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9811000000000001e-05 2023-10-09T04:38:39.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:38:39.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:38:39.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:38:39.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:37.680954+0000", 2023-10-09T04:38:39.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0003999999999998e-05 2023-10-09T04:38:39.165 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.165 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.165 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.165 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:38:39.166 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:38:39.166 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:38:39.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:39.167 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:39.206 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:39.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:39.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:39.467 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:39.544 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:39.641 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:38:39.656 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:38:40.001 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 61652 61636 101.309 68.5625 0.00757178 0.00986566 2023-10-09T04:38:40.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:38:40.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[223858]: pgmap v905: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:38:40.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[223858]: from='client.55687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:40.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:40.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[216903]: pgmap v905: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:38:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[216903]: from='client.55687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:40.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:40.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:41.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 62842 62826 100.613 74.375 0.0109917 0.00993464 2023-10-09T04:38:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:40 smithi177 ceph-mon[195797]: pgmap v905: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-09T04:38:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:40 smithi177 ceph-mon[195797]: from='client.55687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:41.892 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:41.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:41.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:42.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:38:42.004341+0000 min lat: 0.00104865 max lat: 0.325301 avg lat: 0.00998148 2023-10-09T04:38:42.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:38:42.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 64153 64137 100.146 81.9375 0.0102318 0.00998148 2023-10-09T04:38:42.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:42.075 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:42.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:38:42.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:42 smithi029 ceph-mon[223858]: pgmap v906: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T04:38:42.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:38:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:38:42] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.43.0" 2023-10-09T04:38:42.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:42 smithi029 ceph-mon[216903]: pgmap v906: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T04:38:42.980 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:42.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:43.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 64863 64847 98.7864 44.375 0.0255515 0.0101162 2023-10-09T04:38:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:42 smithi177 ceph-mon[195797]: pgmap v906: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-09T04:38:43.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:43.126 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":905,"stamp":"2023-10-09T04:38:39.036761+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3768260512,"num_objects":57514,"num_object_clones":0,"num_object_copies":172542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57514,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":57790,"num_write_kb":3681805,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2952438,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57706,"ondisk_log_size":57706,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16794804,"kb_used_data":11391220,"kb_used_omap":20,"kb_used_meta":5403499,"kb_avail":733166412,"statfs":{"total":767960285184,"available":750762405888,"internally_reserved":0,"allocated":11664609280,"data_stored":11658597024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5533183630},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":763559936,"num_objects":11651,"num_object_clones":0,"num_object_copies":34953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11651,"num_write_kb":745664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006287"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":240,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569548+0000","last_change":"2023-10-09T04:36:47.540352+0000","last_active":"2023-10-09T04:38:21.569548+0000","last_peered":"2023-10-09T04:38:21.569548+0000","last_clean":"2023-10-09T04:38:21.569548+0000","last_became_active":"2023-10-09T04:36:47.538007+0000","last_became_peered":"2023-10-09T04:36:47.538007+0000","last_unstale":"2023-10-09T04:38:21.569548+0000","last_undegraded":"2023-10-09T04:38:21.569548+0000","last_fullsized":"2023-10-09T04:38:21.569548+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:45.054005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":195,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.559177+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:38:21.559177+0000","last_peered":"2023-10-09T04:38:21.559177+0000","last_clean":"2023-10-09T04:38:21.559177+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:38:21.559177+0000","last_undegraded":"2023-10-09T04:38:21.559177+0000","last_fullsized":"2023-10-09T04:38:21.559177+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":210,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.601623+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:38:29.601623+0000","last_peered":"2023-10-09T04:38:29.601623+0000","last_clean":"2023-10-09T04:38:29.601623+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:38:29.601623+0000","last_undegraded":"2023-10-09T04:38:29.601623+0000","last_fullsized":"2023-10-09T04:38:29.601623+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":157,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569895+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:38:21.569895+0000","last_peered":"2023-10-09T04:38:21.569895+0000","last_clean":"2023-10-09T04:38:21.569895+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:38:21.569895+0000","last_undegraded":"2023-10-09T04:38:21.569895+0000","last_fullsized":"2023-10-09T04:38:21.569895+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":231,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.555768+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:38:21.555768+0000","last_peered":"2023-10-09T04:38:21.555768+0000","last_clean":"2023-10-09T04:38:21.555768+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:38:21.555768+0000","last_undegraded":"2023-10-09T04:38:21.555768+0000","last_fullsized":"2023-10-09T04:38:21.555768+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":201,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571803+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:38:21.571803+0000","last_peered":"2023-10-09T04:38:21.571803+0000","last_clean":"2023-10-09T04:38:21.571803+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:38:21.571803+0000","last_undegraded":"2023-10-09T04:38:21.571803+0000","last_fullsized":"2023-10-09T04:38:21.571803+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":156,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571154+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:38:21.571154+0000","last_peered":"2023-10-09T04:38:21.571154+0000","last_clean":"2023-10-09T04:38:21.571154+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:38:21.571154+0000","last_undegraded":"2023-10-09T04:38:21.571154+0000","last_fullsized":"2023-10-09T04:38:21.571154+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.555842+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:38:21.555842+0000","last_peered":"2023-10-09T04:38:21.555842+0000","last_clean":"2023-10-09T04:38:21.555842+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:38:21.555842+0000","last_undegraded":"2023-10-09T04:38:21.555842+0000","last_fullsized":"2023-10-09T04:38:21.555842+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570841+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:38:21.570841+0000","last_peered":"2023-10-09T04:38:21.570841+0000","last_clean":"2023-10-09T04:38:21.570841+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:38:21.570841+0000","last_undegraded":"2023-10-09T04:38:21.570841+0000","last_fullsized":"2023-10-09T04:38:21.570841+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":168,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556466+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:38:21.556466+0000","last_peered":"2023-10-09T04:38:21.556466+0000","last_clean":"2023-10-09T04:38:21.556466+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:38:21.556466+0000","last_undegraded":"2023-10-09T04:38:21.556466+0000","last_fullsized":"2023-10-09T04:38:21.556466+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":206,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570056+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:38:21.570056+0000","last_peered":"2023-10-09T04:38:21.570056+0000","last_clean":"2023-10-09T04:38:21.570056+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:38:21.570056+0000","last_undegraded":"2023-10-09T04:38:21.570056+0000","last_fullsized":"2023-10-09T04:38:21.570056+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.555840+0000","last_change":"2023-10-09T04:36:47.537972+0000","last_active":"2023-10-09T04:38:21.555840+0000","last_peered":"2023-10-09T04:38:21.555840+0000","last_clean":"2023-10-09T04:38:21.555840+0000","last_became_active":"2023-10-09T04:36:47.537663+0000","last_became_peered":"2023-10-09T04:36:47.537663+0000","last_unstale":"2023-10-09T04:38:21.555840+0000","last_undegraded":"2023-10-09T04:38:21.555840+0000","last_fullsized":"2023-10-09T04:38:21.555840+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:37.956271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":210,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570294+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:38:21.570294+0000","last_peered":"2023-10-09T04:38:21.570294+0000","last_clean":"2023-10-09T04:38:21.570294+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:38:21.570294+0000","last_undegraded":"2023-10-09T04:38:21.570294+0000","last_fullsized":"2023-10-09T04:38:21.570294+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569859+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:38:21.569859+0000","last_peered":"2023-10-09T04:38:21.569859+0000","last_clean":"2023-10-09T04:38:21.569859+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:38:21.569859+0000","last_undegraded":"2023-10-09T04:38:21.569859+0000","last_fullsized":"2023-10-09T04:38:21.569859+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570202+0000","last_change":"2023-10-09T04:36:47.543534+0000","last_active":"2023-10-09T04:38:21.570202+0000","last_peered":"2023-10-09T04:38:21.570202+0000","last_clean":"2023-10-09T04:38:21.570202+0000","last_became_active":"2023-10-09T04:36:47.543152+0000","last_became_peered":"2023-10-09T04:36:47.543152+0000","last_unstale":"2023-10-09T04:38:21.570202+0000","last_undegraded":"2023-10-09T04:38:21.570202+0000","last_fullsized":"2023-10-09T04:38:21.570202+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:52.483371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.557530+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:38:21.557530+0000","last_peered":"2023-10-09T04:38:21.557530+0000","last_clean":"2023-10-09T04:38:21.557530+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:38:21.557530+0000","last_undegraded":"2023-10-09T04:38:21.557530+0000","last_fullsized":"2023-10-09T04:38:21.557530+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":156,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569869+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:38:21.569869+0000","last_peered":"2023-10-09T04:38:21.569869+0000","last_clean":"2023-10-09T04:38:21.569869+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:38:21.569869+0000","last_undegraded":"2023-10-09T04:38:21.569869+0000","last_fullsized":"2023-10-09T04:38:21.569869+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":203,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571308+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:38:21.571308+0000","last_peered":"2023-10-09T04:38:21.571308+0000","last_clean":"2023-10-09T04:38:21.571308+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:38:21.571308+0000","last_undegraded":"2023-10-09T04:38:21.571308+0000","last_fullsized":"2023-10-09T04:38:21.571308+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":186,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569627+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:38:21.569627+0000","last_peered":"2023-10-09T04:38:21.569627+0000","last_clean":"2023-10-09T04:38:21.569627+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:38:21.569627+0000","last_undegraded":"2023-10-09T04:38:21.569627+0000","last_fullsized":"2023-10-09T04:38:21.569627+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":156,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.557965+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:38:21.557965+0000","last_peered":"2023-10-09T04:38:21.557965+0000","last_clean":"2023-10-09T04:38:21.557965+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:38:21.557965+0000","last_undegraded":"2023-10-09T04:38:21.557965+0000","last_fullsized":"2023-10-09T04:38:21.557965+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":250,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571410+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:38:21.571410+0000","last_peered":"2023-10-09T04:38:21.571410+0000","last_clean":"2023-10-09T04:38:21.571410+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:38:21.571410+0000","last_undegraded":"2023-10-09T04:38:21.571410+0000","last_fullsized":"2023-10-09T04:38:21.571410+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":195,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.558267+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:38:21.558267+0000","last_peered":"2023-10-09T04:38:21.558267+0000","last_clean":"2023-10-09T04:38:21.558267+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:38:21.558267+0000","last_undegraded":"2023-10-09T04:38:21.558267+0000","last_fullsized":"2023-10-09T04:38:21.558267+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"244'3666","reported_seq":3681,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:35.080884+0000","last_change":"2023-10-09T04:37:52.014620+0000","last_active":"2023-10-09T04:38:35.080884+0000","last_peered":"2023-10-09T04:38:35.080884+0000","last_clean":"2023-10-09T04:38:35.080884+0000","last_became_active":"2023-10-09T04:37:52.014234+0000","last_became_peered":"2023-10-09T04:37:52.014234+0000","last_unstale":"2023-10-09T04:38:35.080884+0000","last_undegraded":"2023-10-09T04:38:35.080884+0000","last_fullsized":"2023-10-09T04:38:35.080884+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3666,"log_dups_size":0,"ondisk_log_size":3666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:14.836376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240254976,"num_objects":3666,"num_object_clones":0,"num_object_copies":10998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3666,"num_write_kb":234624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":232,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570238+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:38:21.570238+0000","last_peered":"2023-10-09T04:38:21.570238+0000","last_clean":"2023-10-09T04:38:21.570238+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:38:21.570238+0000","last_undegraded":"2023-10-09T04:38:21.570238+0000","last_fullsized":"2023-10-09T04:38:21.570238+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":186,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570634+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:38:21.570634+0000","last_peered":"2023-10-09T04:38:21.570634+0000","last_clean":"2023-10-09T04:38:21.570634+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:38:21.570634+0000","last_undegraded":"2023-10-09T04:38:21.570634+0000","last_fullsized":"2023-10-09T04:38:21.570634+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"244'3510","reported_seq":3526,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:35.538267+0000","last_change":"2023-10-09T04:37:52.255599+0000","last_active":"2023-10-09T04:38:35.538267+0000","last_peered":"2023-10-09T04:38:35.538267+0000","last_clean":"2023-10-09T04:38:35.538267+0000","last_became_active":"2023-10-09T04:37:52.255291+0000","last_became_peered":"2023-10-09T04:37:52.255291+0000","last_unstale":"2023-10-09T04:38:35.538267+0000","last_undegraded":"2023-10-09T04:38:35.538267+0000","last_fullsized":"2023-10-09T04:38:35.538267+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3510,"log_dups_size":0,"ondisk_log_size":3510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.990132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230031360,"num_objects":3510,"num_object_clones":0,"num_object_copies":10530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3510,"num_write_kb":224640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":205,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569821+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:38:21.569821+0000","last_peered":"2023-10-09T04:38:21.569821+0000","last_clean":"2023-10-09T04:38:21.569821+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:38:21.569821+0000","last_undegraded":"2023-10-09T04:38:21.569821+0000","last_fullsized":"2023-10-09T04:38:21.569821+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":221,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569206+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:38:21.569206+0000","last_peered":"2023-10-09T04:38:21.569206+0000","last_clean":"2023-10-09T04:38:21.569206+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:38:21.569206+0000","last_undegraded":"2023-10-09T04:38:21.569206+0000","last_fullsized":"2023-10-09T04:38:21.569206+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"244'3598","reported_seq":3613,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:35.056814+0000","last_change":"2023-10-09T04:37:52.013171+0000","last_active":"2023-10-09T04:38:35.056814+0000","last_peered":"2023-10-09T04:38:35.056814+0000","last_clean":"2023-10-09T04:38:35.056814+0000","last_became_active":"2023-10-09T04:37:52.012873+0000","last_became_peered":"2023-10-09T04:37:52.012873+0000","last_unstale":"2023-10-09T04:38:35.056814+0000","last_undegraded":"2023-10-09T04:38:35.056814+0000","last_fullsized":"2023-10-09T04:38:35.056814+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3598,"log_dups_size":0,"ondisk_log_size":3598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:25.833222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235798528,"num_objects":3598,"num_object_clones":0,"num_object_copies":10794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3598,"num_write_kb":230272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":187,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569861+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:38:21.569861+0000","last_peered":"2023-10-09T04:38:21.569861+0000","last_clean":"2023-10-09T04:38:21.569861+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:38:21.569861+0000","last_undegraded":"2023-10-09T04:38:21.569861+0000","last_fullsized":"2023-10-09T04:38:21.569861+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":228,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571294+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:38:21.571294+0000","last_peered":"2023-10-09T04:38:21.571294+0000","last_clean":"2023-10-09T04:38:21.571294+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:38:21.571294+0000","last_undegraded":"2023-10-09T04:38:21.571294+0000","last_fullsized":"2023-10-09T04:38:21.571294+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"244'3504","reported_seq":3519,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:35.041576+0000","last_change":"2023-10-09T04:37:52.012019+0000","last_active":"2023-10-09T04:38:35.041576+0000","last_peered":"2023-10-09T04:38:35.041576+0000","last_clean":"2023-10-09T04:38:35.041576+0000","last_became_active":"2023-10-09T04:37:52.011714+0000","last_became_peered":"2023-10-09T04:37:52.011714+0000","last_unstale":"2023-10-09T04:38:35.041576+0000","last_undegraded":"2023-10-09T04:38:35.041576+0000","last_fullsized":"2023-10-09T04:38:35.041576+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3504,"log_dups_size":0,"ondisk_log_size":3504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:55.324035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229638144,"num_objects":3504,"num_object_clones":0,"num_object_copies":10512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3504,"num_write_kb":224256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570967+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:38:21.570967+0000","last_peered":"2023-10-09T04:38:21.570967+0000","last_clean":"2023-10-09T04:38:21.570967+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:38:21.570967+0000","last_undegraded":"2023-10-09T04:38:21.570967+0000","last_fullsized":"2023-10-09T04:38:21.570967+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":101,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570662+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:38:21.570662+0000","last_peered":"2023-10-09T04:38:21.570662+0000","last_clean":"2023-10-09T04:38:21.570662+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:38:21.570662+0000","last_undegraded":"2023-10-09T04:38:21.570662+0000","last_fullsized":"2023-10-09T04:38:21.570662+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"244'3429","reported_seq":3444,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:34.792949+0000","last_change":"2023-10-09T04:37:52.013681+0000","last_active":"2023-10-09T04:38:34.792949+0000","last_peered":"2023-10-09T04:38:34.792949+0000","last_clean":"2023-10-09T04:38:34.792949+0000","last_became_active":"2023-10-09T04:37:52.013382+0000","last_became_peered":"2023-10-09T04:37:52.013382+0000","last_unstale":"2023-10-09T04:38:34.792949+0000","last_undegraded":"2023-10-09T04:38:34.792949+0000","last_fullsized":"2023-10-09T04:38:34.792949+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3429,"log_dups_size":0,"ondisk_log_size":3429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:30.877521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224722944,"num_objects":3429,"num_object_clones":0,"num_object_copies":10287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3429,"num_write_kb":219456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":239,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.558155+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:38:21.558155+0000","last_peered":"2023-10-09T04:38:21.558155+0000","last_clean":"2023-10-09T04:38:21.558155+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:38:21.558155+0000","last_undegraded":"2023-10-09T04:38:21.558155+0000","last_fullsized":"2023-10-09T04:38:21.558155+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570153+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:38:21.570153+0000","last_peered":"2023-10-09T04:38:21.570153+0000","last_clean":"2023-10-09T04:38:21.570153+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:38:21.570153+0000","last_undegraded":"2023-10-09T04:38:21.570153+0000","last_fullsized":"2023-10-09T04:38:21.570153+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"244'3560","reported_seq":3575,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:35.548458+0000","last_change":"2023-10-09T04:37:52.255728+0000","last_active":"2023-10-09T04:38:35.548458+0000","last_peered":"2023-10-09T04:38:35.548458+0000","last_clean":"2023-10-09T04:38:35.548458+0000","last_became_active":"2023-10-09T04:37:52.255283+0000","last_became_peered":"2023-10-09T04:37:52.255283+0000","last_unstale":"2023-10-09T04:38:35.548458+0000","last_undegraded":"2023-10-09T04:38:35.548458+0000","last_fullsized":"2023-10-09T04:38:35.548458+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3560,"log_dups_size":0,"ondisk_log_size":3560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:18.900085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233308160,"num_objects":3560,"num_object_clones":0,"num_object_copies":10680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3560,"num_write_kb":227840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571432+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:38:21.571432+0000","last_peered":"2023-10-09T04:38:21.571432+0000","last_clean":"2023-10-09T04:38:21.571432+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:38:21.571432+0000","last_undegraded":"2023-10-09T04:38:21.571432+0000","last_fullsized":"2023-10-09T04:38:21.571432+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":238,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570432+0000","last_change":"2023-10-09T04:36:47.537175+0000","last_active":"2023-10-09T04:38:21.570432+0000","last_peered":"2023-10-09T04:38:21.570432+0000","last_clean":"2023-10-09T04:38:21.570432+0000","last_became_active":"2023-10-09T04:36:47.536865+0000","last_became_peered":"2023-10-09T04:36:47.536865+0000","last_unstale":"2023-10-09T04:38:21.570432+0000","last_undegraded":"2023-10-09T04:38:21.570432+0000","last_fullsized":"2023-10-09T04:38:21.570432+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:54.844471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"244'3634","reported_seq":3649,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:34.876787+0000","last_change":"2023-10-09T04:37:52.256288+0000","last_active":"2023-10-09T04:38:34.876787+0000","last_peered":"2023-10-09T04:38:34.876787+0000","last_clean":"2023-10-09T04:38:34.876787+0000","last_became_active":"2023-10-09T04:37:52.256008+0000","last_became_peered":"2023-10-09T04:37:52.256008+0000","last_unstale":"2023-10-09T04:38:34.876787+0000","last_undegraded":"2023-10-09T04:38:34.876787+0000","last_fullsized":"2023-10-09T04:38:34.876787+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3634,"log_dups_size":0,"ondisk_log_size":3634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:34.570497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238157824,"num_objects":3634,"num_object_clones":0,"num_object_copies":10902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3634,"num_write_kb":232576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569075+0000","last_change":"2023-10-09T04:36:47.541552+0000","last_active":"2023-10-09T04:38:21.569075+0000","last_peered":"2023-10-09T04:38:21.569075+0000","last_clean":"2023-10-09T04:38:21.569075+0000","last_became_active":"2023-10-09T04:36:47.541229+0000","last_became_peered":"2023-10-09T04:36:47.541229+0000","last_unstale":"2023-10-09T04:38:21.569075+0000","last_undegraded":"2023-10-09T04:38:21.569075+0000","last_fullsized":"2023-10-09T04:38:21.569075+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:46.140621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":204,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569192+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:38:21.569192+0000","last_peered":"2023-10-09T04:38:21.569192+0000","last_clean":"2023-10-09T04:38:21.569192+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:38:21.569192+0000","last_undegraded":"2023-10-09T04:38:21.569192+0000","last_fullsized":"2023-10-09T04:38:21.569192+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"244'3735","reported_seq":3751,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.632901+0000","last_change":"2023-10-09T04:37:52.011719+0000","last_active":"2023-10-09T04:38:36.632901+0000","last_peered":"2023-10-09T04:38:36.632901+0000","last_clean":"2023-10-09T04:38:36.632901+0000","last_became_active":"2023-10-09T04:37:52.011455+0000","last_became_peered":"2023-10-09T04:37:52.011455+0000","last_unstale":"2023-10-09T04:38:36.632901+0000","last_undegraded":"2023-10-09T04:38:36.632901+0000","last_fullsized":"2023-10-09T04:38:36.632901+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3735,"log_dups_size":0,"ondisk_log_size":3735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:35.397653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244776960,"num_objects":3735,"num_object_clones":0,"num_object_copies":11205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3735,"num_write_kb":239040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":196,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.555683+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:38:21.555683+0000","last_peered":"2023-10-09T04:38:21.555683+0000","last_clean":"2023-10-09T04:38:21.555683+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:38:21.555683+0000","last_undegraded":"2023-10-09T04:38:21.555683+0000","last_fullsized":"2023-10-09T04:38:21.555683+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":210,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.558325+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:38:21.558325+0000","last_peered":"2023-10-09T04:38:21.558325+0000","last_clean":"2023-10-09T04:38:21.558325+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:38:21.558325+0000","last_undegraded":"2023-10-09T04:38:21.558325+0000","last_fullsized":"2023-10-09T04:38:21.558325+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"244'3658","reported_seq":3674,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:35.557621+0000","last_change":"2023-10-09T04:37:52.010535+0000","last_active":"2023-10-09T04:38:35.557621+0000","last_peered":"2023-10-09T04:38:35.557621+0000","last_clean":"2023-10-09T04:38:35.557621+0000","last_became_active":"2023-10-09T04:37:52.009735+0000","last_became_peered":"2023-10-09T04:37:52.009735+0000","last_unstale":"2023-10-09T04:38:35.557621+0000","last_undegraded":"2023-10-09T04:38:35.557621+0000","last_fullsized":"2023-10-09T04:38:35.557621+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3658,"log_dups_size":0,"ondisk_log_size":3658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:31.102036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239730688,"num_objects":3658,"num_object_clones":0,"num_object_copies":10974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3658,"num_write_kb":234112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":219,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556223+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:38:21.556223+0000","last_peered":"2023-10-09T04:38:21.556223+0000","last_clean":"2023-10-09T04:38:21.556223+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:38:21.556223+0000","last_undegraded":"2023-10-09T04:38:21.556223+0000","last_fullsized":"2023-10-09T04:38:21.556223+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":238,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571385+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:38:21.571385+0000","last_peered":"2023-10-09T04:38:21.571385+0000","last_clean":"2023-10-09T04:38:21.571385+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:38:21.571385+0000","last_undegraded":"2023-10-09T04:38:21.571385+0000","last_fullsized":"2023-10-09T04:38:21.571385+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":195,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.555876+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:38:21.555876+0000","last_peered":"2023-10-09T04:38:21.555876+0000","last_clean":"2023-10-09T04:38:21.555876+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:38:21.555876+0000","last_undegraded":"2023-10-09T04:38:21.555876+0000","last_fullsized":"2023-10-09T04:38:21.555876+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"244'3575","reported_seq":3591,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.834687+0000","last_change":"2023-10-09T04:37:52.255930+0000","last_active":"2023-10-09T04:38:36.834687+0000","last_peered":"2023-10-09T04:38:36.834687+0000","last_clean":"2023-10-09T04:38:36.834687+0000","last_became_active":"2023-10-09T04:37:52.255584+0000","last_became_peered":"2023-10-09T04:37:52.255584+0000","last_unstale":"2023-10-09T04:38:36.834687+0000","last_undegraded":"2023-10-09T04:38:36.834687+0000","last_fullsized":"2023-10-09T04:38:36.834687+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3575,"log_dups_size":0,"ondisk_log_size":3575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:52.503096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234291200,"num_objects":3575,"num_object_clones":0,"num_object_copies":10725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3575,"num_write_kb":228800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":208,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571225+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:38:21.571225+0000","last_peered":"2023-10-09T04:38:21.571225+0000","last_clean":"2023-10-09T04:38:21.571225+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:38:21.571225+0000","last_undegraded":"2023-10-09T04:38:21.571225+0000","last_fullsized":"2023-10-09T04:38:21.571225+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569790+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:38:21.569790+0000","last_peered":"2023-10-09T04:38:21.569790+0000","last_clean":"2023-10-09T04:38:21.569790+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:38:21.569790+0000","last_undegraded":"2023-10-09T04:38:21.569790+0000","last_fullsized":"2023-10-09T04:38:21.569790+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":240,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556191+0000","last_change":"2023-10-09T04:36:47.536445+0000","last_active":"2023-10-09T04:38:21.556191+0000","last_peered":"2023-10-09T04:38:21.556191+0000","last_clean":"2023-10-09T04:38:21.556191+0000","last_became_active":"2023-10-09T04:36:47.536098+0000","last_became_peered":"2023-10-09T04:36:47.536098+0000","last_unstale":"2023-10-09T04:38:21.556191+0000","last_undegraded":"2023-10-09T04:38:21.556191+0000","last_fullsized":"2023-10-09T04:38:21.556191+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:56.016452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"244'3532","reported_seq":3548,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.864331+0000","last_change":"2023-10-09T04:37:52.012774+0000","last_active":"2023-10-09T04:38:36.864331+0000","last_peered":"2023-10-09T04:38:36.864331+0000","last_clean":"2023-10-09T04:38:36.864331+0000","last_became_active":"2023-10-09T04:37:52.012402+0000","last_became_peered":"2023-10-09T04:37:52.012402+0000","last_unstale":"2023-10-09T04:38:36.864331+0000","last_undegraded":"2023-10-09T04:38:36.864331+0000","last_fullsized":"2023-10-09T04:38:36.864331+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3532,"log_dups_size":0,"ondisk_log_size":3532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:37.167042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231473152,"num_objects":3532,"num_object_clones":0,"num_object_copies":10596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3532,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":198,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.568446+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:38:21.568446+0000","last_peered":"2023-10-09T04:38:21.568446+0000","last_clean":"2023-10-09T04:38:21.568446+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:38:21.568446+0000","last_undegraded":"2023-10-09T04:38:21.568446+0000","last_fullsized":"2023-10-09T04:38:21.568446+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":226,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.557095+0000","last_change":"2023-10-09T04:36:47.541445+0000","last_active":"2023-10-09T04:38:21.557095+0000","last_peered":"2023-10-09T04:38:21.557095+0000","last_clean":"2023-10-09T04:38:21.557095+0000","last_became_active":"2023-10-09T04:36:47.541142+0000","last_became_peered":"2023-10-09T04:36:47.541142+0000","last_unstale":"2023-10-09T04:38:21.557095+0000","last_undegraded":"2023-10-09T04:38:21.557095+0000","last_fullsized":"2023-10-09T04:38:21.557095+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:53.054914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":214,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.558395+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:38:21.558395+0000","last_peered":"2023-10-09T04:38:21.558395+0000","last_clean":"2023-10-09T04:38:21.558395+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:38:21.558395+0000","last_undegraded":"2023-10-09T04:38:21.558395+0000","last_fullsized":"2023-10-09T04:38:21.558395+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"244'3750","reported_seq":3765,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.853183+0000","last_change":"2023-10-09T04:37:52.013105+0000","last_active":"2023-10-09T04:38:36.853183+0000","last_peered":"2023-10-09T04:38:36.853183+0000","last_clean":"2023-10-09T04:38:36.853183+0000","last_became_active":"2023-10-09T04:37:52.012773+0000","last_became_peered":"2023-10-09T04:37:52.012773+0000","last_unstale":"2023-10-09T04:38:36.853183+0000","last_undegraded":"2023-10-09T04:38:36.853183+0000","last_fullsized":"2023-10-09T04:38:36.853183+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3750,"log_dups_size":0,"ondisk_log_size":3750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.291873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245760000,"num_objects":3750,"num_object_clones":0,"num_object_copies":11250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3750,"num_write_kb":240000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570730+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:38:21.570730+0000","last_peered":"2023-10-09T04:38:21.570730+0000","last_clean":"2023-10-09T04:38:21.570730+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:38:21.570730+0000","last_undegraded":"2023-10-09T04:38:21.570730+0000","last_fullsized":"2023-10-09T04:38:21.570730+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":636,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569527+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:38:21.569527+0000","last_peered":"2023-10-09T04:38:21.569527+0000","last_clean":"2023-10-09T04:38:21.569527+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:38:21.569527+0000","last_undegraded":"2023-10-09T04:38:21.569527+0000","last_fullsized":"2023-10-09T04:38:21.569527+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":168,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.860380+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:38:21.860380+0000","last_peered":"2023-10-09T04:38:21.860380+0000","last_clean":"2023-10-09T04:38:21.860380+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:38:21.860380+0000","last_undegraded":"2023-10-09T04:38:21.860380+0000","last_fullsized":"2023-10-09T04:38:21.860380+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":184,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.572570+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:38:21.572570+0000","last_peered":"2023-10-09T04:38:21.572570+0000","last_clean":"2023-10-09T04:38:21.572570+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:38:21.572570+0000","last_undegraded":"2023-10-09T04:38:21.572570+0000","last_fullsized":"2023-10-09T04:38:21.572570+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"244'3639","reported_seq":3655,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.842280+0000","last_change":"2023-10-09T04:37:52.257119+0000","last_active":"2023-10-09T04:38:36.842280+0000","last_peered":"2023-10-09T04:38:36.842280+0000","last_clean":"2023-10-09T04:38:36.842280+0000","last_became_active":"2023-10-09T04:37:52.256760+0000","last_became_peered":"2023-10-09T04:37:52.256760+0000","last_unstale":"2023-10-09T04:38:36.842280+0000","last_undegraded":"2023-10-09T04:38:36.842280+0000","last_fullsized":"2023-10-09T04:38:36.842280+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3639,"log_dups_size":0,"ondisk_log_size":3639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:16.001253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238485504,"num_objects":3639,"num_object_clones":0,"num_object_copies":10917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3639,"num_write_kb":232896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":244,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569033+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:38:21.569033+0000","last_peered":"2023-10-09T04:38:21.569033+0000","last_clean":"2023-10-09T04:38:21.569033+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:38:21.569033+0000","last_undegraded":"2023-10-09T04:38:21.569033+0000","last_fullsized":"2023-10-09T04:38:21.569033+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":156,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569302+0000","last_change":"2023-10-09T04:34:22.266723+0000","last_active":"2023-10-09T04:38:21.569302+0000","last_peered":"2023-10-09T04:38:21.569302+0000","last_clean":"2023-10-09T04:38:21.569302+0000","last_became_active":"2023-10-09T04:34:22.266261+0000","last_became_peered":"2023-10-09T04:34:22.266261+0000","last_unstale":"2023-10-09T04:38:21.569302+0000","last_undegraded":"2023-10-09T04:38:21.569302+0000","last_fullsized":"2023-10-09T04:38:21.569302+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:54.792194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.600056+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:38:29.600056+0000","last_peered":"2023-10-09T04:38:29.600056+0000","last_clean":"2023-10-09T04:38:29.600056+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:38:29.600056+0000","last_undegraded":"2023-10-09T04:38:29.600056+0000","last_fullsized":"2023-10-09T04:38:29.600056+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"244'3585","reported_seq":3601,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.586245+0000","last_change":"2023-10-09T04:37:52.010038+0000","last_active":"2023-10-09T04:38:36.586245+0000","last_peered":"2023-10-09T04:38:36.586245+0000","last_clean":"2023-10-09T04:38:36.586245+0000","last_became_active":"2023-10-09T04:37:52.009758+0000","last_became_peered":"2023-10-09T04:37:52.009758+0000","last_unstale":"2023-10-09T04:38:36.586245+0000","last_undegraded":"2023-10-09T04:38:36.586245+0000","last_fullsized":"2023-10-09T04:38:36.586245+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3585,"log_dups_size":0,"ondisk_log_size":3585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:03.361014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234946560,"num_objects":3585,"num_object_clones":0,"num_object_copies":10755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3585,"num_write_kb":229440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":227,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569933+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:38:21.569933+0000","last_peered":"2023-10-09T04:38:21.569933+0000","last_clean":"2023-10-09T04:38:21.569933+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:38:21.569933+0000","last_undegraded":"2023-10-09T04:38:21.569933+0000","last_fullsized":"2023-10-09T04:38:21.569933+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556145+0000","last_change":"2023-10-09T04:36:47.543494+0000","last_active":"2023-10-09T04:38:21.556145+0000","last_peered":"2023-10-09T04:38:21.556145+0000","last_clean":"2023-10-09T04:38:21.556145+0000","last_became_active":"2023-10-09T04:36:47.543197+0000","last_became_peered":"2023-10-09T04:36:47.543197+0000","last_unstale":"2023-10-09T04:38:21.556145+0000","last_undegraded":"2023-10-09T04:38:21.556145+0000","last_fullsized":"2023-10-09T04:38:21.556145+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.900185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":88,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570743+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:38:21.570743+0000","last_peered":"2023-10-09T04:38:21.570743+0000","last_clean":"2023-10-09T04:38:21.570743+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:38:21.570743+0000","last_undegraded":"2023-10-09T04:38:21.570743+0000","last_fullsized":"2023-10-09T04:38:21.570743+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"244'3555","reported_seq":3570,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:34.952425+0000","last_change":"2023-10-09T04:37:52.013669+0000","last_active":"2023-10-09T04:38:34.952425+0000","last_peered":"2023-10-09T04:38:34.952425+0000","last_clean":"2023-10-09T04:38:34.952425+0000","last_became_active":"2023-10-09T04:37:52.013386+0000","last_became_peered":"2023-10-09T04:37:52.013386+0000","last_unstale":"2023-10-09T04:38:34.952425+0000","last_undegraded":"2023-10-09T04:38:34.952425+0000","last_fullsized":"2023-10-09T04:38:34.952425+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3555,"log_dups_size":0,"ondisk_log_size":3555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:57.288541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232980480,"num_objects":3555,"num_object_clones":0,"num_object_copies":10665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3555,"num_write_kb":227520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":239,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.558191+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:38:21.558191+0000","last_peered":"2023-10-09T04:38:21.558191+0000","last_clean":"2023-10-09T04:38:21.558191+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:38:21.558191+0000","last_undegraded":"2023-10-09T04:38:21.558191+0000","last_fullsized":"2023-10-09T04:38:21.558191+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":195,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556190+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:38:21.556190+0000","last_peered":"2023-10-09T04:38:21.556190+0000","last_clean":"2023-10-09T04:38:21.556190+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:38:21.556190+0000","last_undegraded":"2023-10-09T04:38:21.556190+0000","last_fullsized":"2023-10-09T04:38:21.556190+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.555707+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:38:21.555707+0000","last_peered":"2023-10-09T04:38:21.555707+0000","last_clean":"2023-10-09T04:38:21.555707+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:38:21.555707+0000","last_undegraded":"2023-10-09T04:38:21.555707+0000","last_fullsized":"2023-10-09T04:38:21.555707+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"244'3560","reported_seq":3575,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:34.931980+0000","last_change":"2023-10-09T04:37:52.016326+0000","last_active":"2023-10-09T04:38:34.931980+0000","last_peered":"2023-10-09T04:38:34.931980+0000","last_clean":"2023-10-09T04:38:34.931980+0000","last_became_active":"2023-10-09T04:37:52.016026+0000","last_became_peered":"2023-10-09T04:37:52.016026+0000","last_unstale":"2023-10-09T04:38:34.931980+0000","last_undegraded":"2023-10-09T04:38:34.931980+0000","last_fullsized":"2023-10-09T04:38:34.931980+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":3560,"log_dups_size":0,"ondisk_log_size":3560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:46.974447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233308160,"num_objects":3560,"num_object_clones":0,"num_object_copies":10680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3560,"num_write_kb":227840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":223,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571733+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:38:21.571733+0000","last_peered":"2023-10-09T04:38:21.571733+0000","last_clean":"2023-10-09T04:38:21.571733+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:38:21.571733+0000","last_undegraded":"2023-10-09T04:38:21.571733+0000","last_fullsized":"2023-10-09T04:38:21.571733+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":98,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556359+0000","last_change":"2023-10-09T04:34:22.266652+0000","last_active":"2023-10-09T04:38:21.556359+0000","last_peered":"2023-10-09T04:38:21.556359+0000","last_clean":"2023-10-09T04:38:21.556359+0000","last_became_active":"2023-10-09T04:34:22.266267+0000","last_became_peered":"2023-10-09T04:34:22.266267+0000","last_unstale":"2023-10-09T04:38:21.556359+0000","last_undegraded":"2023-10-09T04:38:21.556359+0000","last_fullsized":"2023-10-09T04:38:21.556359+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:50.646309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569132+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:38:21.569132+0000","last_peered":"2023-10-09T04:38:21.569132+0000","last_clean":"2023-10-09T04:38:21.569132+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:38:21.569132+0000","last_undegraded":"2023-10-09T04:38:21.569132+0000","last_fullsized":"2023-10-09T04:38:21.569132+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":84,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569714+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:38:21.569714+0000","last_peered":"2023-10-09T04:38:21.569714+0000","last_clean":"2023-10-09T04:38:21.569714+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:38:21.569714+0000","last_undegraded":"2023-10-09T04:38:21.569714+0000","last_fullsized":"2023-10-09T04:38:21.569714+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570432+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:38:21.570432+0000","last_peered":"2023-10-09T04:38:21.570432+0000","last_clean":"2023-10-09T04:38:21.570432+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:38:21.570432+0000","last_undegraded":"2023-10-09T04:38:21.570432+0000","last_fullsized":"2023-10-09T04:38:21.570432+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":206,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.570832+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:38:21.570832+0000","last_peered":"2023-10-09T04:38:21.570832+0000","last_clean":"2023-10-09T04:38:21.570832+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:38:21.570832+0000","last_undegraded":"2023-10-09T04:38:21.570832+0000","last_fullsized":"2023-10-09T04:38:21.570832+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":221,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569988+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:38:21.569988+0000","last_peered":"2023-10-09T04:38:21.569988+0000","last_clean":"2023-10-09T04:38:21.569988+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:38:21.569988+0000","last_undegraded":"2023-10-09T04:38:21.569988+0000","last_fullsized":"2023-10-09T04:38:21.569988+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":189,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571465+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:38:21.571465+0000","last_peered":"2023-10-09T04:38:21.571465+0000","last_clean":"2023-10-09T04:38:21.571465+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:38:21.571465+0000","last_undegraded":"2023-10-09T04:38:21.571465+0000","last_fullsized":"2023-10-09T04:38:21.571465+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":241,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569362+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:38:21.569362+0000","last_peered":"2023-10-09T04:38:21.569362+0000","last_clean":"2023-10-09T04:38:21.569362+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:38:21.569362+0000","last_undegraded":"2023-10-09T04:38:21.569362+0000","last_fullsized":"2023-10-09T04:38:21.569362+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.557834+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:38:21.557834+0000","last_peered":"2023-10-09T04:38:21.557834+0000","last_clean":"2023-10-09T04:38:21.557834+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:38:21.557834+0000","last_undegraded":"2023-10-09T04:38:21.557834+0000","last_fullsized":"2023-10-09T04:38:21.557834+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":262,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.556438+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:38:21.556438+0000","last_peered":"2023-10-09T04:38:21.556438+0000","last_clean":"2023-10-09T04:38:21.556438+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:38:21.556438+0000","last_undegraded":"2023-10-09T04:38:21.556438+0000","last_fullsized":"2023-10-09T04:38:21.556438+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":118,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.571343+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:38:21.571343+0000","last_peered":"2023-10-09T04:38:21.571343+0000","last_clean":"2023-10-09T04:38:21.571343+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:38:21.571343+0000","last_undegraded":"2023-10-09T04:38:21.571343+0000","last_fullsized":"2023-10-09T04:38:21.571343+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":231,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-09T04:38:21.569033+0000","last_change":"2023-10-09T04:36:47.544511+0000","last_active":"2023-10-09T04:38:21.569033+0000","last_peered":"2023-10-09T04:38:21.569033+0000","last_clean":"2023-10-09T04:38:21.569033+0000","last_became_active":"2023-10-09T04:36:47.544123+0000","last_became_peered":"2023-10-09T04:36:47.544123+0000","last_unstale":"2023-10-09T04:38:21.569033+0000","last_undegraded":"2023-10-09T04:38:21.569033+0000","last_fullsized":"2023-10-09T04:38:21.569033+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:56.150751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3767664640,"num_objects":57490,"num_object_clones":0,"num_object_copies":172470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":57490,"num_write_kb":3679360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11324882944,"data_stored":11324882944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57490,"ondisk_log_size":57490,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234380,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2264464,"kb_used_data":1357576,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":91480688,"statfs":{"total":95995035648,"available":93676224512,"internally_reserved":0,"allocated":1390157824,"data_stored":1389572639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":4.1920000000000002,"15min":6.0010000000000003},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.6580000000000004,"5min":111.72199999999999,"15min":629.22500000000002},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.71599999999999997,"5min":4.2530000000000001,"15min":5.9939999999999998},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.7999999999999998,"5min":111.76600000000001,"15min":629.43100000000004},"last":3.5880000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:38:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":1.9710000000000001,"15min":4.2789999999999999},"min":{"1min":0.311,"5min":0.254,"15min":0.22800000000000001},"max":{"1min":39.795999999999999,"5min":39.795999999999999,"15min":224.13499999999999},"last":0.752},{"interface":"front","average":{"1min":3.0329999999999999,"5min":2.157,"15min":4.4000000000000004},"min":{"1min":0.378,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":40.124000000000002,"5min":40.124000000000002,"15min":224.203},"last":7.0579999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:37:49 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.802,"15min":1.802},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.0800000000000001,"5min":6.0800000000000001,"15min":6.0800000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.3029999999999999,"15min":2.3029999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.8379999999999992,"5min":8.8379999999999992,"15min":8.8379999999999992},"last":0.55800000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:38:15 2023","interfaces":[{"interface":"back","average":{"1min":9.9990000000000006,"5min":5.4349999999999996,"15min":6.1600000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":88.75,"5min":133.13499999999999,"15min":249.989},"last":2.8130000000000002},{"interface":"front","average":{"1min":10.470000000000001,"5min":5.7000000000000002,"15min":6.3010000000000002},"min":{"1min":0.40100000000000002,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":88.463999999999999,"5min":133.35900000000001,"15min":250.21700000000001},"last":6.7229999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:37:45 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":1.754,"15min":1.504},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.253},"max":{"1min":11.752000000000001,"5min":11.752000000000001,"15min":11.752000000000001},"last":0.31},{"interface":"front","average":{"1min":1.899,"5min":1.7669999999999999,"15min":1.226},"min":{"1min":0.307,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.0309999999999997,"5min":9.4179999999999993,"15min":10.613},"last":0.34399999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:37:45 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.2749999999999999,"15min":1.1259999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.27900000000000003},"max":{"1min":6.1120000000000001,"5min":6.1120000000000001,"15min":8.8610000000000007},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.8009999999999999,"5min":1.7310000000000001,"15min":1.4390000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27400000000000002},"max":{"1min":8.8089999999999993,"5min":10.326000000000001,"15min":10.895},"last":1.599}]},{"osd":6,"last update":"Mon Oct 9 04:37:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.6220000000000001,"15min":1.3480000000000001},"min":{"1min":0.34300000000000003,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.016,"5min":8.1120000000000001,"15min":12.007},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.157,"5min":2.1709999999999998,"15min":1.581},"min":{"1min":0.42999999999999999,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":8.7789999999999999,"5min":9.3819999999999997,"15min":10.917},"last":0.47799999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496014,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2701440,"kb_used_data":1799160,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":91043712,"statfs":{"total":95995035648,"available":93228761088,"internally_reserved":0,"allocated":1842339840,"data_stored":1841770570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":6.2969999999999997,"15min":8.0120000000000005},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.217},"max":{"1min":1.0469999999999999,"5min":141.14099999999999,"15min":1284.521},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":6.5430000000000001,"15min":8.0630000000000006},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.253},"max":{"1min":2.4049999999999998,"5min":141.167,"15min":1284.4690000000001},"last":0.497}]},{"osd":1,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":6.1059999999999999,"5min":4.391,"15min":3.9660000000000002},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":112.44199999999999,"5min":112.44199999999999,"15min":185.74000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":5.8360000000000003,"5min":4.2629999999999999,"15min":4.0419999999999998},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.24299999999999999},"max":{"1min":112.117,"5min":112.117,"15min":189.03999999999999},"last":0.55300000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:37:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.0990000000000002,"15min":2.0990000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0459999999999994,"5min":9.0459999999999994,"15min":9.0459999999999994},"last":23.75},{"interface":"front","average":{"1min":2.081,"5min":2.081,"15min":2.081},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":23.774000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":7.0410000000000004,"5min":9.7840000000000007,"15min":4.1479999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":121.139,"5min":167.56100000000001,"15min":167.56100000000001},"last":24.736999999999998},{"interface":"front","average":{"1min":7.2510000000000003,"5min":9.9979999999999993,"15min":4.2359999999999998},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.313},"max":{"1min":120.61499999999999,"5min":167.577,"15min":167.577},"last":23.719000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.728,"15min":1.4079999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.246},"max":{"1min":3.6499999999999999,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.2809999999999999,"15min":1.268},"min":{"1min":0.34499999999999997,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":2.7629999999999999,"5min":8.6270000000000007,"15min":12.028},"last":0.40100000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":1.857,"15min":1.873},"min":{"1min":0.34000000000000002,"5min":0.29999999999999999,"15min":0.28100000000000003},"max":{"1min":3.6720000000000002,"5min":14.631,"15min":81.087000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":1.629,"15min":1.8660000000000001},"min":{"1min":0.33800000000000002,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":1.2310000000000001,"5min":11.279999999999999,"15min":81.706999999999994},"last":0.52500000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.623,"15min":1.6919999999999999},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.1950000000000003,"5min":11.226000000000001,"15min":81.674999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.093,"5min":1.9379999999999999,"15min":1.7849999999999999},"min":{"1min":0.35899999999999999,"5min":0.308,"15min":0.308},"max":{"1min":14.390000000000001,"5min":14.390000000000001,"15min":81.055000000000007},"last":0.56799999999999995}]}]},{"osd":1,"up_from":118,"seq":506806141104,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2328676,"kb_used_data":1854620,"kb_used_omap":4,"kb_used_meta":474043,"kb_avail":91416476,"statfs":{"total":95995035648,"available":93610471424,"internally_reserved":0,"allocated":1899130880,"data_stored":1898119716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":485420942},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":3.75,"15min":3.6280000000000001},"min":{"1min":0.19,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":7.7679999999999998,"5min":93.171999999999997,"15min":97.308000000000007},"last":2.0590000000000002},{"interface":"front","average":{"1min":0.751,"5min":3.8330000000000002,"15min":3.9089999999999998},"min":{"1min":0.217,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":2.6000000000000001,"5min":93.147000000000006,"15min":97.055999999999997},"last":0.84199999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:37:47 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":2.637,"15min":2.637},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":40.924999999999997,"5min":40.924999999999997,"15min":40.924999999999997},"last":2.129},{"interface":"front","average":{"1min":2.9489999999999998,"5min":2.9489999999999998,"15min":2.9489999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":42.029000000000003,"5min":42.029000000000003,"15min":42.029000000000003},"last":2.1429999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1669999999999998,"5min":4.6660000000000004,"15min":3.129},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":67.844999999999999,"5min":142.34200000000001,"15min":195.035},"last":2.1019999999999999},{"interface":"front","average":{"1min":4.1180000000000003,"5min":5.2610000000000001,"15min":3.4489999999999998},"min":{"1min":0.30399999999999999,"5min":0.246,"15min":0.19700000000000001},"max":{"1min":67.534000000000006,"5min":141.75200000000001,"15min":195.465},"last":1.044}]},{"osd":4,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":3.359,"15min":2.956},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.9449999999999998,"5min":170.072,"15min":170.072},"last":5.827},{"interface":"front","average":{"1min":1.1699999999999999,"5min":2.766,"15min":2.202},"min":{"1min":0.32800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.2130000000000001,"5min":169.94499999999999,"15min":169.94499999999999},"last":2.2090000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":2.7799999999999998,"15min":2.4489999999999998},"min":{"1min":0.38200000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.6970000000000001,"5min":171.036,"15min":171.036},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.1919999999999999,"5min":3.4009999999999998,"15min":2.7410000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.27500000000000002},"max":{"1min":4.2800000000000002,"5min":170.88,"15min":170.88},"last":5.96}]},{"osd":6,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.778,"15min":2.0070000000000001},"min":{"1min":0.40200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.6440000000000001,"5min":11.548,"15min":13.388},"last":0.872},{"interface":"front","average":{"1min":1.1279999999999999,"5min":2.3610000000000002,"15min":2.419},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.0419999999999998,"5min":12.596,"15min":13.882},"last":5.9130000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":3.6859999999999999,"15min":2.9350000000000001},"min":{"1min":0.36599999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.8680000000000003,"5min":174.75999999999999,"15min":174.75999999999999},"last":5.8760000000000003},{"interface":"front","average":{"1min":1.278,"5min":3.105,"15min":2.1139999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":4.2590000000000003,"5min":174.779,"15min":174.779},"last":2.1739999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024161,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1879648,"kb_used_data":1165400,"kb_used_omap":5,"kb_used_meta":714234,"kb_avail":91865504,"statfs":{"total":95995035648,"available":94070276096,"internally_reserved":0,"allocated":1193369600,"data_stored":1192419454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":731376250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:38:26 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":3.1179999999999999,"15min":4.0209999999999999},"min":{"1min":0.219,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":5.7430000000000003,"5min":101.81100000000001,"15min":167.92699999999999},"last":138.536},{"interface":"front","average":{"1min":1.0609999999999999,"5min":3.371,"15min":4.1079999999999997},"min":{"1min":0.23000000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":7.6369999999999996,"5min":103.444,"15min":169.78999999999999},"last":138.89599999999999}]},{"osd":2,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":9.4489999999999998,"5min":9.4489999999999998,"15min":9.4489999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":131.30699999999999,"5min":131.30699999999999,"15min":131.30699999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":9.8859999999999992,"5min":9.8859999999999992,"15min":9.8859999999999992},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":129.922,"5min":129.922,"15min":129.922},"last":0.497}]},{"osd":3,"last update":"Mon Oct 9 04:38:15 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":3.7269999999999999,"15min":3.0099999999999998},"min":{"1min":0.28699999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.8470000000000004,"5min":91.578999999999994,"15min":213.28399999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.2640000000000002,"15min":3.2229999999999999},"min":{"1min":0.31,"5min":0.222,"15min":0.222},"max":{"1min":11.220000000000001,"5min":92.067999999999998,"15min":212.27000000000001},"last":0.55700000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":3.6099999999999999,"15min":2.3839999999999999},"min":{"1min":0.33200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.7109999999999999,"5min":161.67099999999999,"15min":161.67099999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":3.8279999999999998,"15min":2.5459999999999998},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.4159999999999999,"5min":160.74799999999999,"15min":160.74799999999999},"last":0.68100000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.869,"15min":2.0059999999999998},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.27900000000000003},"max":{"1min":1.9690000000000001,"5min":35.878999999999998,"15min":35.878999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.8360000000000001,"15min":1.712},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.5390000000000001,"5min":35.755000000000003,"15min":35.755000000000003},"last":0.53900000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":2.8500000000000001,"15min":2.3690000000000002},"min":{"1min":0.39300000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.452,"5min":108.63800000000001,"15min":108.63800000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.82799999999999996,"5min":2.726,"15min":2.0880000000000001},"min":{"1min":0.41399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.609,"5min":108.76300000000001,"15min":108.76300000000001},"last":0.61899999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.595,"15min":1.7310000000000001},"min":{"1min":0.45300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.5760000000000001,"5min":6.9770000000000003,"15min":11.890000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.5149999999999999,"15min":1.875},"min":{"1min":0.41199999999999998,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":2.7799999999999998,"5min":8.8260000000000005,"15min":10.765000000000001},"last":0.66100000000000003}]}]},{"osd":2,"up_from":221,"seq":949187772439,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1967384,"kb_used_data":1755216,"kb_used_omap":7,"kb_used_meta":212152,"kb_avail":91777768,"statfs":{"total":95995035648,"available":93980434432,"internally_reserved":0,"allocated":1797341184,"data_stored":1796458835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":217243729},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":10.433999999999999,"5min":10.433999999999999,"15min":10.433999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":72.608000000000004,"5min":72.608000000000004,"15min":72.608000000000004},"last":0.28100000000000003},{"interface":"front","average":{"1min":9.9320000000000004,"5min":9.9320000000000004,"15min":9.9320000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":72.632000000000005,"5min":72.632000000000005,"15min":72.632000000000005},"last":3.431}]},{"osd":1,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":5.3159999999999998,"5min":5.3159999999999998,"15min":5.3159999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":72.807000000000002,"5min":72.807000000000002,"15min":72.807000000000002},"last":0.95199999999999996},{"interface":"front","average":{"1min":5.851,"5min":5.851,"15min":5.851},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":72.664000000000001,"5min":72.664000000000001,"15min":72.664000000000001},"last":0.54300000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":5.4059999999999997,"5min":5.4059999999999997,"15min":5.4059999999999997},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":72.701999999999998,"5min":72.701999999999998,"15min":72.701999999999998},"last":0.84399999999999997},{"interface":"front","average":{"1min":5.2670000000000003,"5min":5.2670000000000003,"15min":5.2670000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":72.822999999999993,"5min":72.822999999999993,"15min":72.822999999999993},"last":2.6709999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":5.0090000000000003,"5min":5.0090000000000003,"15min":5.0090000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":72.944999999999993,"5min":72.944999999999993,"15min":72.944999999999993},"last":0.60099999999999998},{"interface":"front","average":{"1min":5.2690000000000001,"5min":5.2690000000000001,"15min":5.2690000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":73.054000000000002,"5min":73.054000000000002,"15min":73.054000000000002},"last":2.4319999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":4.6280000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":72.986000000000004,"5min":72.986000000000004,"15min":72.986000000000004},"last":4.2480000000000002},{"interface":"front","average":{"1min":5.1840000000000002,"5min":5.1840000000000002,"15min":5.1840000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":72.963999999999999,"5min":72.963999999999999,"15min":72.963999999999999},"last":1.819}]},{"osd":6,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":12.567,"5min":12.567,"15min":12.567},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":135.614,"5min":135.614,"15min":135.614},"last":2.6349999999999998},{"interface":"front","average":{"1min":11.651,"5min":11.651,"15min":11.651},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":130.196,"5min":130.196,"15min":130.196},"last":0.505}]},{"osd":7,"last update":"Mon Oct 9 04:37:48 2023","interfaces":[{"interface":"back","average":{"1min":5.4790000000000001,"5min":5.4790000000000001,"15min":5.4790000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":73.079999999999998,"5min":73.079999999999998,"15min":73.079999999999998},"last":2.476},{"interface":"front","average":{"1min":4.6689999999999996,"5min":4.6689999999999996,"15min":4.6689999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":73.069000000000003,"5min":73.069000000000003,"15min":73.069000000000003},"last":0.80200000000000005}]}]},{"osd":3,"up_from":142,"seq":609885356162,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2042568,"kb_used_data":1635072,"kb_used_omap":2,"kb_used_meta":407485,"kb_avail":91702584,"statfs":{"total":95995035648,"available":93903446016,"internally_reserved":0,"allocated":1674313728,"data_stored":1673442672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":417264693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":2.7050000000000001,"15min":2.048},"min":{"1min":0.23599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.423,"5min":142.14500000000001,"15min":142.14500000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.1279999999999999,"5min":2.8359999999999999,"15min":2.0329999999999999},"min":{"1min":0.28999999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.484,"5min":138.14599999999999,"15min":138.14599999999999},"last":0.27300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":2.8090000000000002,"15min":3.1000000000000001},"min":{"1min":0.255,"5min":0.19800000000000001,"15min":0.192},"max":{"1min":5.0309999999999997,"5min":145.47499999999999,"15min":180.02500000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.873,"5min":2.968,"15min":3.117},"min":{"1min":0.245,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":6.3529999999999998,"5min":144.83799999999999,"15min":184.00899999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6759999999999999,"5min":1.6759999999999999,"15min":1.6759999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":10.119,"5min":10.119,"15min":10.119},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.26,"5min":1.26,"15min":1.26},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":2.6190000000000002,"15min":1.9710000000000001},"min":{"1min":0.379,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.7770000000000001,"5min":103.208,"15min":103.208},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.292,"5min":2.234,"15min":1.6120000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.835,"5min":103.218,"15min":103.218},"last":0.56499999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":2.79,"15min":2.0600000000000001},"min":{"1min":0.35199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.491,"5min":90.972999999999999,"15min":90.972999999999999},"last":2.6440000000000001},{"interface":"front","average":{"1min":1.633,"5min":2.7040000000000002,"15min":2.0329999999999999},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":17.422999999999998,"5min":90.959000000000003,"15min":90.959000000000003},"last":2.0600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":2.8050000000000002,"15min":1.78},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.0890000000000004,"5min":104.422,"15min":104.422},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.0680000000000001,"5min":3.9180000000000001,"15min":2.512},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.30499999999999999},"max":{"1min":11.696999999999999,"5min":123.505,"15min":123.505},"last":0.34499999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":2.036,"15min":2.1579999999999999},"min":{"1min":0.45000000000000001,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":5.2229999999999999,"5min":22.181999999999999,"15min":57.399000000000001},"last":3.3319999999999999},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.696,"15min":1.8440000000000001},"min":{"1min":0.45000000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":6.3920000000000003,"5min":21.812999999999999,"15min":57.447000000000003},"last":2.617}]}]},{"osd":4,"up_from":30,"seq":128849019288,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804616,"kb_used_data":912576,"kb_used_omap":0,"kb_used_meta":892032,"kb_avail":91940536,"statfs":{"total":95995035648,"available":94147108864,"internally_reserved":0,"allocated":934477824,"data_stored":933900217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913440768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:35 2023","interfaces":[{"interface":"back","average":{"1min":9.5199999999999996,"5min":8.1519999999999992,"15min":5.0780000000000003},"min":{"1min":0.28899999999999998,"5min":0.23100000000000001,"15min":0.20499999999999999},"max":{"1min":171.13499999999999,"5min":171.13499999999999,"15min":219.577},"last":0.41599999999999998},{"interface":"front","average":{"1min":9.1720000000000006,"5min":8.2349999999999994,"15min":4.8559999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.247},"max":{"1min":171.11799999999999,"5min":171.11799999999999,"15min":218.96000000000001},"last":0.46700000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":3.081,"15min":4.7430000000000003},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.4590000000000001,"5min":62.518999999999998,"15min":142.74799999999999},"last":56.122},{"interface":"front","average":{"1min":1.0369999999999999,"5min":3.1240000000000001,"15min":4.8869999999999996},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":3.1829999999999998,"5min":62.718000000000004,"15min":139.09700000000001},"last":56.043999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:37:46 2023","interfaces":[{"interface":"back","average":{"1min":3.645,"5min":3.645,"15min":3.645},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":36.737000000000002,"5min":36.737000000000002,"15min":36.737000000000002},"last":0.437},{"interface":"front","average":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":3.0259999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":36.423000000000002,"5min":36.423000000000002,"15min":36.423000000000002},"last":0.38600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:38:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":3.7639999999999998,"15min":4.9219999999999997},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.9859999999999998,"5min":131.869,"15min":262.524},"last":57.228000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":3.6949999999999998,"15min":5.1040000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.694,"5min":131.82300000000001,"15min":262.55099999999999},"last":56.899000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:38:07 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.694,"15min":1.95},"min":{"1min":0.28599999999999998,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":5.165,"5min":107.476,"15min":107.476},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.7509999999999999,"15min":2.1200000000000001},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":6.1609999999999996,"5min":107.414,"15min":107.414},"last":0.29899999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:37:35 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":2.8100000000000001,"15min":1.907},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":4.5439999999999996,"5min":65.75,"15min":65.75},"last":0.37},{"interface":"front","average":{"1min":2.5099999999999998,"5min":2.8559999999999999,"15min":1.8979999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29499999999999998},"max":{"1min":9.3320000000000007,"5min":65.849999999999994,"15min":65.849999999999994},"last":0.312}]},{"osd":7,"last update":"Mon Oct 9 04:37:58 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.839,"15min":1.714},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0759999999999996,"5min":11.093999999999999,"15min":76.283000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.6299999999999999,"15min":1.7050000000000001},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.2450000000000001,"5min":11.214,"15min":76.259},"last":0.40699999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757652,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1806008,"kb_used_data":911600,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":91939144,"statfs":{"total":95995035648,"available":94145683456,"internally_reserved":0,"allocated":933478400,"data_stored":932912921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:37:33 2023","interfaces":[{"interface":"back","average":{"1min":7.444,"5min":7.8120000000000003,"15min":4.8259999999999996},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":88.346000000000004,"5min":210.85499999999999,"15min":210.85499999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":7.8739999999999997,"5min":7.5620000000000003,"15min":4.633},"min":{"1min":0.307,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":88.375,"5min":210.23500000000001,"15min":210.23500000000001},"last":0.52700000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:37:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4770000000000001,"5min":3.9009999999999998,"15min":3.645},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.255},"max":{"1min":3.7690000000000001,"5min":84.030000000000001,"15min":109.95699999999999},"last":3.6280000000000001},{"interface":"front","average":{"1min":1.502,"5min":4.5199999999999996,"15min":4.5190000000000001},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.8559999999999999,"5min":85.373999999999995,"15min":116.04000000000001},"last":4.3689999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:37:47 2023","interfaces":[{"interface":"back","average":{"1min":7.306,"5min":7.306,"15min":7.306},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":90.763999999999996,"5min":90.763999999999996,"15min":90.763999999999996},"last":0.50700000000000001},{"interface":"front","average":{"1min":7.4560000000000004,"5min":7.4560000000000004,"15min":7.4560000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":91.340000000000003,"5min":91.340000000000003,"15min":91.340000000000003},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":4.125,"15min":4.4290000000000003},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9409999999999998,"5min":84.081999999999994,"15min":259.053},"last":0.442},{"interface":"front","average":{"1min":0.94699999999999995,"5min":3.859,"15min":4.3230000000000004},"min":{"1min":0.40400000000000003,"5min":0.31,"15min":0.31},"max":{"1min":4.2060000000000004,"5min":85.540000000000006,"15min":259.108},"last":0.53900000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":5.617,"15min":2.5089999999999999},"min":{"1min":0.32300000000000001,"5min":0.254,"15min":0.248},"max":{"1min":3.206,"5min":361.09399999999999,"15min":361.09399999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.073,"5min":5.9880000000000004,"15min":2.7839999999999998},"min":{"1min":0.28199999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.7599999999999998,"5min":361.71300000000002,"15min":361.71300000000002},"last":0.33100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:37:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":1.8899999999999999,"15min":1.363},"min":{"1min":0.33000000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.476,"5min":10.194000000000001,"15min":11.192},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.0800000000000001,"5min":1.8280000000000001,"15min":1.4339999999999999},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":7.8719999999999999,"5min":11.974,"15min":14.334},"last":0.372}]},{"osd":7,"last update":"Mon Oct 9 04:38:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.49,"15min":1.409},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.899,"5min":11.093,"15min":11.093},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.607,"15min":1.3600000000000001},"min":{"1min":0.41199999999999998,"5min":0.315,"15min":0.315},"max":{"1min":6.2380000000000004,"5min":15.257999999999999,"15min":15.257999999999999},"last":0.40000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1211301888,"data_stored":1211301888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1900150784,"data_stored":1900150784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1404043264,"data_stored":1404043264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1677524992,"data_stored":1677524992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":938999808,"data_stored":938999808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":939327488,"data_stored":939327488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1856241664,"data_stored":1856241664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1397293056,"data_stored":1397293056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:38:43.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:38:43.187 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:43.393 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:43.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:43.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:43.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:43.574 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:44.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 65840 65824 97.8886 61.0625 0.00999271 0.0102101 2023-10-09T04:38:44.052 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:44.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:44.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:44.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:44.241 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:44.509 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:44 smithi029 ceph-mon[216903]: pgmap v907: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:38:44.509 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:44.510 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:44 smithi177 ceph-mon[195797]: pgmap v907: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:38:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:44 smithi029 ceph-mon[223858]: pgmap v907: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:38:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:38:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:45.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 66737 66721 96.9164 56.0625 0.00269084 0.0102762 2023-10-09T04:38:45.864 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:45.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:45.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:45.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:46.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 67318 67302 95.5397 36.3125 0.0130055 0.0104613 2023-10-09T04:38:46.047 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:46.176 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:46.181 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:46.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:46.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:46.225 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:46.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:46.373 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:46.420 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:46.598 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:46.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:46.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:46.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:46.770 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:46 smithi177 ceph-mon[195797]: pgmap v908: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-10-09T04:38:46.790 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:46 smithi029 ceph-mon[223858]: pgmap v908: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-10-09T04:38:46.790 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:46 smithi029 ceph-mon[216903]: pgmap v908: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2023-10-09T04:38:47.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 68302 68286 94.7835 61.5 0.00626584 0.0105436 2023-10-09T04:38:47.218 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:47.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:47.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:38:47.418 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:47.806 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:38:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:47.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:38:47.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:47.963 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:48.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 69384 69368 94.1932 67.625 0.0158303 0.0106118 2023-10-09T04:38:48.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:48.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:48.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:48.232 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:48.541 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:38:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:48 smithi029 ceph-mon[216903]: pgmap v909: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:38:48.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2320660104' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:48.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:48.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:48 smithi029 ceph-mon[223858]: pgmap v909: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:38:48.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2320660104' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:48.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 69944 69928 92.9344 35 0.0143805 0.0107547 2023-10-09T04:38:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:48 smithi177 ceph-mon[195797]: pgmap v909: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-09T04:38:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2320660104' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:38:49.563 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:49.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:49.578 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:49.578 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[216903]: failed to encode map e245 with expected crc 2023-10-09T04:38:49.578 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:49.578 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[216903]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:38:49.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[216903]: failed to encode map e245 with expected crc 2023-10-09T04:38:49.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:49.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:49.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[223858]: failed to encode map e245 with expected crc 2023-10-09T04:38:49.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:49.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[223858]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:38:49.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:49 smithi029 ceph-mon[223858]: failed to encode map e245 with expected crc 2023-10-09T04:38:49.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:49.739 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:50.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 70922 70906 92.2719 61.125 0.0229587 0.010832 2023-10-09T04:38:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:49 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:38:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:49 smithi177 ceph-mon[195797]: failed to encode map e245 with expected crc 2023-10-09T04:38:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:49 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:38:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:49 smithi177 ceph-mon[195797]: osdmap e245: 8 total, 8 up, 8 in 2023-10-09T04:38:50.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:49 smithi177 ceph-mon[195797]: failed to encode map e245 with expected crc 2023-10-09T04:38:50.213 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:50.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:50.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:50.359 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:50.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:50.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:50.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:38:50.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:50.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:50 smithi177 ceph-mon[195797]: pgmap v911: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:38:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:50 smithi029 ceph-mon[216903]: pgmap v911: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:38:50.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:50 smithi029 ceph-mon[223858]: pgmap v911: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-10-09T04:38:51.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 71632 71616 91.2945 44.375 0.0140743 0.0109482 2023-10-09T04:38:52.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 72748 72732 90.8637 69.75 0.013271 0.0109999 2023-10-09T04:38:52.181 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:38:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:52.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:38:52.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:52.374 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:52.566 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:38:52.566 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:38:52.566 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:38:52.566 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:38:52.624 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:52.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:52.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:52.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:52 smithi029 ceph-mon[216903]: pgmap v912: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-10-09T04:38:52.809 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:52 smithi029 ceph-mon[223858]: pgmap v912: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-10-09T04:38:52.824 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:53.007 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 73713 73697 90.2608 60.3125 0.0123102 0.0110737 2023-10-09T04:38:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:52 smithi177 ceph-mon[195797]: pgmap v912: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-10-09T04:38:53.042 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:53.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:53.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:53.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:38:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:38:52] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-10-09T04:38:53.188 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:53.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:53.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:53.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:53.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:53.401 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:53.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:53.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:53.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:53.525 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:38:53.534 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:38:53.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:53.633 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:54.007 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 74724 74708 89.7403 63.1875 0.00847929 0.0111391 2023-10-09T04:38:54.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:38:54.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.564 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:38:54.742 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:54.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:54.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:54.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:38:54.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:54 smithi029 ceph-mon[223858]: pgmap v913: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-09T04:38:54.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:54 smithi029 ceph-mon[223858]: from='client.35961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:54.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1123810015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:54.912 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:54 smithi029 ceph-mon[216903]: pgmap v913: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-09T04:38:54.912 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:54 smithi029 ceph-mon[216903]: from='client.35961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:54.912 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1123810015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:54.915 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:55.007 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 75373 75357 88.8128 40.5625 0.00484038 0.0112464 2023-10-09T04:38:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:54 smithi177 ceph-mon[195797]: pgmap v913: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-09T04:38:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:54 smithi177 ceph-mon[195797]: from='client.35961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:38:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1123810015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:38:56.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 76155 76139 88.0735 48.875 0.00321405 0.0113155 2023-10-09T04:38:56.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:38:56.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:56 smithi029 ceph-mon[216903]: pgmap v914: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T04:38:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:56 smithi029 ceph-mon[223858]: pgmap v914: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T04:38:57.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 76882 76866 87.2986 45.4375 0.0146948 0.0114491 2023-10-09T04:38:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:56 smithi177 ceph-mon[195797]: pgmap v914: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-10-09T04:38:57.152 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:38:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:57.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:38:57.203 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":913,"stamp":"2023-10-09T04:38:53.041264+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4695398304,"num_objects":71661,"num_object_clones":0,"num_object_copies":214983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71661,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":71937,"num_write_kb":4587213,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":2952438,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71853,"ondisk_log_size":71853,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19250564,"kb_used_data":13839556,"kb_used_omap":20,"kb_used_meta":5410923,"kb_avail":730710652,"statfs":{"total":767960285184,"available":748247707648,"internally_reserved":0,"allocated":14171705344,"data_stored":14165676620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5540785806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,5,8,4,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":591724544,"num_objects":9029,"num_object_clones":0,"num_object_copies":27087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9029,"num_write_kb":577856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003405"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":241,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616197+0000","last_change":"2023-10-09T04:36:47.540352+0000","last_active":"2023-10-09T04:38:29.616197+0000","last_peered":"2023-10-09T04:38:29.616197+0000","last_clean":"2023-10-09T04:38:29.616197+0000","last_became_active":"2023-10-09T04:36:47.538007+0000","last_became_peered":"2023-10-09T04:36:47.538007+0000","last_unstale":"2023-10-09T04:38:29.616197+0000","last_undegraded":"2023-10-09T04:38:29.616197+0000","last_fullsized":"2023-10-09T04:38:29.616197+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:45.054005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":196,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.609613+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:38:29.609613+0000","last_peered":"2023-10-09T04:38:29.609613+0000","last_clean":"2023-10-09T04:38:29.609613+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:38:29.609613+0000","last_undegraded":"2023-10-09T04:38:29.609613+0000","last_fullsized":"2023-10-09T04:38:29.609613+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:36.723910+0000","last_change":"2023-10-09T04:27:57.982645+0000","last_active":"2023-10-09T04:38:36.723910+0000","last_peered":"2023-10-09T04:38:36.723910+0000","last_clean":"2023-10-09T04:38:36.723910+0000","last_became_active":"2023-10-09T04:27:57.982333+0000","last_became_peered":"2023-10-09T04:27:57.982333+0000","last_unstale":"2023-10-09T04:38:36.723910+0000","last_undegraded":"2023-10-09T04:38:36.723910+0000","last_fullsized":"2023-10-09T04:38:36.723910+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:26.274533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":158,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.622014+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:38:29.622014+0000","last_peered":"2023-10-09T04:38:29.622014+0000","last_clean":"2023-10-09T04:38:29.622014+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:38:29.622014+0000","last_undegraded":"2023-10-09T04:38:29.622014+0000","last_fullsized":"2023-10-09T04:38:29.622014+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.608369+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:38:29.608369+0000","last_peered":"2023-10-09T04:38:29.608369+0000","last_clean":"2023-10-09T04:38:29.608369+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:38:29.608369+0000","last_undegraded":"2023-10-09T04:38:29.608369+0000","last_fullsized":"2023-10-09T04:38:29.608369+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":202,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.620785+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:38:29.620785+0000","last_peered":"2023-10-09T04:38:29.620785+0000","last_clean":"2023-10-09T04:38:29.620785+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:38:29.620785+0000","last_undegraded":"2023-10-09T04:38:29.620785+0000","last_fullsized":"2023-10-09T04:38:29.620785+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":157,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614073+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:38:29.614073+0000","last_peered":"2023-10-09T04:38:29.614073+0000","last_clean":"2023-10-09T04:38:29.614073+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:38:29.614073+0000","last_undegraded":"2023-10-09T04:38:29.614073+0000","last_fullsized":"2023-10-09T04:38:29.614073+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":196,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.608489+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:38:29.608489+0000","last_peered":"2023-10-09T04:38:29.608489+0000","last_clean":"2023-10-09T04:38:29.608489+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:38:29.608489+0000","last_undegraded":"2023-10-09T04:38:29.608489+0000","last_fullsized":"2023-10-09T04:38:29.608489+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614414+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:38:29.614414+0000","last_peered":"2023-10-09T04:38:29.614414+0000","last_clean":"2023-10-09T04:38:29.614414+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:38:29.614414+0000","last_undegraded":"2023-10-09T04:38:29.614414+0000","last_fullsized":"2023-10-09T04:38:29.614414+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":169,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.609205+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:38:29.609205+0000","last_peered":"2023-10-09T04:38:29.609205+0000","last_clean":"2023-10-09T04:38:29.609205+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:38:29.609205+0000","last_undegraded":"2023-10-09T04:38:29.609205+0000","last_fullsized":"2023-10-09T04:38:29.609205+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":207,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617413+0000","last_change":"2023-10-09T04:27:57.986830+0000","last_active":"2023-10-09T04:38:29.617413+0000","last_peered":"2023-10-09T04:38:29.617413+0000","last_clean":"2023-10-09T04:38:29.617413+0000","last_became_active":"2023-10-09T04:27:57.986510+0000","last_became_peered":"2023-10-09T04:27:57.986510+0000","last_unstale":"2023-10-09T04:38:29.617413+0000","last_undegraded":"2023-10-09T04:38:29.617413+0000","last_fullsized":"2023-10-09T04:38:29.617413+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:07:26.959878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":226,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.599502+0000","last_change":"2023-10-09T04:36:47.537972+0000","last_active":"2023-10-09T04:38:29.599502+0000","last_peered":"2023-10-09T04:38:29.599502+0000","last_clean":"2023-10-09T04:38:29.599502+0000","last_became_active":"2023-10-09T04:36:47.537663+0000","last_became_peered":"2023-10-09T04:36:47.537663+0000","last_unstale":"2023-10-09T04:38:29.599502+0000","last_undegraded":"2023-10-09T04:38:29.599502+0000","last_fullsized":"2023-10-09T04:38:29.599502+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:37.956271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.618243+0000","last_change":"2023-10-09T04:07:34.630127+0000","last_active":"2023-10-09T04:38:29.618243+0000","last_peered":"2023-10-09T04:38:29.618243+0000","last_clean":"2023-10-09T04:38:29.618243+0000","last_became_active":"2023-10-09T04:07:34.629799+0000","last_became_peered":"2023-10-09T04:07:34.629799+0000","last_unstale":"2023-10-09T04:38:29.618243+0000","last_undegraded":"2023-10-09T04:38:29.618243+0000","last_fullsized":"2023-10-09T04:38:29.618243+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:26.957792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616988+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:38:29.616988+0000","last_peered":"2023-10-09T04:38:29.616988+0000","last_clean":"2023-10-09T04:38:29.616988+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:38:29.616988+0000","last_undegraded":"2023-10-09T04:38:29.616988+0000","last_fullsized":"2023-10-09T04:38:29.616988+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":223,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.618377+0000","last_change":"2023-10-09T04:36:47.543534+0000","last_active":"2023-10-09T04:38:29.618377+0000","last_peered":"2023-10-09T04:38:29.618377+0000","last_clean":"2023-10-09T04:38:29.618377+0000","last_became_active":"2023-10-09T04:36:47.543152+0000","last_became_peered":"2023-10-09T04:36:47.543152+0000","last_unstale":"2023-10-09T04:38:29.618377+0000","last_undegraded":"2023-10-09T04:38:29.618377+0000","last_fullsized":"2023-10-09T04:38:29.618377+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:52.483371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":196,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.608640+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:38:29.608640+0000","last_peered":"2023-10-09T04:38:29.608640+0000","last_clean":"2023-10-09T04:38:29.608640+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:38:29.608640+0000","last_undegraded":"2023-10-09T04:38:29.608640+0000","last_fullsized":"2023-10-09T04:38:29.608640+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":157,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617925+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:38:29.617925+0000","last_peered":"2023-10-09T04:38:29.617925+0000","last_clean":"2023-10-09T04:38:29.617925+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:38:29.617925+0000","last_undegraded":"2023-10-09T04:38:29.617925+0000","last_fullsized":"2023-10-09T04:38:29.617925+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614689+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:38:29.614689+0000","last_peered":"2023-10-09T04:38:29.614689+0000","last_clean":"2023-10-09T04:38:29.614689+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:38:29.614689+0000","last_undegraded":"2023-10-09T04:38:29.614689+0000","last_fullsized":"2023-10-09T04:38:29.614689+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":187,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617002+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:38:29.617002+0000","last_peered":"2023-10-09T04:38:29.617002+0000","last_clean":"2023-10-09T04:38:29.617002+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:38:29.617002+0000","last_undegraded":"2023-10-09T04:38:29.617002+0000","last_fullsized":"2023-10-09T04:38:29.617002+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":157,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.609097+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:38:29.609097+0000","last_peered":"2023-10-09T04:38:29.609097+0000","last_clean":"2023-10-09T04:38:29.609097+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:38:29.609097+0000","last_undegraded":"2023-10-09T04:38:29.609097+0000","last_fullsized":"2023-10-09T04:38:29.609097+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":251,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616899+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:38:29.616899+0000","last_peered":"2023-10-09T04:38:29.616899+0000","last_clean":"2023-10-09T04:38:29.616899+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:38:29.616899+0000","last_undegraded":"2023-10-09T04:38:29.616899+0000","last_fullsized":"2023-10-09T04:38:29.616899+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":196,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.609508+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:38:29.609508+0000","last_peered":"2023-10-09T04:38:29.609508+0000","last_clean":"2023-10-09T04:38:29.609508+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:38:29.609508+0000","last_undegraded":"2023-10-09T04:38:29.609508+0000","last_fullsized":"2023-10-09T04:38:29.609508+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"245'4585","reported_seq":4601,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.320602+0000","last_change":"2023-10-09T04:37:52.014620+0000","last_active":"2023-10-09T04:38:50.320602+0000","last_peered":"2023-10-09T04:38:50.320602+0000","last_clean":"2023-10-09T04:38:50.320602+0000","last_became_active":"2023-10-09T04:37:52.014234+0000","last_became_peered":"2023-10-09T04:37:52.014234+0000","last_unstale":"2023-10-09T04:38:50.320602+0000","last_undegraded":"2023-10-09T04:38:50.320602+0000","last_fullsized":"2023-10-09T04:38:50.320602+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4585,"log_dups_size":0,"ondisk_log_size":4585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:00:14.836376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300482560,"num_objects":4585,"num_object_clones":0,"num_object_copies":13755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4585,"num_write_kb":293440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":233,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617151+0000","last_change":"2023-10-09T04:29:51.070720+0000","last_active":"2023-10-09T04:38:29.617151+0000","last_peered":"2023-10-09T04:38:29.617151+0000","last_clean":"2023-10-09T04:38:29.617151+0000","last_became_active":"2023-10-09T04:29:51.058644+0000","last_became_peered":"2023-10-09T04:29:51.058644+0000","last_unstale":"2023-10-09T04:38:29.617151+0000","last_undegraded":"2023-10-09T04:38:29.617151+0000","last_fullsized":"2023-10-09T04:38:29.617151+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:09.713565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":187,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.613976+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:38:29.613976+0000","last_peered":"2023-10-09T04:38:29.613976+0000","last_clean":"2023-10-09T04:38:29.613976+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:38:29.613976+0000","last_undegraded":"2023-10-09T04:38:29.613976+0000","last_fullsized":"2023-10-09T04:38:29.613976+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"245'4411","reported_seq":4428,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.409361+0000","last_change":"2023-10-09T04:37:52.255599+0000","last_active":"2023-10-09T04:38:50.409361+0000","last_peered":"2023-10-09T04:38:50.409361+0000","last_clean":"2023-10-09T04:38:50.409361+0000","last_became_active":"2023-10-09T04:37:52.255291+0000","last_became_peered":"2023-10-09T04:37:52.255291+0000","last_unstale":"2023-10-09T04:38:50.409361+0000","last_undegraded":"2023-10-09T04:38:50.409361+0000","last_fullsized":"2023-10-09T04:38:50.409361+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4411,"log_dups_size":0,"ondisk_log_size":4411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.990132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289079296,"num_objects":4411,"num_object_clones":0,"num_object_copies":13233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4411,"num_write_kb":282304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":206,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.621924+0000","last_change":"2023-10-09T04:24:07.926681+0000","last_active":"2023-10-09T04:38:29.621924+0000","last_peered":"2023-10-09T04:38:29.621924+0000","last_clean":"2023-10-09T04:38:29.621924+0000","last_became_active":"2023-10-09T04:24:07.926106+0000","last_became_peered":"2023-10-09T04:24:07.926106+0000","last_unstale":"2023-10-09T04:38:29.621924+0000","last_undegraded":"2023-10-09T04:38:29.621924+0000","last_fullsized":"2023-10-09T04:38:29.621924+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:25.509079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":222,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616142+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:38:29.616142+0000","last_peered":"2023-10-09T04:38:29.616142+0000","last_clean":"2023-10-09T04:38:29.616142+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:38:29.616142+0000","last_undegraded":"2023-10-09T04:38:29.616142+0000","last_fullsized":"2023-10-09T04:38:29.616142+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"245'4480","reported_seq":4496,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.061927+0000","last_change":"2023-10-09T04:37:52.013171+0000","last_active":"2023-10-09T04:38:50.061927+0000","last_peered":"2023-10-09T04:38:50.061927+0000","last_clean":"2023-10-09T04:38:50.061927+0000","last_became_active":"2023-10-09T04:37:52.012873+0000","last_became_peered":"2023-10-09T04:37:52.012873+0000","last_unstale":"2023-10-09T04:38:50.061927+0000","last_undegraded":"2023-10-09T04:38:50.061927+0000","last_fullsized":"2023-10-09T04:38:50.061927+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4480,"log_dups_size":0,"ondisk_log_size":4480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:25.833222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293601280,"num_objects":4480,"num_object_clones":0,"num_object_copies":13440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4480,"num_write_kb":286720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":188,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617254+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:38:29.617254+0000","last_peered":"2023-10-09T04:38:29.617254+0000","last_clean":"2023-10-09T04:38:29.617254+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:38:29.617254+0000","last_undegraded":"2023-10-09T04:38:29.617254+0000","last_fullsized":"2023-10-09T04:38:29.617254+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":229,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617180+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:38:29.617180+0000","last_peered":"2023-10-09T04:38:29.617180+0000","last_clean":"2023-10-09T04:38:29.617180+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:38:29.617180+0000","last_undegraded":"2023-10-09T04:38:29.617180+0000","last_fullsized":"2023-10-09T04:38:29.617180+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"245'4375","reported_seq":4390,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.046096+0000","last_change":"2023-10-09T04:37:52.012019+0000","last_active":"2023-10-09T04:38:50.046096+0000","last_peered":"2023-10-09T04:38:50.046096+0000","last_clean":"2023-10-09T04:38:50.046096+0000","last_became_active":"2023-10-09T04:37:52.011714+0000","last_became_peered":"2023-10-09T04:37:52.011714+0000","last_unstale":"2023-10-09T04:38:50.046096+0000","last_undegraded":"2023-10-09T04:38:50.046096+0000","last_fullsized":"2023-10-09T04:38:50.046096+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4375,"log_dups_size":0,"ondisk_log_size":4375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:55.324035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286720000,"num_objects":4375,"num_object_clones":0,"num_object_copies":13125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4375,"num_write_kb":280000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614322+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:38:29.614322+0000","last_peered":"2023-10-09T04:38:29.614322+0000","last_clean":"2023-10-09T04:38:29.614322+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:38:29.614322+0000","last_undegraded":"2023-10-09T04:38:29.614322+0000","last_fullsized":"2023-10-09T04:38:29.614322+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":102,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.613972+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:38:29.613972+0000","last_peered":"2023-10-09T04:38:29.613972+0000","last_clean":"2023-10-09T04:38:29.613972+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:38:29.613972+0000","last_undegraded":"2023-10-09T04:38:29.613972+0000","last_fullsized":"2023-10-09T04:38:29.613972+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"245'4322","reported_seq":4338,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.850293+0000","last_change":"2023-10-09T04:37:52.013681+0000","last_active":"2023-10-09T04:38:49.850293+0000","last_peered":"2023-10-09T04:38:49.850293+0000","last_clean":"2023-10-09T04:38:49.850293+0000","last_became_active":"2023-10-09T04:37:52.013382+0000","last_became_peered":"2023-10-09T04:37:52.013382+0000","last_unstale":"2023-10-09T04:38:49.850293+0000","last_undegraded":"2023-10-09T04:38:49.850293+0000","last_fullsized":"2023-10-09T04:38:49.850293+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:30.877521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283246592,"num_objects":4322,"num_object_clones":0,"num_object_copies":12966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4322,"num_write_kb":276608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":240,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.609298+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:38:29.609298+0000","last_peered":"2023-10-09T04:38:29.609298+0000","last_clean":"2023-10-09T04:38:29.609298+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:38:29.609298+0000","last_undegraded":"2023-10-09T04:38:29.609298+0000","last_fullsized":"2023-10-09T04:38:29.609298+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.618095+0000","last_change":"2023-10-09T04:27:57.988243+0000","last_active":"2023-10-09T04:38:29.618095+0000","last_peered":"2023-10-09T04:38:29.618095+0000","last_clean":"2023-10-09T04:38:29.618095+0000","last_became_active":"2023-10-09T04:27:57.987954+0000","last_became_peered":"2023-10-09T04:27:57.987954+0000","last_unstale":"2023-10-09T04:38:29.618095+0000","last_undegraded":"2023-10-09T04:38:29.618095+0000","last_fullsized":"2023-10-09T04:38:29.618095+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:58.233723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"245'4456","reported_seq":4472,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.706679+0000","last_change":"2023-10-09T04:37:52.255728+0000","last_active":"2023-10-09T04:38:50.706679+0000","last_peered":"2023-10-09T04:38:50.706679+0000","last_clean":"2023-10-09T04:38:50.706679+0000","last_became_active":"2023-10-09T04:37:52.255283+0000","last_became_peered":"2023-10-09T04:37:52.255283+0000","last_unstale":"2023-10-09T04:38:50.706679+0000","last_undegraded":"2023-10-09T04:38:50.706679+0000","last_fullsized":"2023-10-09T04:38:50.706679+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4456,"log_dups_size":0,"ondisk_log_size":4456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:18.900085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292028416,"num_objects":4456,"num_object_clones":0,"num_object_copies":13368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4456,"num_write_kb":285184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":203,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617326+0000","last_change":"2023-10-09T04:24:07.944546+0000","last_active":"2023-10-09T04:38:29.617326+0000","last_peered":"2023-10-09T04:38:29.617326+0000","last_clean":"2023-10-09T04:38:29.617326+0000","last_became_active":"2023-10-09T04:24:07.944198+0000","last_became_peered":"2023-10-09T04:24:07.944198+0000","last_unstale":"2023-10-09T04:38:29.617326+0000","last_undegraded":"2023-10-09T04:38:29.617326+0000","last_fullsized":"2023-10-09T04:38:29.617326+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:14.277706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617539+0000","last_change":"2023-10-09T04:36:47.537175+0000","last_active":"2023-10-09T04:38:29.617539+0000","last_peered":"2023-10-09T04:38:29.617539+0000","last_clean":"2023-10-09T04:38:29.617539+0000","last_became_active":"2023-10-09T04:36:47.536865+0000","last_became_peered":"2023-10-09T04:36:47.536865+0000","last_unstale":"2023-10-09T04:38:29.617539+0000","last_undegraded":"2023-10-09T04:38:29.617539+0000","last_fullsized":"2023-10-09T04:38:29.617539+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:35:54.844471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"245'4485","reported_seq":4501,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:49.841499+0000","last_change":"2023-10-09T04:37:52.256288+0000","last_active":"2023-10-09T04:38:49.841499+0000","last_peered":"2023-10-09T04:38:49.841499+0000","last_clean":"2023-10-09T04:38:49.841499+0000","last_became_active":"2023-10-09T04:37:52.256008+0000","last_became_peered":"2023-10-09T04:37:52.256008+0000","last_unstale":"2023-10-09T04:38:49.841499+0000","last_undegraded":"2023-10-09T04:38:49.841499+0000","last_fullsized":"2023-10-09T04:38:49.841499+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4485,"log_dups_size":0,"ondisk_log_size":4485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:34.570497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293928960,"num_objects":4485,"num_object_clones":0,"num_object_copies":13455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4485,"num_write_kb":287040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":223,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.615661+0000","last_change":"2023-10-09T04:36:47.541552+0000","last_active":"2023-10-09T04:38:29.615661+0000","last_peered":"2023-10-09T04:38:29.615661+0000","last_clean":"2023-10-09T04:38:29.615661+0000","last_became_active":"2023-10-09T04:36:47.541229+0000","last_became_peered":"2023-10-09T04:36:47.541229+0000","last_unstale":"2023-10-09T04:38:29.615661+0000","last_undegraded":"2023-10-09T04:38:29.615661+0000","last_fullsized":"2023-10-09T04:38:29.615661+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:00:46.140621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617373+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:38:29.617373+0000","last_peered":"2023-10-09T04:38:29.617373+0000","last_clean":"2023-10-09T04:38:29.617373+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:38:29.617373+0000","last_undegraded":"2023-10-09T04:38:29.617373+0000","last_fullsized":"2023-10-09T04:38:29.617373+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"245'4608","reported_seq":4625,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.716176+0000","last_change":"2023-10-09T04:37:52.011719+0000","last_active":"2023-10-09T04:38:51.716176+0000","last_peered":"2023-10-09T04:38:51.716176+0000","last_clean":"2023-10-09T04:38:51.716176+0000","last_became_active":"2023-10-09T04:37:52.011455+0000","last_became_peered":"2023-10-09T04:37:52.011455+0000","last_unstale":"2023-10-09T04:38:51.716176+0000","last_undegraded":"2023-10-09T04:38:51.716176+0000","last_fullsized":"2023-10-09T04:38:51.716176+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4608,"log_dups_size":0,"ondisk_log_size":4608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:04:35.397653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301989888,"num_objects":4608,"num_object_clones":0,"num_object_copies":13824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4608,"num_write_kb":294912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":198,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:46.879641+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:38:46.879641+0000","last_peered":"2023-10-09T04:38:46.879641+0000","last_clean":"2023-10-09T04:38:46.879641+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:38:46.879641+0000","last_undegraded":"2023-10-09T04:38:46.879641+0000","last_fullsized":"2023-10-09T04:38:46.879641+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.600679+0000","last_change":"2023-10-09T04:27:57.982752+0000","last_active":"2023-10-09T04:38:29.600679+0000","last_peered":"2023-10-09T04:38:29.600679+0000","last_clean":"2023-10-09T04:38:29.600679+0000","last_became_active":"2023-10-09T04:27:57.982433+0000","last_became_peered":"2023-10-09T04:27:57.982433+0000","last_unstale":"2023-10-09T04:38:29.600679+0000","last_undegraded":"2023-10-09T04:38:29.600679+0000","last_fullsized":"2023-10-09T04:38:29.600679+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:53.029579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"245'4482","reported_seq":4499,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.706484+0000","last_change":"2023-10-09T04:37:52.010535+0000","last_active":"2023-10-09T04:38:50.706484+0000","last_peered":"2023-10-09T04:38:50.706484+0000","last_clean":"2023-10-09T04:38:50.706484+0000","last_became_active":"2023-10-09T04:37:52.009735+0000","last_became_peered":"2023-10-09T04:37:52.009735+0000","last_unstale":"2023-10-09T04:38:50.706484+0000","last_undegraded":"2023-10-09T04:38:50.706484+0000","last_fullsized":"2023-10-09T04:38:50.706484+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4482,"log_dups_size":0,"ondisk_log_size":4482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:31.102036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293732352,"num_objects":4482,"num_object_clones":0,"num_object_copies":13446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4482,"num_write_kb":286848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":221,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:41.726057+0000","last_change":"2023-10-09T04:29:51.055161+0000","last_active":"2023-10-09T04:38:41.726057+0000","last_peered":"2023-10-09T04:38:41.726057+0000","last_clean":"2023-10-09T04:38:41.726057+0000","last_became_active":"2023-10-09T04:29:51.054822+0000","last_became_peered":"2023-10-09T04:29:51.054822+0000","last_unstale":"2023-10-09T04:38:41.726057+0000","last_undegraded":"2023-10-09T04:38:41.726057+0000","last_fullsized":"2023-10-09T04:38:41.726057+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:40:37.202273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":239,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614766+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:38:29.614766+0000","last_peered":"2023-10-09T04:38:29.614766+0000","last_clean":"2023-10-09T04:38:29.614766+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:38:29.614766+0000","last_undegraded":"2023-10-09T04:38:29.614766+0000","last_fullsized":"2023-10-09T04:38:29.614766+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":197,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:41.725987+0000","last_change":"2023-10-09T04:24:07.920815+0000","last_active":"2023-10-09T04:38:41.725987+0000","last_peered":"2023-10-09T04:38:41.725987+0000","last_clean":"2023-10-09T04:38:41.725987+0000","last_became_active":"2023-10-09T04:24:07.920465+0000","last_became_peered":"2023-10-09T04:24:07.920465+0000","last_unstale":"2023-10-09T04:38:41.725987+0000","last_undegraded":"2023-10-09T04:38:41.725987+0000","last_fullsized":"2023-10-09T04:38:41.725987+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:12.557959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"245'4462","reported_seq":4479,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.869115+0000","last_change":"2023-10-09T04:37:52.255930+0000","last_active":"2023-10-09T04:38:51.869115+0000","last_peered":"2023-10-09T04:38:51.869115+0000","last_clean":"2023-10-09T04:38:51.869115+0000","last_became_active":"2023-10-09T04:37:52.255584+0000","last_became_peered":"2023-10-09T04:37:52.255584+0000","last_unstale":"2023-10-09T04:38:51.869115+0000","last_undegraded":"2023-10-09T04:38:51.869115+0000","last_fullsized":"2023-10-09T04:38:51.869115+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4462,"log_dups_size":0,"ondisk_log_size":4462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:52.503096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292421632,"num_objects":4462,"num_object_clones":0,"num_object_copies":13386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4462,"num_write_kb":285568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":209,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.615024+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:38:29.615024+0000","last_peered":"2023-10-09T04:38:29.615024+0000","last_clean":"2023-10-09T04:38:29.615024+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:38:29.615024+0000","last_undegraded":"2023-10-09T04:38:29.615024+0000","last_fullsized":"2023-10-09T04:38:29.615024+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":226,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616348+0000","last_change":"2023-10-09T04:29:51.053908+0000","last_active":"2023-10-09T04:38:29.616348+0000","last_peered":"2023-10-09T04:38:29.616348+0000","last_clean":"2023-10-09T04:38:29.616348+0000","last_became_active":"2023-10-09T04:29:51.053535+0000","last_became_peered":"2023-10-09T04:29:51.053535+0000","last_unstale":"2023-10-09T04:38:29.616348+0000","last_undegraded":"2023-10-09T04:38:29.616348+0000","last_fullsized":"2023-10-09T04:38:29.616348+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:37.050568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":242,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:41.726040+0000","last_change":"2023-10-09T04:36:47.536445+0000","last_active":"2023-10-09T04:38:41.726040+0000","last_peered":"2023-10-09T04:38:41.726040+0000","last_clean":"2023-10-09T04:38:41.726040+0000","last_became_active":"2023-10-09T04:36:47.536098+0000","last_became_peered":"2023-10-09T04:36:47.536098+0000","last_unstale":"2023-10-09T04:38:41.726040+0000","last_undegraded":"2023-10-09T04:38:41.726040+0000","last_fullsized":"2023-10-09T04:38:41.726040+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:56.016452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"245'4449","reported_seq":4466,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.854400+0000","last_change":"2023-10-09T04:37:52.012774+0000","last_active":"2023-10-09T04:38:51.854400+0000","last_peered":"2023-10-09T04:38:51.854400+0000","last_clean":"2023-10-09T04:38:51.854400+0000","last_became_active":"2023-10-09T04:37:52.012402+0000","last_became_peered":"2023-10-09T04:37:52.012402+0000","last_unstale":"2023-10-09T04:38:51.854400+0000","last_undegraded":"2023-10-09T04:38:51.854400+0000","last_fullsized":"2023-10-09T04:38:51.854400+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4449,"log_dups_size":0,"ondisk_log_size":4449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:37.167042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291569664,"num_objects":4449,"num_object_clones":0,"num_object_copies":13347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4449,"num_write_kb":284736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":200,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:48.653299+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:38:48.653299+0000","last_peered":"2023-10-09T04:38:48.653299+0000","last_clean":"2023-10-09T04:38:48.653299+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:38:48.653299+0000","last_undegraded":"2023-10-09T04:38:48.653299+0000","last_fullsized":"2023-10-09T04:38:48.653299+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":227,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.599811+0000","last_change":"2023-10-09T04:36:47.541445+0000","last_active":"2023-10-09T04:38:29.599811+0000","last_peered":"2023-10-09T04:38:29.599811+0000","last_clean":"2023-10-09T04:38:29.599811+0000","last_became_active":"2023-10-09T04:36:47.541142+0000","last_became_peered":"2023-10-09T04:36:47.541142+0000","last_unstale":"2023-10-09T04:38:29.599811+0000","last_undegraded":"2023-10-09T04:38:29.599811+0000","last_fullsized":"2023-10-09T04:38:29.599811+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:53.054914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":215,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.600747+0000","last_change":"2023-10-09T04:29:51.055190+0000","last_active":"2023-10-09T04:38:29.600747+0000","last_peered":"2023-10-09T04:38:29.600747+0000","last_clean":"2023-10-09T04:38:29.600747+0000","last_became_active":"2023-10-09T04:29:51.054850+0000","last_became_peered":"2023-10-09T04:29:51.054850+0000","last_unstale":"2023-10-09T04:38:29.600747+0000","last_undegraded":"2023-10-09T04:38:29.600747+0000","last_fullsized":"2023-10-09T04:38:29.600747+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:27:15.387346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"245'4667","reported_seq":4684,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.879027+0000","last_change":"2023-10-09T04:37:52.013105+0000","last_active":"2023-10-09T04:38:51.879027+0000","last_peered":"2023-10-09T04:38:51.879027+0000","last_clean":"2023-10-09T04:38:51.879027+0000","last_became_active":"2023-10-09T04:37:52.012773+0000","last_became_peered":"2023-10-09T04:37:52.012773+0000","last_unstale":"2023-10-09T04:38:51.879027+0000","last_undegraded":"2023-10-09T04:38:51.879027+0000","last_fullsized":"2023-10-09T04:38:51.879027+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4667,"log_dups_size":0,"ondisk_log_size":4667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.291873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305856512,"num_objects":4667,"num_object_clones":0,"num_object_copies":14001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4667,"num_write_kb":298688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":226,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614463+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:38:29.614463+0000","last_peered":"2023-10-09T04:38:29.614463+0000","last_clean":"2023-10-09T04:38:29.614463+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:38:29.614463+0000","last_undegraded":"2023-10-09T04:38:29.614463+0000","last_fullsized":"2023-10-09T04:38:29.614463+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":637,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617616+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:38:29.617616+0000","last_peered":"2023-10-09T04:38:29.617616+0000","last_clean":"2023-10-09T04:38:29.617616+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:38:29.617616+0000","last_undegraded":"2023-10-09T04:38:29.617616+0000","last_fullsized":"2023-10-09T04:38:29.617616+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":169,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.613067+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:38:29.613067+0000","last_peered":"2023-10-09T04:38:29.613067+0000","last_clean":"2023-10-09T04:38:29.613067+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:38:29.613067+0000","last_undegraded":"2023-10-09T04:38:29.613067+0000","last_fullsized":"2023-10-09T04:38:29.613067+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":185,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.620860+0000","last_change":"2023-10-09T04:07:34.676960+0000","last_active":"2023-10-09T04:38:29.620860+0000","last_peered":"2023-10-09T04:38:29.620860+0000","last_clean":"2023-10-09T04:38:29.620860+0000","last_became_active":"2023-10-09T04:07:34.676669+0000","last_became_peered":"2023-10-09T04:07:34.676669+0000","last_unstale":"2023-10-09T04:38:29.620860+0000","last_undegraded":"2023-10-09T04:38:29.620860+0000","last_fullsized":"2023-10-09T04:38:29.620860+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:49.774814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"245'4500","reported_seq":4515,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.861327+0000","last_change":"2023-10-09T04:37:52.257119+0000","last_active":"2023-10-09T04:38:51.861327+0000","last_peered":"2023-10-09T04:38:51.861327+0000","last_clean":"2023-10-09T04:38:51.861327+0000","last_became_active":"2023-10-09T04:37:52.256760+0000","last_became_peered":"2023-10-09T04:37:52.256760+0000","last_unstale":"2023-10-09T04:38:51.861327+0000","last_undegraded":"2023-10-09T04:38:51.861327+0000","last_fullsized":"2023-10-09T04:38:51.861327+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4500,"log_dups_size":0,"ondisk_log_size":4500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:16.001253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912000,"num_objects":4500,"num_object_clones":0,"num_object_copies":13500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4500,"num_write_kb":288000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":245,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.615236+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:38:29.615236+0000","last_peered":"2023-10-09T04:38:29.615236+0000","last_clean":"2023-10-09T04:38:29.615236+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:38:29.615236+0000","last_undegraded":"2023-10-09T04:38:29.615236+0000","last_fullsized":"2023-10-09T04:38:29.615236+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":157,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616213+0000","last_change":"2023-10-09T04:34:22.266723+0000","last_active":"2023-10-09T04:38:29.616213+0000","last_peered":"2023-10-09T04:38:29.616213+0000","last_clean":"2023-10-09T04:38:29.616213+0000","last_became_active":"2023-10-09T04:34:22.266261+0000","last_became_peered":"2023-10-09T04:34:22.266261+0000","last_unstale":"2023-10-09T04:38:29.616213+0000","last_undegraded":"2023-10-09T04:38:29.616213+0000","last_fullsized":"2023-10-09T04:38:29.616213+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:54.792194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":120,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:33.029438+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:38:33.029438+0000","last_peered":"2023-10-09T04:38:33.029438+0000","last_clean":"2023-10-09T04:38:33.029438+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:38:33.029438+0000","last_undegraded":"2023-10-09T04:38:33.029438+0000","last_fullsized":"2023-10-09T04:38:33.029438+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"245'4480","reported_seq":4497,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:51.625570+0000","last_change":"2023-10-09T04:37:52.010038+0000","last_active":"2023-10-09T04:38:51.625570+0000","last_peered":"2023-10-09T04:38:51.625570+0000","last_clean":"2023-10-09T04:38:51.625570+0000","last_became_active":"2023-10-09T04:37:52.009758+0000","last_became_peered":"2023-10-09T04:37:52.009758+0000","last_unstale":"2023-10-09T04:38:51.625570+0000","last_undegraded":"2023-10-09T04:38:51.625570+0000","last_fullsized":"2023-10-09T04:38:51.625570+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4480,"log_dups_size":0,"ondisk_log_size":4480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:49:03.361014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293601280,"num_objects":4480,"num_object_clones":0,"num_object_copies":13440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4480,"num_write_kb":286720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":228,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617331+0000","last_change":"2023-10-09T04:29:51.058903+0000","last_active":"2023-10-09T04:38:29.617331+0000","last_peered":"2023-10-09T04:38:29.617331+0000","last_clean":"2023-10-09T04:38:29.617331+0000","last_became_active":"2023-10-09T04:29:51.058599+0000","last_became_peered":"2023-10-09T04:29:51.058599+0000","last_unstale":"2023-10-09T04:38:29.617331+0000","last_undegraded":"2023-10-09T04:38:29.617331+0000","last_fullsized":"2023-10-09T04:38:29.617331+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:53.477685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":227,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:41.726022+0000","last_change":"2023-10-09T04:36:47.543494+0000","last_active":"2023-10-09T04:38:41.726022+0000","last_peered":"2023-10-09T04:38:41.726022+0000","last_clean":"2023-10-09T04:38:41.726022+0000","last_became_active":"2023-10-09T04:36:47.543197+0000","last_became_peered":"2023-10-09T04:36:47.543197+0000","last_unstale":"2023-10-09T04:38:41.726022+0000","last_undegraded":"2023-10-09T04:38:41.726022+0000","last_fullsized":"2023-10-09T04:38:41.726022+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:10.900185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":89,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614116+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:38:29.614116+0000","last_peered":"2023-10-09T04:38:29.614116+0000","last_clean":"2023-10-09T04:38:29.614116+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:38:29.614116+0000","last_undegraded":"2023-10-09T04:38:29.614116+0000","last_fullsized":"2023-10-09T04:38:29.614116+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"245'4427","reported_seq":4443,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.008010+0000","last_change":"2023-10-09T04:37:52.013669+0000","last_active":"2023-10-09T04:38:50.008010+0000","last_peered":"2023-10-09T04:38:50.008010+0000","last_clean":"2023-10-09T04:38:50.008010+0000","last_became_active":"2023-10-09T04:37:52.013386+0000","last_became_peered":"2023-10-09T04:37:52.013386+0000","last_unstale":"2023-10-09T04:38:50.008010+0000","last_undegraded":"2023-10-09T04:38:50.008010+0000","last_fullsized":"2023-10-09T04:38:50.008010+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4427,"log_dups_size":0,"ondisk_log_size":4427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:57.288541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290127872,"num_objects":4427,"num_object_clones":0,"num_object_copies":13281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4427,"num_write_kb":283328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":241,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:46.879814+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:38:46.879814+0000","last_peered":"2023-10-09T04:38:46.879814+0000","last_clean":"2023-10-09T04:38:46.879814+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:38:46.879814+0000","last_undegraded":"2023-10-09T04:38:46.879814+0000","last_fullsized":"2023-10-09T04:38:46.879814+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":196,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.599618+0000","last_change":"2023-10-09T04:24:07.917198+0000","last_active":"2023-10-09T04:38:29.599618+0000","last_peered":"2023-10-09T04:38:29.599618+0000","last_clean":"2023-10-09T04:38:29.599618+0000","last_became_active":"2023-10-09T04:24:07.914922+0000","last_became_peered":"2023-10-09T04:24:07.914922+0000","last_unstale":"2023-10-09T04:38:29.599618+0000","last_undegraded":"2023-10-09T04:38:29.599618+0000","last_fullsized":"2023-10-09T04:38:29.599618+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:12.444769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":230,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.608322+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:38:29.608322+0000","last_peered":"2023-10-09T04:38:29.608322+0000","last_clean":"2023-10-09T04:38:29.608322+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:38:29.608322+0000","last_undegraded":"2023-10-09T04:38:29.608322+0000","last_fullsized":"2023-10-09T04:38:29.608322+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"245'4448","reported_seq":4464,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-09T04:38:50.000804+0000","last_change":"2023-10-09T04:37:52.016326+0000","last_active":"2023-10-09T04:38:50.000804+0000","last_peered":"2023-10-09T04:38:50.000804+0000","last_clean":"2023-10-09T04:38:50.000804+0000","last_became_active":"2023-10-09T04:37:52.016026+0000","last_became_peered":"2023-10-09T04:37:52.016026+0000","last_unstale":"2023-10-09T04:38:50.000804+0000","last_undegraded":"2023-10-09T04:38:50.000804+0000","last_fullsized":"2023-10-09T04:38:50.000804+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":4448,"log_dups_size":0,"ondisk_log_size":4448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:46.974447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504128,"num_objects":4448,"num_object_clones":0,"num_object_copies":13344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4448,"num_write_kb":284672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":224,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.620602+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:38:29.620602+0000","last_peered":"2023-10-09T04:38:29.620602+0000","last_clean":"2023-10-09T04:38:29.620602+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:38:29.620602+0000","last_undegraded":"2023-10-09T04:38:29.620602+0000","last_fullsized":"2023-10-09T04:38:29.620602+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":100,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:41.726085+0000","last_change":"2023-10-09T04:34:22.266652+0000","last_active":"2023-10-09T04:38:41.726085+0000","last_peered":"2023-10-09T04:38:41.726085+0000","last_clean":"2023-10-09T04:38:41.726085+0000","last_became_active":"2023-10-09T04:34:22.266267+0000","last_became_peered":"2023-10-09T04:34:22.266267+0000","last_unstale":"2023-10-09T04:38:41.726085+0000","last_undegraded":"2023-10-09T04:38:41.726085+0000","last_fullsized":"2023-10-09T04:38:41.726085+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:50.646309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":188,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616063+0000","last_change":"2023-10-09T04:07:34.676160+0000","last_active":"2023-10-09T04:38:29.616063+0000","last_peered":"2023-10-09T04:38:29.616063+0000","last_clean":"2023-10-09T04:38:29.616063+0000","last_became_active":"2023-10-09T04:07:34.675863+0000","last_became_peered":"2023-10-09T04:07:34.675863+0000","last_unstale":"2023-10-09T04:38:29.616063+0000","last_undegraded":"2023-10-09T04:38:29.616063+0000","last_fullsized":"2023-10-09T04:38:29.616063+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:13.732246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":85,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616351+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:38:29.616351+0000","last_peered":"2023-10-09T04:38:29.616351+0000","last_clean":"2023-10-09T04:38:29.616351+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:38:29.616351+0000","last_undegraded":"2023-10-09T04:38:29.616351+0000","last_fullsized":"2023-10-09T04:38:29.616351+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":192,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.604877+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:38:29.604877+0000","last_peered":"2023-10-09T04:38:29.604877+0000","last_clean":"2023-10-09T04:38:29.604877+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:38:29.604877+0000","last_undegraded":"2023-10-09T04:38:29.604877+0000","last_fullsized":"2023-10-09T04:38:29.604877+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":207,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614195+0000","last_change":"2023-10-09T04:24:07.927104+0000","last_active":"2023-10-09T04:38:29.614195+0000","last_peered":"2023-10-09T04:38:29.614195+0000","last_clean":"2023-10-09T04:38:29.614195+0000","last_became_active":"2023-10-09T04:24:07.926155+0000","last_became_peered":"2023-10-09T04:24:07.926155+0000","last_unstale":"2023-10-09T04:38:29.614195+0000","last_undegraded":"2023-10-09T04:38:29.614195+0000","last_fullsized":"2023-10-09T04:38:29.614195+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:11.880940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":222,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.617129+0000","last_change":"2023-10-09T04:27:57.988397+0000","last_active":"2023-10-09T04:38:29.617129+0000","last_peered":"2023-10-09T04:38:29.617129+0000","last_clean":"2023-10-09T04:38:29.617129+0000","last_became_active":"2023-10-09T04:27:57.987993+0000","last_became_peered":"2023-10-09T04:27:57.987993+0000","last_unstale":"2023-10-09T04:38:29.617129+0000","last_undegraded":"2023-10-09T04:38:29.617129+0000","last_fullsized":"2023-10-09T04:38:29.617129+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:59:14.351883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":190,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.614855+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:38:29.614855+0000","last_peered":"2023-10-09T04:38:29.614855+0000","last_clean":"2023-10-09T04:38:29.614855+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:38:29.614855+0000","last_undegraded":"2023-10-09T04:38:29.614855+0000","last_fullsized":"2023-10-09T04:38:29.614855+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":242,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.616248+0000","last_change":"2023-10-09T04:29:51.053956+0000","last_active":"2023-10-09T04:38:29.616248+0000","last_peered":"2023-10-09T04:38:29.616248+0000","last_clean":"2023-10-09T04:38:29.616248+0000","last_became_active":"2023-10-09T04:29:51.053488+0000","last_became_peered":"2023-10-09T04:29:51.053488+0000","last_unstale":"2023-10-09T04:38:29.616248+0000","last_undegraded":"2023-10-09T04:38:29.616248+0000","last_fullsized":"2023-10-09T04:38:29.616248+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:23:56.990937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":231,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.608942+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:38:29.608942+0000","last_peered":"2023-10-09T04:38:29.608942+0000","last_clean":"2023-10-09T04:38:29.608942+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:38:29.608942+0000","last_undegraded":"2023-10-09T04:38:29.608942+0000","last_fullsized":"2023-10-09T04:38:29.608942+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":263,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.609135+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:38:29.609135+0000","last_peered":"2023-10-09T04:38:29.609135+0000","last_clean":"2023-10-09T04:38:29.609135+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:38:29.609135+0000","last_undegraded":"2023-10-09T04:38:29.609135+0000","last_fullsized":"2023-10-09T04:38:29.609135+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":119,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.615406+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:38:29.615406+0000","last_peered":"2023-10-09T04:38:29.615406+0000","last_clean":"2023-10-09T04:38:29.615406+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:38:29.615406+0000","last_undegraded":"2023-10-09T04:38:29.615406+0000","last_fullsized":"2023-10-09T04:38:29.615406+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-09T04:38:29.615658+0000","last_change":"2023-10-09T04:36:47.544511+0000","last_active":"2023-10-09T04:38:29.615658+0000","last_peered":"2023-10-09T04:38:29.615658+0000","last_clean":"2023-10-09T04:38:29.615658+0000","last_became_active":"2023-10-09T04:36:47.544123+0000","last_became_peered":"2023-10-09T04:36:47.544123+0000","last_unstale":"2023-10-09T04:38:29.615658+0000","last_undegraded":"2023-10-09T04:38:29.615658+0000","last_fullsized":"2023-10-09T04:38:29.615658+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:56.150751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4694802432,"num_objects":71637,"num_object_clones":0,"num_object_copies":214911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":71637,"num_write_kb":4584768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14115799040,"data_stored":14115799040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71637,"ondisk_log_size":71637,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234383,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2601624,"kb_used_data":1694736,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":91143528,"statfs":{"total":95995035648,"available":93330972672,"internally_reserved":0,"allocated":1735409664,"data_stored":1734824303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":4.1920000000000002,"15min":6.0010000000000003},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.6580000000000004,"5min":111.72199999999999,"15min":629.22500000000002},"last":2.25},{"interface":"front","average":{"1min":0.71599999999999997,"5min":4.2530000000000001,"15min":5.9939999999999998},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.7999999999999998,"5min":111.76600000000001,"15min":629.43100000000004},"last":0.433}]},{"osd":1,"last update":"Mon Oct 9 04:38:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9689999999999999,"5min":1.9710000000000001,"15min":4.2789999999999999},"min":{"1min":0.311,"5min":0.254,"15min":0.22800000000000001},"max":{"1min":39.795999999999999,"5min":39.795999999999999,"15min":224.13499999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":3.0329999999999999,"5min":2.157,"15min":4.4000000000000004},"min":{"1min":0.378,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":40.124000000000002,"5min":40.124000000000002,"15min":224.203},"last":1.6870000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:37:49 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.802,"15min":1.802},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.0800000000000001,"5min":6.0800000000000001,"15min":6.0800000000000001},"last":3.1800000000000002},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.3029999999999999,"15min":2.3029999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.8379999999999992,"5min":8.8379999999999992,"15min":8.8379999999999992},"last":0.29899999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:38:15 2023","interfaces":[{"interface":"back","average":{"1min":9.9990000000000006,"5min":5.4349999999999996,"15min":6.1600000000000001},"min":{"1min":0.34499999999999997,"5min":0.33100000000000002,"15min":0.32900000000000001},"max":{"1min":88.75,"5min":133.13499999999999,"15min":249.989},"last":2.423},{"interface":"front","average":{"1min":10.470000000000001,"5min":5.7000000000000002,"15min":6.3010000000000002},"min":{"1min":0.40100000000000002,"5min":0.307,"15min":0.29799999999999999},"max":{"1min":88.463999999999999,"5min":133.35900000000001,"15min":250.21700000000001},"last":2.2050000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.655,"15min":1.5049999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.024,"5min":11.752000000000001,"15min":11.752000000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.663,"15min":1.2490000000000001},"min":{"1min":0.28399999999999997,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":3.528,"5min":9.4179999999999993,"15min":10.613},"last":1.899}]},{"osd":5,"last update":"Mon Oct 9 04:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.21,"15min":1.127},"min":{"1min":0.32400000000000001,"5min":0.28199999999999997,"15min":0.27900000000000003},"max":{"1min":1.837,"5min":6.1120000000000001,"15min":8.8610000000000007},"last":0.372},{"interface":"front","average":{"1min":1.167,"5min":1.696,"15min":1.4510000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.9199999999999999,"5min":10.326000000000001,"15min":10.895},"last":0.32200000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.5249999999999999,"15min":1.3620000000000001},"min":{"1min":0.34300000000000003,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.1890000000000001,"5min":8.1120000000000001,"15min":12.007},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.95299999999999996,"5min":2.0419999999999998,"15min":1.5820000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.0259999999999998,"5min":9.3819999999999997,"15min":10.917},"last":0.34200000000000003}]}]},{"osd":6,"up_from":46,"seq":197568496017,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3162504,"kb_used_data":2252032,"kb_used_omap":0,"kb_used_meta":910464,"kb_avail":90582648,"statfs":{"total":95995035648,"available":92756631552,"internally_reserved":0,"allocated":2306080768,"data_stored":2305511322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932315136},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":6.2969999999999997,"15min":8.0120000000000005},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.217},"max":{"1min":1.0469999999999999,"5min":141.14099999999999,"15min":1284.521},"last":3.4420000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":6.5430000000000001,"15min":8.0630000000000006},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.253},"max":{"1min":2.4049999999999998,"5min":141.167,"15min":1284.4690000000001},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":6.1059999999999999,"5min":4.391,"15min":3.9660000000000002},"min":{"1min":0.28000000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":112.44199999999999,"5min":112.44199999999999,"15min":185.74000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":5.8360000000000003,"5min":4.2629999999999999,"15min":4.0419999999999998},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.24299999999999999},"max":{"1min":112.117,"5min":112.117,"15min":189.03999999999999},"last":2.302}]},{"osd":2,"last update":"Mon Oct 9 04:37:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":2.0990000000000002,"15min":2.0990000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0459999999999994,"5min":9.0459999999999994,"15min":9.0459999999999994},"last":3.5979999999999999},{"interface":"front","average":{"1min":2.081,"5min":2.081,"15min":2.081},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":3.5899999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:38:22 2023","interfaces":[{"interface":"back","average":{"1min":7.0410000000000004,"5min":9.7840000000000007,"15min":4.1479999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":121.139,"5min":167.56100000000001,"15min":167.56100000000001},"last":0.78900000000000003},{"interface":"front","average":{"1min":7.2510000000000003,"5min":9.9979999999999993,"15min":4.2359999999999998},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.313},"max":{"1min":120.61499999999999,"5min":167.577,"15min":167.577},"last":3.1909999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.728,"15min":1.4079999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.246},"max":{"1min":3.6499999999999999,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.317},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.2809999999999999,"15min":1.268},"min":{"1min":0.34499999999999997,"5min":0.27500000000000002,"15min":0.26000000000000001},"max":{"1min":2.7629999999999999,"5min":8.6270000000000007,"15min":12.028},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":1.857,"15min":1.873},"min":{"1min":0.34000000000000002,"5min":0.29999999999999999,"15min":0.28100000000000003},"max":{"1min":3.6720000000000002,"5min":14.631,"15min":81.087000000000003},"last":0.498},{"interface":"front","average":{"1min":0.52500000000000002,"5min":1.629,"15min":1.8660000000000001},"min":{"1min":0.33800000000000002,"5min":0.311,"15min":0.28000000000000003},"max":{"1min":1.2310000000000001,"5min":11.279999999999999,"15min":81.706999999999994},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:37:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.623,"15min":1.6919999999999999},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.1950000000000003,"5min":11.226000000000001,"15min":81.674999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.093,"5min":1.9379999999999999,"15min":1.7849999999999999},"min":{"1min":0.35899999999999999,"5min":0.308,"15min":0.308},"max":{"1min":14.390000000000001,"5min":14.390000000000001,"15min":81.055000000000007},"last":0.48099999999999998}]}]},{"osd":1,"up_from":118,"seq":506806141107,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2765808,"kb_used_data":2285992,"kb_used_omap":4,"kb_used_meta":479803,"kb_avail":90979344,"statfs":{"total":95995035648,"available":93162848256,"internally_reserved":0,"allocated":2340855808,"data_stored":2339840703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":491319182},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":3.75,"15min":3.6280000000000001},"min":{"1min":0.19,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":7.7679999999999998,"5min":93.171999999999997,"15min":97.308000000000007},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.751,"5min":3.8330000000000002,"15min":3.9089999999999998},"min":{"1min":0.217,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":2.6000000000000001,"5min":93.147000000000006,"15min":97.055999999999997},"last":4.6669999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":2.5219999999999998,"15min":2.5979999999999999},"min":{"1min":0.24299999999999999,"5min":0.222,"15min":0.222},"max":{"1min":9.6470000000000002,"5min":40.924999999999997,"15min":40.924999999999997},"last":0.87},{"interface":"front","average":{"1min":1.1080000000000001,"5min":2.5800000000000001,"15min":2.8260000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.9609999999999999,"5min":42.029000000000003,"15min":42.029000000000003},"last":4.7320000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1669999999999998,"5min":4.6660000000000004,"15min":3.129},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":67.844999999999999,"5min":142.34200000000001,"15min":195.035},"last":1.1579999999999999},{"interface":"front","average":{"1min":4.1180000000000003,"5min":5.2610000000000001,"15min":3.4489999999999998},"min":{"1min":0.30399999999999999,"5min":0.246,"15min":0.19700000000000001},"max":{"1min":67.534000000000006,"5min":141.75200000000001,"15min":195.465},"last":6.0419999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":3.359,"15min":2.956},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.245},"max":{"1min":3.9449999999999998,"5min":170.072,"15min":170.072},"last":0.313},{"interface":"front","average":{"1min":1.1699999999999999,"5min":2.766,"15min":2.202},"min":{"1min":0.32800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.2130000000000001,"5min":169.94499999999999,"15min":169.94499999999999},"last":0.93100000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":2.7799999999999998,"15min":2.4489999999999998},"min":{"1min":0.38200000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.6970000000000001,"5min":171.036,"15min":171.036},"last":4.6189999999999998},{"interface":"front","average":{"1min":1.1919999999999999,"5min":3.4009999999999998,"15min":2.7410000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.27500000000000002},"max":{"1min":4.2800000000000002,"5min":170.88,"15min":170.88},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.778,"15min":2.0070000000000001},"min":{"1min":0.40200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.6440000000000001,"5min":11.548,"15min":13.388},"last":4.7009999999999996},{"interface":"front","average":{"1min":1.1279999999999999,"5min":2.3610000000000002,"15min":2.419},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.0419999999999998,"5min":12.596,"15min":13.882},"last":0.60199999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":3.6859999999999999,"15min":2.9350000000000001},"min":{"1min":0.36599999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.8680000000000003,"5min":174.75999999999999,"15min":174.75999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.278,"5min":3.105,"15min":2.1139999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":4.2590000000000003,"5min":174.779,"15min":174.779},"last":0.89900000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024164,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2170604,"kb_used_data":1456356,"kb_used_omap":5,"kb_used_meta":714234,"kb_avail":91574548,"statfs":{"total":95995035648,"available":93772337152,"internally_reserved":0,"allocated":1491308544,"data_stored":1490354457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":731376250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:38:26 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":3.1179999999999999,"15min":4.0209999999999999},"min":{"1min":0.219,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":5.7430000000000003,"5min":101.81100000000001,"15min":167.92699999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.0609999999999999,"5min":3.371,"15min":4.1079999999999997},"min":{"1min":0.23000000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":7.6369999999999996,"5min":103.444,"15min":169.78999999999999},"last":0.53900000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:38:47 2023","interfaces":[{"interface":"back","average":{"1min":8.0099999999999998,"5min":9.1609999999999996,"15min":9.3529999999999998},"min":{"1min":0.245,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":122.983,"5min":131.30699999999999,"15min":131.30699999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":8.5779999999999994,"5min":9.6240000000000006,"15min":9.798},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":122.715,"5min":129.922,"15min":129.922},"last":0.48599999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:38:15 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":3.7269999999999999,"15min":3.0099999999999998},"min":{"1min":0.28699999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.8470000000000004,"5min":91.578999999999994,"15min":213.28399999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.2640000000000002,"15min":3.2229999999999999},"min":{"1min":0.31,"5min":0.222,"15min":0.222},"max":{"1min":11.220000000000001,"5min":92.067999999999998,"15min":212.27000000000001},"last":0.436}]},{"osd":4,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":3.6099999999999999,"15min":2.3839999999999999},"min":{"1min":0.33200000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.7109999999999999,"5min":161.67099999999999,"15min":161.67099999999999},"last":4.8019999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":3.8279999999999998,"15min":2.5459999999999998},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.4159999999999999,"5min":160.74799999999999,"15min":160.74799999999999},"last":2.0800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":1.869,"15min":2.0059999999999998},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.27900000000000003},"max":{"1min":1.9690000000000001,"5min":35.878999999999998,"15min":35.878999999999998},"last":0.749},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.8360000000000001,"15min":1.712},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.5390000000000001,"5min":35.755000000000003,"15min":35.755000000000003},"last":1.349}]},{"osd":6,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":2.8500000000000001,"15min":2.3690000000000002},"min":{"1min":0.39300000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.452,"5min":108.63800000000001,"15min":108.63800000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.82799999999999996,"5min":2.726,"15min":2.0880000000000001},"min":{"1min":0.41399999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.609,"5min":108.76300000000001,"15min":108.76300000000001},"last":1.3759999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.595,"15min":1.7310000000000001},"min":{"1min":0.45300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.5760000000000001,"5min":6.9770000000000003,"15min":11.890000000000001},"last":1.3120000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.5149999999999999,"15min":1.875},"min":{"1min":0.41199999999999998,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":2.7799999999999998,"5min":8.8260000000000005,"15min":10.765000000000001},"last":0.72699999999999998}]}]},{"osd":2,"up_from":221,"seq":949187772442,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2048548,"kb_used_data":1834716,"kb_used_omap":7,"kb_used_meta":213816,"kb_avail":91696604,"statfs":{"total":95995035648,"available":93897322496,"internally_reserved":0,"allocated":1878749184,"data_stored":1877862894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":218947665},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":8.7370000000000001,"15min":9.8680000000000003},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.488,"5min":72.608000000000004,"15min":72.608000000000004},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.1319999999999999,"5min":8.1720000000000006,"15min":9.3450000000000006},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.0469999999999997,"5min":72.632000000000005,"15min":72.632000000000005},"last":1.458}]},{"osd":1,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":4.4349999999999996,"15min":5.0220000000000002},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.464,"5min":72.807000000000002,"15min":72.807000000000002},"last":1.423},{"interface":"front","average":{"1min":1.583,"5min":4.9969999999999999,"15min":5.5659999999999998},"min":{"1min":0.254,"5min":0.248,"15min":0.248},"max":{"1min":8.0510000000000002,"5min":72.664000000000001,"15min":72.664000000000001},"last":4.3780000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":4.6420000000000003,"15min":5.1509999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.8220000000000001,"5min":72.701999999999998,"15min":72.701999999999998},"last":4.6699999999999999},{"interface":"front","average":{"1min":2.677,"5min":4.7489999999999997,"15min":5.0940000000000003},"min":{"1min":0.34200000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":11.548999999999999,"5min":72.822999999999993,"15min":72.822999999999993},"last":5.2450000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":4.3010000000000002,"15min":4.7729999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":5.5650000000000004,"5min":72.944999999999993,"15min":72.944999999999993},"last":4.298},{"interface":"front","average":{"1min":1.157,"5min":4.4459999999999997,"15min":4.9939999999999998},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.378},"max":{"1min":2.4319999999999999,"5min":73.054000000000002,"15min":73.054000000000002},"last":0.77100000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":3.972,"15min":4.4089999999999998},"min":{"1min":0.371,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.2480000000000002,"5min":72.986000000000004,"15min":72.986000000000004},"last":1.353},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.4820000000000002,"15min":4.9500000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.6109999999999998,"5min":72.963999999999999,"15min":72.963999999999999},"last":4.3460000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":10.289999999999999,"15min":11.808},"min":{"1min":0.433,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.6349999999999998,"5min":135.614,"15min":135.614},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.149,"5min":9.5500000000000007,"15min":10.949999999999999},"min":{"1min":0.505,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.5720000000000001,"5min":130.196,"15min":130.196},"last":1.3799999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:38:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":4.6210000000000004,"15min":5.1929999999999996},"min":{"1min":0.40300000000000002,"5min":0.378,"15min":0.378},"max":{"1min":2.476,"5min":73.079999999999998,"15min":73.079999999999998},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.1839999999999999,"5min":3.972,"15min":4.4359999999999999},"min":{"1min":0.54800000000000004,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":3.6120000000000001,"5min":73.069000000000003,"15min":73.069000000000003},"last":1.3120000000000001}]}]},{"osd":3,"up_from":142,"seq":609885356165,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2438868,"kb_used_data":2031372,"kb_used_omap":2,"kb_used_meta":407485,"kb_avail":91306284,"statfs":{"total":95995035648,"available":93497634816,"internally_reserved":0,"allocated":2080124928,"data_stored":2079249931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":417264693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":2.7050000000000001,"15min":2.048},"min":{"1min":0.23599999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.423,"5min":142.14500000000001,"15min":142.14500000000001},"last":0.20000000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":2.8359999999999999,"15min":2.0329999999999999},"min":{"1min":0.28999999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.484,"5min":138.14599999999999,"15min":138.14599999999999},"last":0.40200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":2.8090000000000002,"15min":3.1000000000000001},"min":{"1min":0.255,"5min":0.19800000000000001,"15min":0.192},"max":{"1min":5.0309999999999997,"5min":145.47499999999999,"15min":180.02500000000001},"last":1.919},{"interface":"front","average":{"1min":0.873,"5min":2.968,"15min":3.117},"min":{"1min":0.245,"5min":0.24199999999999999,"15min":0.20699999999999999},"max":{"1min":6.3529999999999998,"5min":144.83799999999999,"15min":184.00899999999999},"last":0.29399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:38:50 2023","interfaces":[{"interface":"back","average":{"1min":9.4429999999999996,"5min":3.2290000000000001,"15min":2.1930000000000001},"min":{"1min":0.29899999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":177.28399999999999,"5min":177.28399999999999,"15min":177.28399999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":9.7230000000000008,"5min":2.952,"15min":1.8240000000000001},"min":{"1min":0.32200000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":177.53999999999999,"5min":177.53999999999999,"15min":177.53999999999999},"last":0.442}]},{"osd":4,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":2.6190000000000002,"15min":1.9710000000000001},"min":{"1min":0.379,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.7770000000000001,"5min":103.208,"15min":103.208},"last":1.3500000000000001},{"interface":"front","average":{"1min":1.292,"5min":2.234,"15min":1.6120000000000001},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.835,"5min":103.218,"15min":103.218},"last":0.37}]},{"osd":5,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":2.79,"15min":2.0600000000000001},"min":{"1min":0.35199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.491,"5min":90.972999999999999,"15min":90.972999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.633,"5min":2.7040000000000002,"15min":2.0329999999999999},"min":{"1min":0.38400000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":17.422999999999998,"5min":90.959000000000003,"15min":90.959000000000003},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":2.8050000000000002,"15min":1.78},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.0890000000000004,"5min":104.422,"15min":104.422},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.0680000000000001,"5min":3.9180000000000001,"15min":2.512},"min":{"1min":0.377,"5min":0.34599999999999997,"15min":0.30499999999999999},"max":{"1min":11.696999999999999,"5min":123.505,"15min":123.505},"last":1.411}]},{"osd":7,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":2.036,"15min":2.1579999999999999},"min":{"1min":0.45000000000000001,"5min":0.34999999999999998,"15min":0.32300000000000001},"max":{"1min":5.2229999999999999,"5min":22.181999999999999,"15min":57.399000000000001},"last":1.3859999999999999},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.696,"15min":1.8440000000000001},"min":{"1min":0.45000000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":6.3920000000000003,"5min":21.812999999999999,"15min":57.447000000000003},"last":0.60599999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019291,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2029200,"kb_used_data":1141256,"kb_used_omap":0,"kb_used_meta":887936,"kb_avail":91715952,"statfs":{"total":95995035648,"available":93917134848,"internally_reserved":0,"allocated":1168646144,"data_stored":1168068361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909246464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:37 2023","interfaces":[{"interface":"back","average":{"1min":4.194,"5min":8.2710000000000008,"15min":5.2329999999999997},"min":{"1min":0.26200000000000001,"5min":0.23100000000000001,"15min":0.20499999999999999},"max":{"1min":73.471000000000004,"5min":171.13499999999999,"15min":219.577},"last":0.26600000000000001},{"interface":"front","average":{"1min":4.4669999999999996,"5min":8.3399999999999999,"15min":5.0309999999999997},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.247},"max":{"1min":73.364999999999995,"5min":171.11799999999999,"15min":218.96000000000001},"last":0.371}]},{"osd":1,"last update":"Mon Oct 9 04:38:30 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":3.081,"15min":4.7430000000000003},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.4590000000000001,"5min":62.518999999999998,"15min":142.74799999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":3.1240000000000001,"15min":4.8869999999999996},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":3.1829999999999998,"5min":62.718000000000004,"15min":139.09700000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":3.1040000000000001,"15min":3.464},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.1850000000000001,"5min":36.737000000000002,"15min":36.737000000000002},"last":6.6959999999999997},{"interface":"front","average":{"1min":1.3120000000000001,"5min":2.6829999999999998,"15min":2.911},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0549999999999997,"5min":36.423000000000002,"15min":36.423000000000002},"last":3.9820000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:38:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":3.7639999999999998,"15min":4.9219999999999997},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.9859999999999998,"5min":131.869,"15min":262.524},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.97699999999999998,"5min":3.6949999999999998,"15min":5.1040000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.694,"5min":131.82300000000001,"15min":262.55099999999999},"last":2.9089999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:38:07 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.694,"15min":1.95},"min":{"1min":0.28599999999999998,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":5.165,"5min":107.476,"15min":107.476},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.7509999999999999,"15min":2.1200000000000001},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":6.1609999999999996,"5min":107.414,"15min":107.414},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":2.5699999999999998,"15min":1.8779999999999999},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":4.375,"5min":65.75,"15min":65.75},"last":0.31},{"interface":"front","average":{"1min":0.66700000000000004,"5min":2.5329999999999999,"15min":1.8560000000000001},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.9289999999999998,"5min":65.849999999999994,"15min":65.849999999999994},"last":0.44800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:37:58 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.839,"15min":1.714},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0759999999999996,"5min":11.093999999999999,"15min":76.283000000000001},"last":0.439},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.6299999999999999,"15min":1.7050000000000001},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.2450000000000001,"5min":11.214,"15min":76.259},"last":0.52100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757655,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2033408,"kb_used_data":1143096,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":91711744,"statfs":{"total":95995035648,"available":93912825856,"internally_reserved":0,"allocated":1170530304,"data_stored":1169964649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:38:35 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":7.5990000000000002,"15min":4.9900000000000002},"min":{"1min":0.248,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":76.587999999999994,"5min":210.85499999999999,"15min":210.85499999999999},"last":56.869999999999997},{"interface":"front","average":{"1min":3.9359999999999999,"5min":7.8739999999999997,"15min":4.782},"min":{"1min":0.27000000000000002,"5min":0.249,"15min":0.24099999999999999},"max":{"1min":76.682000000000002,"5min":210.23500000000001,"15min":210.23500000000001},"last":56.787999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:38:35 2023","interfaces":[{"interface":"back","average":{"1min":7.2160000000000002,"5min":4.8579999999999997,"15min":4.0099999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":145.577,"5min":145.577,"15min":145.577},"last":0.45500000000000002},{"interface":"front","average":{"1min":7.1260000000000003,"5min":5.1959999999999997,"15min":4.8620000000000001},"min":{"1min":0.28999999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":145.792,"5min":145.792,"15min":145.792},"last":0.49399999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":6.2430000000000003,"15min":6.9509999999999996},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":18.276,"5min":90.763999999999996,"15min":90.763999999999996},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.7869999999999999,"5min":6.3220000000000001,"15min":7.0780000000000003},"min":{"1min":0.36599999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":18.48,"5min":91.340000000000003,"15min":91.340000000000003},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:38:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":4.125,"15min":4.4290000000000003},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9409999999999998,"5min":84.081999999999994,"15min":259.053},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.94699999999999995,"5min":3.859,"15min":4.3230000000000004},"min":{"1min":0.40400000000000003,"5min":0.31,"15min":0.31},"max":{"1min":4.2060000000000004,"5min":85.540000000000006,"15min":259.108},"last":0.53100000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":5.617,"15min":2.5089999999999999},"min":{"1min":0.32300000000000001,"5min":0.254,"15min":0.248},"max":{"1min":3.206,"5min":361.09399999999999,"15min":361.09399999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.073,"5min":5.9880000000000004,"15min":2.7839999999999998},"min":{"1min":0.28199999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.7599999999999998,"5min":361.71300000000002,"15min":361.71300000000002},"last":0.39900000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:38:40 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.8009999999999999,"15min":1.379},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.609,"5min":10.194000000000001,"15min":11.192},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.75,"5min":1.653,"15min":1.4299999999999999},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":4.5369999999999999,"5min":11.974,"15min":14.334},"last":0.38900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:38:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.49,"15min":1.409},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.899,"5min":11.093,"15min":11.093},"last":0.438},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.607,"15min":1.3600000000000001},"min":{"1min":0.41199999999999998,"5min":0.315,"15min":0.315},"max":{"1min":6.2380000000000004,"5min":15.257999999999999,"15min":15.257999999999999},"last":0.377}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1502740480,"data_stored":1502740480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2368602112,"data_stored":2368602112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1745223680,"data_stored":1745223680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2086010880,"data_stored":2086010880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1169686528,"data_stored":1169686528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1182531584,"data_stored":1182531584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2318204928,"data_stored":2318204928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1742798848,"data_stored":1742798848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:38:57.214 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:38:57.215 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:38:57.215 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.2523973080117444 2023-10-09T04:38:57.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 0.2523973080117444 2023-10-09T04:38:57.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:38:57.308 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:57.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:38:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:57.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:38:57.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:57.523 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:57.586 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:57.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:57.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:57.597 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:57.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:57.730 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:57.773 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:38:57.805 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:57.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:57.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:57.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:38:57.963 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:58.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 77833 77817 86.8005 59.4375 0.0129641 0.0115157 2023-10-09T04:38:58.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:38:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:58.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:38:58.135 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:38:58.174 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:58.246 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:38:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:58.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:38:58.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:58.428 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:58.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:58 smithi029 ceph-mon[216903]: pgmap v915: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2023-10-09T04:38:58.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:38:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:58.611 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:38:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:38:58.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:38:58.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:38:58.787 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:38:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:58 smithi029 ceph-mon[223858]: pgmap v915: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2023-10-09T04:38:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:38:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:59.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 78814 78798 86.3535 61.3125 0.01519 0.0115747 2023-10-09T04:38:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:58 smithi177 ceph-mon[195797]: pgmap v915: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s 2023-10-09T04:38:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:38:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:38:59.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:39:00.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 79638 79622 85.7528 51.5 0.0124307 0.0116336 2023-10-09T04:39:00.631 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:00.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:00.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:00.771 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:00.831 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:00.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:00 smithi029 ceph-mon[216903]: pgmap v916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T04:39:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:00 smithi029 ceph-mon[223858]: pgmap v916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T04:39:00.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:00.983 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:01.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 80419 80403 85.1268 48.8125 0.00875705 0.0117418 2023-10-09T04:39:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:00 smithi177 ceph-mon[195797]: pgmap v916: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-10-09T04:39:01.645 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:01.709 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:01.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:01.828 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:01.898 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:02.014 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:39:02.015019+0000 min lat: 0.00104865 max lat: 0.531777 avg lat: 0.0117964 2023-10-09T04:39:02.015 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:39:02.015 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 81410 81394 84.7321 61.9375 0.0120829 0.0117964 2023-10-09T04:39:02.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:02.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:02.352 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.391 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:02.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:02.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:02.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:02.526 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:02.570 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:02.608 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:02.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:02.688 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:02 smithi177 ceph-mon[195797]: pgmap v917: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:39:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:02.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:02.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:02.826 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:02.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:02.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:02.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:02.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:02 smithi029 ceph-mon[216903]: pgmap v917: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:39:02.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:39:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:39:02] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T04:39:02.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:02 smithi029 ceph-mon[223858]: pgmap v917: 89 pgs: 89 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-09T04:39:03.014 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 82305 82289 84.2602 55.9375 0.0315823 0.0118628 2023-10-09T04:39:03.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:03.086 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:03.295 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:39:03.295 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:39:03.669 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:39:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4149977318' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4149977318' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:03 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:04.015 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 83204 83188 83.8073 56.1875 0.016581 0.0119276 2023-10-09T04:39:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4149977318' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:39:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:03 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: pgmap v918: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[216903]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: pgmap v918: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: failed to encode map e246 with expected crc 2023-10-09T04:39:04.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:04 smithi029 ceph-mon[223858]: failed to encode map e246 with expected crc 2023-10-09T04:39:05.015 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 84310 84294 83.5735 69.125 0.0145852 0.01196 2023-10-09T04:39:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: pgmap v918: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-09T04:39:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: failed to encode map e246 with expected crc 2023-10-09T04:39:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: failed to encode map e246 with expected crc 2023-10-09T04:39:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:39:05.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: osdmap e246: 8 total, 8 up, 8 in 2023-10-09T04:39:05.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: failed to encode map e246 with expected crc 2023-10-09T04:39:05.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:04 smithi177 ceph-mon[195797]: failed to encode map e246 with expected crc 2023-10-09T04:39:06.016 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 85138 85122 83.0764 51.75 0.0156252 0.0120318 2023-10-09T04:39:06.165 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:06.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:06.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:06.355 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:06 smithi029 ceph-mon[223858]: pgmap v920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T04:39:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:06 smithi029 ceph-mon[216903]: pgmap v920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T04:39:07.016 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 85998 85982 82.6254 53.75 0.0148371 0.0120983 2023-10-09T04:39:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:06 smithi177 ceph-mon[195797]: pgmap v920: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s 2023-10-09T04:39:07.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:07.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:07.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:07.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:07.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:07.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:08.016 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 86779 86763 82.1133 48.8125 0.0150998 0.0121724 2023-10-09T04:39:08.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:39:08.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:39:08.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:39:08.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:08.175 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:08.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:08.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:08.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:08.377 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:08.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:08 smithi029 ceph-mon[216903]: pgmap v921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:39:08.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:08 smithi029 ceph-mon[223858]: pgmap v921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:39:09.016 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 87741 87725 81.7852 60.125 0.0144816 0.0122223 2023-10-09T04:39:09.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:08 smithi177 ceph-mon[195797]: pgmap v921: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:39:09.062 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:09.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:09.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:09.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:09.257 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:09.522 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:09.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:09.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:09.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:09.675 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:09.707 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:09.742 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:09.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:09.826 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:39:09.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:09.892 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:39:09.954 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:39:09.965 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:10.000 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:39:10.019 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 88625 88609 81.3909 55.25 0.0311675 0.0122803 2023-10-09T04:39:10.047 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:39:10.113 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:39:10.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:39:10.151 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:39:10.194 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:10.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:10.204 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:39:10.217 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.217 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.218 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.218 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.218 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.218 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.218 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.218 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:39:10.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:10.407 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:10.756 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:10.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:10.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:10.914 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:11.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:11.020 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 89452 89436 80.9595 51.6875 0.0194807 0.012346 2023-10-09T04:39:11.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:11.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:11.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:10 smithi177 ceph-mon[195797]: pgmap v922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:39:11.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/238951217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:11.023 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1322143032' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.2523973080117444}]: dispatch 2023-10-09T04:39:11.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:10 smithi029 ceph-mon[216903]: pgmap v922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:39:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/238951217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1322143032' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.2523973080117444}]: dispatch 2023-10-09T04:39:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:10 smithi029 ceph-mon[223858]: pgmap v922: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-09T04:39:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/238951217' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1322143032' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.2523973080117444}]: dispatch 2023-10-09T04:39:11.201 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:11.414 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 0.252397 (409d) 2023-10-09T04:39:12.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 89800 89784 80.1139 21.75 0.00344655 0.0123704 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: pgmap v923: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1322143032' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.2523973080117444}]': finished 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[216903]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: pgmap v923: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-10-09T04:39:12.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1322143032' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.2523973080117444}]': finished 2023-10-09T04:39:12.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:39:12.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.635 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:12 smithi029 ceph-mon[223858]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.665 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:12.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:12.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:12.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: pgmap v923: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2023-10-09T04:39:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1322143032' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.2523973080117444}]': finished 2023-10-09T04:39:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: osdmap e247: 8 total, 8 up, 8 in 2023-10-09T04:39:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:12 smithi177 ceph-mon[195797]: failed to encode map e247 with expected crc 2023-10-09T04:39:12.795 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:12.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:12.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:12.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:39:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:39:12] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.43.0" 2023-10-09T04:39:12.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:12.969 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:13.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:13.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:13.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:13.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 89800 89784 78.9862 0 - 0.0123704 2023-10-09T04:39:13.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:13.151 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:13.311 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:13.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:13.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:39:13.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[223858]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.409 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:13 smithi029 ceph-mon[216903]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:13.459 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:13.551 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:13.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:13.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:13.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: osdmap e248: 8 total, 8 up, 8 in 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.684 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.685 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:13 smithi177 ceph-mon[195797]: failed to encode map e248 with expected crc 2023-10-09T04:39:13.687 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:13.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:13.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:13.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:13.949 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:14.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 89955 89939 78.024 4.84375 0.0151189 0.0128108 2023-10-09T04:39:14.284 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:14.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:14.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:14.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: pgmap v926: 89 pgs: 1 unknown, 10 peering, 78 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:39:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:39:14.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: osdmap e249: 8 total, 8 up, 8 in 2023-10-09T04:39:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:14.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: pgmap v926: 89 pgs: 1 unknown, 10 peering, 78 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:39:14.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:39:14.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: osdmap e249: 8 total, 8 up, 8 in 2023-10-09T04:39:14.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:14.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:14.429 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:61972 9.9 9:90f3cb0e:::benchmark_data_smithi029_299184_object61971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.529067+0000", 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 537.903614662, 2023-10-09T04:39:14.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1577070090000001, 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 61972 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529067+0000", 2023-10-09T04:39:14.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529067+0000", 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529066+0000", 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:14.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529094+0000", 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7983999999999998e-05 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529108+0000", 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3859999999999999e-05 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529116+0000", 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5699999999999993e-06 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529174+0000", 2023-10-09T04:39:14.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7540999999999997e-05 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.529182+0000", 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0900000000000005e-06 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.530789+0000", 2023-10-09T04:39:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016072059999999999 2023-10-09T04:39:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.530800+0000", 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0720000000000001e-05 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.681698+0000", 2023-10-09T04:39:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1508984250000001 2023-10-09T04:39:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:39:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682029+0000", 2023-10-09T04:39:14.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033048000000000002 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686711+0000", 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046828160000000002 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686774+0000", 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3068e-05 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:61980 9.9 9:9f2d6e30:::benchmark_data_smithi029_299184_object61979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:39:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.536369+0000", 2023-10-09T04:39:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 537.89631342500002, 2023-10-09T04:39:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.154441914, 2023-10-09T04:39:14.455 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 61980 2023-10-09T04:39:14.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536369+0000", 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536369+0000", 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536367+0000", 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536393+0000", 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5503999999999998e-05 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536411+0000", 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8919999999999998e-05 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536421+0000", 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2580000000000001e-06 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536486+0000", 2023-10-09T04:39:14.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5320000000000005e-05 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.536497+0000", 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1276e-05 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682269+0000", 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1457712170000001 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682371+0000", 2023-10-09T04:39:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102947 2023-10-09T04:39:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.682476+0000", 2023-10-09T04:39:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010462 2023-10-09T04:39:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686667+0000", 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041908800000000001 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690531+0000", 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038639249999999998 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690651+0000", 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011974200000000001 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690710+0000", 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9570000000000001e-05 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690811+0000", 2023-10-09T04:39:14.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100449 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:61982 9.9 9:9c6412a6:::benchmark_data_smithi029_299184_object61981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.542493+0000", 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 537.89018947399995, 2023-10-09T04:39:14.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1484788960000001, 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 61982 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542493+0000", 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542493+0000", 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542491+0000", 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542520+0000", 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9142e-05 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542534+0000", 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4059999999999999e-05 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542543+0000", 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7450000000000001e-06 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542673+0000", 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012984300000000001 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.542688+0000", 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5449999999999999e-05 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.684067+0000", 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1413787470000001 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.684210+0000", 2023-10-09T04:39:14.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014304300000000001 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.684405+0000", 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019457200000000001 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690490+0000", 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060849470000000003 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690595+0000", 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105111 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690866+0000", 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00027154099999999998 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690898+0000", 2023-10-09T04:39:14.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2054999999999997e-05 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690972+0000", 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3209999999999999e-05 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.474 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:62008 9.9 9:9ba51e8c:::benchmark_data_smithi029_299184_object62007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.566286+0000", 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 537.86639574399999, 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.129050973, 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.475 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62008 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566286+0000", 2023-10-09T04:39:14.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566286+0000", 2023-10-09T04:39:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566284+0000", 2023-10-09T04:39:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566318+0000", 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3426000000000002e-05 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566336+0000", 2023-10-09T04:39:14.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7604e-05 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566345+0000", 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5100000000000004e-06 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566429+0000", 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3690999999999995e-05 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.566440+0000", 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.114e-05 2023-10-09T04:39:14.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686796+0000", 2023-10-09T04:39:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1203562330000001 2023-10-09T04:39:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.686945+0000", 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000149128 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.687135+0000", 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000189657 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.690609+0000", 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034738770000000002 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695192+0000", 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045837070000000002 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695231+0000", 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8741000000000001e-05 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695266+0000", 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4295000000000003e-05 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695337+0000", 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1872999999999994e-05 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:62039 9.9 9:9b50b364:::benchmark_data_smithi029_299184_object62038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.603585+0000", 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 537.82909696000002, 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.091967184, 2023-10-09T04:39:14.486 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62039 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603585+0000", 2023-10-09T04:39:14.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603585+0000", 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603583+0000", 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603608+0000", 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4511e-05 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603625+0000", 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7382000000000001e-05 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603635+0000", 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7990000000000005e-06 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603752+0000", 2023-10-09T04:39:14.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011654000000000001 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.603767+0000", 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5027e-05 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.688633+0000", 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0848658689999999 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.688747+0000", 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011479799999999999 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.688908+0000", 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016042399999999999 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695208+0000", 2023-10-09T04:39:14.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062998819999999997 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695379+0000", 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017120500000000001 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695422+0000", 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3010999999999999e-05 2023-10-09T04:39:14.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695483+0000", 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0869999999999998e-05 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695552+0000", 2023-10-09T04:39:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9579000000000006e-05 2023-10-09T04:39:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:62051 9.9 9:92149584:::benchmark_data_smithi029_299184_object62050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:30:16.616339+0000", 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 537.81634333700003, 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.086473722, 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.496 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62051 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616339+0000", 2023-10-09T04:39:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616339+0000", 2023-10-09T04:39:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616337+0000", 2023-10-09T04:39:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616361+0000", 2023-10-09T04:39:14.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4099000000000001e-05 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616375+0000", 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3662000000000001e-05 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616383+0000", 2023-10-09T04:39:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4579999999999993e-06 2023-10-09T04:39:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616449+0000", 2023-10-09T04:39:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5829999999999998e-05 2023-10-09T04:39:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:39:14.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:16.616462+0000", 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2737e-05 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.691033+0000", 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0745713690000001 2023-10-09T04:39:14.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.691164+0000", 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130483 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.505 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.691296+0000", 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131746 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.695568+0000", 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042726090000000001 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702529+0000", 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069606370000000004 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702648+0000", 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011957 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702718+0000", 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9638e-05 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:30:17.702813+0000", 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4545999999999999e-05 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166771 9.b 9:dd78fb9c:::benchmark_data_smithi029_299184_object68468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.077564+0000", 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.355117759, 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3448709380000001, 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166771 2023-10-09T04:39:14.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077564+0000", 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077564+0000", 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077563+0000", 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077570+0000", 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2880000000000004e-06 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077583+0000", 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3049e-05 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077592+0000", 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5779999999999999e-06 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077645+0000", 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2834000000000003e-05 2023-10-09T04:39:14.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077763+0000", 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118306 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.077877+0000", 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114071 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422189+0000", 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3443123450000001 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422208+0000", 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8799000000000001e-05 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422325+0000", 2023-10-09T04:39:14.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116734 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422381+0000", 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5961000000000002e-05 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422435+0000", 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4348999999999998e-05 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.515 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166776 9.5 9:ab9ec60c:::benchmark_data_smithi029_299184_object68473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081143+0000", 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.35153880299998, 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.346525757, 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.516 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166776 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081143+0000", 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081143+0000", 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081142+0000", 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081149+0000", 2023-10-09T04:39:14.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2899999999999997e-06 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081169+0000", 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9813000000000002e-05 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081178+0000", 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8119999999999997e-06 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.085605+0000", 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044272540000000003 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.085733+0000", 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012755500000000001 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.085838+0000", 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105432 2023-10-09T04:39:14.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427505+0000", 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3416669649999999 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427520+0000", 2023-10-09T04:39:14.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5014000000000001e-05 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427561+0000", 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1162999999999999e-05 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427622+0000", 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0359000000000003e-05 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427669+0000", 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7559999999999999e-05 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166777 9.b 9:d7522bda:::benchmark_data_smithi029_299184_object68474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081193+0000", 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.35148911300001, 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3414661510000001, 2023-10-09T04:39:14.523 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166777 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081193+0000", 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081193+0000", 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081192+0000", 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081196+0000", 2023-10-09T04:39:14.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4259999999999999e-06 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081203+0000", 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.374e-06 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081209+0000", 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0430000000000004e-06 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082783+0000", 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015734099999999999 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082897+0000", 2023-10-09T04:39:14.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114535 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.083005+0000", 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000107649 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422253+0000", 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.339248386 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422261+0000", 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1030000000000002e-06 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422564+0000", 2023-10-09T04:39:14.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00030216800000000001 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422617+0000", 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.304e-05 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422659+0000", 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2553999999999999e-05 2023-10-09T04:39:14.530 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166778 9.5 9:ac5d0fe7:::benchmark_data_smithi029_299184_object68475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.35146285000002, 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.357283987, 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.531 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166778 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081219+0000", 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081222+0000", 2023-10-09T04:39:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9979999999999999e-06 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081228+0000", 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3509999999999998e-06 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081233+0000", 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0950000000000003e-06 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090131+0000", 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088972760000000008 2023-10-09T04:39:14.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090227+0000", 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6126999999999996e-05 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090322+0000", 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5367999999999999e-05 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427723+0000", 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3374010250000001 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.427756+0000", 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3222999999999997e-05 2023-10-09T04:39:14.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.438410+0000", 2023-10-09T04:39:14.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010653856999999999 2023-10-09T04:39:14.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.438462+0000", 2023-10-09T04:39:14.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1761000000000001e-05 2023-10-09T04:39:14.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.438503+0000", 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1337000000000001e-05 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166782 9.b 9:de3f3061:::benchmark_data_smithi029_299184_object68479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081559+0000", 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.35112294300001, 2023-10-09T04:39:14.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3412489839999999, 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166782 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081559+0000", 2023-10-09T04:39:14.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081559+0000", 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081558+0000", 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081564+0000", 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.22e-06 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081574+0000", 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.08e-05 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081583+0000", 2023-10-09T04:39:14.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2430000000000005e-06 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.087862+0000", 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062788369999999998 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.087948+0000", 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6804000000000004e-05 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.088029+0000", 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0216000000000006e-05 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422267+0000", 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3342385990000001 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422497+0000", 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022963700000000001 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422721+0000", 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022408899999999999 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422771+0000", 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0076e-05 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422808+0000", 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7255999999999997e-05 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.548 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:166781 9.8 e182/167 9:1ce571f8:::benchmark_data_smithi029_299184_object68478:head v 182'10397, mlcod=182'10395)", 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.081697+0000", 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.35098507700002, 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3404559030000001, 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081697+0000", 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081697+0000", 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081696+0000", 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081703+0000", 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4289999999999998e-06 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081712+0000", 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5210000000000008e-06 2023-10-09T04:39:14.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.081718+0000", 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1129999999999997e-06 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082774+0000", 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001055865 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.082795+0000", 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0959999999999999e-05 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422101+0000", 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3393052329999999 2023-10-09T04:39:14.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422153+0000", 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2463999999999998e-05 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:166771 9.b e182/163 ondisk, result = 0)", 2023-10-09T04:39:14.554 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.086822+0000", 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.34585999000001, 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.335388005, 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086822+0000", 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086822+0000", 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086821+0000", 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086828+0000", 2023-10-09T04:39:14.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086840+0000", 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2448e-05 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.086848+0000", 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8669999999999994e-06 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422168+0000", 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.335319393 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422180+0000", 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.293e-05 2023-10-09T04:39:14.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422210+0000", 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9728000000000001e-05 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.35373.0:166784 9.8 e182/167 9:1040dc07:::benchmark_data_smithi029_299184_object68481:head v 182'10398, mlcod=182'10395)", 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.089440+0000", 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.34324185999998, 2023-10-09T04:39:14.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3327767939999999, 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089440+0000", 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089440+0000", 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089439+0000", 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089450+0000", 2023-10-09T04:39:14.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0657e-05 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089459+0000", 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7839999999999992e-06 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.089465+0000", 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0769999999999996e-06 2023-10-09T04:39:14.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090074+0000", 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00060908399999999997 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.090094+0000", 2023-10-09T04:39:14.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0406000000000001e-05 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422161+0000", 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3320668019999999 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422217+0000", 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5782000000000002e-05 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.564 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.35373.0:166787 9.9 9:9b506fbe:::benchmark_data_smithi029_299184_object68484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.093964+0000", 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.33871781599998, 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3390893639999999, 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.35373", 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 166787 2023-10-09T04:39:14.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093964+0000", 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093964+0000", 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093963+0000", 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.093969+0000", 2023-10-09T04:39:14.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0739999999999998e-06 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422069+0000", 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.328100367 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422094+0000", 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4117e-05 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422212+0000", 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118548 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422393+0000", 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000181349 2023-10-09T04:39:14.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422546+0000", 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000152614 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432118+0000", 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095720019999999996 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432933+0000", 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00081524799999999997 2023-10-09T04:39:14.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432942+0000", 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.432994+0000", 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2791000000000003e-05 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.433054+0000", 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9372e-05 2023-10-09T04:39:14.573 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.574 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.574 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.574 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:166771 9.b e182/163 ondisk, result = 0)", 2023-10-09T04:39:14.574 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.095374+0000", 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.33730788899999, 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.327075148, 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095374+0000", 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095374+0000", 2023-10-09T04:39:14.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095373+0000", 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095379+0000", 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3180000000000003e-06 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422081+0000", 2023-10-09T04:39:14.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3267017489999999 2023-10-09T04:39:14.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422093+0000", 2023-10-09T04:39:14.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2282e-05 2023-10-09T04:39:14.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422308+0000", 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021445599999999999 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422318+0000", 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7629999999999995e-06 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422449+0000", 2023-10-09T04:39:14.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131755 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.35373.0:166777 9.b e182/163 ondisk, result = 0)", 2023-10-09T04:39:14.582 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 386.33688401500001, 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3264704890000001, 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:39:14.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095798+0000", 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:48.095802+0000", 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2590000000000002e-06 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422075+0000", 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3262729289999999 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422092+0000", 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6778000000000001e-05 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422237+0000", 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014545199999999999 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422246+0000", 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9369999999999993e-06 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:32:49.422269+0000", 2023-10-09T04:39:14.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2693000000000001e-05 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:196406 10.18 10:1f858c40:::benchmark_data_smithi029_344401_object88035:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:04.313876+0000", 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 130.11880599200001, 2023-10-09T04:39:14.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.31330416100000003, 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 196406 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.590 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313876+0000", 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313876+0000", 2023-10-09T04:39:14.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313875+0000", 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313881+0000", 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9909999999999999e-06 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313891+0000", 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0821e-05 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313899+0000", 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5209999999999997e-06 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.313940+0000", 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1007999999999998e-05 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.314131+0000", 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190998 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.314684+0000", 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00055255099999999998 2023-10-09T04:39:14.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.622633+0000", 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.307949162 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.622666+0000", 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3284999999999997e-05 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.626485+0000", 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038185430000000002 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.626547+0000", 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1974e-05 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.627180+0000", 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00063377699999999997 2023-10-09T04:39:14.596 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45542.0:196409 10.c e225/216 10:367521f5:::benchmark_data_smithi029_344401_object88038:head v 225'10513, mlcod=225'10510)", 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:04.317271+0000", 2023-10-09T04:39:14.597 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 130.11541135900001, 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30909826200000001, 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317271+0000", 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317271+0000", 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317270+0000", 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317280+0000", 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0261000000000001e-05 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.600 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.619967+0000", 2023-10-09T04:39:14.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30268719900000002 2023-10-09T04:39:14.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.600 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.619979+0000", 2023-10-09T04:39:14.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1817999999999999e-05 2023-10-09T04:39:14.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.620059+0000", 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0356000000000001e-05 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.620077+0000", 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8048999999999999e-05 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.626333+0000", 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062551710000000003 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.626369+0000", 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6400999999999999e-05 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.603 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:196412 10.5 10:a63430ae:::benchmark_data_smithi029_344401_object88041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:04.317642+0000", 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 130.11504029599999, 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.321884055, 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 196412 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.604 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317642+0000", 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317642+0000", 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:39:14.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317640+0000", 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.317649+0000", 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2520000000000005e-06 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.619941+0000", 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.302291845 2023-10-09T04:39:14.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.619962+0000", 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0628e-05 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.620017+0000", 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5133999999999998e-05 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.620201+0000", 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000184303 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.620886+0000", 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00068490499999999998 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.628982+0000", 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080963909999999997 2023-10-09T04:39:14.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.628993+0000", 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0394000000000001e-05 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.638863+0000", 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098700909999999992 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.638908+0000", 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5315999999999998e-05 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.639526+0000", 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00061768200000000004 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:39:14.610 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:39:14.611 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:39:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:14.612 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:39:14.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:14.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: pgmap v926: 89 pgs: 1 unknown, 10 peering, 78 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-10-09T04:39:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-10-09T04:39:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: osdmap e249: 8 total, 8 up, 8 in 2023-10-09T04:39:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: failed to encode map e249 with expected crc 2023-10-09T04:39:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:14.836 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:15.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 90543 90527 77.4588 36.75 0.0130206 0.0129039 2023-10-09T04:39:15.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:39:16.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 91202 91186 76.9689 41.1875 0.022538 0.0129852 2023-10-09T04:39:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:16 smithi029 ceph-mon[216903]: pgmap v928: 89 pgs: 1 unknown, 10 peering, 78 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T04:39:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:16 smithi029 ceph-mon[223858]: pgmap v928: 89 pgs: 1 unknown, 10 peering, 78 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T04:39:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:16 smithi177 ceph-mon[195797]: pgmap v928: 89 pgs: 1 unknown, 10 peering, 78 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2023-10-09T04:39:17.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 91858 91842 76.4895 41 0.0176879 0.0130676 2023-10-09T04:39:17.814 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:17.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:17.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:17.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:17.956 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:18.018 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:18.021 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 92527 92511 76.0334 41.8125 0.0118517 0.0131448 2023-10-09T04:39:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:18.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:18.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:18.176 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:18.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:39:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:18.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:18.346 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:18.382 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:18.435 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:39:18.435 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:39:18.435 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:39:18.435 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.138736 2023-10-09T04:39:18.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.1387358026870562 2023-10-09T04:39:18.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:18.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:18 smithi177 ceph-mon[195797]: pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 11386/273537 objects degraded (4.163%); 11054/273537 objects misplaced (4.041%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-09T04:39:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:18 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 11386/273537 objects degraded (4.163%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:18 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-10-09T04:39:18.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:18.658 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:18 smithi029 ceph-mon[216903]: pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 11386/273537 objects degraded (4.163%); 11054/273537 objects misplaced (4.041%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-09T04:39:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:18 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 11386/273537 objects degraded (4.163%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:18.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:18 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-10-09T04:39:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:18 smithi029 ceph-mon[223858]: pgmap v929: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 11386/273537 objects degraded (4.163%); 11054/273537 objects misplaced (4.041%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-09T04:39:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:18 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 11386/273537 objects degraded (4.163%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:18 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-10-09T04:39:19.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 93150 93134 75.5518 38.9375 0.0215087 0.0132292 2023-10-09T04:39:19.123 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:19.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:19.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:19.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:19.260 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:19.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:19.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:19.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:19.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:19.521 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:20.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 93795 93779 75.1002 40.3125 0.049023 0.0133096 2023-10-09T04:39:20.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:20 smithi177 ceph-mon[195797]: pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 28 MiB/s wr, 453 op/s; 11386/273537 objects degraded (4.163%); 11054/273537 objects misplaced (4.041%); 1.1 MiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T04:39:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:20 smithi029 ceph-mon[216903]: pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 28 MiB/s wr, 453 op/s; 11386/273537 objects degraded (4.163%); 11054/273537 objects misplaced (4.041%); 1.1 MiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T04:39:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:20 smithi029 ceph-mon[223858]: pgmap v930: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 28 MiB/s wr, 453 op/s; 11386/273537 objects degraded (4.163%); 11054/273537 objects misplaced (4.041%); 1.1 MiB/s, 1 keys/s, 18 objects/s recovering 2023-10-09T04:39:21.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 94304 94288 74.5524 31.8125 0.0289006 0.0134073 2023-10-09T04:39:21.056 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:21.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:21.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:21.162 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:21.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:39:21.191 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:21.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:21.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:21.344 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:21.349 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:21.377 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:21.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:21.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:21.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:21.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:21.579 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:21.625 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:22.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:39:22.022388+0000 min lat: 0.00104865 max lat: 2.50447 avg lat: 0.013505 2023-10-09T04:39:22.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:39:22.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 94795 94779 74.0043 30.6875 0.0258399 0.013505 2023-10-09T04:39:22.045 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:22.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:22.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:22.194 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:22.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:22.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:22.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:22.354 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:22.385 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:22.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:22.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:22.542 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:22.644 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:22 smithi177 ceph-mon[195797]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 27 MiB/s wr, 431 op/s; 11386/279039 objects degraded (4.080%); 10945/279039 objects misplaced (3.922%); 1.8 MiB/s, 1 keys/s, 29 objects/s recovering 2023-10-09T04:39:22.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:22 smithi029 ceph-mon[216903]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 27 MiB/s wr, 431 op/s; 11386/279039 objects degraded (4.080%); 10945/279039 objects misplaced (3.922%); 1.8 MiB/s, 1 keys/s, 29 objects/s recovering 2023-10-09T04:39:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:22 smithi029 ceph-mon[223858]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 27 MiB/s wr, 431 op/s; 11386/279039 objects degraded (4.080%); 10945/279039 objects misplaced (3.922%); 1.8 MiB/s, 1 keys/s, 29 objects/s recovering 2023-10-09T04:39:22.821 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:39:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:39:22] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.43.0" 2023-10-09T04:39:23.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 95321 95305 73.4967 32.875 0.00759546 0.0135989 2023-10-09T04:39:23.540 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:39:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1461801958' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:23 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1461801958' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:23 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1461801958' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:23 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:39:24.022 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 95853 95837 73.0061 33.25 0.0104272 0.0136879 2023-10-09T04:39:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 31 MiB/s wr, 500 op/s; 11386/282828 objects degraded (4.026%); 10803/282828 objects misplaced (3.820%); 2.4 MiB/s, 0 keys/s, 39 objects/s recovering 2023-10-09T04:39:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11386/282828 objects degraded (4.026%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:39:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: failed to encode map e250 with expected crc 2023-10-09T04:39:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:39:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: osdmap e250: 8 total, 8 up, 8 in 2023-10-09T04:39:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: failed to encode map e250 with expected crc 2023-10-09T04:39:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 31 MiB/s wr, 500 op/s; 11386/282828 objects degraded (4.026%); 10803/282828 objects misplaced (3.820%); 2.4 MiB/s, 0 keys/s, 39 objects/s recovering 2023-10-09T04:39:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11386/282828 objects degraded (4.026%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:39:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: failed to encode map e250 with expected crc 2023-10-09T04:39:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: osdmap e250: 8 total, 8 up, 8 in 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: failed to encode map e250 with expected crc 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 31 MiB/s wr, 500 op/s; 11386/282828 objects degraded (4.026%); 10803/282828 objects misplaced (3.820%); 2.4 MiB/s, 0 keys/s, 39 objects/s recovering 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11386/282828 objects degraded (4.026%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: failed to encode map e250 with expected crc 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:39:24.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: osdmap e250: 8 total, 8 up, 8 in 2023-10-09T04:39:24.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: failed to encode map e250 with expected crc 2023-10-09T04:39:24.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:25.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 96476 96460 72.5932 38.9375 0.0262342 0.0137693 2023-10-09T04:39:26.025 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 97041 97025 72.1496 35.3125 0.00947208 0.0138547 2023-10-09T04:39:26.073 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:26.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:26.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:26.211 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:26.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:26.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:26.375 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:26.418 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:26.472 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:26.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:26.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:26.678 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:26 smithi177 ceph-mon[195797]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 32 MiB/s wr, 505 op/s; 11386/283728 objects degraded (4.013%); 10803/283728 objects misplaced (3.808%); 2.4 MiB/s, 0 keys/s, 39 objects/s recovering 2023-10-09T04:39:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:26 smithi029 ceph-mon[216903]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 32 MiB/s wr, 505 op/s; 11386/283728 objects degraded (4.013%); 10803/283728 objects misplaced (3.808%); 2.4 MiB/s, 0 keys/s, 39 objects/s recovering 2023-10-09T04:39:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:26 smithi029 ceph-mon[223858]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 32 MiB/s wr, 505 op/s; 11386/283728 objects degraded (4.013%); 10803/283728 objects misplaced (3.808%); 2.4 MiB/s, 0 keys/s, 39 objects/s recovering 2023-10-09T04:39:27.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 97694 97678 71.7804 40.8125 0.0279663 0.0139261 2023-10-09T04:39:27.650 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:27.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:27.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:27.793 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:27.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:27.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:27.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:27.991 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:28.026 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 98304 98288 71.3892 38.125 0.00973584 0.0140013 2023-10-09T04:39:28.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:39:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:28.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:28.154 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.181 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:28.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:28.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:28.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.388 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:28.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:28.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:28.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:28 smithi029 ceph-mon[216903]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 11386/291198 objects degraded (3.910%); 10600/291198 objects misplaced (3.640%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:28.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:28.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:28.532 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:28 smithi029 ceph-mon[223858]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 11386/291198 objects degraded (3.910%); 10600/291198 objects misplaced (3.640%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:28.532 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:28.533 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:28.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:28.583 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:28.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:28.647 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:28 smithi177 ceph-mon[195797]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 11386/291198 objects degraded (3.910%); 10600/291198 objects misplaced (3.640%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:28.647 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:28.647 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:28.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:28.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:28.852 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:29.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 98873 98857 70.9749 35.5625 0.0289523 0.0140831 2023-10-09T04:39:29.386 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:29.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:29.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:29.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:29.596 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:29.747 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:29.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:29.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:29.926 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:30.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 99500 99484 70.6139 39.1875 0.0478567 0.014155 2023-10-09T04:39:30.110 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:30.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:30.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:30.295 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:30.500 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:30.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:30.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:30.660 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:30.712 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:30.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:30.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:30.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:30 smithi029 ceph-mon[223858]: pgmap v936: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 11386/291198 objects degraded (3.910%); 10600/291198 objects misplaced (3.640%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:30.892 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11386/291198 objects degraded (3.910%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:30.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:30 smithi029 ceph-mon[216903]: pgmap v936: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 11386/291198 objects degraded (3.910%); 10600/291198 objects misplaced (3.640%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:30.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11386/291198 objects degraded (3.910%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:30.910 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:30 smithi177 ceph-mon[195797]: pgmap v936: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 37 MiB/s wr, 588 op/s; 11386/291198 objects degraded (3.910%); 10600/291198 objects misplaced (3.640%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11386/291198 objects degraded (3.910%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:31.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 100072 100056 70.2223 35.75 0.0185358 0.0142328 2023-10-09T04:39:31.093 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:31.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:31.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:31.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:31.289 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:31.336 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:31.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:31.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:31.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:31.467 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:31.755 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:31.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:31.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:31.939 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:31.997 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:32.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:32.029 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 15 100553 100538 69.777 30.125 0.0115611 0.0143157 2023-10-09T04:39:32.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:32.128 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:32.170 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:32.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:32.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:32.305 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:32.914 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:32.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:32 smithi029 ceph-mon[216903]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 579 op/s; 11386/296424 objects degraded (3.841%); 10488/296424 objects misplaced (3.538%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:39:32.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:32 smithi029 ceph-mon[223858]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 579 op/s; 11386/296424 objects degraded (3.841%); 10488/296424 objects misplaced (3.538%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:39:32.917 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:39:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:39:32] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-10-09T04:39:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:32.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:33.019 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:32 smithi177 ceph-mon[195797]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 579 op/s; 11386/296424 objects degraded (3.841%); 10488/296424 objects misplaced (3.538%); 2.9 MiB/s, 45 objects/s recovering 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.1455 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 100553 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 69.7158 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 39.7878 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 194.75 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1115 2023-10-09T04:39:33.030 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 636.615 2023-10-09T04:39:33.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3116 2023-10-09T04:39:33.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2023-10-09T04:39:33.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0143385 2023-10-09T04:39:33.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0358154 2023-10-09T04:39:33.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.50447 2023-10-09T04:39:33.031 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00104865 2023-10-09T04:39:33.045 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:33.054 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:39:33.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:33.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:33.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:33.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:33.249 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:33.307 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:33.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:33.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:33.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:33.527 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:34 smithi029 ceph-mon[216903]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 11386/300198 objects degraded (3.793%); 10349/300198 objects misplaced (3.447%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:34.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:34 smithi029 ceph-mon[223858]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 11386/300198 objects degraded (3.793%); 10349/300198 objects misplaced (3.447%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:34 smithi177 ceph-mon[195797]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 11386/300198 objects degraded (3.793%); 10349/300198 objects misplaced (3.447%); 2.8 MiB/s, 45 objects/s recovering 2023-10-09T04:39:35.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11386/300198 objects degraded (3.793%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:35.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11386/300198 objects degraded (3.793%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11386/300198 objects degraded (3.793%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:36 smithi029 ceph-mon[223858]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 32 MiB/s wr, 523 op/s; 11386/300132 objects degraded (3.794%); 10349/300132 objects misplaced (3.448%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:39:36.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:36 smithi029 ceph-mon[216903]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 32 MiB/s wr, 523 op/s; 11386/300132 objects degraded (3.794%); 10349/300132 objects misplaced (3.448%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:39:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:36 smithi177 ceph-mon[195797]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 32 MiB/s wr, 523 op/s; 11386/300132 objects degraded (3.794%); 10349/300132 objects misplaced (3.448%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T04:39:37.035 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:37.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:37.099 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:37.164 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:37.279 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:37.633 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:37.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:37.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:37.778 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:37.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:37.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:37.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:37.950 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:39:38.007 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:39:38.048 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:38.072 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:39:38.114 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:39:38.147 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:39:38.194 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:39:38.228 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:39:38.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:38.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:38.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:38.263 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:39:38.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.300 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.300 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.300 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:39:38.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:38.445 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:38.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:38 smithi029 ceph-mon[216903]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 605 op/s; 11386/297936 objects degraded (3.822%); 10010/297936 objects misplaced (3.360%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T04:39:38.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/159824263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:38.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:38 smithi029 ceph-mon[223858]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 605 op/s; 11386/297936 objects degraded (3.822%); 10010/297936 objects misplaced (3.360%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T04:39:38.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/159824263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:38.826 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:38.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:38.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:38.963 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:38 smithi177 ceph-mon[195797]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 605 op/s; 11386/297936 objects degraded (3.822%); 10010/297936 objects misplaced (3.360%); 4.1 MiB/s, 66 objects/s recovering 2023-10-09T04:39:38.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/159824263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:39:38.966 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:39.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:39.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:39.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:39:39.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:39.214 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:39.499 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:39.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:39.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:39.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:39.671 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.1 primary-affinity to 0.138736 (890922) 2023-10-09T04:39:39.697 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3266807469' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]: dispatch 2023-10-09T04:39:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:39 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]: dispatch 2023-10-09T04:39:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3266807469' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]: dispatch 2023-10-09T04:39:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:39 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]: dispatch 2023-10-09T04:39:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3266807469' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]: dispatch 2023-10-09T04:39:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:39 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]: dispatch 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 451 op/s; 11386/295998 objects degraded (3.847%); 10010/295998 objects misplaced (3.382%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]': finished 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: osdmap e251: 8 total, 8 up, 8 in 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11386/295998 objects degraded (3.847%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[216903]: osdmap e252: 8 total, 8 up, 8 in 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 451 op/s; 11386/295998 objects degraded (3.847%); 10010/295998 objects misplaced (3.382%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]': finished 2023-10-09T04:39:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: osdmap e251: 8 total, 8 up, 8 in 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: failed to encode map e251 with expected crc 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11386/295998 objects degraded (3.847%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:40.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:40 smithi029 ceph-mon[223858]: osdmap e252: 8 total, 8 up, 8 in 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 451 op/s; 11386/295998 objects degraded (3.847%); 10010/295998 objects misplaced (3.382%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.1387358026870562}]': finished 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: osdmap e251: 8 total, 8 up, 8 in 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: failed to encode map e251 with expected crc 2023-10-09T04:39:41.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11386/295998 objects degraded (3.847%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:41.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:40 smithi177 ceph-mon[195797]: osdmap e252: 8 total, 8 up, 8 in 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:41 smithi177 ceph-mon[195797]: osdmap e253: 8 total, 8 up, 8 in 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[223858]: osdmap e253: 8 total, 8 up, 8 in 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: failed to encode map e252 with expected crc 2023-10-09T04:39:42.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:41 smithi029 ceph-mon[216903]: osdmap e253: 8 total, 8 up, 8 in 2023-10-09T04:39:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: pgmap v944: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 4.0 MiB/s wr, 501 op/s; 11385/291243 objects degraded (3.909%); 4983/291243 objects misplaced (1.711%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:39:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:42 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: pgmap v944: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 4.0 MiB/s wr, 501 op/s; 11385/291243 objects degraded (3.909%); 4983/291243 objects misplaced (1.711%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: pgmap v944: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 4.0 MiB/s wr, 501 op/s; 11385/291243 objects degraded (3.909%); 4983/291243 objects misplaced (1.711%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: failed to encode map e253 with expected crc 2023-10-09T04:39:43.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:42 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:39:43.177 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:39:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:39:42] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-10-09T04:39:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:44 smithi177 ceph-mon[195797]: pgmap v946: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 11385/289359 objects degraded (3.935%); 4983/289359 objects misplaced (1.722%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:39:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:44 smithi029 ceph-mon[216903]: pgmap v946: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 11385/289359 objects degraded (3.935%); 4983/289359 objects misplaced (1.722%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:39:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:44.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:44.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:44 smithi029 ceph-mon[223858]: pgmap v946: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 11385/289359 objects degraded (3.935%); 4983/289359 objects misplaced (1.722%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:39:44.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:44.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11385/289359 objects degraded (3.935%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11385/289359 objects degraded (3.935%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:46.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11385/289359 objects degraded (3.935%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:46.700 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:46.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:46.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:46.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:46.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:46 smithi177 ceph-mon[195797]: pgmap v947: 89 pgs: 4 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 11385/288504 objects degraded (3.946%); 4983/288504 objects misplaced (1.727%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:39:46.843 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:46.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:39:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:46.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:46.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:46 smithi029 ceph-mon[223858]: pgmap v947: 89 pgs: 4 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 11385/288504 objects degraded (3.946%); 4983/288504 objects misplaced (1.727%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:39:46.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:46 smithi029 ceph-mon[216903]: pgmap v947: 89 pgs: 4 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 416 op/s; 11385/288504 objects degraded (3.946%); 4983/288504 objects misplaced (1.727%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:39:47.015 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:47.103 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:47.378 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:47.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:47.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:39:47.513 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:47.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:39:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:47.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:39:47.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:47 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:39:47.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:47 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:39:47.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:47.736 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:47 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:39:48.646 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:48.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:48.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:48 smithi029 ceph-mon[223858]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16035/283719 objects degraded (5.652%); 4823/283719 objects misplaced (1.700%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:39:48.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:48 smithi029 ceph-mon[216903]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16035/283719 objects degraded (5.652%); 4823/283719 objects misplaced (1.700%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:39:48.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:48.875 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:48 smithi177 ceph-mon[195797]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16035/283719 objects degraded (5.652%); 4823/283719 objects misplaced (1.700%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:39:49.066 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:49.164 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:49.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:49.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:49.383 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:49.607 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:39:49.607 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:39:49.612 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:49.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:49.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:49.808 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:50.197 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:50.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:50.322 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:50.359 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:50.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:39:50.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:50.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:50.536 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:39:50.624 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:50.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:50 smithi029 ceph-mon[216903]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 16035/280794 objects degraded (5.711%); 4823/280794 objects misplaced (1.718%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:39:50.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 16035/280794 objects degraded (5.711%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:50 smithi029 ceph-mon[223858]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 16035/280794 objects degraded (5.711%); 4823/280794 objects misplaced (1.718%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:39:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 16035/280794 objects degraded (5.711%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:51.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:50 smithi177 ceph-mon[195797]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 461 op/s; 16035/280794 objects degraded (5.711%); 4823/280794 objects misplaced (1.718%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:39:51.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 16035/280794 objects degraded (5.711%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:51.860 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:39:51.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:51.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:39:51.969 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:52.002 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:52.052 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:52.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:52.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:52.280 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:52.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:52 smithi029 ceph-mon[216903]: pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 15831/274905 objects degraded (5.759%); 4710/274905 objects misplaced (1.713%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:39:52.660 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:52 smithi029 ceph-mon[223858]: pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 15831/274905 objects degraded (5.759%); 4710/274905 objects misplaced (1.713%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:39:52.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:39:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:39:52] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-10-09T04:39:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:52 smithi177 ceph-mon[195797]: pgmap v950: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 15831/274905 objects degraded (5.759%); 4710/274905 objects misplaced (1.713%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:39:53.819 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:39:53.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:39:53.911 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:53.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:54.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:54.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:39:54.135 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:54.335 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:54.338 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:54.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:54.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:54.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:54.555 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:54.704 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:54.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:54.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:54.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:54 smithi029 ceph-mon[216903]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 15831/272094 objects degraded (5.818%); 4710/272094 objects misplaced (1.731%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:39:54.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:54 smithi029 ceph-mon[223858]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 15831/272094 objects degraded (5.818%); 4710/272094 objects misplaced (1.731%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:39:54.895 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:55.013 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:54 smithi177 ceph-mon[195797]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 15831/272094 objects degraded (5.818%); 4710/272094 objects misplaced (1.731%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:39:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:55.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:55.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:55.161 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:55.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:39:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:55.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:55.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:55.452 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:55.559 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:39:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:55.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:39:55.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:39:55.723 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:55.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:39:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:55.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:39:55.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 15831/272094 objects degraded (5.818%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:55.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1911131402' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:39:55.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:55 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:39:55.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 15831/272094 objects degraded (5.818%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:55.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1911131402' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:39:55.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:55 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:39:55.954 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 15831/272094 objects degraded (5.818%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:39:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1911131402' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:39:56.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:55 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:39:56.052 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:56.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:56.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:56.201 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:56.256 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:56.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:56.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:56.450 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:39:56.458 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:56.589 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:56.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:56.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:39:56.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[216903]: pgmap v952: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 15831/270993 objects degraded (5.842%); 4710/270993 objects misplaced (1.738%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T04:39:56.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:56.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:39:56.779 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[216903]: osdmap e254: 8 total, 8 up, 8 in 2023-10-09T04:39:56.780 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[223858]: pgmap v952: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 15831/270993 objects degraded (5.842%); 4710/270993 objects misplaced (1.738%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T04:39:56.780 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:56.780 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:39:56.780 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:56 smithi029 ceph-mon[223858]: osdmap e254: 8 total, 8 up, 8 in 2023-10-09T04:39:56.791 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:56.973 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:39:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:56.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:39:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:56 smithi177 ceph-mon[195797]: pgmap v952: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 15831/270993 objects degraded (5.842%); 4710/270993 objects misplaced (1.738%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T04:39:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:56 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:39:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:39:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:56 smithi177 ceph-mon[195797]: osdmap e254: 8 total, 8 up, 8 in 2023-10-09T04:39:57.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:39:57.150 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:57 smithi029 ceph-mon[216903]: failed to encode map e254 with expected crc 2023-10-09T04:39:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:57 smithi029 ceph-mon[216903]: failed to encode map e254 with expected crc 2023-10-09T04:39:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:57 smithi029 ceph-mon[216903]: failed to encode map e254 with expected crc 2023-10-09T04:39:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:57 smithi029 ceph-mon[223858]: failed to encode map e254 with expected crc 2023-10-09T04:39:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:57 smithi029 ceph-mon[223858]: failed to encode map e254 with expected crc 2023-10-09T04:39:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:57 smithi029 ceph-mon[223858]: failed to encode map e254 with expected crc 2023-10-09T04:39:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:57 smithi177 ceph-mon[195797]: failed to encode map e254 with expected crc 2023-10-09T04:39:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:57 smithi177 ceph-mon[195797]: failed to encode map e254 with expected crc 2023-10-09T04:39:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:57 smithi177 ceph-mon[195797]: failed to encode map e254 with expected crc 2023-10-09T04:39:58.697 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:58.700 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:39:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:58 smithi177 ceph-mon[195797]: pgmap v954: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 15639/266157 objects degraded (5.876%); 4601/266157 objects misplaced (1.729%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:39:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:39:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:59.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:58 smithi029 ceph-mon[216903]: pgmap v954: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 15639/266157 objects degraded (5.876%); 4601/266157 objects misplaced (1.729%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:39:59.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:59.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:39:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:59.045 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:58 smithi029 ceph-mon[223858]: pgmap v954: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 15639/266157 objects degraded (5.876%); 4601/266157 objects misplaced (1.729%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:39:59.045 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:39:59.045 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:39:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:39:59.058 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:39:59.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:59.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:39:59.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:39:59.195 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:59.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:39:59.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:59.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:39:59.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:39:59.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:39:59.461 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:39:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:39:59.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:39:59.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:39:59.676 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:40:00.491 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.492 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: pgmap v955: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15639/263625 objects degraded (5.932%); 4601/263625 objects misplaced (1.745%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: [WRN] PG_DEGRADED: Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: pg 11.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: pg 11.c is active+recovery_wait+degraded, acting [2,1,7] 2023-10-09T04:40:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: pg 11.e is active+recovery_wait+degraded, acting [7,1,0] 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3687674963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: pgmap v955: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15639/263625 objects degraded (5.932%); 4601/263625 objects misplaced (1.745%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: [WRN] PG_DEGRADED: Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: pg 11.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: pg 11.c is active+recovery_wait+degraded, acting [2,1,7] 2023-10-09T04:40:00.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: pg 11.e is active+recovery_wait+degraded, acting [7,1,0] 2023-10-09T04:40:00.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3687674963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:00.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:01.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: pgmap v955: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15639/263625 objects degraded (5.932%); 4601/263625 objects misplaced (1.745%); 3.9 MiB/s, 61 objects/s recovering 2023-10-09T04:40:01.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded 2023-10-09T04:40:01.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:40:01.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: [WRN] PG_DEGRADED: Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded 2023-10-09T04:40:01.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: pg 11.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2023-10-09T04:40:01.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: pg 11.c is active+recovery_wait+degraded, acting [2,1,7] 2023-10-09T04:40:01.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: pg 11.e is active+recovery_wait+degraded, acting [7,1,0] 2023-10-09T04:40:01.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3687674963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:01.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 15639/263625 objects degraded (5.932%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:01.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:40:02.478 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:02.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:02.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:02.670 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:02 smithi177 ceph-mon[195797]: pgmap v956: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 15315/258051 objects degraded (5.935%); 4473/258051 objects misplaced (1.733%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:40:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:40:02] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-10-09T04:40:03.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:02 smithi029 ceph-mon[223858]: pgmap v956: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 15315/258051 objects degraded (5.935%); 4473/258051 objects misplaced (1.733%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:03.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:02 smithi029 ceph-mon[216903]: pgmap v956: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 15315/258051 objects degraded (5.935%); 4473/258051 objects misplaced (1.733%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:04.443 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:04.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:04.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:04.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:04.613 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:04 smithi029 ceph-mon[216903]: pgmap v957: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 15315/255639 objects degraded (5.991%); 4473/255639 objects misplaced (1.750%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:04 smithi029 ceph-mon[223858]: pgmap v957: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 15315/255639 objects degraded (5.991%); 4473/255639 objects misplaced (1.750%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:04 smithi177 ceph-mon[195797]: pgmap v957: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 15315/255639 objects degraded (5.991%); 4473/255639 objects misplaced (1.750%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:05.420 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:05.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:05.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:05.570 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:05.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:05.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:05.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:05.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:05.836 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 15315/255639 objects degraded (5.991%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:05.836 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 15315/255639 objects degraded (5.991%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:05.853 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:06.020 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:06.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 15315/255639 objects degraded (5.991%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:06.080 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:06.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:06.176 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:40:06.176 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:40:06.176 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:40:06.176 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:40:06.176 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:40:06.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:06.271 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:06.404 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:06.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:06.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:06.550 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:06.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:06.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:06.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:06.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:06.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:06.995 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:07.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:07.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:07.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:06 smithi177 ceph-mon[195797]: pgmap v958: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 15315/254727 objects degraded (6.012%); 4473/254727 objects misplaced (1.756%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:07.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:06 smithi029 ceph-mon[223858]: pgmap v958: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 15315/254727 objects degraded (6.012%); 4473/254727 objects misplaced (1.756%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:07.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:06 smithi029 ceph-mon[216903]: pgmap v958: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 15315/254727 objects degraded (6.012%); 4473/254727 objects misplaced (1.756%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:40:07.193 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:07.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:40:07.329 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:07.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:07.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:07.465 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:07.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:07.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:07.614 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:07.660 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:07.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:40:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:07.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:07.834 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:40:07.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:07.921 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4265958746' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:40:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:08.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4265958746' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:40:08.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:08.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4265958746' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:40:08.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: pgmap v959: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 15006/250497 objects degraded (5.990%); 4343/250497 objects misplaced (1.734%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4265958746' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: osdmap e255: 8 total, 8 up, 8 in 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[223858]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: pgmap v959: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 15006/250497 objects degraded (5.990%); 4343/250497 objects misplaced (1.734%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4265958746' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: osdmap e255: 8 total, 8 up, 8 in 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:08 smithi029 ceph-mon[216903]: failed to encode map e255 with expected crc 2023-10-09T04:40:08.985 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:08.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:09.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:09.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: pgmap v959: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 15006/250497 objects degraded (5.990%); 4343/250497 objects misplaced (1.734%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4265958746' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:40:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: osdmap e255: 8 total, 8 up, 8 in 2023-10-09T04:40:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: failed to encode map e255 with expected crc 2023-10-09T04:40:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: failed to encode map e255 with expected crc 2023-10-09T04:40:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: failed to encode map e255 with expected crc 2023-10-09T04:40:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:08 smithi177 ceph-mon[195797]: failed to encode map e255 with expected crc 2023-10-09T04:40:09.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:40:10.054 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:10.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:10.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:10.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:10.202 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:10.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:40:10.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:40:10.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:10.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:10.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:10.409 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:10.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:10.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:10.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:10.607 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:10.631 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:10.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:10.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:10.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:10.870 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:10 smithi029 ceph-mon[216903]: pgmap v961: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 15006/248322 objects degraded (6.043%); 4343/248322 objects misplaced (1.749%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:40:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 15006/248322 objects degraded (6.043%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:10 smithi029 ceph-mon[223858]: pgmap v961: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 15006/248322 objects degraded (6.043%); 4343/248322 objects misplaced (1.749%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:40:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 15006/248322 objects degraded (6.043%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:10 smithi177 ceph-mon[195797]: pgmap v961: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 15006/248322 objects degraded (6.043%); 4343/248322 objects misplaced (1.749%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:40:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 15006/248322 objects degraded (6.043%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:11.989 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:11.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:11.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:12.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:12.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:12.387 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:12.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:12.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:12.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:12.530 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:12.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:12.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:12.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:12.848 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:13.072 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:12 smithi029 ceph-mon[216903]: pgmap v962: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 14673/243285 objects degraded (6.031%); 4229/243285 objects misplaced (1.738%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:12 smithi029 ceph-mon[223858]: pgmap v962: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 14673/243285 objects degraded (6.031%); 4229/243285 objects misplaced (1.738%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:13.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:40:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:40:12] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-10-09T04:40:13.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:12 smithi177 ceph-mon[195797]: pgmap v962: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 14673/243285 objects degraded (6.031%); 4229/243285 objects misplaced (1.738%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:13.285 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:13.518 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:13.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:13.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:13.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:13.724 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:14.128 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:40:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[216903]: pgmap v963: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 14673/241071 objects degraded (6.087%); 4229/241071 objects misplaced (1.754%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3536300988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[223858]: pgmap v963: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 14673/241071 objects degraded (6.087%); 4229/241071 objects misplaced (1.754%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3536300988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:14 smithi177 ceph-mon[195797]: pgmap v963: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 14673/241071 objects degraded (6.087%); 4229/241071 objects misplaced (1.754%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3536300988' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:15.408 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:15.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:15.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:15.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:15.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:15.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[223858]: osdmap e256: 8 total, 8 up, 8 in 2023-10-09T04:40:15.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[223858]: failed to encode map e256 with expected crc 2023-10-09T04:40:15.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[223858]: failed to encode map e256 with expected crc 2023-10-09T04:40:15.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:15.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:15.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[216903]: osdmap e256: 8 total, 8 up, 8 in 2023-10-09T04:40:15.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[216903]: failed to encode map e256 with expected crc 2023-10-09T04:40:15.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:15 smithi029 ceph-mon[216903]: failed to encode map e256 with expected crc 2023-10-09T04:40:15.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:15 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:15 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:15 smithi177 ceph-mon[195797]: osdmap e256: 8 total, 8 up, 8 in 2023-10-09T04:40:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:15 smithi177 ceph-mon[195797]: failed to encode map e256 with expected crc 2023-10-09T04:40:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:15 smithi177 ceph-mon[195797]: failed to encode map e256 with expected crc 2023-10-09T04:40:15.561 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:15.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:15.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:15.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:15.772 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:15.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:15.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:15.955 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:16.049 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:16 smithi177 ceph-mon[195797]: failed to encode map e256 with expected crc 2023-10-09T04:40:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:16 smithi177 ceph-mon[195797]: failed to encode map e256 with expected crc 2023-10-09T04:40:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:16 smithi177 ceph-mon[195797]: pgmap v965: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 14673/240195 objects degraded (6.109%); 4229/240195 objects misplaced (1.761%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:40:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 14673/241071 objects degraded (6.087%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[223858]: failed to encode map e256 with expected crc 2023-10-09T04:40:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[223858]: failed to encode map e256 with expected crc 2023-10-09T04:40:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[223858]: pgmap v965: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 14673/240195 objects degraded (6.109%); 4229/240195 objects misplaced (1.761%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:40:16.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 14673/241071 objects degraded (6.087%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[216903]: failed to encode map e256 with expected crc 2023-10-09T04:40:16.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[216903]: failed to encode map e256 with expected crc 2023-10-09T04:40:16.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[216903]: pgmap v965: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 429 op/s; 14673/240195 objects degraded (6.109%); 4229/240195 objects misplaced (1.761%); 3.5 MiB/s, 55 objects/s recovering 2023-10-09T04:40:16.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 14673/241071 objects degraded (6.087%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:40:17.536 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:17.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:17.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:17.759 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:18.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:18 smithi029 ceph-mon[216903]: pgmap v966: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 14398/236388 objects degraded (6.091%); 4109/236388 objects misplaced (1.738%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:40:18.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:18 smithi029 ceph-mon[223858]: pgmap v966: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 14398/236388 objects degraded (6.091%); 4109/236388 objects misplaced (1.738%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:40:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:18 smithi177 ceph-mon[195797]: pgmap v966: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 14398/236388 objects degraded (6.091%); 4109/236388 objects misplaced (1.738%); 5.7 MiB/s, 90 objects/s recovering 2023-10-09T04:40:19.265 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:19.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:19.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:19.404 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:19.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:40:19.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:19.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:19.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:19.713 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:20.216 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:20.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:20.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:20.355 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:20 smithi029 ceph-mon[216903]: pgmap v967: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 14398/234009 objects degraded (6.153%); 4109/234009 objects misplaced (1.756%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:40:20.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:20 smithi029 ceph-mon[223858]: pgmap v967: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 14398/234009 objects degraded (6.153%); 4109/234009 objects misplaced (1.756%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:40:20.456 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:20.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:20 smithi177 ceph-mon[195797]: pgmap v967: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 14398/234009 objects degraded (6.153%); 4109/234009 objects misplaced (1.756%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:40:20.566 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:40:20.643 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:20.644 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:40:20.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:20.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:20.696 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:40:20.752 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:20.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:40:20.780 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:20.815 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:40:20.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:40:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:20.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:20.866 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:40:20.903 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:20.921 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:40:20.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:20.979 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:40:20.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:40:21.068 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:21.081 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:40:21.082 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:40:21.334 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 14398/234009 objects degraded (6.153%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:21.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1741230902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:21.335 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 14398/234009 objects degraded (6.153%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:21.335 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1741230902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 14398/234009 objects degraded (6.153%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1741230902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:22.174 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:22.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:22.283 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:22.325 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:22.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:22.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:22.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:22.460 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:40:22.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:22 smithi029 ceph-mon[216903]: pgmap v968: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 14248/230847 objects degraded (6.172%); 4109/230847 objects misplaced (1.780%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:40:22.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1301619001' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:40:22.482 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:22 smithi029 ceph-mon[223858]: pgmap v968: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 14248/230847 objects degraded (6.172%); 4109/230847 objects misplaced (1.780%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:40:22.483 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1301619001' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:40:22.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:22.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:22 smithi177 ceph-mon[195797]: pgmap v968: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 14248/230847 objects degraded (6.172%); 4109/230847 objects misplaced (1.780%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:40:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1301619001' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:40:23.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:40:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:40:22] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-10-09T04:40:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1301619001' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:40:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:23 smithi177 ceph-mon[195797]: osdmap e257: 8 total, 8 up, 8 in 2023-10-09T04:40:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:23 smithi177 ceph-mon[195797]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:23 smithi177 ceph-mon[195797]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:23 smithi177 ceph-mon[195797]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:23 smithi177 ceph-mon[195797]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1301619001' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:40:23.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[216903]: osdmap e257: 8 total, 8 up, 8 in 2023-10-09T04:40:23.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[216903]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[216903]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[216903]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[216903]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1301619001' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[223858]: osdmap e257: 8 total, 8 up, 8 in 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[223858]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[223858]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[223858]: failed to encode map e257 with expected crc 2023-10-09T04:40:23.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:23 smithi029 ceph-mon[223858]: failed to encode map e257 with expected crc 2023-10-09T04:40:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:24 smithi177 ceph-mon[195797]: pgmap v970: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 14117/226977 objects degraded (6.220%); 3997/226977 objects misplaced (1.761%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:40:24.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:24 smithi029 ceph-mon[216903]: pgmap v970: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 14117/226977 objects degraded (6.220%); 3997/226977 objects misplaced (1.761%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:40:24.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:24 smithi029 ceph-mon[223858]: pgmap v970: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 14117/226977 objects degraded (6.220%); 3997/226977 objects misplaced (1.761%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:40:25.313 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:25.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:25.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:25.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:25.475 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:25.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:25.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:25.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:25.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 14117/226977 objects degraded (6.220%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:25.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 14117/226977 objects degraded (6.220%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:25.776 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 14117/226977 objects degraded (6.220%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:26.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:26 smithi177 ceph-mon[195797]: pgmap v971: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 14117/225987 objects degraded (6.247%); 3997/225987 objects misplaced (1.769%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:40:26.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:26 smithi029 ceph-mon[223858]: pgmap v971: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 14117/225987 objects degraded (6.247%); 3997/225987 objects misplaced (1.769%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:40:26.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:26 smithi029 ceph-mon[216903]: pgmap v971: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 14117/225987 objects degraded (6.247%); 3997/225987 objects misplaced (1.769%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:40:28.397 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:28.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:28.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:28.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:28.543 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:28.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:28.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:28.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:28.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:28 smithi029 ceph-mon[216903]: pgmap v972: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 13789/221553 objects degraded (6.224%); 3856/221553 objects misplaced (1.740%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:28.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:28 smithi029 ceph-mon[223858]: pgmap v972: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 13789/221553 objects degraded (6.224%); 3856/221553 objects misplaced (1.740%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:28.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:28.740 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:28 smithi177 ceph-mon[195797]: pgmap v972: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 13789/221553 objects degraded (6.224%); 3856/221553 objects misplaced (1.740%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:28.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:28.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:28.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:28.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:28.946 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:29.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:29.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:29.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:29.319 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:29.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:29.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:29.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:30 smithi177 ceph-mon[195797]: pgmap v973: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 13789/219018 objects degraded (6.296%); 3856/219018 objects misplaced (1.761%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 13789/219018 objects degraded (6.296%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:30 smithi029 ceph-mon[216903]: pgmap v973: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 13789/219018 objects degraded (6.296%); 3856/219018 objects misplaced (1.761%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:30.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 13789/219018 objects degraded (6.296%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:30 smithi029 ceph-mon[223858]: pgmap v973: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 13789/219018 objects degraded (6.296%); 3856/219018 objects misplaced (1.761%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 13789/219018 objects degraded (6.296%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:32.648 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:40:32.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:32 smithi029 ceph-mon[216903]: pgmap v974: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 13624/215544 objects degraded (6.321%); 3856/215544 objects misplaced (1.789%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:40:32.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:32 smithi029 ceph-mon[223858]: pgmap v974: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 13624/215544 objects degraded (6.321%); 3856/215544 objects misplaced (1.789%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:40:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:32 smithi177 ceph-mon[195797]: pgmap v974: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 13624/215544 objects degraded (6.321%); 3856/215544 objects misplaced (1.789%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:40:32.792 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:32.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:32.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:32.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:32.932 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:32.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:32.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:32.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:33.063 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:40:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:40:32] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-10-09T04:40:33.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T04:40:33.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:33.126 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:33.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:33.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:33.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:33.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:33.394 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:34.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:40:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:34 smithi177 ceph-mon[195797]: pgmap v975: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 13489/211095 objects degraded (6.390%); 3685/211095 objects misplaced (1.746%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:34.824 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:34.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:34.836 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:34 smithi029 ceph-mon[216903]: pgmap v975: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 13489/211095 objects degraded (6.390%); 3685/211095 objects misplaced (1.746%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:34.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:34 smithi029 ceph-mon[223858]: pgmap v975: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 13489/211095 objects degraded (6.390%); 3685/211095 objects misplaced (1.746%); 5.5 MiB/s, 88 objects/s recovering 2023-10-09T04:40:34.937 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:35.108 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:35.237 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:35.272 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:35.403 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:35.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:35.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:35.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:40:35.621 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:35.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 13489/211095 objects degraded (6.390%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:35.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 13489/211095 objects degraded (6.390%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 13489/211095 objects degraded (6.390%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:36.194 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:36.289 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:40:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:36.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:40:36.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:36.426 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:36.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:36.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:36.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:36.599 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:40:36.622 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:36.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:36 smithi029 ceph-mon[216903]: pgmap v976: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 13489/210117 objects degraded (6.420%); 3685/210117 objects misplaced (1.754%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:40:36.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2042929126' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:40:36.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:36 smithi029 ceph-mon[223858]: pgmap v976: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 13489/210117 objects degraded (6.420%); 3685/210117 objects misplaced (1.754%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:40:36.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2042929126' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:40:36.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:36.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:36 smithi177 ceph-mon[195797]: pgmap v976: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 13489/210117 objects degraded (6.420%); 3685/210117 objects misplaced (1.754%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:40:36.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2042929126' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:40:36.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:36.802 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:36.832 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:36.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:36.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:36.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:37.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:37.061 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2042929126' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: osdmap e258: 8 total, 8 up, 8 in 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[216903]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2042929126' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: osdmap e258: 8 total, 8 up, 8 in 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: failed to encode map e258 with expected crc 2023-10-09T04:40:37.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:37 smithi029 ceph-mon[223858]: failed to encode map e258 with expected crc 2023-10-09T04:40:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:40:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2042929126' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:40:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: osdmap e258: 8 total, 8 up, 8 in 2023-10-09T04:40:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: failed to encode map e258 with expected crc 2023-10-09T04:40:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: failed to encode map e258 with expected crc 2023-10-09T04:40:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: failed to encode map e258 with expected crc 2023-10-09T04:40:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:37 smithi177 ceph-mon[195797]: failed to encode map e258 with expected crc 2023-10-09T04:40:38.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:38 smithi029 ceph-mon[216903]: pgmap v978: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 13215/206124 objects degraded (6.411%); 3561/206124 objects misplaced (1.728%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:38.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:38 smithi029 ceph-mon[223858]: pgmap v978: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 13215/206124 objects degraded (6.411%); 3561/206124 objects misplaced (1.728%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:38 smithi177 ceph-mon[195797]: pgmap v978: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 514 op/s; 13215/206124 objects degraded (6.411%); 3561/206124 objects misplaced (1.728%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:39.863 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:39.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:39.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:40.011 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:40.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:40.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:40.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:40.230 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:40.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:40.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:40.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:40.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:40.427 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:40.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:40.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:40.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:40.597 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:40 smithi177 ceph-mon[195797]: pgmap v979: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 13215/203997 objects degraded (6.478%); 3561/203997 objects misplaced (1.746%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:40.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 13215/203997 objects degraded (6.478%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:40.678 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:40.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:40 smithi029 ceph-mon[216903]: pgmap v979: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 13215/203997 objects degraded (6.478%); 3561/203997 objects misplaced (1.746%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:40.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 13215/203997 objects degraded (6.478%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:40.723 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:40 smithi029 ceph-mon[223858]: pgmap v979: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 13215/203997 objects degraded (6.478%); 3561/203997 objects misplaced (1.746%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:40.723 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 13215/203997 objects degraded (6.478%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:42.132 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:42.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:42.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:42.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:42.280 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:42.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:42.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:42.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:42.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:42.476 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:42 smithi029 ceph-mon[223858]: pgmap v980: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13058/200592 objects degraded (6.510%); 3561/200592 objects misplaced (1.775%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:42.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:42 smithi029 ceph-mon[216903]: pgmap v980: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13058/200592 objects degraded (6.510%); 3561/200592 objects misplaced (1.775%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:42.821 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:40:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:40:42] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-10-09T04:40:42.980 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:42.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:42 smithi177 ceph-mon[195797]: pgmap v980: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13058/200592 objects degraded (6.510%); 3561/200592 objects misplaced (1.775%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:43.090 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:43.121 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:43.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:43.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:43.287 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:43.317 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:43.371 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:43.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:43.482 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:43.523 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:43.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:43.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:43.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:43.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:43.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:43.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:43.772 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:43.783 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:43.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:40:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:44.095 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:44.117 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:40:44.117 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:40:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:44.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:44.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:44.369 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:44.467 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:44.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:44.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:44.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:44.681 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:44.697 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T04:40:44.723 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:40:44.835 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:40:44.835 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:40:44.890 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:40:44.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[223858]: pgmap v981: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 12942/196104 objects degraded (6.600%); 3373/196104 objects misplaced (1.720%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T04:40:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2425463162' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:40:44.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:40:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[216903]: pgmap v981: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 12942/196104 objects degraded (6.600%); 3373/196104 objects misplaced (1.720%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T04:40:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2425463162' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:40:44.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:44 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:40:44.929 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:40:44.962 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:40:45.003 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:40:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:44 smithi177 ceph-mon[195797]: pgmap v981: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 12942/196104 objects degraded (6.600%); 3373/196104 objects misplaced (1.720%); 5.4 MiB/s, 85 objects/s recovering 2023-10-09T04:40:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2425463162' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:40:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:44 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:40:45.054 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:40:45.086 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.087 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: osdmap e259: 8 total, 8 up, 8 in 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2083863011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:45.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 12942/196104 objects degraded (6.600%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: osdmap e259: 8 total, 8 up, 8 in 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2083863011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: failed to encode map e259 with expected crc 2023-10-09T04:40:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:45.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 12942/196104 objects degraded (6.600%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: osdmap e259: 8 total, 8 up, 8 in 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: failed to encode map e259 with expected crc 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: failed to encode map e259 with expected crc 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2083863011' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: failed to encode map e259 with expected crc 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: failed to encode map e259 with expected crc 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 12942/196104 objects degraded (6.600%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:46.804 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:46.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:46.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:46.993 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:47.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:46 smithi029 ceph-mon[223858]: pgmap v983: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 12942/195147 objects degraded (6.632%); 3373/195147 objects misplaced (1.728%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:40:47.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:46 smithi029 ceph-mon[216903]: pgmap v983: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 12942/195147 objects degraded (6.632%); 3373/195147 objects misplaced (1.728%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:40:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:46 smithi177 ceph-mon[195797]: pgmap v983: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 12942/195147 objects degraded (6.632%); 3373/195147 objects misplaced (1.728%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T04:40:48.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:40:49.055 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:49.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:49.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:49.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:49.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:48 smithi029 ceph-mon[216903]: pgmap v984: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 12712/190947 objects degraded (6.657%); 3194/190947 objects misplaced (1.673%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:49.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:48 smithi029 ceph-mon[223858]: pgmap v984: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 12712/190947 objects degraded (6.657%); 3194/190947 objects misplaced (1.673%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:49.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:40:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:48 smithi177 ceph-mon[195797]: pgmap v984: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 12712/190947 objects degraded (6.657%); 3194/190947 objects misplaced (1.673%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:49.298 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:49.347 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:49.424 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:49.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:49.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:49.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:49.640 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:49.773 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:49.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:49.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:49.916 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:49.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:49.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:50.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:40:50.110 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:50.119 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:50.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:40:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:50.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:40:50.272 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:50.351 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:50.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:40:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:50 smithi029 ceph-mon[216903]: pgmap v985: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 12712/188493 objects degraded (6.744%); 3194/188493 objects misplaced (1.694%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 12712/188493 objects degraded (6.744%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:50 smithi029 ceph-mon[223858]: pgmap v985: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 12712/188493 objects degraded (6.744%); 3194/188493 objects misplaced (1.694%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 12712/188493 objects degraded (6.744%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:50 smithi177 ceph-mon[195797]: pgmap v985: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 12712/188493 objects degraded (6.744%); 3194/188493 objects misplaced (1.694%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 12712/188493 objects degraded (6.744%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:51.872 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:51.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:51.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:40:52.007 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:52.017 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:52.064 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:40:52.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:52.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:40:52.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:52.203 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:52.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:52.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:52.386 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:40:52.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:52.418 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:52.476 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:52.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:52.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:52.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:40:52.672 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:53.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:52 smithi029 ceph-mon[216903]: pgmap v986: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12558/185316 objects degraded (6.777%); 3194/185316 objects misplaced (1.724%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:53.137 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:40:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:40:52] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-10-09T04:40:53.137 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:52 smithi029 ceph-mon[223858]: pgmap v986: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12558/185316 objects degraded (6.777%); 3194/185316 objects misplaced (1.724%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:53.148 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:40:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:53.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:40:53.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:52 smithi177 ceph-mon[195797]: pgmap v986: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12558/185316 objects degraded (6.777%); 3194/185316 objects misplaced (1.724%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:40:53.294 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:54.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/476498133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:54.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/476498133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:54.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/476498133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:40:55.152 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:40:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:54 smithi177 ceph-mon[195797]: pgmap v987: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 12435/181359 objects degraded (6.857%); 3058/181359 objects misplaced (1.686%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:40:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:54 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:40:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:54 smithi177 ceph-mon[195797]: failed to encode map e260 with expected crc 2023-10-09T04:40:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/476498133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:54 smithi177 ceph-mon[195797]: osdmap e260: 8 total, 8 up, 8 in 2023-10-09T04:40:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:54 smithi177 ceph-mon[195797]: failed to encode map e260 with expected crc 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[216903]: pgmap v987: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 12435/181359 objects degraded (6.857%); 3058/181359 objects misplaced (1.686%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[216903]: failed to encode map e260 with expected crc 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/476498133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[216903]: osdmap e260: 8 total, 8 up, 8 in 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[216903]: failed to encode map e260 with expected crc 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[223858]: pgmap v987: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 12435/181359 objects degraded (6.857%); 3058/181359 objects misplaced (1.686%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:40:55.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:40:55.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[223858]: failed to encode map e260 with expected crc 2023-10-09T04:40:55.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/476498133' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:40:55.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[223858]: osdmap e260: 8 total, 8 up, 8 in 2023-10-09T04:40:55.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:54 smithi029 ceph-mon[223858]: failed to encode map e260 with expected crc 2023-10-09T04:40:55.584 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:55.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:55.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:40:55.732 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:55.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:40:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:55.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:40:55.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:40:55.941 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:56.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:40:56.008 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:55 smithi177 ceph-mon[195797]: failed to encode map e260 with expected crc 2023-10-09T04:40:56.008 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 12435/181359 objects degraded (6.857%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:56.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:40:56.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:40:56.217 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:55 smithi029 ceph-mon[216903]: failed to encode map e260 with expected crc 2023-10-09T04:40:56.217 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 12435/181359 objects degraded (6.857%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:56.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:55 smithi029 ceph-mon[223858]: failed to encode map e260 with expected crc 2023-10-09T04:40:56.218 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 12435/181359 objects degraded (6.857%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:40:56.218 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:57.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:56 smithi177 ceph-mon[195797]: pgmap v989: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 12435/180498 objects degraded (6.889%); 3058/180498 objects misplaced (1.694%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:40:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:56 smithi029 ceph-mon[216903]: pgmap v989: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 12435/180498 objects degraded (6.889%); 3058/180498 objects misplaced (1.694%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:40:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:56 smithi029 ceph-mon[223858]: pgmap v989: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 12435/180498 objects degraded (6.889%); 3058/180498 objects misplaced (1.694%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:40:57.904 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:40:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:57.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:40:58.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:40:58.090 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:40:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:57 smithi029 ceph-mon[223858]: pgmap v990: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 12185/176271 objects degraded (6.913%); 2881/176271 objects misplaced (1.634%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:57 smithi029 ceph-mon[216903]: pgmap v990: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 12185/176271 objects degraded (6.913%); 2881/176271 objects misplaced (1.634%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:57 smithi177 ceph-mon[195797]: pgmap v990: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 12185/176271 objects degraded (6.913%); 2881/176271 objects misplaced (1.634%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:40:59.793 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:40:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:40:59.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:40:59.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:59.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:40:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:59.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:40:59.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:40:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:40:59.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:00.023 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:40:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:00.217 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:00.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:00.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:00.447 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:00.455 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:41:00.455 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:41:00.455 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:41:00.455 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:41:00.455 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:41:00.769 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:00.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:00 smithi029 ceph-mon[216903]: pgmap v991: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 12185/173994 objects degraded (7.003%); 2881/173994 objects misplaced (1.656%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:41:00.774 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 12185/173994 objects degraded (7.003%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:00.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:00 smithi029 ceph-mon[223858]: pgmap v991: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 12185/173994 objects degraded (7.003%); 2881/173994 objects misplaced (1.656%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:41:00.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 12185/173994 objects degraded (7.003%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:00.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:00.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:00.986 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:00 smithi177 ceph-mon[195797]: pgmap v991: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 12185/173994 objects degraded (7.003%); 2881/173994 objects misplaced (1.656%); 5.2 MiB/s, 83 objects/s recovering 2023-10-09T04:41:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 12185/173994 objects degraded (7.003%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:01.613 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:41:01.818 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:41:01.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3041634566' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:41:01.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:01 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:41:01.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3041634566' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:41:01.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:01 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:41:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3041634566' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:41:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:01 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:41:02.621 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:02.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:02.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:02.761 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:02.822 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:02.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:02.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:02.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: pgmap v992: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 12018/170574 objects degraded (7.046%); 2881/170574 objects misplaced (1.689%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:41:02.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:41:02.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: osdmap e261: 8 total, 8 up, 8 in 2023-10-09T04:41:02.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[216903]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: pgmap v992: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 12018/170574 objects degraded (7.046%); 2881/170574 objects misplaced (1.689%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: osdmap e261: 8 total, 8 up, 8 in 2023-10-09T04:41:02.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:02 smithi029 ceph-mon[223858]: failed to encode map e261 with expected crc 2023-10-09T04:41:02.842 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:41:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:41:02] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2023-10-09T04:41:02.929 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:02.964 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: pgmap v992: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 85 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 12018/170574 objects degraded (7.046%); 2881/170574 objects misplaced (1.689%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:41:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:41:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: osdmap e261: 8 total, 8 up, 8 in 2023-10-09T04:41:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: failed to encode map e261 with expected crc 2023-10-09T04:41:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: failed to encode map e261 with expected crc 2023-10-09T04:41:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: failed to encode map e261 with expected crc 2023-10-09T04:41:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:02 smithi177 ceph-mon[195797]: failed to encode map e261 with expected crc 2023-10-09T04:41:03.213 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:03.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:03.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:03.345 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:03.547 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:41:03.576 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:41:03.576 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:41:03.576 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:41:03.576 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:41:03.576 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:41:03.577 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:41:03.577 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:41:03.579 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:41:03.581 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.582 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:41:03.873 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: osdmap e262: 8 total, 8 up, 8 in 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1595081459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[223858]: osdmap e263: 8 total, 8 up, 8 in 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: osdmap e262: 8 total, 8 up, 8 in 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: failed to encode map e262 with expected crc 2023-10-09T04:41:03.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1595081459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:03.877 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:03 smithi029 ceph-mon[216903]: osdmap e263: 8 total, 8 up, 8 in 2023-10-09T04:41:03.877 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:03.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:03.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:41:03.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: osdmap e262: 8 total, 8 up, 8 in 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: failed to encode map e262 with expected crc 2023-10-09T04:41:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1595081459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:03 smithi177 ceph-mon[195797]: osdmap e263: 8 total, 8 up, 8 in 2023-10-09T04:41:04.037 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:04.667 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:04.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:04.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:04.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:04.810 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:04.827 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:04.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:04.877 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:04 smithi177 ceph-mon[195797]: pgmap v995: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 11882/166605 objects degraded (7.132%); 2716/166605 objects misplaced (1.630%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:41:04.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:04 smithi177 ceph-mon[195797]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:04 smithi177 ceph-mon[195797]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:04 smithi177 ceph-mon[195797]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.878 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:04 smithi177 ceph-mon[195797]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:04.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:04.897 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:04.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[216903]: pgmap v995: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 11882/166605 objects degraded (7.132%); 2716/166605 objects misplaced (1.630%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:41:04.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[216903]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[216903]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[216903]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[216903]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[223858]: pgmap v995: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 11882/166605 objects degraded (7.132%); 2716/166605 objects misplaced (1.630%); 7.0 MiB/s, 111 objects/s recovering 2023-10-09T04:41:04.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[223858]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[223858]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[223858]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:04 smithi029 ceph-mon[223858]: failed to encode map e263 with expected crc 2023-10-09T04:41:04.947 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:04.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:05.058 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:05.210 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:05.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:05.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:05.406 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:05.651 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 11882/166605 objects degraded (7.132%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:05.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 11882/166605 objects degraded (7.132%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 11882/166605 objects degraded (7.132%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:41:06.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:41:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:06 smithi029 ceph-mon[223858]: pgmap v997: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 11882/166002 objects degraded (7.158%); 2716/166002 objects misplaced (1.636%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:41:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:06 smithi029 ceph-mon[216903]: pgmap v997: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 11882/166002 objects degraded (7.158%); 2716/166002 objects misplaced (1.636%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:41:07.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:06 smithi177 ceph-mon[195797]: pgmap v997: 89 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 83 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 11882/166002 objects degraded (7.158%); 2716/166002 objects misplaced (1.636%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:41:07.623 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:07.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:07.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:07.767 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:07.848 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:07.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:07.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:07.979 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:08.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:08.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:08.131 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:08.186 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:08.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:08.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:08.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:08.468 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:08.640 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:08 smithi177 ceph-mon[195797]: pgmap v998: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 84 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 10511/163365 objects degraded (6.434%); 452/163365 objects misplaced (0.277%); 5.7 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T04:41:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:08 smithi029 ceph-mon[216903]: pgmap v998: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 84 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 10511/163365 objects degraded (6.434%); 452/163365 objects misplaced (0.277%); 5.7 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T04:41:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:08 smithi029 ceph-mon[223858]: pgmap v998: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 84 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 399 op/s; 10511/163365 objects degraded (6.434%); 452/163365 objects misplaced (0.277%); 5.7 MiB/s, 1 keys/s, 92 objects/s recovering 2023-10-09T04:41:09.395 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:09.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:09.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:09.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:09.563 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:09.650 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:09.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:09.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:09.869 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:10.242 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:10.244 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:10.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:10.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:10.456 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:10.833 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:10.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:10.921 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:10.921 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[216903]: pgmap v999: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 85 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 10511/161334 objects degraded (6.515%); 452/161334 objects misplaced (0.280%); 2.0 MiB/s, 1 keys/s, 33 objects/s recovering 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/161334 objects degraded (6.515%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1628077410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[223858]: pgmap v999: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 85 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 10511/161334 objects degraded (6.515%); 452/161334 objects misplaced (0.280%); 2.0 MiB/s, 1 keys/s, 33 objects/s recovering 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/161334 objects degraded (6.515%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1628077410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:10.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:11.010 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:11.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:10 smithi177 ceph-mon[195797]: pgmap v999: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 85 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 323 op/s; 10511/161334 objects degraded (6.515%); 452/161334 objects misplaced (0.280%); 2.0 MiB/s, 1 keys/s, 33 objects/s recovering 2023-10-09T04:41:11.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/161334 objects degraded (6.515%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:11.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1628077410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:11.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:11.228 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:11.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:11.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:11.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:11.386 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:11.506 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:41:11.512 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:41:11.645 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:11.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:11.764 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:11.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1695211878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:41:11.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:41:11.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:11.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:41:11.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:41:11.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[216903]: osdmap e264: 8 total, 8 up, 8 in 2023-10-09T04:41:11.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1695211878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:41:11.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:41:11.836 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:11.836 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:41:11.836 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:41:11.836 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:11 smithi029 ceph-mon[223858]: osdmap e264: 8 total, 8 up, 8 in 2023-10-09T04:41:11.848 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1695211878' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:41:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:41:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:11 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:41:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:41:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:11 smithi177 ceph-mon[195797]: osdmap e264: 8 total, 8 up, 8 in 2023-10-09T04:41:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:12 smithi177 ceph-mon[195797]: pgmap v1000: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 10511/158049 objects degraded (6.650%); 151/158049 objects misplaced (0.096%); 4.0 MiB/s, 1 keys/s, 65 objects/s recovering 2023-10-09T04:41:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:12 smithi177 ceph-mon[195797]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:12 smithi177 ceph-mon[195797]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:12 smithi177 ceph-mon[195797]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:12 smithi177 ceph-mon[195797]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:12 smithi177 ceph-mon[195797]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[216903]: pgmap v1000: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 10511/158049 objects degraded (6.650%); 151/158049 objects misplaced (0.096%); 4.0 MiB/s, 1 keys/s, 65 objects/s recovering 2023-10-09T04:41:13.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[216903]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[216903]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[216903]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[216903]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[216903]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:41:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:41:12] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2023-10-09T04:41:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[223858]: pgmap v1000: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 10511/158049 objects degraded (6.650%); 151/158049 objects misplaced (0.096%); 4.0 MiB/s, 1 keys/s, 65 objects/s recovering 2023-10-09T04:41:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[223858]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[223858]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[223858]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[223858]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:12 smithi029 ceph-mon[223858]: failed to encode map e264 with expected crc 2023-10-09T04:41:13.535 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:13.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:13.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:13.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:13.711 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:14.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:41:14.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:14 smithi029 ceph-mon[216903]: pgmap v1002: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 10511/152391 objects degraded (6.897%); 151/152391 objects misplaced (0.099%); 3.6 MiB/s, 1 keys/s, 58 objects/s recovering 2023-10-09T04:41:14.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:14.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:14 smithi177 ceph-mon[195797]: pgmap v1002: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 10511/152391 objects degraded (6.897%); 151/152391 objects misplaced (0.099%); 3.6 MiB/s, 1 keys/s, 58 objects/s recovering 2023-10-09T04:41:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:14.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:14.856 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:14 smithi029 ceph-mon[223858]: pgmap v1002: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 10511/152391 objects degraded (6.897%); 151/152391 objects misplaced (0.099%); 3.6 MiB/s, 1 keys/s, 58 objects/s recovering 2023-10-09T04:41:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:15.267 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:15.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:15.373 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:15.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T04:41:15.437 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:15.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:15.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:15.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/152391 objects degraded (6.897%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/152391 objects degraded (6.897%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:15.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:15.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/152391 objects degraded (6.897%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:15.811 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:15.910 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:41:15.920 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:15.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:15.956 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:41:15.994 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:41:16.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:16.045 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:41:16.078 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:16.094 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:41:16.141 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:41:16.161 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:16.197 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:41:16.257 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:41:16.272 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.272 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.273 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.273 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.273 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.273 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.273 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.273 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:41:16.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:16.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:16.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:16.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:16 smithi029 ceph-mon[216903]: pgmap v1003: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 10511/151206 objects degraded (6.951%); 151/151206 objects misplaced (0.100%); 3.4 MiB/s, 0 keys/s, 55 objects/s recovering 2023-10-09T04:41:16.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3397591193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:16.599 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:16 smithi029 ceph-mon[223858]: pgmap v1003: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 10511/151206 objects degraded (6.951%); 151/151206 objects misplaced (0.100%); 3.4 MiB/s, 0 keys/s, 55 objects/s recovering 2023-10-09T04:41:16.599 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3397591193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:16.622 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:16.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:16 smithi177 ceph-mon[195797]: pgmap v1003: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 10511/151206 objects degraded (6.951%); 151/151206 objects misplaced (0.100%); 3.4 MiB/s, 0 keys/s, 55 objects/s recovering 2023-10-09T04:41:16.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3397591193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:16.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:16.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:16.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:16.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:17.043 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:17.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:41:17.574 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:17.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:17.680 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:17.709 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:18.401 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:18.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:18.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:18.602 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:18 smithi029 ceph-mon[216903]: pgmap v1004: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 10511/146736 objects degraded (7.163%); 2.8 MiB/s, 58 objects/s recovering 2023-10-09T04:41:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:18 smithi029 ceph-mon[223858]: pgmap v1004: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 10511/146736 objects degraded (7.163%); 2.8 MiB/s, 58 objects/s recovering 2023-10-09T04:41:18.777 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:18 smithi177 ceph-mon[195797]: pgmap v1004: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 10511/146736 objects degraded (7.163%); 2.8 MiB/s, 58 objects/s recovering 2023-10-09T04:41:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:18.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:18.889 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:18.962 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:19.043 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:41:19.043 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:41:19.043 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:41:19.043 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:41:19.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:19.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:19.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:19.296 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:19.423 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:19.457 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:19.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:19.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:19.638 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:19.704 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:19.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:19.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:19.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:19.834 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:19.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:19.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:19.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:19.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:20.122 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:20.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:41:20.357 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:20.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:20.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:41:20.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:20.528 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:20.589 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:20 smithi177 ceph-mon[195797]: pgmap v1005: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 10511/143250 objects degraded (7.338%); 2.8 MiB/s, 58 objects/s recovering 2023-10-09T04:41:20.589 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2659118319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:41:20.589 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:20 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:41:20.589 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/143250 objects degraded (7.338%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:20.619 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T04:41:20.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[223858]: pgmap v1005: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 10511/143250 objects degraded (7.338%); 2.8 MiB/s, 58 objects/s recovering 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2659118319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/143250 objects degraded (7.338%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[216903]: pgmap v1005: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 10511/143250 objects degraded (7.338%); 2.8 MiB/s, 58 objects/s recovering 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2659118319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T04:41:20.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/143250 objects degraded (7.338%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:20.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:20.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:20.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:20.820 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:20.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:20.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:20.904 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:20.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:21.049 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:21.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:21.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:21.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:21.429 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:21.772 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:21.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:41:21.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:41:21.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[223858]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:21 smithi029 ceph-mon[216903]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:21.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:21.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T04:41:21.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: osdmap e265: 8 total, 8 up, 8 in 2023-10-09T04:41:21.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:21 smithi177 ceph-mon[195797]: failed to encode map e265 with expected crc 2023-10-09T04:41:21.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:21.904 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:22.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:22.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:22.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:22.607 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:22.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:41:22.708 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:22.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:22.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:22.876 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:41:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:41:22] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T04:41:22.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:22 smithi029 ceph-mon[223858]: pgmap v1007: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 10511/139842 objects degraded (7.516%); 1004 KiB/s, 57 objects/s recovering 2023-10-09T04:41:22.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:22 smithi029 ceph-mon[216903]: pgmap v1007: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 10511/139842 objects degraded (7.516%); 1004 KiB/s, 57 objects/s recovering 2023-10-09T04:41:22.908 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:23.023 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:22 smithi177 ceph-mon[195797]: pgmap v1007: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 10511/139842 objects degraded (7.516%); 1004 KiB/s, 57 objects/s recovering 2023-10-09T04:41:23.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:23.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:23.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:23.263 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:23.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:23.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:23.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:23.495 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:41:23.495 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:41:23.495 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:41:23.495 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:41:23.495 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:41:23.564 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:23.644 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/783733040' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:23.645 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:23 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:23.667 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:41:23.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/783733040' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:23.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:23 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:23.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/783733040' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:23.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:23 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:23.751 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:23.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:23.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:23.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:23.913 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:23.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:24.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:24.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:24.134 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:24.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: pgmap v1008: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 10511/134673 objects degraded (7.805%); 966 KiB/s, 54 objects/s recovering 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: failed to encode map e266 with expected crc 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: failed to encode map e266 with expected crc 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[223858]: failed to encode map e266 with expected crc 2023-10-09T04:41:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: pgmap v1008: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 10511/134673 objects degraded (7.805%); 966 KiB/s, 54 objects/s recovering 2023-10-09T04:41:24.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:24.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:24.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:41:24.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: failed to encode map e266 with expected crc 2023-10-09T04:41:24.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: failed to encode map e266 with expected crc 2023-10-09T04:41:24.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:24 smithi029 ceph-mon[216903]: failed to encode map e266 with expected crc 2023-10-09T04:41:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: pgmap v1008: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 10511/134673 objects degraded (7.805%); 966 KiB/s, 54 objects/s recovering 2023-10-09T04:41:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: osdmap e266: 8 total, 8 up, 8 in 2023-10-09T04:41:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: failed to encode map e266 with expected crc 2023-10-09T04:41:25.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: failed to encode map e266 with expected crc 2023-10-09T04:41:25.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:24 smithi177 ceph-mon[195797]: failed to encode map e266 with expected crc 2023-10-09T04:41:25.286 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:25.890 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/134673 objects degraded (7.805%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:25.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/134673 objects degraded (7.805%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:25.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/134673 objects degraded (7.805%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:26.251 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:26.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:26.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:26.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:26.429 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:26.541 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:26.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:26.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:26.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:26 smithi029 ceph-mon[216903]: pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 10511/133689 objects degraded (7.862%); 0 B/s, 33 objects/s recovering 2023-10-09T04:41:26.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:26 smithi029 ceph-mon[223858]: pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 10511/133689 objects degraded (7.862%); 0 B/s, 33 objects/s recovering 2023-10-09T04:41:26.684 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:26 smithi177 ceph-mon[195797]: pgmap v1010: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 10511/133689 objects degraded (7.862%); 0 B/s, 33 objects/s recovering 2023-10-09T04:41:27.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:41:27.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:41:27.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:41:27.774 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:41:27.779 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:27.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:27.796 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:41:27.841 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:41:27.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:27.897 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:41:27.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1097874158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:27.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1097874158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:27.952 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.971 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.972 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:41:27.974 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:28.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1097874158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:28.051 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:28.051 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:28.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:41:28.629 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:28.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:28.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:28.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:28.799 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:28.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:28.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:28.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:28 smithi029 ceph-mon[216903]: pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 10511/129618 objects degraded (8.109%); 0 B/s, 70 objects/s recovering 2023-10-09T04:41:28.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:28.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:28 smithi029 ceph-mon[223858]: pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 10511/129618 objects degraded (8.109%); 0 B/s, 70 objects/s recovering 2023-10-09T04:41:28.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:28.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:28 smithi177 ceph-mon[195797]: pgmap v1011: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 10511/129618 objects degraded (8.109%); 0 B/s, 70 objects/s recovering 2023-10-09T04:41:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:29.046 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:29.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:29.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:41:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:29.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:29.423 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:29.488 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:29.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:29.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:29.719 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:29.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:29.827 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:29.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:30.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:30.031 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:30.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:30.166 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:30.336 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:30.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:30.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:30.511 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:30.691 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:41:30.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:30.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:30.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:30.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:30.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:30 smithi177 ceph-mon[195797]: pgmap v1012: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 10511/126507 objects degraded (8.309%); 0 B/s, 34 objects/s recovering 2023-10-09T04:41:30.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2816156404' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:41:30.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/126507 objects degraded (8.309%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:30.992 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:31.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:31.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:30 smithi029 ceph-mon[216903]: pgmap v1012: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 10511/126507 objects degraded (8.309%); 0 B/s, 34 objects/s recovering 2023-10-09T04:41:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2816156404' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:41:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/126507 objects degraded (8.309%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:30 smithi029 ceph-mon[223858]: pgmap v1012: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 10511/126507 objects degraded (8.309%); 0 B/s, 34 objects/s recovering 2023-10-09T04:41:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2816156404' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:41:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/126507 objects degraded (8.309%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:31.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:31.270 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:31.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:41:32.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2816156404' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:41:32.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:31 smithi177 ceph-mon[195797]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:41:32.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:31 smithi177 ceph-mon[195797]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:31 smithi177 ceph-mon[195797]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:31 smithi177 ceph-mon[195797]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:31 smithi177 ceph-mon[195797]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2816156404' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:41:32.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[216903]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:41:32.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[216903]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[216903]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[216903]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[216903]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.167 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2816156404' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:41:32.167 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[223858]: osdmap e267: 8 total, 8 up, 8 in 2023-10-09T04:41:32.167 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[223858]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[223858]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[223858]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:31 smithi029 ceph-mon[223858]: failed to encode map e267 with expected crc 2023-10-09T04:41:32.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:32 smithi177 ceph-mon[195797]: pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 10511/123321 objects degraded (8.523%); 0 B/s, 79 objects/s recovering 2023-10-09T04:41:32.881 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:32.881 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:32.899 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:32.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:32 smithi029 ceph-mon[216903]: pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 10511/123321 objects degraded (8.523%); 0 B/s, 79 objects/s recovering 2023-10-09T04:41:32.904 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:41:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:41:32] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-10-09T04:41:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:32.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:32.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:32 smithi029 ceph-mon[223858]: pgmap v1014: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 10511/123321 objects degraded (8.523%); 0 B/s, 79 objects/s recovering 2023-10-09T04:41:32.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T04:41:33.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:33.144 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:33.285 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:33.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:33.381 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:33.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:33.460 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:33.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:33.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:33.655 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:33.782 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:33.897 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:33.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:34.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:34.109 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:34.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:41:34.292 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:34.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:34.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:34.473 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:34.588 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:34.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:34.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:34.693 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:34.740 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:41:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:34 smithi029 ceph-mon[216903]: pgmap v1015: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 10511/118053 objects degraded (8.904%); 0 B/s, 67 objects/s recovering 2023-10-09T04:41:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3706357195' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:34.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:34 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:34.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:34 smithi029 ceph-mon[223858]: pgmap v1015: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 10511/118053 objects degraded (8.904%); 0 B/s, 67 objects/s recovering 2023-10-09T04:41:34.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3706357195' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:34.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:34 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:34.769 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:34.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:34.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:34.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:34.941 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:34.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:35.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:34 smithi177 ceph-mon[195797]: pgmap v1015: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 10511/118053 objects degraded (8.904%); 0 B/s, 67 objects/s recovering 2023-10-09T04:41:35.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3706357195' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:35.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:34 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:41:35.050 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:35.170 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:35.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:35.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:35.359 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:35.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:35.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/118053 objects degraded (8.904%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:35.974 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:35.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:35.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/118053 objects degraded (8.904%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:35.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: osdmap e268: 8 total, 8 up, 8 in 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: failed to encode map e268 with expected crc 2023-10-09T04:41:35.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/118053 objects degraded (8.904%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:35.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:36.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:36.114 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:36.572 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:36.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:36.674 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:36.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:36.773 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:36.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:36 smithi029 ceph-mon[216903]: pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 10511/117042 objects degraded (8.981%); 0 B/s, 42 objects/s recovering 2023-10-09T04:41:36.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:36 smithi029 ceph-mon[223858]: pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 10511/117042 objects degraded (8.981%); 0 B/s, 42 objects/s recovering 2023-10-09T04:41:37.023 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:37.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:36 smithi177 ceph-mon[195797]: pgmap v1017: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 10511/117042 objects degraded (8.981%); 0 B/s, 42 objects/s recovering 2023-10-09T04:41:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:37.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:37.139 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:37.258 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:37.388 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:41:37.396 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:37.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:37.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:37.445 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:41:37.496 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:41:37.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:37.545 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:41:37.554 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:37.554 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:37.584 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:41:37.585 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:37.636 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:41:37.692 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:41:37.709 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:37.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:37.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:37.732 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:41:37.745 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.745 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.745 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.745 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.745 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.746 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.746 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.746 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:41:37.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:37.869 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2561389188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:37.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2561389188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:37.917 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2561389188' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:38.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:41:38.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:41:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:38 smithi177 ceph-mon[195797]: pgmap v1018: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 10511/112749 objects degraded (9.322%); 0 B/s, 91 objects/s recovering 2023-10-09T04:41:39.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:38 smithi029 ceph-mon[216903]: pgmap v1018: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 10511/112749 objects degraded (9.322%); 0 B/s, 91 objects/s recovering 2023-10-09T04:41:39.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:38 smithi029 ceph-mon[223858]: pgmap v1018: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 10511/112749 objects degraded (9.322%); 0 B/s, 91 objects/s recovering 2023-10-09T04:41:39.249 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:39.333 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:39.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:39.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:39.524 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:39.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:39.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:39.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:39.737 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:39.766 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:39.871 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:39.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:39.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:40.080 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:40.232 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:40.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:40.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:40.396 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:40.491 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:40.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:40.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:40.682 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:40.790 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:40.818 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:40.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:40 smithi029 ceph-mon[216903]: pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 10511/109443 objects degraded (9.604%); 0 B/s, 46 objects/s recovering 2023-10-09T04:41:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/109443 objects degraded (9.604%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:40 smithi029 ceph-mon[223858]: pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 10511/109443 objects degraded (9.604%); 0 B/s, 46 objects/s recovering 2023-10-09T04:41:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/109443 objects degraded (9.604%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:40.923 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:40.966 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:41.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:40 smithi177 ceph-mon[195797]: pgmap v1019: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 10511/109443 objects degraded (9.604%); 0 B/s, 46 objects/s recovering 2023-10-09T04:41:41.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/109443 objects degraded (9.604%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:41.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:41.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:41.175 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:41.275 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:41.655 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:41:41.655 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:120129 10.19 10:9dbedecc:::benchmark_data_smithi029_344401_object11758:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:47.593087+0000", 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 354.05511893200003, 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0070726539999999, 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:41:41.656 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120129 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593087+0000", 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593087+0000", 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593085+0000", 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593094+0000", 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3999999999999992e-06 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593120+0000", 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5780999999999999e-05 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.593131+0000", 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1008e-05 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.735165+0000", 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14203438700000001 2023-10-09T04:41:41.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:47.735172+0000", 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9399999999999996e-06 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.585926+0000", 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85075447699999995 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.590090+0000", 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041638459999999997 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 7", 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.590193+0000", 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102508 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.593257+0000", 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030638290000000001 2023-10-09T04:41:41.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600029+0000", 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067728179999999999 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600080+0000", 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0160000000000001e-05 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:48.600160+0000", 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0267999999999998e-05 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143839 10.23 10:c6eaaf06:::benchmark_data_smithi029_344401_object35468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.418089+0000", 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 330.23011763300002, 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96140360199999997, 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.663 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143839 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418089+0000", 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418089+0000", 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418088+0000", 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418096+0000", 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3059999999999998e-06 2023-10-09T04:41:41.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418110+0000", 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4487999999999999e-05 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.418117+0000", 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6229999999999999e-06 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.531432+0000", 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.11331483000000001 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.531439+0000", 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9630000000000001e-06 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.379415+0000", 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.84797630999999996 2023-10-09T04:41:41.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.379492+0000", 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6917999999999998e-05 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:66689 11.7 11:e9352e84:::benchmark_data_smithi029_392508_object66688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.826953+0000", 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.82125291099999, 2023-10-09T04:41:41.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.517259574, 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 66689 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.826953+0000", 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.826953+0000", 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.826952+0000", 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827032+0000", 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9770000000000004e-05 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827043+0000", 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1239e-05 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827048+0000", 2023-10-09T04:41:41.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9450000000000001e-06 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827094+0000", 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5633000000000002e-05 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827250+0000", 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000156296 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827407+0000", 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015663799999999999 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829917+0000", 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025108840000000001 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.830705+0000", 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000787362 2023-10-09T04:41:41.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344108+0000", 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51340283099999995 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344165+0000", 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7386999999999998e-05 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344213+0000", 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7788999999999999e-05 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:66681 11.a e244/237 11:5a1f7244:::benchmark_data_smithi029_392508_object66680:head v 244'4280, mlcod=244'4278)", 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.827074+0000", 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.821132654, 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51699164399999997, 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.675 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827074+0000", 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827074+0000", 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827073+0000", 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827124+0000", 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1149000000000001e-05 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827135+0000", 2023-10-09T04:41:41.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0963e-05 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827139+0000", 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3599999999999998e-06 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827194+0000", 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5009999999999997e-05 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827219+0000", 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4198000000000001e-05 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344013+0000", 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51679458199999995 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344065+0000", 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1959000000000002e-05 2023-10-09T04:41:41.679 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:66685 11.a e244/237 11:5330dcfa:::benchmark_data_smithi029_392508_object66684:head v 244'4281, mlcod=244'4278)", 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.827214+0000", 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.82099239999999, 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51688367000000002, 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827214+0000", 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827214+0000", 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827214+0000", 2023-10-09T04:41:41.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827281+0000", 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7941999999999994e-05 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827293+0000", 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.137e-05 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.827297+0000", 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3590000000000001e-06 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829486+0000", 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002189304 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829507+0000", 2023-10-09T04:41:41.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0227000000000001e-05 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344075+0000", 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51456821500000005 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344097+0000", 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2521999999999999e-05 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.684 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:66688 11.a e244/237 11:573ebdb5:::benchmark_data_smithi029_392508_object66687:head v 244'4282, mlcod=244'4278)", 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.829456+0000", 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.81875037399999, 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.52546579000000004, 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829456+0000", 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829456+0000", 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829455+0000", 2023-10-09T04:41:41.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829578+0000", 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123058 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829590+0000", 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1409e-05 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.829596+0000", 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3080000000000004e-06 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344138+0000", 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51454169699999996 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344160+0000", 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2489000000000001e-05 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.354886+0000", 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01072648 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.354922+0000", 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5108000000000003e-05 2023-10-09T04:41:41.689 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:66696 11.7 11:ec837d6d:::benchmark_data_smithi029_392508_object66695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.834481+0000", 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.81372534900001, 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51832118800000004, 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.690 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 66696 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834481+0000", 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834481+0000", 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834479+0000", 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834571+0000", 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2010999999999997e-05 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.343992+0000", 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50942123100000003 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344011+0000", 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8085e-05 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344261+0000", 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025067999999999998 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344418+0000", 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015658800000000001 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344574+0000", 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000156496 2023-10-09T04:41:41.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.349807+0000", 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052328130000000002 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.349845+0000", 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7703000000000001e-05 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.352712+0000", 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002867667 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.352763+0000", 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0982000000000001e-05 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.352802+0000", 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8566000000000003e-05 2023-10-09T04:41:41.696 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:66695 11.5 e244/237 11:acda587b:::benchmark_data_smithi029_392508_object66694:head v 244'4205, mlcod=244'4203)", 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.836696+0000", 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.81151029599999, 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51423856499999998, 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.836696+0000", 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.836696+0000", 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.836695+0000", 2023-10-09T04:41:41.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.836727+0000", 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2020000000000002e-05 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344011+0000", 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50728443300000003 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344020+0000", 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7730000000000005e-06 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344062+0000", 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1956999999999998e-05 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.344081+0000", 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9281000000000001e-05 2023-10-09T04:41:41.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.350883+0000", 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068013650000000002 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.350934+0000", 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1765000000000003e-05 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.701 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:89742 11.7 11:e1ab25f6:::benchmark_data_smithi029_392508_object89741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:39:11.369839+0000", 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.27836741600001, 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4462705859999998, 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.702 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89742 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369839+0000", 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369839+0000", 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369837+0000", 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369877+0000", 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0191000000000002e-05 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369893+0000", 2023-10-09T04:41:41.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6186000000000001e-05 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369900+0000", 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.759e-06 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.369950+0000", 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9563e-05 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.370064+0000", 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113888 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.370198+0000", 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000134666 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.373600+0000", 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034020600000000002 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.373822+0000", 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00022169099999999999 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.396991+0000", 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023168635 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.397002+0000", 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0769e-05 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.404718+0000", 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.007716912 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.404729+0000", 2023-10-09T04:41:41.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0710000000000001e-05 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.816028+0000", 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.411298562 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.816109+0000", 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1646999999999999e-05 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.709 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:89762 11.7 11:ed097274:::benchmark_data_smithi029_392508_object89761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:39:11.389715+0000", 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.258491158, 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4600571219999998, 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.710 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89762 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389715+0000", 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389715+0000", 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389714+0000", 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389765+0000", 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1195e-05 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389778+0000", 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3227000000000001e-05 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389784+0000", 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2490000000000001e-06 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389818+0000", 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4854999999999997e-05 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.389979+0000", 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000160891 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.390079+0000", 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9877999999999995e-05 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.397360+0000", 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00728029 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.397367+0000", 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7049999999999999e-06 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.405153+0000", 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0077857050000001 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.405161+0000", 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6930000000000007e-06 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.817922+0000", 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4127618370000001 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:41:41.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.818618+0000", 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00069566300000000001 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.849668+0000", 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.031050376000000001 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.849772+0000", 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103659 2023-10-09T04:41:41.717 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:89775 11.7 11:e1185992:::benchmark_data_smithi029_392508_object89774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:39:11.414067+0000", 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.23413914299999, 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4380177769999998, 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.718 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89775 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414067+0000", 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414067+0000", 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414066+0000", 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414119+0000", 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2978e-05 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414133+0000", 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4141999999999999e-05 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414138+0000", 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3539999999999999e-06 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414197+0000", 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8876999999999999e-05 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.414207+0000", 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6350000000000001e-06 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.405180+0000", 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.990973416 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.405186+0000", 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7339999999999996e-06 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.818685+0000", 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.413499402 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.818835+0000", 2023-10-09T04:41:41.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015016300000000001 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.819032+0000", 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000196245 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.827100+0000", 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080681680000000006 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.844397+0000", 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017297290999999999 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.849818+0000", 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054203860000000001 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.849873+0000", 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5889e-05 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.852085+0000", 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022113720000000001 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.726 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:89787 11.7 11:ec0facdf:::benchmark_data_smithi029_392508_object89786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:39:11.428843+0000", 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.21936285500001, 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4270114899999999, 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 89787 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.428843+0000", 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.428843+0000", 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.428842+0000", 2023-10-09T04:41:41.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.428954+0000", 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111489 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.428965+0000", 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1318000000000001e-05 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.428971+0000", 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8170000000000004e-06 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.429030+0000", 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9512999999999998e-05 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:11.429040+0000", 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3640000000000005e-06 2023-10-09T04:41:41.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.405360+0000", 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97632006400000004 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:12.405367+0000", 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.114e-06 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.821571+0000", 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.416203783 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.821686+0000", 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011567 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.821810+0000", 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123764 2023-10-09T04:41:41.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.827131+0000", 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053205500000000003 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.852157+0000", 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025026203 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.853942+0000", 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017854640000000001 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.853986+0000", 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3597999999999998e-05 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:39:13.855855+0000", 2023-10-09T04:41:41.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018688579999999999 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:116145 11.7 11:ef915afd:::benchmark_data_smithi029_392508_object15589:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e254)", 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:01.295017+0000", 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 100.353189323, 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.22893092800000001, 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.735 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 116145 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295017+0000", 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295017+0000", 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295016+0000", 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295021+0000", 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5269999999999998e-06 2023-10-09T04:41:41.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295035+0000", 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.347e-05 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295042+0000", 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3849999999999996e-06 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295102+0000", 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9524999999999997e-05 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295250+0000", 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014802899999999999 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.295375+0000", 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012498399999999999 2023-10-09T04:41:41.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.298713+0000", 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033385770000000001 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.308355+0000", 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096418119999999996 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.520572+0000", 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21221742699999999 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.520649+0000", 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6311999999999997e-05 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:01.523948+0000", 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032990699999999999 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.741 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:122459 11.7 11:edda5a0f:::benchmark_data_smithi029_392508_object21903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e256)", 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:14.427344+0000", 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 87.220862660999998, 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21970826900000001, 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.742 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122459 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427344+0000", 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427344+0000", 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427342+0000", 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427349+0000", 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8340000000000001e-06 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427372+0000", 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.283e-05 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427400+0000", 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8337e-05 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427440+0000", 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0067000000000001e-05 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427614+0000", 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017417100000000001 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.427731+0000", 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116322 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.431025+0000", 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032946500000000001 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.431400+0000", 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00037429199999999997 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.644388+0000", 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21298837000000001 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.644431+0000", 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3414999999999997e-05 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:14.647052+0000", 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026203229999999999 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:140736 11.7 11:ee0f11ab:::benchmark_data_smithi029_392508_object40180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:51.058280+0000", 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 50.589926179000003, 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26133637999999998, 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 140736 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.749 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058280+0000", 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058280+0000", 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058279+0000", 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058286+0000", 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5410000000000003e-06 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058303+0000", 2023-10-09T04:41:41.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6401000000000001e-05 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058315+0000", 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2775999999999999e-05 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058366+0000", 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0677999999999998e-05 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058498+0000", 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013217 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058616+0000", 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117798 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.065563+0000", 2023-10-09T04:41:41.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069472639999999999 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.067021+0000", 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014573050000000001 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.317094+0000", 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25007307200000001 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.754 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.317139+0000", 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4898000000000003e-05 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.319616+0000", 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00247781 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:140739 11.7 11:e379a624:::benchmark_data_smithi029_392508_object40183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:51.058733+0000", 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 50.589472733000001, 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26257959199999997, 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 140739 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.756 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058733+0000", 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058733+0000", 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058733+0000", 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058740+0000", 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0060000000000003e-06 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058750+0000", 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.961e-06 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058755+0000", 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8029999999999996e-06 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.062395+0000", 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003640505 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.062516+0000", 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000121437 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.062624+0000", 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107741 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.065702+0000", 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030774940000000001 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.067096+0000", 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001394319 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.319666+0000", 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.25257011600000001 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.319708+0000", 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2193000000000001e-05 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.321313+0000", 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016047189999999999 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.762 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145940 11.7 11:ed08b1b0:::benchmark_data_smithi029_392508_object45384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.860794+0000", 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 39.787412400000001, 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90692854300000003, 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145940 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860794+0000", 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860794+0000", 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860792+0000", 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860800+0000", 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7330000000000003e-06 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860818+0000", 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8258999999999999e-05 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860826+0000", 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7470000000000005e-06 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860892+0000", 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5526999999999998e-05 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860904+0000", 2023-10-09T04:41:41.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2452999999999999e-05 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.757260+0000", 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89635559200000003 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.757432+0000", 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017239499999999999 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.757601+0000", 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016914299999999999 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.763593+0000", 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059920989999999999 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.766613+0000", 2023-10-09T04:41:41.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030199979999999999 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.766746+0000", 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000133084 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.766789+0000", 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2710000000000003e-05 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.767722+0000", 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00093332400000000002 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDeleteReply(11.7 e265,261 11:e95a2464:::benchmark_data_smithi029_392508_object7663:head,253'6864)", 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:22.053939+0000", 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 19.594267164000001, 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21709225600000001, 2023-10-09T04:41:41.770 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053939+0000", 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053939+0000", 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053938+0000", 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053943+0000", 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9799999999999998e-06 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053953+0000", 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7890000000000006e-06 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053958+0000", 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.853e-06 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.053992+0000", 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4269e-05 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.271031+0000", 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21703893699999999 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:156605 11.5 e265/237 11:a1a4213a:::benchmark_data_smithi029_392508_object56049:head v 265'9886, mlcod=265'9884)", 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:22.064155+0000", 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 19.584051502000001, 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21523849, 2023-10-09T04:41:41.774 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.064155+0000", 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.064155+0000", 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.775 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.064154+0000", 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.064163+0000", 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.38e-06 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270619+0000", 2023-10-09T04:41:41.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.20645608800000001 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270627+0000", 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5979999999999999e-06 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270691+0000", 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3844999999999996e-05 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270713+0000", 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1735000000000001e-05 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.279351+0000", 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086389759999999996 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.279393+0000", 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1677000000000001e-05 2023-10-09T04:41:41.778 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:156606 11.a e265/237 11:5caa797e:::benchmark_data_smithi029_392508_object56050:head v 265'10024, mlcod=265'10022)", 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:22.070481+0000", 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 19.577725041000001, 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21296361899999999, 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.779 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.070481+0000", 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.070481+0000", 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.070480+0000", 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:41:41.780 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.070490+0000", 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0083e-05 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270607+0000", 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.200117131 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.781 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270620+0000", 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2424e-05 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270676+0000", 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6348000000000003e-05 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.270703+0000", 2023-10-09T04:41:41.782 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6718000000000001e-05 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.283412+0000", 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012709399999999999 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:22.283445+0000", 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2774999999999997e-05 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.783 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.784 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:41.784 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:41.784 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:41:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:41.785 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:41.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:41.819 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T04:41:41.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:41.938 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:42.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/529609683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:41:42.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:41 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:41:42.038 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:42.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/529609683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:41:42.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:41 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:41:42.048 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/529609683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:41:42.048 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:41 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T04:41:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:42.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:42.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:42.193 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:42.237 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:42.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:42.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:41:42.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:42.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:42.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:42.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:42.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:41:42.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:42.989 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: pgmap v1020: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 10511/106431 objects degraded (9.876%); 0 B/s, 84 objects/s recovering 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:42 smithi177 ceph-mon[195797]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:43.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:41:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:41:42] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-10-09T04:41:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: pgmap v1020: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 10511/106431 objects degraded (9.876%); 0 B/s, 84 objects/s recovering 2023-10-09T04:41:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:41:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[223858]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: pgmap v1020: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 10511/106431 objects degraded (9.876%); 0 B/s, 84 objects/s recovering 2023-10-09T04:41:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T04:41:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: osdmap e269: 8 total, 8 up, 8 in 2023-10-09T04:41:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:42 smithi029 ceph-mon[216903]: failed to encode map e269 with expected crc 2023-10-09T04:41:43.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:43.308 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:43.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:43.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:43.473 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:43.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:43.614 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:43.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:43.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:43.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:44.072 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:44.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:44.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:44.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:44.215 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:41:44.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:41:44.399 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:44.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:44.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:44.620 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:44.648 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:44.680 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:44.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:44.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:44.837 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:45.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:44 smithi177 ceph-mon[195797]: pgmap v1022: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 10511/100983 objects degraded (10.409%); 0 B/s, 101 objects/s recovering 2023-10-09T04:41:45.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:45.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:45.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:44 smithi029 ceph-mon[216903]: pgmap v1022: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 10511/100983 objects degraded (10.409%); 0 B/s, 101 objects/s recovering 2023-10-09T04:41:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:44 smithi029 ceph-mon[223858]: pgmap v1022: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 10511/100983 objects degraded (10.409%); 0 B/s, 101 objects/s recovering 2023-10-09T04:41:45.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:45.333 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:45.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:45.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:45.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:45.685 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:45.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:45.876 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:41:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:45.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:45.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:41:46.005 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:170091 11.3 11:c18dcff7:::benchmark_data_smithi029_392508_object69535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:45.986953+0000", 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.018396440999999999, 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018421544000000002, 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:41:46.006 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 170091 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.986953+0000", 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.986953+0000", 2023-10-09T04:41:46.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.986951+0000", 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.986960+0000", 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4910000000000005e-06 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.986980+0000", 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0517e-05 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.986989+0000", 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7180000000000002e-06 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:46.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.987070+0000", 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1581000000000004e-05 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.987189+0000", 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118322 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:45.987302+0000", 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011302999999999999 2023-10-09T04:41:46.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.000018+0000", 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012715822999999999 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001675+0000", 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001657197 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:170081 11.d e270/261 11:b9d35f30:::benchmark_data_smithi029_392508_object69525:head v 270'10681, mlcod=246'5604)", 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:46.001800+0000", 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0035491979999999999, 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036317099999999998, 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001800+0000", 2023-10-09T04:41:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001800+0000", 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001799+0000", 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001814+0000", 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.486e-05 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001828+0000", 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4045e-05 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:46.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.001833+0000", 2023-10-09T04:41:46.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4149999999999998e-06 2023-10-09T04:41:46.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004526+0000", 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026920550000000001 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004559+0000", 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3680999999999998e-05 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:46.016 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.45869.0:170093 11.d e270/261)", 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:46.004404+0000", 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.00094494200000000005, 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001060458, 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "queued for pg", 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004404+0000", 2023-10-09T04:41:46.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004404+0000", 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004403+0000", 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:41:46.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004415+0000", 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.173e-05 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004435+0000", 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0106999999999999e-05 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:41:46.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:46.004444+0000", 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5310000000000001e-06 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 3 2023-10-09T04:41:46.020 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:41:46.049 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/796171852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:46.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/100983 objects degraded (10.409%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:46.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:46.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/796171852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:46.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:46.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/100983 objects degraded (10.409%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:46.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:41:46.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/796171852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:46.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:45 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:41:46.217 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/100983 objects degraded (10.409%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:46.781 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:41:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:46.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:41:46.810 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:46.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:46.956 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:47.063 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:47.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:47.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: pgmap v1023: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 10511/99912 objects degraded (10.520%); 0 B/s, 84 objects/s recovering 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.115 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:46 smithi177 ceph-mon[195797]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.116 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: pgmap v1023: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 10511/99912 objects degraded (10.520%); 0 B/s, 84 objects/s recovering 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[216903]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: pgmap v1023: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 10511/99912 objects degraded (10.520%); 0 B/s, 84 objects/s recovering 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: osdmap e270: 8 total, 8 up, 8 in 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:46 smithi029 ceph-mon[223858]: failed to encode map e270 with expected crc 2023-10-09T04:41:47.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:47.298 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:47.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:47.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:47.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:47.835 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:47.971 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:48.165 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:48.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:48.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:48.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:48.360 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:48.628 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:41:48.666 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:41:48.699 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:41:48.738 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:41:48.795 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:41:48.849 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:41:48.881 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:41:48.919 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:41:48.927 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.927 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.927 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.927 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.927 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.928 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.928 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.928 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:41:48.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:48.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:49.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:41:49.129 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:48 smithi029 ceph-mon[223858]: pgmap v1025: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 10511/95520 objects degraded (11.004%); 0 B/s, 107 objects/s recovering 2023-10-09T04:41:49.129 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2466410948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:49.129 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:48 smithi029 ceph-mon[216903]: pgmap v1025: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 10511/95520 objects degraded (11.004%); 0 B/s, 107 objects/s recovering 2023-10-09T04:41:49.129 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2466410948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:49.138 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:48 smithi177 ceph-mon[195797]: pgmap v1025: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 10511/95520 objects degraded (11.004%); 0 B/s, 107 objects/s recovering 2023-10-09T04:41:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2466410948' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:41:49.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:41:50.012 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:41:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:50.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:41:50.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:50.212 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:50.474 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:50.474 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:51.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:50 smithi177 ceph-mon[195797]: pgmap v1026: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 10511/92199 objects degraded (11.400%); 0 B/s, 49 objects/s recovering 2023-10-09T04:41:51.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/92199 objects degraded (11.400%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:51.041 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:51.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:50 smithi029 ceph-mon[216903]: pgmap v1026: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 10511/92199 objects degraded (11.400%); 0 B/s, 49 objects/s recovering 2023-10-09T04:41:51.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/92199 objects degraded (11.400%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:51.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:50 smithi029 ceph-mon[223858]: pgmap v1026: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 10511/92199 objects degraded (11.400%); 0 B/s, 49 objects/s recovering 2023-10-09T04:41:51.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/92199 objects degraded (11.400%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:51.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:51.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.206 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:51.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:51.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:51.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:41:51.797 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:51.903 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:51.908 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:51.998 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:52.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:52.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:52.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:52.222 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:52.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:52.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:52.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:52.436 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:41:52.467 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:41:52.533 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:52.536 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:52.635 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:52.806 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:52.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:52.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:52 smithi029 ceph-mon[216903]: pgmap v1027: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 10511/88716 objects degraded (11.848%); 0 B/s, 85 objects/s recovering 2023-10-09T04:41:52.900 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:41:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:41:52] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.43.0" 2023-10-09T04:41:52.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:52 smithi029 ceph-mon[223858]: pgmap v1027: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 10511/88716 objects degraded (11.848%); 0 B/s, 85 objects/s recovering 2023-10-09T04:41:52.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:52.992 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:53.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:41:53.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:52 smithi177 ceph-mon[195797]: pgmap v1027: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 10511/88716 objects degraded (11.848%); 0 B/s, 85 objects/s recovering 2023-10-09T04:41:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:53 smithi029 ceph-mon[216903]: from='client.36114 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:41:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:53 smithi029 ceph-mon[223858]: from='client.36114 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:41:54.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:53 smithi177 ceph-mon[195797]: from='client.36114 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:41:54.832 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:54.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:54.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:54.971 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:55.125 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:55.125 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:55.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: pgmap v1028: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10511/82947 objects degraded (12.672%); 0 B/s, 80 objects/s recovering 2023-10-09T04:41:55.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.156 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:41:55.157 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.157 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.157 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:55.157 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:54 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: pgmap v1028: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10511/82947 objects degraded (12.672%); 0 B/s, 80 objects/s recovering 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: pgmap v1028: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 10511/82947 objects degraded (12.672%); 0 B/s, 80 objects/s recovering 2023-10-09T04:41:55.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: osdmap e271: 8 total, 8 up, 8 in 2023-10-09T04:41:55.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: failed to encode map e271 with expected crc 2023-10-09T04:41:55.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:55.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:41:55.918 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1027,"stamp":"2023-10-09T04:41:51.151271+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1936988088,"num_objects":29572,"num_object_clones":0,"num_object_copies":88716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29572,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":171860,"num_write_kb":6437838,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11412,"num_bytes_recovered":571154584,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139278,"ondisk_log_size":139278,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11967780,"kb_used_data":6329956,"kb_used_omap":20,"kb_used_meta":5637675,"kb_avail":737993436,"statfs":{"total":767960285184,"available":755705278464,"internally_reserved":0,"allocated":6481874944,"data_stored":6475500679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5772979854},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,9,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-338690048,"num_objects":-5168,"num_object_clones":0,"num_object_copies":-15504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5168,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.372208"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805030+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:41:41.805030+0000","last_peered":"2023-10-09T04:41:41.805030+0000","last_clean":"2023-10-09T04:41:41.805030+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:41:41.805030+0000","last_undegraded":"2023-10-09T04:41:41.805030+0000","last_fullsized":"2023-10-09T04:41:41.805030+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":222,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805883+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:41:41.805883+0000","last_peered":"2023-10-09T04:41:41.805883+0000","last_clean":"2023-10-09T04:41:41.805883+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:41:41.805883+0000","last_undegraded":"2023-10-09T04:41:41.805883+0000","last_fullsized":"2023-10-09T04:41:41.805883+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":255,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.790801+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:41:41.790801+0000","last_peered":"2023-10-09T04:41:41.790801+0000","last_clean":"2023-10-09T04:41:41.790801+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:41:41.790801+0000","last_undegraded":"2023-10-09T04:41:41.790801+0000","last_fullsized":"2023-10-09T04:41:41.790801+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":183,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.811204+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:41:41.811204+0000","last_peered":"2023-10-09T04:41:41.811204+0000","last_clean":"2023-10-09T04:41:41.811204+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:41:41.811204+0000","last_undegraded":"2023-10-09T04:41:41.811204+0000","last_fullsized":"2023-10-09T04:41:41.811204+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806369+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:41:41.806369+0000","last_peered":"2023-10-09T04:41:41.806369+0000","last_clean":"2023-10-09T04:41:41.806369+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:41:41.806369+0000","last_undegraded":"2023-10-09T04:41:41.806369+0000","last_fullsized":"2023-10-09T04:41:41.806369+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":227,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.808169+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:41:41.808169+0000","last_peered":"2023-10-09T04:41:41.808169+0000","last_clean":"2023-10-09T04:41:41.808169+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:41:41.808169+0000","last_undegraded":"2023-10-09T04:41:41.808169+0000","last_fullsized":"2023-10-09T04:41:41.808169+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":182,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805933+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:41:41.805933+0000","last_peered":"2023-10-09T04:41:41.805933+0000","last_clean":"2023-10-09T04:41:41.805933+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:41:41.805933+0000","last_undegraded":"2023-10-09T04:41:41.805933+0000","last_fullsized":"2023-10-09T04:41:41.805933+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":222,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806519+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:41:41.806519+0000","last_peered":"2023-10-09T04:41:41.806519+0000","last_clean":"2023-10-09T04:41:41.806519+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:41:41.806519+0000","last_undegraded":"2023-10-09T04:41:41.806519+0000","last_fullsized":"2023-10-09T04:41:41.806519+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804719+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:41:41.804719+0000","last_peered":"2023-10-09T04:41:41.804719+0000","last_clean":"2023-10-09T04:41:41.804719+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:41:41.804719+0000","last_undegraded":"2023-10-09T04:41:41.804719+0000","last_fullsized":"2023-10-09T04:41:41.804719+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":195,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.807190+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:41:41.807190+0000","last_peered":"2023-10-09T04:41:41.807190+0000","last_clean":"2023-10-09T04:41:41.807190+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:41:41.807190+0000","last_undegraded":"2023-10-09T04:41:41.807190+0000","last_fullsized":"2023-10-09T04:41:41.807190+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":264,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805559+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:41:41.805559+0000","last_peered":"2023-10-09T04:41:41.805559+0000","last_clean":"2023-10-09T04:41:41.805559+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:41:41.805559+0000","last_undegraded":"2023-10-09T04:41:41.805559+0000","last_fullsized":"2023-10-09T04:41:41.805559+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":224,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.813199+0000","last_change":"2023-10-09T04:39:40.406537+0000","last_active":"2023-10-09T04:41:41.813199+0000","last_peered":"2023-10-09T04:41:41.813199+0000","last_clean":"2023-10-09T04:41:41.813199+0000","last_became_active":"2023-10-09T04:39:40.406281+0000","last_became_peered":"2023-10-09T04:39:40.406281+0000","last_unstale":"2023-10-09T04:41:41.813199+0000","last_undegraded":"2023-10-09T04:41:41.813199+0000","last_fullsized":"2023-10-09T04:41:41.813199+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:09.883302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":28,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806333+0000","last_change":"2023-10-09T04:41:02.634559+0000","last_active":"2023-10-09T04:41:41.806333+0000","last_peered":"2023-10-09T04:41:41.806333+0000","last_clean":"2023-10-09T04:41:41.806333+0000","last_became_active":"2023-10-09T04:41:02.624597+0000","last_became_peered":"2023-10-09T04:41:02.624597+0000","last_unstale":"2023-10-09T04:41:41.806333+0000","last_undegraded":"2023-10-09T04:41:41.806333+0000","last_fullsized":"2023-10-09T04:41:41.806333+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:57.939757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":230,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805576+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:41:41.805576+0000","last_peered":"2023-10-09T04:41:41.805576+0000","last_clean":"2023-10-09T04:41:41.805576+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:41:41.805576+0000","last_undegraded":"2023-10-09T04:41:41.805576+0000","last_fullsized":"2023-10-09T04:41:41.805576+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":280,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805502+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:41:41.805502+0000","last_peered":"2023-10-09T04:41:41.805502+0000","last_clean":"2023-10-09T04:41:41.805502+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:41:41.805502+0000","last_undegraded":"2023-10-09T04:41:41.805502+0000","last_fullsized":"2023-10-09T04:41:41.805502+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":222,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805521+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:41:41.805521+0000","last_peered":"2023-10-09T04:41:41.805521+0000","last_clean":"2023-10-09T04:41:41.805521+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:41:41.805521+0000","last_undegraded":"2023-10-09T04:41:41.805521+0000","last_fullsized":"2023-10-09T04:41:41.805521+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":182,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805137+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:41:41.805137+0000","last_peered":"2023-10-09T04:41:41.805137+0000","last_clean":"2023-10-09T04:41:41.805137+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:41:41.805137+0000","last_undegraded":"2023-10-09T04:41:41.805137+0000","last_fullsized":"2023-10-09T04:41:41.805137+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805750+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:41:41.805750+0000","last_peered":"2023-10-09T04:41:41.805750+0000","last_clean":"2023-10-09T04:41:41.805750+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:41:41.805750+0000","last_undegraded":"2023-10-09T04:41:41.805750+0000","last_fullsized":"2023-10-09T04:41:41.805750+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":212,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804932+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:41:41.804932+0000","last_peered":"2023-10-09T04:41:41.804932+0000","last_clean":"2023-10-09T04:41:41.804932+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:41:41.804932+0000","last_undegraded":"2023-10-09T04:41:41.804932+0000","last_fullsized":"2023-10-09T04:41:41.804932+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":184,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806144+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:41:41.806144+0000","last_peered":"2023-10-09T04:41:41.806144+0000","last_clean":"2023-10-09T04:41:41.806144+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:41:41.806144+0000","last_undegraded":"2023-10-09T04:41:41.806144+0000","last_fullsized":"2023-10-09T04:41:41.806144+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":276,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.808485+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:41:41.808485+0000","last_peered":"2023-10-09T04:41:41.808485+0000","last_clean":"2023-10-09T04:41:41.808485+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:41:41.808485+0000","last_undegraded":"2023-10-09T04:41:41.808485+0000","last_fullsized":"2023-10-09T04:41:41.808485+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":222,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806431+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:41:41.806431+0000","last_peered":"2023-10-09T04:41:41.806431+0000","last_clean":"2023-10-09T04:41:41.806431+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:41:41.806431+0000","last_undegraded":"2023-10-09T04:41:41.806431+0000","last_fullsized":"2023-10-09T04:41:41.806431+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"270'10989","reported_seq":17752,"reported_epoch":270,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:41:50.420925+0000","last_change":"2023-10-09T04:41:02.787388+0000","last_active":"2023-10-09T04:41:50.420925+0000","last_peered":"2023-10-09T04:41:50.420925+0000","last_clean":"2023-10-09T04:39:11.356784+0000","last_became_active":"2023-10-09T04:41:02.727633+0000","last_became_peered":"2023-10-09T04:41:02.727633+0000","last_unstale":"2023-10-09T04:41:50.420925+0000","last_undegraded":"2023-10-09T04:41:50.420925+0000","last_fullsized":"2023-10-09T04:41:02.591500+0000","mapping_epoch":261,"log_start":"242'900","ondisk_log_start":"242'900","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10089,"log_dups_size":0,"ondisk_log_size":10089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:39.473622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112394240,"num_objects":1715,"num_object_clones":0,"num_object_copies":5145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10989,"num_write_kb":406528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3351,"num_bytes_recovered":43122688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1715}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":286,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806949+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:41:41.806949+0000","last_peered":"2023-10-09T04:41:41.806949+0000","last_clean":"2023-10-09T04:41:41.806949+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:41:41.806949+0000","last_undegraded":"2023-10-09T04:41:41.806949+0000","last_fullsized":"2023-10-09T04:41:41.806949+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":212,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805321+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:41:41.805321+0000","last_peered":"2023-10-09T04:41:41.805321+0000","last_clean":"2023-10-09T04:41:41.805321+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:41:41.805321+0000","last_undegraded":"2023-10-09T04:41:41.805321+0000","last_fullsized":"2023-10-09T04:41:41.805321+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"270'10574","reported_seq":10617,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:46.708005+0000","last_change":"2023-10-09T04:37:52.255599+0000","last_active":"2023-10-09T04:41:46.708005+0000","last_peered":"2023-10-09T04:41:46.708005+0000","last_clean":"2023-10-09T04:41:46.708005+0000","last_became_active":"2023-10-09T04:37:52.255291+0000","last_became_peered":"2023-10-09T04:37:52.255291+0000","last_unstale":"2023-10-09T04:41:46.708005+0000","last_undegraded":"2023-10-09T04:41:46.708005+0000","last_fullsized":"2023-10-09T04:41:46.708005+0000","mapping_epoch":237,"log_start":"242'500","ondisk_log_start":"242'500","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.990132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125698048,"num_objects":1918,"num_object_clones":0,"num_object_copies":5754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10574,"num_write_kb":399744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":259,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.811133+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:41:41.811133+0000","last_peered":"2023-10-09T04:41:41.811133+0000","last_clean":"2023-10-09T04:41:41.811133+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:41:41.811133+0000","last_undegraded":"2023-10-09T04:41:41.811133+0000","last_fullsized":"2023-10-09T04:41:41.811133+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":247,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804808+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:41:41.804808+0000","last_peered":"2023-10-09T04:41:41.804808+0000","last_clean":"2023-10-09T04:41:41.804808+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:41:41.804808+0000","last_undegraded":"2023-10-09T04:41:41.804808+0000","last_fullsized":"2023-10-09T04:41:41.804808+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"270'10886","reported_seq":10927,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:50.088771+0000","last_change":"2023-10-09T04:37:52.013171+0000","last_active":"2023-10-09T04:41:50.088771+0000","last_peered":"2023-10-09T04:41:50.088771+0000","last_clean":"2023-10-09T04:41:50.088771+0000","last_became_active":"2023-10-09T04:37:52.012873+0000","last_became_peered":"2023-10-09T04:37:52.012873+0000","last_unstale":"2023-10-09T04:41:50.088771+0000","last_undegraded":"2023-10-09T04:41:50.088771+0000","last_fullsized":"2023-10-09T04:41:50.088771+0000","mapping_epoch":237,"log_start":"243'2700","ondisk_log_start":"243'2700","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8186,"log_dups_size":0,"ondisk_log_size":8186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:25.833222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116654080,"num_objects":1780,"num_object_clones":0,"num_object_copies":5340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10886,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":213,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805390+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:41:41.805390+0000","last_peered":"2023-10-09T04:41:41.805390+0000","last_clean":"2023-10-09T04:41:41.805390+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:41:41.805390+0000","last_undegraded":"2023-10-09T04:41:41.805390+0000","last_fullsized":"2023-10-09T04:41:41.805390+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":254,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.812849+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:41:41.812849+0000","last_peered":"2023-10-09T04:41:41.812849+0000","last_clean":"2023-10-09T04:41:41.812849+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:41:41.812849+0000","last_undegraded":"2023-10-09T04:41:41.812849+0000","last_fullsized":"2023-10-09T04:41:41.812849+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"270'10787","reported_seq":10828,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:50.087683+0000","last_change":"2023-10-09T04:37:52.012019+0000","last_active":"2023-10-09T04:41:50.087683+0000","last_peered":"2023-10-09T04:41:50.087683+0000","last_clean":"2023-10-09T04:41:50.087683+0000","last_became_active":"2023-10-09T04:37:52.011714+0000","last_became_peered":"2023-10-09T04:37:52.011714+0000","last_unstale":"2023-10-09T04:41:50.087683+0000","last_undegraded":"2023-10-09T04:41:50.087683+0000","last_fullsized":"2023-10-09T04:41:50.087683+0000","mapping_epoch":237,"log_start":"243'2600","ondisk_log_start":"243'2600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:55.324035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122355712,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10787,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":230,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805411+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:41:41.805411+0000","last_peered":"2023-10-09T04:41:41.805411+0000","last_clean":"2023-10-09T04:41:41.805411+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:41:41.805411+0000","last_undegraded":"2023-10-09T04:41:41.805411+0000","last_fullsized":"2023-10-09T04:41:41.805411+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804672+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:41:41.804672+0000","last_peered":"2023-10-09T04:41:41.804672+0000","last_clean":"2023-10-09T04:41:41.804672+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:41:41.804672+0000","last_undegraded":"2023-10-09T04:41:41.804672+0000","last_fullsized":"2023-10-09T04:41:41.804672+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"270'10589","reported_seq":10630,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:49.940468+0000","last_change":"2023-10-09T04:37:52.013681+0000","last_active":"2023-10-09T04:41:49.940468+0000","last_peered":"2023-10-09T04:41:49.940468+0000","last_clean":"2023-10-09T04:41:49.940468+0000","last_became_active":"2023-10-09T04:37:52.013382+0000","last_became_peered":"2023-10-09T04:37:52.013382+0000","last_unstale":"2023-10-09T04:41:49.940468+0000","last_undegraded":"2023-10-09T04:41:49.940468+0000","last_fullsized":"2023-10-09T04:41:49.940468+0000","mapping_epoch":237,"log_start":"242'2000","ondisk_log_start":"242'2000","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8589,"log_dups_size":0,"ondisk_log_size":8589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:30.877521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117374976,"num_objects":1791,"num_object_clones":0,"num_object_copies":5373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10589,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":267,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805432+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:41:41.805432+0000","last_peered":"2023-10-09T04:41:41.805432+0000","last_clean":"2023-10-09T04:41:41.805432+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:41:41.805432+0000","last_undegraded":"2023-10-09T04:41:41.805432+0000","last_fullsized":"2023-10-09T04:41:41.805432+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806408+0000","last_change":"2023-10-09T04:41:02.624962+0000","last_active":"2023-10-09T04:41:41.806408+0000","last_peered":"2023-10-09T04:41:41.806408+0000","last_clean":"2023-10-09T04:41:41.806408+0000","last_became_active":"2023-10-09T04:41:02.624555+0000","last_became_peered":"2023-10-09T04:41:02.624555+0000","last_unstale":"2023-10-09T04:41:41.806408+0000","last_undegraded":"2023-10-09T04:41:41.806408+0000","last_fullsized":"2023-10-09T04:41:41.806408+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:48.506794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"270'10608","reported_seq":10651,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:46.684430+0000","last_change":"2023-10-09T04:37:52.255728+0000","last_active":"2023-10-09T04:41:46.684430+0000","last_peered":"2023-10-09T04:41:46.684430+0000","last_clean":"2023-10-09T04:41:46.684430+0000","last_became_active":"2023-10-09T04:37:52.255283+0000","last_became_peered":"2023-10-09T04:37:52.255283+0000","last_unstale":"2023-10-09T04:41:46.684430+0000","last_undegraded":"2023-10-09T04:41:46.684430+0000","last_fullsized":"2023-10-09T04:41:46.684430+0000","mapping_epoch":237,"log_start":"242'600","ondisk_log_start":"242'600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:18.900085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124387328,"num_objects":1898,"num_object_clones":0,"num_object_copies":5694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10608,"num_write_kb":400192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.812985+0000","last_change":"2023-10-09T04:41:02.603773+0000","last_active":"2023-10-09T04:41:41.812985+0000","last_peered":"2023-10-09T04:41:41.812985+0000","last_clean":"2023-10-09T04:41:41.812985+0000","last_became_active":"2023-10-09T04:41:02.603407+0000","last_became_peered":"2023-10-09T04:41:02.603407+0000","last_unstale":"2023-10-09T04:41:41.812985+0000","last_undegraded":"2023-10-09T04:41:41.812985+0000","last_fullsized":"2023-10-09T04:41:41.812985+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:18.401503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.810689+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:41:41.810689+0000","last_peered":"2023-10-09T04:41:41.810689+0000","last_clean":"2023-10-09T04:41:41.810689+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:41:41.810689+0000","last_undegraded":"2023-10-09T04:41:41.810689+0000","last_fullsized":"2023-10-09T04:41:41.810689+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"270'10764","reported_seq":10805,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:49.918355+0000","last_change":"2023-10-09T04:37:52.256288+0000","last_active":"2023-10-09T04:41:49.918355+0000","last_peered":"2023-10-09T04:41:49.918355+0000","last_clean":"2023-10-09T04:41:49.918355+0000","last_became_active":"2023-10-09T04:37:52.256008+0000","last_became_peered":"2023-10-09T04:37:52.256008+0000","last_unstale":"2023-10-09T04:41:49.918355+0000","last_undegraded":"2023-10-09T04:41:49.918355+0000","last_fullsized":"2023-10-09T04:41:49.918355+0000","mapping_epoch":237,"log_start":"242'2100","ondisk_log_start":"242'2100","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8664,"log_dups_size":0,"ondisk_log_size":8664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:34.570497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107347968,"num_objects":1638,"num_object_clones":0,"num_object_copies":4914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10764,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804581+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:41:41.804581+0000","last_peered":"2023-10-09T04:41:41.804581+0000","last_clean":"2023-10-09T04:41:41.804581+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:41:41.804581+0000","last_undegraded":"2023-10-09T04:41:41.804581+0000","last_fullsized":"2023-10-09T04:41:41.804581+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":230,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806371+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:41:41.806371+0000","last_peered":"2023-10-09T04:41:41.806371+0000","last_clean":"2023-10-09T04:41:41.806371+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:41:41.806371+0000","last_undegraded":"2023-10-09T04:41:41.806371+0000","last_fullsized":"2023-10-09T04:41:41.806371+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"270'10918","reported_seq":10969,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:47.569385+0000","last_change":"2023-10-09T04:39:40.403627+0000","last_active":"2023-10-09T04:41:47.569385+0000","last_peered":"2023-10-09T04:41:47.569385+0000","last_clean":"2023-10-09T04:41:47.569385+0000","last_became_active":"2023-10-09T04:39:40.393228+0000","last_became_peered":"2023-10-09T04:39:40.393228+0000","last_unstale":"2023-10-09T04:41:47.569385+0000","last_undegraded":"2023-10-09T04:41:47.569385+0000","last_fullsized":"2023-10-09T04:41:47.569385+0000","mapping_epoch":251,"log_start":"243'3000","ondisk_log_start":"243'3000","created":237,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7918,"log_dups_size":0,"ondisk_log_size":7918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:22.432701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125960192,"num_objects":1922,"num_object_clones":0,"num_object_copies":5766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10918,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":224,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806230+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:41:41.806230+0000","last_peered":"2023-10-09T04:41:41.806230+0000","last_clean":"2023-10-09T04:41:41.806230+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:41:41.806230+0000","last_undegraded":"2023-10-09T04:41:41.806230+0000","last_fullsized":"2023-10-09T04:41:41.806230+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":253,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806278+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:41:41.806278+0000","last_peered":"2023-10-09T04:41:41.806278+0000","last_clean":"2023-10-09T04:41:41.806278+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:41:41.806278+0000","last_undegraded":"2023-10-09T04:41:41.806278+0000","last_fullsized":"2023-10-09T04:41:41.806278+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"270'10718","reported_seq":16085,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:41:46.753573+0000","last_change":"2023-10-09T04:41:02.761588+0000","last_active":"2023-10-09T04:41:46.753573+0000","last_peered":"2023-10-09T04:41:46.753573+0000","last_clean":"2023-10-09T04:39:11.377114+0000","last_became_active":"2023-10-09T04:41:02.731395+0000","last_became_peered":"2023-10-09T04:41:02.731395+0000","last_unstale":"2023-10-09T04:41:46.753573+0000","last_undegraded":"2023-10-09T04:41:02.602613+0000","last_fullsized":"2023-10-09T04:41:02.586350+0000","mapping_epoch":261,"log_start":"242'700","ondisk_log_start":"242'700","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:11.256631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124715032,"num_objects":1904,"num_object_clones":0,"num_object_copies":5712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3595,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1904,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10718,"num_write_kb":403841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2653,"num_bytes_recovered":173867008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,6","objects":707},{"shards":"2,6","objects":1197}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":147,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805322+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:41:41.805322+0000","last_peered":"2023-10-09T04:41:41.805322+0000","last_clean":"2023-10-09T04:41:41.805322+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:41:41.805322+0000","last_undegraded":"2023-10-09T04:41:41.805322+0000","last_fullsized":"2023-10-09T04:41:41.805322+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":264,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805826+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:41:41.805826+0000","last_peered":"2023-10-09T04:41:41.805826+0000","last_clean":"2023-10-09T04:41:41.805826+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:41:41.805826+0000","last_undegraded":"2023-10-09T04:41:41.805826+0000","last_fullsized":"2023-10-09T04:41:41.805826+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":241,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804903+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:41:41.804903+0000","last_peered":"2023-10-09T04:41:41.804903+0000","last_clean":"2023-10-09T04:41:41.804903+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:41:41.804903+0000","last_undegraded":"2023-10-09T04:41:41.804903+0000","last_fullsized":"2023-10-09T04:41:41.804903+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"270'10525","reported_seq":10568,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:47.578393+0000","last_change":"2023-10-09T04:37:52.255930+0000","last_active":"2023-10-09T04:41:47.578393+0000","last_peered":"2023-10-09T04:41:47.578393+0000","last_clean":"2023-10-09T04:41:47.578393+0000","last_became_active":"2023-10-09T04:37:52.255584+0000","last_became_peered":"2023-10-09T04:37:52.255584+0000","last_unstale":"2023-10-09T04:41:47.578393+0000","last_undegraded":"2023-10-09T04:41:47.578393+0000","last_fullsized":"2023-10-09T04:41:47.578393+0000","mapping_epoch":237,"log_start":"243'2600","ondisk_log_start":"243'2600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7925,"log_dups_size":0,"ondisk_log_size":7925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:52.503096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122748928,"num_objects":1873,"num_object_clones":0,"num_object_copies":5619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10525,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":234,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804968+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:41:41.804968+0000","last_peered":"2023-10-09T04:41:41.804968+0000","last_clean":"2023-10-09T04:41:41.804968+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:41:41.804968+0000","last_undegraded":"2023-10-09T04:41:41.804968+0000","last_fullsized":"2023-10-09T04:41:41.804968+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805298+0000","last_change":"2023-10-09T04:41:02.608773+0000","last_active":"2023-10-09T04:41:41.805298+0000","last_peered":"2023-10-09T04:41:41.805298+0000","last_clean":"2023-10-09T04:41:41.805298+0000","last_became_active":"2023-10-09T04:41:02.608455+0000","last_became_peered":"2023-10-09T04:41:02.608455+0000","last_unstale":"2023-10-09T04:41:41.805298+0000","last_undegraded":"2023-10-09T04:41:41.805298+0000","last_fullsized":"2023-10-09T04:41:41.805298+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:29.807000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":240,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:45.846515+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:41:45.846515+0000","last_peered":"2023-10-09T04:41:45.846515+0000","last_clean":"2023-10-09T04:41:45.846515+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:41:45.846515+0000","last_undegraded":"2023-10-09T04:41:45.846515+0000","last_fullsized":"2023-10-09T04:41:45.846515+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"270'10501","reported_seq":10544,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:47.601040+0000","last_change":"2023-10-09T04:37:52.012774+0000","last_active":"2023-10-09T04:41:47.601040+0000","last_peered":"2023-10-09T04:41:47.601040+0000","last_clean":"2023-10-09T04:41:47.601040+0000","last_became_active":"2023-10-09T04:37:52.012402+0000","last_became_peered":"2023-10-09T04:37:52.012402+0000","last_unstale":"2023-10-09T04:41:47.601040+0000","last_undegraded":"2023-10-09T04:41:47.601040+0000","last_fullsized":"2023-10-09T04:41:47.601040+0000","mapping_epoch":237,"log_start":"243'2600","ondisk_log_start":"243'2600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7901,"log_dups_size":0,"ondisk_log_size":7901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:37.167042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120389632,"num_objects":1837,"num_object_clones":0,"num_object_copies":5511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10501,"num_write_kb":394816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":226,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.793517+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:41:41.793517+0000","last_peered":"2023-10-09T04:41:41.793517+0000","last_clean":"2023-10-09T04:41:41.793517+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:41:41.793517+0000","last_undegraded":"2023-10-09T04:41:41.793517+0000","last_fullsized":"2023-10-09T04:41:41.793517+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":225,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.793364+0000","last_change":"2023-10-09T04:39:40.406420+0000","last_active":"2023-10-09T04:41:41.793364+0000","last_peered":"2023-10-09T04:41:41.793364+0000","last_clean":"2023-10-09T04:41:41.793364+0000","last_became_active":"2023-10-09T04:39:40.406072+0000","last_became_peered":"2023-10-09T04:39:40.406072+0000","last_unstale":"2023-10-09T04:41:41.793364+0000","last_undegraded":"2023-10-09T04:41:41.793364+0000","last_fullsized":"2023-10-09T04:41:41.793364+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:08.385113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806001+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:41:41.806001+0000","last_peered":"2023-10-09T04:41:41.806001+0000","last_clean":"2023-10-09T04:41:41.806001+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:41:41.806001+0000","last_undegraded":"2023-10-09T04:41:41.806001+0000","last_fullsized":"2023-10-09T04:41:41.806001+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"270'10958","reported_seq":11001,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:47.591146+0000","last_change":"2023-10-09T04:37:52.013105+0000","last_active":"2023-10-09T04:41:47.591146+0000","last_peered":"2023-10-09T04:41:47.591146+0000","last_clean":"2023-10-09T04:41:47.591146+0000","last_became_active":"2023-10-09T04:37:52.012773+0000","last_became_peered":"2023-10-09T04:37:52.012773+0000","last_unstale":"2023-10-09T04:41:47.591146+0000","last_undegraded":"2023-10-09T04:41:47.591146+0000","last_fullsized":"2023-10-09T04:41:47.591146+0000","mapping_epoch":237,"log_start":"243'3000","ondisk_log_start":"243'3000","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.291873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123731968,"num_objects":1888,"num_object_clones":0,"num_object_copies":5664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10958,"num_write_kb":411072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":251,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804753+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:41:41.804753+0000","last_peered":"2023-10-09T04:41:41.804753+0000","last_clean":"2023-10-09T04:41:41.804753+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:41:41.804753+0000","last_undegraded":"2023-10-09T04:41:41.804753+0000","last_fullsized":"2023-10-09T04:41:41.804753+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":662,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804952+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:41:41.804952+0000","last_peered":"2023-10-09T04:41:41.804952+0000","last_clean":"2023-10-09T04:41:41.804952+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:41:41.804952+0000","last_undegraded":"2023-10-09T04:41:41.804952+0000","last_fullsized":"2023-10-09T04:41:41.804952+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":195,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805375+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:41:41.805375+0000","last_peered":"2023-10-09T04:41:41.805375+0000","last_clean":"2023-10-09T04:41:41.805375+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:41:41.805375+0000","last_undegraded":"2023-10-09T04:41:41.805375+0000","last_fullsized":"2023-10-09T04:41:41.805375+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":230,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.811215+0000","last_change":"2023-10-09T04:41:02.612580+0000","last_active":"2023-10-09T04:41:41.811215+0000","last_peered":"2023-10-09T04:41:41.811215+0000","last_clean":"2023-10-09T04:41:41.811215+0000","last_became_active":"2023-10-09T04:41:02.612258+0000","last_became_peered":"2023-10-09T04:41:02.612258+0000","last_unstale":"2023-10-09T04:41:41.811215+0000","last_undegraded":"2023-10-09T04:41:41.811215+0000","last_fullsized":"2023-10-09T04:41:41.811215+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:53.656800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"270'10638","reported_seq":10681,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:47.608200+0000","last_change":"2023-10-09T04:37:52.257119+0000","last_active":"2023-10-09T04:41:47.608200+0000","last_peered":"2023-10-09T04:41:47.608200+0000","last_clean":"2023-10-09T04:41:47.608200+0000","last_became_active":"2023-10-09T04:37:52.256760+0000","last_became_peered":"2023-10-09T04:37:52.256760+0000","last_unstale":"2023-10-09T04:41:47.608200+0000","last_undegraded":"2023-10-09T04:41:47.608200+0000","last_fullsized":"2023-10-09T04:41:47.608200+0000","mapping_epoch":237,"log_start":"243'2700","ondisk_log_start":"243'2700","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:16.001253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125304832,"num_objects":1912,"num_object_clones":0,"num_object_copies":5736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10638,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":270,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804339+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:41:41.804339+0000","last_peered":"2023-10-09T04:41:41.804339+0000","last_clean":"2023-10-09T04:41:41.804339+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:41:41.804339+0000","last_undegraded":"2023-10-09T04:41:41.804339+0000","last_fullsized":"2023-10-09T04:41:41.804339+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804732+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:41:41.804732+0000","last_peered":"2023-10-09T04:41:41.804732+0000","last_clean":"2023-10-09T04:41:41.804732+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:41:41.804732+0000","last_undegraded":"2023-10-09T04:41:41.804732+0000","last_fullsized":"2023-10-09T04:41:41.804732+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":146,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.809212+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:41:41.809212+0000","last_peered":"2023-10-09T04:41:41.809212+0000","last_clean":"2023-10-09T04:41:41.809212+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:41:41.809212+0000","last_undegraded":"2023-10-09T04:41:41.809212+0000","last_fullsized":"2023-10-09T04:41:41.809212+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"270'10553","reported_seq":16130,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:41:46.690102+0000","last_change":"2023-10-09T04:41:03.882872+0000","last_active":"2023-10-09T04:41:46.690102+0000","last_peered":"2023-10-09T04:41:46.690102+0000","last_clean":"2023-10-09T04:39:11.375310+0000","last_became_active":"2023-10-09T04:41:03.851493+0000","last_became_peered":"2023-10-09T04:41:03.851493+0000","last_unstale":"2023-10-09T04:41:46.690102+0000","last_undegraded":"2023-10-09T04:41:03.605177+0000","last_fullsized":"2023-10-09T04:41:03.588185+0000","mapping_epoch":262,"log_start":"242'500","ondisk_log_start":"242'500","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10053,"log_dups_size":0,"ondisk_log_size":10053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:51.273314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3394,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10553,"num_write_kb":398016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2757,"num_bytes_recovered":180682752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":1249},{"shards":"1,2,7","objects":635}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":273,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805471+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:41:41.805471+0000","last_peered":"2023-10-09T04:41:41.805471+0000","last_clean":"2023-10-09T04:41:41.805471+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:41:41.805471+0000","last_undegraded":"2023-10-09T04:41:41.805471+0000","last_fullsized":"2023-10-09T04:41:41.805471+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":225,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:45.846435+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:41:45.846435+0000","last_peered":"2023-10-09T04:41:45.846435+0000","last_clean":"2023-10-09T04:41:45.846435+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:41:45.846435+0000","last_undegraded":"2023-10-09T04:41:45.846435+0000","last_fullsized":"2023-10-09T04:41:45.846435+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":114,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804827+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:41:41.804827+0000","last_peered":"2023-10-09T04:41:41.804827+0000","last_clean":"2023-10-09T04:41:41.804827+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:41:41.804827+0000","last_undegraded":"2023-10-09T04:41:41.804827+0000","last_fullsized":"2023-10-09T04:41:41.804827+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"270'10764","reported_seq":10804,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:50.139971+0000","last_change":"2023-10-09T04:37:52.013669+0000","last_active":"2023-10-09T04:41:50.139971+0000","last_peered":"2023-10-09T04:41:50.139971+0000","last_clean":"2023-10-09T04:41:50.139971+0000","last_became_active":"2023-10-09T04:37:52.013386+0000","last_became_peered":"2023-10-09T04:37:52.013386+0000","last_unstale":"2023-10-09T04:41:50.139971+0000","last_undegraded":"2023-10-09T04:41:50.139971+0000","last_fullsized":"2023-10-09T04:41:50.139971+0000","mapping_epoch":237,"log_start":"243'2600","ondisk_log_start":"243'2600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8164,"log_dups_size":0,"ondisk_log_size":8164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:57.288541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115736576,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10764,"num_write_kb":400960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":267,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806358+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:41:41.806358+0000","last_peered":"2023-10-09T04:41:41.806358+0000","last_clean":"2023-10-09T04:41:41.806358+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:41:41.806358+0000","last_undegraded":"2023-10-09T04:41:41.806358+0000","last_fullsized":"2023-10-09T04:41:41.806358+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":240,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.808025+0000","last_change":"2023-10-09T04:41:02.629667+0000","last_active":"2023-10-09T04:41:41.808025+0000","last_peered":"2023-10-09T04:41:41.808025+0000","last_clean":"2023-10-09T04:41:41.808025+0000","last_became_active":"2023-10-09T04:41:02.629305+0000","last_became_peered":"2023-10-09T04:41:02.629305+0000","last_unstale":"2023-10-09T04:41:41.808025+0000","last_undegraded":"2023-10-09T04:41:41.808025+0000","last_fullsized":"2023-10-09T04:41:41.808025+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:36.600453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806299+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:41:41.806299+0000","last_peered":"2023-10-09T04:41:41.806299+0000","last_clean":"2023-10-09T04:41:41.806299+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:41:41.806299+0000","last_undegraded":"2023-10-09T04:41:41.806299+0000","last_fullsized":"2023-10-09T04:41:41.806299+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"270'10788","reported_seq":16059,"reported_epoch":270,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:41:47.150459+0000","last_change":"2023-10-09T04:41:03.880649+0000","last_active":"2023-10-09T04:41:47.150459+0000","last_peered":"2023-10-09T04:41:47.150459+0000","last_clean":"2023-10-09T04:39:11.359026+0000","last_became_active":"2023-10-09T04:41:03.853327+0000","last_became_peered":"2023-10-09T04:41:03.853327+0000","last_unstale":"2023-10-09T04:41:47.150459+0000","last_undegraded":"2023-10-09T04:41:03.604660+0000","last_fullsized":"2023-10-09T04:41:03.586462+0000","mapping_epoch":262,"log_start":"244'3398","ondisk_log_start":"244'3398","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:51.288086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128057344,"num_objects":1954,"num_object_clones":0,"num_object_copies":5862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3522,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10788,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2607,"num_bytes_recovered":170524672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":680},{"shards":"1,7","objects":1274}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":249,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.808106+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:41:41.808106+0000","last_peered":"2023-10-09T04:41:41.808106+0000","last_clean":"2023-10-09T04:41:41.808106+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:41:41.808106+0000","last_undegraded":"2023-10-09T04:41:41.808106+0000","last_fullsized":"2023-10-09T04:41:41.808106+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":42,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806507+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:41:41.806507+0000","last_peered":"2023-10-09T04:41:41.806507+0000","last_clean":"2023-10-09T04:41:41.806507+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:41:41.806507+0000","last_undegraded":"2023-10-09T04:41:41.806507+0000","last_fullsized":"2023-10-09T04:41:41.806507+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804493+0000","last_change":"2023-10-09T04:41:02.614004+0000","last_active":"2023-10-09T04:41:41.804493+0000","last_peered":"2023-10-09T04:41:41.804493+0000","last_clean":"2023-10-09T04:41:41.804493+0000","last_became_active":"2023-10-09T04:41:02.612300+0000","last_became_peered":"2023-10-09T04:41:02.612300+0000","last_unstale":"2023-10-09T04:41:41.804493+0000","last_undegraded":"2023-10-09T04:41:41.804493+0000","last_fullsized":"2023-10-09T04:41:41.804493+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:04.375059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":110,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805499+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:41:41.805499+0000","last_peered":"2023-10-09T04:41:41.805499+0000","last_clean":"2023-10-09T04:41:41.805499+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:41:41.805499+0000","last_undegraded":"2023-10-09T04:41:41.805499+0000","last_fullsized":"2023-10-09T04:41:41.805499+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":219,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-10-09T04:41:45.846598+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:41:45.846598+0000","last_peered":"2023-10-09T04:41:45.846598+0000","last_clean":"2023-10-09T04:41:45.846598+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:41:45.846598+0000","last_undegraded":"2023-10-09T04:41:45.846598+0000","last_fullsized":"2023-10-09T04:41:45.846598+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":260,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804990+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:41:41.804990+0000","last_peered":"2023-10-09T04:41:41.804990+0000","last_clean":"2023-10-09T04:41:41.804990+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:41:41.804990+0000","last_undegraded":"2023-10-09T04:41:41.804990+0000","last_fullsized":"2023-10-09T04:41:41.804990+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805370+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:41:41.805370+0000","last_peered":"2023-10-09T04:41:41.805370+0000","last_clean":"2023-10-09T04:41:41.805370+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:41:41.805370+0000","last_undegraded":"2023-10-09T04:41:41.805370+0000","last_fullsized":"2023-10-09T04:41:41.805370+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":215,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805907+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:41:41.805907+0000","last_peered":"2023-10-09T04:41:41.805907+0000","last_clean":"2023-10-09T04:41:41.805907+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:41:41.805907+0000","last_undegraded":"2023-10-09T04:41:41.805907+0000","last_fullsized":"2023-10-09T04:41:41.805907+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.805162+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:41:41.805162+0000","last_peered":"2023-10-09T04:41:41.805162+0000","last_clean":"2023-10-09T04:41:41.805162+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:41:41.805162+0000","last_undegraded":"2023-10-09T04:41:41.805162+0000","last_fullsized":"2023-10-09T04:41:41.805162+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":257,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806026+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:41:41.806026+0000","last_peered":"2023-10-09T04:41:41.806026+0000","last_clean":"2023-10-09T04:41:41.806026+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:41:41.806026+0000","last_undegraded":"2023-10-09T04:41:41.806026+0000","last_fullsized":"2023-10-09T04:41:41.806026+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":289,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.807124+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:41:41.807124+0000","last_peered":"2023-10-09T04:41:41.807124+0000","last_clean":"2023-10-09T04:41:41.807124+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:41:41.807124+0000","last_undegraded":"2023-10-09T04:41:41.807124+0000","last_fullsized":"2023-10-09T04:41:41.807124+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":144,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.806138+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:41:41.806138+0000","last_peered":"2023-10-09T04:41:41.806138+0000","last_clean":"2023-10-09T04:41:41.806138+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:41:41.806138+0000","last_undegraded":"2023-10-09T04:41:41.806138+0000","last_fullsized":"2023-10-09T04:41:41.806138+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":18,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-10-09T04:41:41.804541+0000","last_change":"2023-10-09T04:41:02.608725+0000","last_active":"2023-10-09T04:41:41.804541+0000","last_peered":"2023-10-09T04:41:41.804541+0000","last_clean":"2023-10-09T04:41:41.804541+0000","last_became_active":"2023-10-09T04:41:02.606509+0000","last_became_peered":"2023-10-09T04:41:02.606509+0000","last_unstale":"2023-10-09T04:41:41.804541+0000","last_undegraded":"2023-10-09T04:41:41.804541+0000","last_fullsized":"2023-10-09T04:41:41.804541+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:49.430101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1936392216,"num_objects":29548,"num_object_clones":0,"num_object_copies":88644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10511,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29548,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171560,"num_write_kb":6435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11368,"num_bytes_recovered":568197120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6384332800,"data_stored":6384320584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139062,"ondisk_log_size":139062,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":5624,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234419,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1652128,"kb_used_data":741144,"kb_used_omap":0,"kb_used_meta":910976,"kb_avail":92093024,"statfs":{"total":95995035648,"available":94303256576,"internally_reserved":0,"allocated":758931456,"data_stored":758343783,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932839424},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":2.6349999999999998,"15min":4.1559999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":14.571999999999999,"5min":44.122999999999998,"15min":186.446},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.992,"5min":3.0579999999999998,"15min":4.3620000000000001},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":14.170999999999999,"5min":43.313000000000002,"15min":186.429},"last":1.5960000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:41:35 2023","interfaces":[{"interface":"back","average":{"1min":3.4449999999999998,"5min":6.1289999999999996,"15min":5.3879999999999999},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":19.298999999999999,"5min":264.62,"15min":264.62},"last":0.85599999999999998},{"interface":"front","average":{"1min":2.9220000000000002,"5min":6.4630000000000001,"15min":5.4969999999999999},"min":{"1min":0.29899999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":19.251999999999999,"5min":264.65899999999999,"15min":264.65899999999999},"last":3.3860000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:40:54 2023","interfaces":[{"interface":"back","average":{"1min":12.686999999999999,"5min":5.6609999999999996,"15min":3.0880000000000001},"min":{"1min":0.33600000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":108.878,"5min":184.72,"15min":184.72},"last":2.1019999999999999},{"interface":"front","average":{"1min":12.831,"5min":5.9039999999999999,"15min":3.5030000000000001},"min":{"1min":0.33800000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":108.81,"5min":184.697,"15min":184.697},"last":1.653}]},{"osd":3,"last update":"Mon Oct 9 04:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.4220000000000002,"5min":7.0350000000000001,"15min":7.5499999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":20.469999999999999,"5min":182.839,"15min":249.989},"last":0.71099999999999997},{"interface":"front","average":{"1min":4.8120000000000003,"5min":7.1369999999999996,"15min":7.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":20.765000000000001,"5min":182.678,"15min":250.21700000000001},"last":7.2409999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.284,"15min":1.395},"min":{"1min":0.27100000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.3819999999999999,"5min":11.752000000000001,"15min":11.752000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.349,"15min":1.163},"min":{"1min":0.31900000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":6.319,"5min":9.4179999999999993,"15min":10.613},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":1,"15min":1.004},"min":{"1min":0.29399999999999998,"5min":0.28199999999999997,"15min":0.27900000000000003},"max":{"1min":1.3540000000000001,"5min":6.1120000000000001,"15min":6.6749999999999998},"last":3.911},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.304,"15min":1.331},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.7429999999999999,"5min":8.8089999999999993,"15min":10.895},"last":0.55000000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":1.1990000000000001,"15min":1.232},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":0.86099999999999999,"5min":8.1120000000000001,"15min":12.007},"last":3.8199999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.464,"15min":1.476},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.5640000000000001,"5min":8.7789999999999999,"15min":10.917},"last":0.68300000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496053,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1828052,"kb_used_data":929868,"kb_used_omap":0,"kb_used_meta":898176,"kb_avail":91917100,"statfs":{"total":95995035648,"available":94123110400,"internally_reserved":0,"allocated":952184832,"data_stored":951609002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919732224},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":4.5620000000000003,"15min":8.1639999999999997},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":5.383,"5min":113.86799999999999,"15min":1284.521},"last":5.1139999999999999},{"interface":"front","average":{"1min":2.3540000000000001,"5min":5.008,"15min":8.2889999999999997},"min":{"1min":0.26600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.0050000000000008,"5min":109.63500000000001,"15min":1284.4690000000001},"last":1.9870000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:41:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9660000000000002,"5min":7.1600000000000001,"15min":5.3419999999999996},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":21.939,"5min":150.75,"15min":185.74000000000001},"last":5.2789999999999999},{"interface":"front","average":{"1min":3.5760000000000001,"5min":7.3040000000000003,"15min":5.4509999999999996},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.26100000000000001},"max":{"1min":21.751999999999999,"5min":150.81999999999999,"15min":189.03999999999999},"last":5.085}]},{"osd":2,"last update":"Mon Oct 9 04:40:57 2023","interfaces":[{"interface":"back","average":{"1min":5.9180000000000001,"5min":4.3700000000000001,"15min":2.8559999999999999},"min":{"1min":0.32000000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":75.364999999999995,"5min":112.35899999999999,"15min":112.35899999999999},"last":2.0419999999999998},{"interface":"front","average":{"1min":6.327,"5min":4.4530000000000003,"15min":2.871},"min":{"1min":0.35999999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":75.329999999999998,"5min":112.331,"15min":112.331},"last":5.2439999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:41:26 2023","interfaces":[{"interface":"back","average":{"1min":16.07,"5min":10.82,"15min":5.8230000000000004},"min":{"1min":0.34499999999999997,"5min":0.253,"15min":0.253},"max":{"1min":157.93100000000001,"5min":167.56100000000001,"15min":167.56100000000001},"last":5.1470000000000002},{"interface":"front","average":{"1min":16.552,"5min":11.131,"15min":5.9630000000000001},"min":{"1min":0.38500000000000001,"5min":0.33000000000000002,"15min":0.313},"max":{"1min":157.95099999999999,"5min":167.577,"15min":167.577},"last":2.0190000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.169,"15min":1.3560000000000001},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":6.8380000000000001,"5min":14.430999999999999,"15min":14.430999999999999},"last":5.1840000000000002},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.0389999999999999,"15min":1.1299999999999999},"min":{"1min":0.36499999999999999,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":3.5209999999999999,"5min":6.0430000000000001,"15min":12.028},"last":5.024}]},{"osd":5,"last update":"Mon Oct 9 04:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.1990000000000001,"15min":1.798},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.28100000000000003},"max":{"1min":3.8250000000000002,"5min":14.631,"15min":81.087000000000003},"last":5.2169999999999996},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.127,"15min":1.7470000000000001},"min":{"1min":0.39200000000000002,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":3.7890000000000001,"5min":8.6479999999999997,"15min":81.706999999999994},"last":5.0570000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:41:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":1.222,"15min":1.633},"min":{"1min":0.35299999999999998,"5min":0.34000000000000002,"15min":0.32000000000000001},"max":{"1min":4.0270000000000001,"5min":16.512,"15min":81.674999999999997},"last":5.1719999999999997},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.393,"15min":1.716},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":4.7629999999999999,"5min":17.715,"15min":81.055000000000007},"last":5.3040000000000003}]}]},{"osd":1,"up_from":118,"seq":506806141142,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1578144,"kb_used_data":1026136,"kb_used_omap":4,"kb_used_meta":551995,"kb_avail":92167008,"statfs":{"total":95995035648,"available":94379016192,"internally_reserved":0,"allocated":1050763264,"data_stored":1049649254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":565243790},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9340000000000002,"5min":4.516,"15min":4.0359999999999996},"min":{"1min":0.14799999999999999,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":14.407999999999999,"5min":93.171999999999997,"15min":97.308000000000007},"last":3.5720000000000001},{"interface":"front","average":{"1min":5.4749999999999996,"5min":4.1600000000000001,"15min":4.1479999999999997},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":27.321000000000002,"5min":93.147000000000006,"15min":97.055999999999997},"last":8.1560000000000006}]},{"osd":2,"last update":"Mon Oct 9 04:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.3570000000000002,"5min":2.7160000000000002,"15min":2.6629999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":17.148,"5min":40.924999999999997,"15min":40.924999999999997},"last":9.5459999999999994},{"interface":"front","average":{"1min":8.593,"5min":3.7090000000000001,"15min":3.202},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":75.730000000000004,"5min":75.730000000000004,"15min":75.730000000000004},"last":14.002000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:41:20 2023","interfaces":[{"interface":"back","average":{"1min":14.061,"5min":8.8849999999999998,"15min":5.1139999999999999},"min":{"1min":0.38,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":118.837,"5min":142.34200000000001,"15min":195.035},"last":6.8200000000000003},{"interface":"front","average":{"1min":15.640000000000001,"5min":10.092000000000001,"15min":5.75},"min":{"1min":0.39300000000000002,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":117.88500000000001,"5min":141.75200000000001,"15min":195.465},"last":8.0879999999999992}]},{"osd":4,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.383,"15min":2.4889999999999999},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.245},"max":{"1min":9.593,"5min":14.449,"15min":170.072},"last":8.0489999999999995},{"interface":"front","average":{"1min":2.343,"5min":2.5150000000000001,"15min":2.2130000000000001},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.28799999999999998},"max":{"1min":7.8010000000000002,"5min":75.513999999999996,"15min":169.94499999999999},"last":6.8570000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1680000000000001,"5min":2.113,"15min":2.4399999999999999},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":11.499000000000001,"5min":11.499000000000001,"15min":171.036},"last":8.3979999999999997},{"interface":"front","average":{"1min":2.6469999999999998,"5min":2.375,"15min":2.6299999999999999},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":9.4900000000000002,"5min":14.557,"15min":170.88},"last":13.031000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":2.137,"15min":1.9850000000000001},"min":{"1min":0.34300000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.545,"5min":11.548,"15min":13.388},"last":8.125},{"interface":"front","average":{"1min":2.6629999999999998,"5min":2.6760000000000002,"15min":2.3069999999999999},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":9.4459999999999997,"5min":14.500999999999999,"15min":14.500999999999999},"last":3.8849999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1930000000000001,"5min":2.7719999999999998,"15min":2.8769999999999998},"min":{"1min":0.434,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.5180000000000007,"5min":14.528,"15min":174.75999999999999},"last":7.9729999999999999},{"interface":"front","average":{"1min":3.2149999999999999,"5min":2.9119999999999999,"15min":2.5449999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":8.4309999999999992,"5min":75.471999999999994,"15min":174.779},"last":7.6660000000000004}]}]},{"osd":0,"up_from":91,"seq":390842024199,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1466492,"kb_used_data":675956,"kb_used_omap":5,"kb_used_meta":790522,"kb_avail":92278660,"statfs":{"total":95995035648,"available":94493347840,"internally_reserved":0,"allocated":692178944,"data_stored":691155724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":809495162},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:41:31 2023","interfaces":[{"interface":"back","average":{"1min":7.4660000000000002,"5min":6.5350000000000001,"15min":5.0620000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":119.392,"5min":138.536,"15min":167.92699999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":9.0709999999999997,"5min":7.4199999999999999,"15min":5.4500000000000002},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.19500000000000001},"max":{"1min":119.34399999999999,"5min":138.89599999999999,"15min":169.78999999999999},"last":1.2669999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":7.375,"15min":8.7569999999999997},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.925000000000001,"5min":131.30699999999999,"15min":131.30699999999999},"last":1.371},{"interface":"front","average":{"1min":4.2990000000000004,"5min":7.7480000000000002,"15min":9.173},"min":{"1min":0.33900000000000002,"5min":0.219,"15min":0.219},"max":{"1min":14.074999999999999,"5min":129.922,"15min":129.922},"last":1.516}]},{"osd":3,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":2.919,"5min":4.9589999999999996,"15min":4.1219999999999999},"min":{"1min":0.27000000000000002,"5min":0.216,"15min":0.20399999999999999},"max":{"1min":11.949,"5min":91.578999999999994,"15min":213.28399999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":4.6360000000000001,"5min":5.9779999999999998,"15min":4.5549999999999997},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":28.696999999999999,"5min":92.067999999999998,"15min":212.27000000000001},"last":7.2590000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":3.7509999999999999,"15min":2.419},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.25700000000000001},"max":{"1min":4.7670000000000003,"5min":161.67099999999999,"15min":161.67099999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.9990000000000001,"5min":4.1189999999999998,"15min":2.6299999999999999},"min":{"1min":0.39000000000000001,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":8.0589999999999993,"5min":160.74799999999999,"15min":160.74799999999999},"last":1.327}]},{"osd":5,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":8.8889999999999993,"5min":3.673,"15min":2.4809999999999999},"min":{"1min":0.36599999999999999,"5min":0.32100000000000001,"15min":0.27900000000000003},"max":{"1min":136.322,"5min":136.322,"15min":136.322},"last":1.357},{"interface":"front","average":{"1min":8.7119999999999997,"5min":3.4220000000000002,"15min":2.2069999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.29099999999999998},"max":{"1min":136.15799999999999,"5min":136.15799999999999,"15min":136.15799999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":2.4060000000000001,"15min":2.4889999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":8.1039999999999992,"5min":14.162000000000001,"15min":108.63800000000001},"last":1.381},{"interface":"front","average":{"1min":2.5910000000000002,"5min":1.8720000000000001,"15min":2.1829999999999998},"min":{"1min":0.40400000000000003,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":7.2300000000000004,"5min":9.9510000000000005,"15min":108.76300000000001},"last":0.47999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.901,"15min":1.8240000000000001},"min":{"1min":0.50700000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.2960000000000003,"5min":9.9809999999999999,"15min":11.890000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.587,"5min":1.909,"15min":1.9139999999999999},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":4.133,"5min":11.260999999999999,"15min":11.260999999999999},"last":0.63100000000000001}]}]},{"osd":2,"up_from":221,"seq":949187772477,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1016964,"kb_used_data":748668,"kb_used_omap":7,"kb_used_meta":268280,"kb_avail":92728188,"statfs":{"total":95995035648,"available":94953664512,"internally_reserved":0,"allocated":766636032,"data_stored":765665356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":274718801},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":5.7290000000000001,"15min":8.8650000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":13.287000000000001,"5min":72.608000000000004,"15min":72.608000000000004},"last":6.4889999999999999},{"interface":"front","average":{"1min":2.5289999999999999,"5min":5.5309999999999997,"15min":8.4649999999999999},"min":{"1min":0.254,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.566000000000001,"5min":72.632000000000005,"15min":72.632000000000005},"last":0.253}]},{"osd":1,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":3.3780000000000001,"15min":4.6699999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.537000000000001,"5min":72.807000000000002,"15min":72.807000000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":3.7519999999999998,"5min":4.0860000000000003,"15min":5.2619999999999996},"min":{"1min":0.27800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.243,"5min":72.664000000000001,"15min":72.664000000000001},"last":4.1050000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":21.677,"5min":7.6310000000000002,"15min":6.1470000000000002},"min":{"1min":0.371,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":335.928,"5min":335.928,"15min":335.928},"last":6.375},{"interface":"front","average":{"1min":22.667999999999999,"5min":7.9359999999999999,"15min":6.1559999999999997},"min":{"1min":0.43099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":335.68599999999998,"5min":335.68599999999998,"15min":335.68599999999998},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":3.2789999999999999,"15min":4.4320000000000004},"min":{"1min":0.40300000000000002,"5min":0.315,"15min":0.315},"max":{"1min":6.0049999999999999,"5min":72.944999999999993,"15min":72.944999999999993},"last":3.4060000000000001},{"interface":"front","average":{"1min":2.1480000000000001,"5min":3.194,"15min":4.577},"min":{"1min":0.41099999999999998,"5min":0.378,"15min":0.378},"max":{"1min":7.4850000000000003,"5min":73.054000000000002,"15min":73.054000000000002},"last":0.52600000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":3.0390000000000001,"15min":4.0979999999999999},"min":{"1min":0.45800000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.478,"5min":72.986000000000004,"15min":72.986000000000004},"last":7.1660000000000004},{"interface":"front","average":{"1min":2.3300000000000001,"5min":3.298,"15min":4.5549999999999997},"min":{"1min":0.35299999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.9690000000000003,"5min":72.963999999999999,"15min":72.963999999999999},"last":3.4550000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":6.1929999999999996,"15min":10.442},"min":{"1min":0.39000000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":7.4569999999999999,"5min":135.614,"15min":135.614},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.9039999999999999,"5min":5.8899999999999997,"15min":9.7300000000000004},"min":{"1min":0.42599999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.509,"5min":130.196,"15min":130.196},"last":0.55200000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":3.2890000000000001,"15min":4.7489999999999997},"min":{"1min":0.38500000000000001,"5min":0.378,"15min":0.378},"max":{"1min":7.4939999999999998,"5min":73.079999999999998,"15min":73.079999999999998},"last":6.4530000000000003},{"interface":"front","average":{"1min":1.835,"5min":3.052,"15min":4.1299999999999999},"min":{"1min":0.45700000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":11.425000000000001,"5min":73.069000000000003,"15min":73.069000000000003},"last":7.133}]}]},{"osd":3,"up_from":142,"seq":609885356200,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1290288,"kb_used_data":850728,"kb_used_omap":2,"kb_used_meta":439485,"kb_avail":92454864,"statfs":{"total":95995035648,"available":94673780736,"internally_reserved":0,"allocated":871145472,"data_stored":870190678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":450032693},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":1.734,"15min":2.327},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.648,"5min":17.648,"15min":142.14500000000001},"last":0.26400000000000001},{"interface":"front","average":{"1min":4.1529999999999996,"5min":2.327,"15min":2.52},"min":{"1min":0.26800000000000002,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":14.951000000000001,"5min":14.951000000000001,"15min":138.14599999999999},"last":0.80200000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1059999999999999,"5min":2.4409999999999998,"15min":3.464},"min":{"1min":0.254,"5min":0.19800000000000001,"15min":0.192},"max":{"1min":21.879000000000001,"5min":21.879000000000001,"15min":180.02500000000001},"last":1.4970000000000001},{"interface":"front","average":{"1min":4.3979999999999997,"5min":2.0720000000000001,"15min":3.5},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":21.911999999999999,"5min":21.911999999999999,"15min":184.00899999999999},"last":2.048}]},{"osd":2,"last update":"Mon Oct 9 04:40:53 2023","interfaces":[{"interface":"back","average":{"1min":10.102,"5min":4.8550000000000004,"15min":2.7349999999999999},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":112.613,"5min":177.28399999999999,"15min":177.28399999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":5.4779999999999998,"5min":4.1310000000000002,"15min":2.2170000000000001},"min":{"1min":0.29499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":19.465,"5min":177.53999999999999,"15min":177.53999999999999},"last":1.3100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.9570000000000001,"15min":2.0179999999999998},"min":{"1min":0.373,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":12.144,"5min":12.144,"15min":103.208},"last":0.93600000000000005},{"interface":"front","average":{"1min":2.4089999999999998,"5min":1.613,"15min":1.7470000000000001},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.307},"max":{"1min":17.082000000000001,"5min":17.082000000000001,"15min":103.218},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":2.488,"5min":1.8320000000000001,"15min":2.2850000000000001},"min":{"1min":0.315,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.045000000000002,"5min":24.675999999999998,"15min":90.972999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":3.056,"5min":2.2090000000000001,"15min":2.371},"min":{"1min":0.434,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":14.867000000000001,"5min":17.422999999999998,"15min":90.959000000000003},"last":0.95399999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":3.077,"5min":2.3170000000000002,"15min":2.161},"min":{"1min":0.40000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.916,"5min":14.916,"15min":104.422},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.98,"5min":3.2890000000000001,"15min":2.577},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":12.177,"5min":123.505,"15min":123.505},"last":0.873}]},{"osd":7,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":2.1179999999999999,"15min":2.2770000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32300000000000001},"max":{"1min":12.096,"5min":12.096,"15min":57.399000000000001},"last":1.883},{"interface":"front","average":{"1min":2.698,"5min":1.702,"15min":2.0139999999999998},"min":{"1min":0.45900000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":16.984999999999999,"5min":16.984999999999999,"15min":57.447000000000003},"last":1.8680000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019327,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1788696,"kb_used_data":900752,"kb_used_omap":0,"kb_used_meta":887936,"kb_avail":91956456,"statfs":{"total":95995035648,"available":94163410944,"internally_reserved":0,"allocated":922370048,"data_stored":921789953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909246464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":4.2530000000000001,"15min":5.3250000000000002},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":23.114999999999998,"5min":171.13499999999999,"15min":219.577},"last":11.221},{"interface":"front","average":{"1min":3.0529999999999999,"5min":4.2450000000000001,"15min":5.1150000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.247},"max":{"1min":13.992000000000001,"5min":171.11799999999999,"15min":218.96000000000001},"last":2.0579999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:41:35 2023","interfaces":[{"interface":"back","average":{"1min":3.7040000000000002,"5min":3.3260000000000001,"15min":5.0640000000000001},"min":{"1min":0.316,"5min":0.25,"15min":0.23400000000000001},"max":{"1min":33.322000000000003,"5min":61.764000000000003,"15min":142.74799999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.1469999999999998,"5min":3.294,"15min":5.1100000000000003},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":33.418999999999997,"5min":61.747999999999998,"15min":139.09700000000001},"last":1.9890000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:40:52 2023","interfaces":[{"interface":"back","average":{"1min":8.3960000000000008,"5min":3.9140000000000001,"15min":3.734},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":65.381,"5min":65.381,"15min":65.381},"last":11.266},{"interface":"front","average":{"1min":7.952,"5min":3.7679999999999998,"15min":3.2730000000000001},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":63.515999999999998,"5min":63.515999999999998,"15min":63.515999999999998},"last":2.0329999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:41:14 2023","interfaces":[{"interface":"back","average":{"1min":23.756,"5min":6.9429999999999996,"15min":6.8600000000000003},"min":{"1min":0.38900000000000001,"5min":0.32600000000000001,"15min":0.27900000000000003},"max":{"1min":357.125,"5min":357.125,"15min":357.125},"last":2.3330000000000002},{"interface":"front","average":{"1min":23.155000000000001,"5min":6.7270000000000003,"15min":7.0039999999999996},"min":{"1min":0.34300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":359.02800000000002,"5min":359.02800000000002,"15min":359.02800000000002},"last":1.3069999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.1819999999999999,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":9.5609999999999999,"5min":9.5609999999999999,"15min":107.476},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.137,"5min":1.286,"15min":2.089},"min":{"1min":0.311,"5min":0.26900000000000002,"15min":0.23300000000000001},"max":{"1min":6.694,"5min":9.0210000000000008,"15min":107.414},"last":1.966}]},{"osd":6,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1479999999999999,"15min":1.869},"min":{"1min":0.32200000000000001,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":9.6099999999999994,"5min":9.6099999999999994,"15min":65.75},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.2270000000000001,"15min":1.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.1630000000000003,"5min":9.3320000000000007,"15min":65.849999999999994},"last":1.9350000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.2470000000000001,"15min":1.6120000000000001},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.28000000000000003},"max":{"1min":5.0590000000000002,"5min":6.8259999999999996,"15min":76.283000000000001},"last":2.0070000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.1899999999999999,"15min":1.583},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":76.259},"last":7.1020000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757691,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1347016,"kb_used_data":456704,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":92398136,"statfs":{"total":95995035648,"available":94615691264,"internally_reserved":0,"allocated":467664896,"data_stored":467096929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":3.992,"15min":5.117},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.22},"max":{"1min":7.7249999999999996,"5min":88.346000000000004,"15min":210.85499999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":3.278,"5min":4.5339999999999998,"15min":5.1429999999999998},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":18.606999999999999,"5min":88.375,"15min":210.23500000000001},"last":3.5139999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:41:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":4.0810000000000004,"15min":4.3440000000000003},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.255},"max":{"1min":11.345000000000001,"5min":145.577,"15min":145.577},"last":73.296000000000006},{"interface":"front","average":{"1min":2.2029999999999998,"5min":4.0250000000000004,"15min":5.1840000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.547000000000001,"5min":145.792,"15min":145.792},"last":73.045000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:40:51 2023","interfaces":[{"interface":"back","average":{"1min":5.1360000000000001,"5min":4.5620000000000003,"15min":6.391},"min":{"1min":0.33600000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":49.058,"5min":90.763999999999996,"15min":90.763999999999996},"last":1.169},{"interface":"front","average":{"1min":5.173,"5min":4.7270000000000003,"15min":6.5460000000000003},"min":{"1min":0.42799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":49.521000000000001,"5min":91.340000000000003,"15min":91.340000000000003},"last":0.80400000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:41:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":8.9920000000000009,"15min":6.6029999999999998},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28100000000000003},"max":{"1min":8.4100000000000001,"5min":268.63299999999998,"15min":268.63299999999998},"last":3.5960000000000001},{"interface":"front","average":{"1min":2.4620000000000002,"5min":8.8689999999999998,"15min":6.4640000000000004},"min":{"1min":0.46600000000000003,"5min":0.32200000000000001,"15min":0.31},"max":{"1min":9.0899999999999999,"5min":268.92500000000001,"15min":268.92500000000001},"last":0.76500000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.76400000000000001,"15min":2.448},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.248},"max":{"1min":2.077,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.823,"5min":1.242,"15min":2.798},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.6280000000000001,"5min":8.7599999999999998,"15min":361.71300000000002},"last":0.55800000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.97299999999999998,"15min":1.2869999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.7640000000000002,"5min":5.7640000000000002,"15min":11.192},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.409,"15min":1.4490000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.231999999999999,"5min":13.231999999999999,"15min":14.334},"last":3.6219999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.163,"15min":1.4039999999999999},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.5899999999999999,"5min":8.5589999999999993,"15min":11.093},"last":3.5659999999999998},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.0349999999999999,"15min":1.2989999999999999},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.4429999999999996,"5min":6.5129999999999999,"15min":15.257999999999999},"last":1.3300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":685113344,"data_stored":685113344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1040060416,"data_stored":1040056344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":754716672,"data_stored":754712600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":853082112,"data_stored":853082112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":909377536,"data_stored":909377536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":457179136,"data_stored":457179136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":931532800,"data_stored":931528728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":753270784,"data_stored":753270784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:41:55.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:41:55.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:41:56.074 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:56.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[216903]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:41:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[216903]: failed to encode map e272 with expected crc 2023-10-09T04:41:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[216903]: failed to encode map e272 with expected crc 2023-10-09T04:41:56.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10511/82947 objects degraded (12.672%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[223858]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:41:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[223858]: failed to encode map e272 with expected crc 2023-10-09T04:41:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[223858]: failed to encode map e272 with expected crc 2023-10-09T04:41:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10511/82947 objects degraded (12.672%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:56.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:56.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:55 smithi177 ceph-mon[195797]: osdmap e272: 8 total, 8 up, 8 in 2023-10-09T04:41:56.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:55 smithi177 ceph-mon[195797]: failed to encode map e272 with expected crc 2023-10-09T04:41:56.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:55 smithi177 ceph-mon[195797]: failed to encode map e272 with expected crc 2023-10-09T04:41:56.230 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10511/82947 objects degraded (12.672%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:41:56.233 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:56.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:56.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:56.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:41:56.425 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:41:56.455 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:56.503 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:41:56.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:41:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:56.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:41:56.629 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:56.676 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:56.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:56.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:56.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:41:56.993 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:57.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:56 smithi177 ceph-mon[195797]: pgmap v1031: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10511/82083 objects degraded (12.805%); 0 B/s, 51 objects/s recovering 2023-10-09T04:41:57.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:56 smithi029 ceph-mon[216903]: pgmap v1031: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10511/82083 objects degraded (12.805%); 0 B/s, 51 objects/s recovering 2023-10-09T04:41:57.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:56 smithi029 ceph-mon[223858]: pgmap v1031: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 10511/82083 objects degraded (12.805%); 0 B/s, 51 objects/s recovering 2023-10-09T04:41:57.303 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:41:57.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:57.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:41:57.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:57.484 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:57.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:41:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:57.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:41:57.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:57.966 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1570924611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:57.966 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:57 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:57.997 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:57.998 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:41:58.142 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1570924611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:58.142 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:57 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:58.145 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:58.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:58.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:58.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:41:58.299 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:58.414 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:41:58.414 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:41:58.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1570924611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:58.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:57 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:41:58.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:41:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:58.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:41:58.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:41:58.634 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:59.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: pgmap v1032: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6916/78261 objects degraded (8.837%); 547/78261 objects misplaced (0.699%); 1.2 MiB/s, 104 objects/s recovering 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:41:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:59.364 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: pgmap v1032: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6916/78261 objects degraded (8.837%); 547/78261 objects misplaced (0.699%); 1.2 MiB/s, 104 objects/s recovering 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:59.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: pgmap v1032: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6916/78261 objects degraded (8.837%); 547/78261 objects misplaced (0.699%); 1.2 MiB/s, 104 objects/s recovering 2023-10-09T04:41:59.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:41:59.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: osdmap e273: 8 total, 8 up, 8 in 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: failed to encode map e273 with expected crc 2023-10-09T04:41:59.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:41:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:41:59.469 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:41:59.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:59.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:41:59.575 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:41:59.687 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:41:59.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:41:59.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:41:59.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:41:59.891 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:41:59.944 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:00.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:00.043 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:00.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:00.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:00.210 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:00.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:00.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:00.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:00.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:00.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:00 smithi177 ceph-mon[195797]: pgmap v1034: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 6916/75465 objects degraded (9.165%); 547/75465 objects misplaced (0.725%); 1.7 MiB/s, 71 objects/s recovering 2023-10-09T04:42:00.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/75465 objects degraded (9.165%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:00 smithi029 ceph-mon[216903]: pgmap v1034: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 6916/75465 objects degraded (9.165%); 547/75465 objects misplaced (0.725%); 1.7 MiB/s, 71 objects/s recovering 2023-10-09T04:42:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/75465 objects degraded (9.165%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:00 smithi029 ceph-mon[223858]: pgmap v1034: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 6916/75465 objects degraded (9.165%); 547/75465 objects misplaced (0.725%); 1.7 MiB/s, 71 objects/s recovering 2023-10-09T04:42:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/75465 objects degraded (9.165%), 2 pgs degraded (PG_DEGRADED) 2023-10-09T04:42:01.950 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:42:01.950 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:42:02.371 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:02.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:02.410 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:02.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:02.549 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:02.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:42:02.810 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:02.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:02.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:03.009 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:03 smithi029 ceph-mon[216903]: pgmap v1035: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 6916/72297 objects degraded (9.566%); 547/72297 objects misplaced (0.757%); 1.4 MiB/s, 59 objects/s recovering 2023-10-09T04:42:03.009 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:42:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:42:02] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T04:42:03.018 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:03 smithi177 ceph-mon[195797]: pgmap v1035: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 6916/72297 objects degraded (9.566%); 547/72297 objects misplaced (0.757%); 1.4 MiB/s, 59 objects/s recovering 2023-10-09T04:42:03.157 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:03.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:03.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:03.270 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:03.276 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:03 smithi029 ceph-mon[223858]: pgmap v1035: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 6916/72297 objects degraded (9.566%); 547/72297 objects misplaced (0.757%); 1.4 MiB/s, 59 objects/s recovering 2023-10-09T04:42:03.360 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:03.367 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:42:03.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:42:03.601 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:42:03.638 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:03.643 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:42:03.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:03.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:03.692 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:42:03.742 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:42:03.751 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:03.774 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:42:03.815 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:42:03.845 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:42:03.857 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:03.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.864 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:42:03.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:42:04.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2532007792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2532007792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:04.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2532007792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:04.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:42:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:05 smithi029 ceph-mon[216903]: pgmap v1036: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 6916/67257 objects degraded (10.283%); 338/67257 objects misplaced (0.503%); 2.8 MiB/s, 77 objects/s recovering 2023-10-09T04:42:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:05 smithi029 ceph-mon[223858]: pgmap v1036: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 6916/67257 objects degraded (10.283%); 338/67257 objects misplaced (0.503%); 2.8 MiB/s, 77 objects/s recovering 2023-10-09T04:42:05.241 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:42:05.241 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:165843 9.3 9:c89d6233:::benchmark_data_smithi029_299184_object67540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:32:46.612549+0000", 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 558.62631292599997, 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38172534299999999, 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165843 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612549+0000", 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612549+0000", 2023-10-09T04:42:05.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612548+0000", 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612555+0000", 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.374e-06 2023-10-09T04:42:05.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612566+0000", 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1039000000000001e-05 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612571+0000", 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9490000000000004e-06 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612613+0000", 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1940000000000002e-05 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612743+0000", 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012974500000000001 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.612860+0000", 2023-10-09T04:42:05.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011655 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.621718+0000", 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0088587300000000004 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.621725+0000", 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4489999999999996e-06 2023-10-09T04:42:05.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.994172+0000", 2023-10-09T04:42:05.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37244685799999999 2023-10-09T04:42:05.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.994209+0000", 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7703000000000001e-05 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.994275+0000", 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5469999999999995e-05 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.35373.0:165852 9.3 9:c4f9f1bf:::benchmark_data_smithi029_299184_object67549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:32:46.625676+0000", 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 558.61318644999994, 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.368716763, 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.35373", 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/1694207949", 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 165852 2023-10-09T04:42:05.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.625676+0000", 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.625676+0000", 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.625674+0000", 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.625683+0000", 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2240000000000001e-06 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.625703+0000", 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0060000000000001e-05 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.625711+0000", 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9859999999999994e-06 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.626951+0000", 2023-10-09T04:42:05.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012397790000000001 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.627107+0000", 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015613899999999999 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.627239+0000", 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013199899999999999 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.631043+0000", 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003803481 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.631179+0000", 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000135819 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.994319+0000", 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36314069900000001 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.994368+0000", 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.897e-05 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:32:46.994393+0000", 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4522999999999999e-05 2023-10-09T04:42:05.256 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:103478 10.c 10:3505fe57:::benchmark_data_smithi029_344401_object103477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:26.564434+0000", 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 398.67442807100002, 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36597732999999999, 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.257 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 103478 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564434+0000", 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564434+0000", 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564433+0000", 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564694+0000", 2023-10-09T04:42:05.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026092500000000001 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564718+0000", 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3767000000000001e-05 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564726+0000", 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2500000000000006e-06 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564791+0000", 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5164000000000002e-05 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.564939+0000", 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014777200000000001 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.565087+0000", 2023-10-09T04:42:05.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014835400000000001 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.568953+0000", 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038660080000000002 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.571222+0000", 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022691579999999999 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.930310+0000", 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.359087818 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.930368+0000", 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7670999999999997e-05 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.930412+0000", 2023-10-09T04:42:05.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3890999999999997e-05 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:103485 10.c 10:3d9ed386:::benchmark_data_smithi029_344401_object103484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:26.572499+0000", 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 398.66636376500003, 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36789768900000003, 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.264 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 103485 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572499+0000", 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572499+0000", 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572498+0000", 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572626+0000", 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000128773 2023-10-09T04:42:05.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572649+0000", 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2989e-05 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572656+0000", 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7379999999999997e-06 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572704+0000", 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7793999999999997e-05 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572858+0000", 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015434499999999999 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.573021+0000", 2023-10-09T04:42:05.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000162812 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.579586+0000", 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065651939999999999 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.580671+0000", 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001084996 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.940307+0000", 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.35963546800000001 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.940359+0000", 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2491999999999998e-05 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.940396+0000", 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7146e-05 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:103486 10.c 10:34207258:::benchmark_data_smithi029_344401_object103485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:35:26.572673+0000", 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 398.66618942999997, 2023-10-09T04:42:05.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37670843199999998, 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 103486 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572673+0000", 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572673+0000", 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572673+0000", 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572761+0000", 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8156000000000001e-05 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572778+0000", 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6983e-05 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.572783+0000", 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8209999999999998e-06 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.576373+0000", 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035896180000000001 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.576513+0000", 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013952600000000001 2023-10-09T04:42:05.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.576687+0000", 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017439500000000001 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.579744+0000", 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030567340000000002 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.581900+0000", 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021566210000000001 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.949292+0000", 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36739125 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.949355+0000", 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2990000000000005e-05 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:35:26.949381+0000", 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6818e-05 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:143852 10.1c 10:3cb42b9b:::benchmark_data_smithi029_344401_object35481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e216)", 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:11.551676+0000", 2023-10-09T04:42:05.278 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 353.68718645500002, 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.87610185600000001, 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 143852 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.551676+0000", 2023-10-09T04:42:05.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.551676+0000", 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.551673+0000", 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999971 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.551690+0000", 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6288999999999999e-05 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.551729+0000", 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9288000000000001e-05 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.551742+0000", 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3512e-05 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.675117+0000", 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.123374368 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:11.675126+0000", 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5540000000000003e-06 2023-10-09T04:42:05.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.416620+0000", 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.74149363199999996 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.416792+0000", 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017206599999999999 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.417159+0000", 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036717899999999998 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.419067+0000", 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019079629999999999 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.419549+0000", 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00048203200000000002 2023-10-09T04:42:05.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.427662+0000", 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081131310000000009 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.427734+0000", 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1410999999999998e-05 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:12.427778+0000", 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4129999999999999e-05 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:66695 11.5 11:acda587b:::benchmark_data_smithi029_392508_object66694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.834408+0000", 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 200.40445417800001, 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51682144799999996, 2023-10-09T04:42:05.286 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 66695 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834408+0000", 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834408+0000", 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834407+0000", 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834460+0000", 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2787999999999997e-05 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834475+0000", 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4832e-05 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834491+0000", 2023-10-09T04:42:05.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6629999999999998e-05 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834542+0000", 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0844000000000003e-05 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834661+0000", 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119249 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.834788+0000", 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012627 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.837259+0000", 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024712319999999999 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.839782+0000", 2023-10-09T04:42:05.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025228820000000002 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.351129+0000", 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51134759900000004 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.351181+0000", 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1384999999999997e-05 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.351230+0000", 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8908e-05 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:66708 11.5 11:acb8a4f4:::benchmark_data_smithi029_392508_object66707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.843422+0000", 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 200.395440243, 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50791751900000004, 2023-10-09T04:42:05.293 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 66708 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843422+0000", 2023-10-09T04:42:05.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843422+0000", 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843421+0000", 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843483+0000", 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2076999999999995e-05 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843497+0000", 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4688999999999999e-05 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843503+0000", 2023-10-09T04:42:05.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9610000000000002e-06 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843547+0000", 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3940999999999998e-05 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843686+0000", 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000138233 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843831+0000", 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000145208 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.847886+0000", 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040556790000000004 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.850999+0000", 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031122210000000001 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.351268+0000", 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50026910099999999 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.351307+0000", 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9768999999999998e-05 2023-10-09T04:42:05.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.351340+0000", 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2212000000000003e-05 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:66711 11.5 11:ab967f16:::benchmark_data_smithi029_392508_object66710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.843554+0000", 2023-10-09T04:42:05.300 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 200.395308548, 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50993285499999996, 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 66711 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843554+0000", 2023-10-09T04:42:05.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843554+0000", 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843553+0000", 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:05.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843614+0000", 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1209000000000003e-05 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843622+0000", 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.221e-06 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.843626+0000", 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1899999999999997e-06 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.845948+0000", 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023223599999999999 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.846054+0000", 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105834 2023-10-09T04:42:05.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.846163+0000", 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010848700000000001 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.847948+0000", 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017850050000000001 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.851624+0000", 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003676042 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.353404+0000", 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50178027800000002 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.353447+0000", 2023-10-09T04:42:05.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3414000000000001e-05 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.353487+0000", 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9409000000000002e-05 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:66715 11.5 11:a8389492:::benchmark_data_smithi029_392508_object66714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:42:05.307 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:38:44.848661+0000", 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 200.39020110999999, 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50492438100000003, 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 66715 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848661+0000", 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848661+0000", 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848660+0000", 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848747+0000", 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7318000000000007e-05 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848759+0000", 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1977999999999999e-05 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848764+0000", 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9799999999999998e-06 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848805+0000", 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1029000000000003e-05 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.848945+0000", 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014013099999999999 2023-10-09T04:42:05.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.849105+0000", 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000159195 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.851094+0000", 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019895479999999998 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:44.851753+0000", 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00065895899999999998 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.353528+0000", 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50177459800000002 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.353561+0000", 2023-10-09T04:42:05.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3773999999999999e-05 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:38:45.353586+0000", 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4289000000000001e-05 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:120913 11.4 11:2a176aab:::benchmark_data_smithi029_392508_object20357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:42:05.314 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:11.196192+0000", 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 114.042670089, 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39291548100000001, 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120913 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196192+0000", 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196192+0000", 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196192+0000", 2023-10-09T04:42:05.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196195+0000", 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0960000000000001e-06 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196203+0000", 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5109999999999997e-06 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196211+0000", 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5499999999999995e-06 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196260+0000", 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9036000000000003e-05 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196374+0000", 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011340099999999999 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.196462+0000", 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8913999999999996e-05 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.199891+0000", 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034280729999999998 2023-10-09T04:42:05.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.200161+0000", 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027043299999999999 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.589015+0000", 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38885418500000002 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.589062+0000", 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6733000000000001e-05 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.589108+0000", 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5948000000000001e-05 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.321 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:120920 11.f 11:ff1b57ae:::benchmark_data_smithi029_392508_object20364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:11.205052+0000", 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 114.033810059, 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37417450000000002, 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120920 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205052+0000", 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205052+0000", 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205051+0000", 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205058+0000", 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.233e-06 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205078+0000", 2023-10-09T04:42:05.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9950999999999999e-05 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205086+0000", 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3389999999999993e-06 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205112+0000", 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5748000000000001e-05 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205200+0000", 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7867000000000004e-05 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.205301+0000", 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010108200000000001 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.208033+0000", 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027314470000000001 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.213454+0000", 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054210389999999999 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.579155+0000", 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36570148600000002 2023-10-09T04:42:05.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.579197+0000", 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1527999999999999e-05 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.579227+0000", 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0165999999999999e-05 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.328 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:120940 11.4 11:2adb9a94:::benchmark_data_smithi029_392508_object20384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:11.231242+0000", 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 114.00762063400001, 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37010720000000003, 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120940 2023-10-09T04:42:05.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231242+0000", 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231242+0000", 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231241+0000", 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231250+0000", 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8030000000000008e-06 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231263+0000", 2023-10-09T04:42:05.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2961e-05 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231269+0000", 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7209999999999999e-06 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231327+0000", 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7768e-05 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231449+0000", 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012227700000000001 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.231562+0000", 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011267199999999999 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.234874+0000", 2023-10-09T04:42:05.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033125089999999999 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.235832+0000", 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00095817700000000005 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.601283+0000", 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36545098799999998 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.601321+0000", 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7608999999999999e-05 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:11.601349+0000", 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7919000000000002e-05 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:140720 11.4 11:296f2f17:::benchmark_data_smithi029_392508_object40164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:51.019254+0000", 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 74.219608332000007, 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.482252397, 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 140720 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.336 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019254+0000", 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019254+0000", 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019253+0000", 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019259+0000", 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7599999999999999e-06 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019274+0000", 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4528e-05 2023-10-09T04:42:05.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019280+0000", 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4289999999999998e-06 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019343+0000", 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3501000000000001e-05 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019523+0000", 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017955400000000001 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019645+0000", 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012169599999999999 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.026950+0000", 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073055710000000003 2023-10-09T04:42:05.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.027178+0000", 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00022792400000000001 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501446+0000", 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47426804900000002 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501479+0000", 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3123000000000001e-05 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501506+0000", 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7081e-05 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.342 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:140722 11.4 11:22387037:::benchmark_data_smithi029_392508_object40166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:51.019295+0000", 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 74.219567878000007, 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48232274400000003, 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 140722 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.343 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019295+0000", 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019295+0000", 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019294+0000", 2023-10-09T04:42:05.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019297+0000", 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8629999999999999e-06 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019448+0000", 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015078199999999999 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.019456+0000", 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7870000000000002e-06 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.023695+0000", 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042392669999999997 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.023812+0000", 2023-10-09T04:42:05.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117459 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.023926+0000", 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113428 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.027144+0000", 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032179660000000001 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.027325+0000", 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000181745 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501549+0000", 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.47422376500000002 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501582+0000", 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3076e-05 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501617+0000", 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5008000000000001e-05 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.349 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:140729 11.f 11:f1124be6:::benchmark_data_smithi029_392508_object40173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:51.042484+0000", 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 74.196378011999997, 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.453173037, 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 140729 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.350 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042484+0000", 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042484+0000", 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042483+0000", 2023-10-09T04:42:05.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042491+0000", 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8089999999999993e-06 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042507+0000", 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5891000000000001e-05 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042514+0000", 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8650000000000003e-06 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042566+0000", 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2933999999999998e-05 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042707+0000", 2023-10-09T04:42:05.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014039599999999999 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2", 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.042829+0000", 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012222599999999999 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.046620+0000", 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037905730000000002 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.047600+0000", 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00098045300000000005 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.495589+0000", 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44798842300000002 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.495623+0000", 2023-10-09T04:42:05.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4652e-05 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.495657+0000", 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4270000000000002e-05 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:140737 11.4 11:2b379a82:::benchmark_data_smithi029_392508_object40181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:42:05.356 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:51.058611+0000", 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 74.180251491000007, 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44314634400000003, 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 140737 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058611+0000", 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058611+0000", 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058609+0000", 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058616+0000", 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8310000000000003e-06 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058634+0000", 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7265000000000002e-05 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058640+0000", 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8140000000000003e-06 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058693+0000", 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2249000000000001e-05 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058803+0000", 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110042 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.058873+0000", 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0266999999999997e-05 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.062242+0000", 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003369019 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.062522+0000", 2023-10-09T04:42:05.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027964 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501677+0000", 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43915538100000001 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501720+0000", 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3158999999999998e-05 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:51.501757+0000", 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7169999999999998e-05 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:144656 11.e 11:70117d8a:::benchmark_data_smithi029_392508_object44100:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:59.018948+0000", 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 66.219914488000001, 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.263205405, 2023-10-09T04:42:05.363 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 144656 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.018948+0000", 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.018948+0000", 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.018947+0000", 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.018953+0000", 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1580000000000002e-06 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.018967+0000", 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4232e-05 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.018973+0000", 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2890000000000003e-06 2023-10-09T04:42:05.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.029848+0000", 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010874287 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030460+0000", 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00061260799999999995 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.367 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.054037+0000", 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023577058000000001 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.054125+0000", 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8294999999999994e-05 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.054246+0000", 2023-10-09T04:42:05.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012057699999999999 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.077929+0000", 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023683216 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.083326+0000", 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053969259999999998 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.279786+0000", 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.19645974899999999 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.279831+0000", 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4793000000000003e-05 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.282153+0000", 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023226050000000002 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:144661 11.e 11:74707e8d:::benchmark_data_smithi029_392508_object44105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:59.019031+0000", 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 66.219830999999999, 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26943424300000002, 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.371 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 144661 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.019031+0000", 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.019031+0000", 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.019031+0000", 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.019033+0000", 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0999999999999998e-06 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.019038+0000", 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1549999999999998e-06 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.019042+0000", 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.366e-06 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.374 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030508+0000", 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011466575999999999 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.031152+0000", 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00064407199999999998 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.061094+0000", 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029941437000000001 2023-10-09T04:42:05.375 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.061167+0000", 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2941000000000005e-05 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.061277+0000", 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011061300000000001 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.376 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.077941+0000", 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016663194999999999 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.083380+0000", 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054390940000000002 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.286008+0000", 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.202628265 2023-10-09T04:42:05.377 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.286061+0000", 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3492999999999998e-05 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.288466+0000", 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002404247 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.378 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:144667 11.e 11:76f0c046:::benchmark_data_smithi029_392508_object44111:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:40:59.030300+0000", 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 66.208562259999994, 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26225256499999999, 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 144667 2023-10-09T04:42:05.379 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030300+0000", 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030300+0000", 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030300+0000", 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030302+0000", 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9580000000000001e-06 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030306+0000", 2023-10-09T04:42:05.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.318e-06 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.030309+0000", 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.729e-06 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.031855+0000", 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001546375 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.032414+0000", 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055919599999999998 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.074293+0000", 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.041878406 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.074400+0000", 2023-10-09T04:42:05.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010713399999999999 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.074568+0000", 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016811699999999999 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.083334+0000", 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0087658149999999997 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.084922+0000", 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00158814 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.290677+0000", 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.205754934 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.290735+0000", 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8158999999999997e-05 2023-10-09T04:42:05.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:40:59.292553+0000", 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018175509999999999 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:42:05.386 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:42:05.387 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:05 smithi177 ceph-mon[195797]: pgmap v1036: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 6916/67257 objects degraded (10.283%); 338/67257 objects misplaced (0.503%); 2.8 MiB/s, 77 objects/s recovering 2023-10-09T04:42:06.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:42:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/67257 objects degraded (10.283%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:42:06.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/67257 objects degraded (10.283%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:42:06.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/67257 objects degraded (10.283%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:42:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:07 smithi029 ceph-mon[216903]: pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 6916/66363 objects degraded (10.421%); 338/66363 objects misplaced (0.509%); 2.3 MiB/s, 63 objects/s recovering 2023-10-09T04:42:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:07 smithi029 ceph-mon[223858]: pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 6916/66363 objects degraded (10.421%); 338/66363 objects misplaced (0.509%); 2.3 MiB/s, 63 objects/s recovering 2023-10-09T04:42:07.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:07 smithi177 ceph-mon[195797]: pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 6916/66363 objects degraded (10.421%); 338/66363 objects misplaced (0.509%); 2.3 MiB/s, 63 objects/s recovering 2023-10-09T04:42:07.793 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:07.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:07.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:07.944 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:08.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:08 smithi029 ceph-mon[223858]: pgmap v1038: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 6916/62301 objects degraded (11.101%); 84/62301 objects misplaced (0.135%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:42:08.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:08 smithi029 ceph-mon[216903]: pgmap v1038: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 6916/62301 objects degraded (11.101%); 84/62301 objects misplaced (0.135%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:42:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:08 smithi177 ceph-mon[195797]: pgmap v1038: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 6916/62301 objects degraded (11.101%); 84/62301 objects misplaced (0.135%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T04:42:09.517 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:09.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:09.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:09.659 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:09.673 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:09.688 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:09.744 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:09.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:09.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:09.972 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:10.379 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:10.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:10.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:10.502 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:10 smithi029 ceph-mon[216903]: pgmap v1039: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 6916/59058 objects degraded (11.711%); 84/59058 objects misplaced (0.142%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:42:10.503 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:10 smithi029 ceph-mon[223858]: pgmap v1039: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 6916/59058 objects degraded (11.711%); 84/59058 objects misplaced (0.142%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:42:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:10 smithi177 ceph-mon[195797]: pgmap v1039: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 6916/59058 objects degraded (11.711%); 84/59058 objects misplaced (0.142%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:42:10.596 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:10.955 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:10.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:11.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:11.115 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:42:11.169 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:42:11.178 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:11.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:11.284 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:11.331 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:11.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:11.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:11.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:11.540 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:11.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:42:11.618 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:11.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:11.626 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:11.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:11.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/59058 objects degraded (11.711%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:11.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:11.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:11.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/59058 objects degraded (11.711%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:11.724 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:11.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/59058 objects degraded (11.711%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:11.835 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:12 smithi177 ceph-mon[195797]: from='client.36126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:42:12.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:12 smithi177 ceph-mon[195797]: pgmap v1040: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 6916/55887 objects degraded (12.375%); 84/55887 objects misplaced (0.150%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:42:12.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:12 smithi029 ceph-mon[216903]: from='client.36126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:42:12.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:12 smithi029 ceph-mon[216903]: pgmap v1040: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 6916/55887 objects degraded (12.375%); 84/55887 objects misplaced (0.150%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:42:12.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:12 smithi029 ceph-mon[223858]: from='client.36126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:42:12.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:12 smithi029 ceph-mon[223858]: pgmap v1040: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 6916/55887 objects degraded (12.375%); 84/55887 objects misplaced (0.150%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T04:42:13.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:42:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:42:12] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T04:42:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:13.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:13.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:14.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:14.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:14.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:42:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:42:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:42:14.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:14.182 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:14.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:14.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:14.415 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:14.507 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:14 smithi029 ceph-mon[216903]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6916/50286 objects degraded (13.753%); 2.8 MiB/s, 59 objects/s recovering 2023-10-09T04:42:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:14 smithi029 ceph-mon[223858]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6916/50286 objects degraded (13.753%); 2.8 MiB/s, 59 objects/s recovering 2023-10-09T04:42:14.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:15.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:14 smithi177 ceph-mon[195797]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6916/50286 objects degraded (13.753%); 2.8 MiB/s, 59 objects/s recovering 2023-10-09T04:42:15.035 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:15.742 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:15.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:15.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:15.885 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/50286 objects degraded (13.753%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:15.891 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/50286 objects degraded (13.753%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/50286 objects degraded (13.753%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:15.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:15.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:16.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:16.209 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:16.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:16 smithi029 ceph-mon[216903]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1022 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 6916/49080 objects degraded (14.091%); 1.8 MiB/s, 41 objects/s recovering 2023-10-09T04:42:16.609 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:16.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:16.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:16.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:16 smithi029 ceph-mon[223858]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1022 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 6916/49080 objects degraded (14.091%); 1.8 MiB/s, 41 objects/s recovering 2023-10-09T04:42:16.826 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:17.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:16 smithi177 ceph-mon[195797]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1022 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 6916/49080 objects degraded (14.091%); 1.8 MiB/s, 41 objects/s recovering 2023-10-09T04:42:17.373 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:17.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:17.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:17.512 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:17.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:17.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:17.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:17.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:17.718 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:17.831 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:17.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:17.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:17.989 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:18.038 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:18.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:18.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:18.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:18.185 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:18.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:18.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:18.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:18.426 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:18.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:18.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:18.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:18.668 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:18.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:18 smithi177 ceph-mon[195797]: pgmap v1043: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 925 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 6916/44451 objects degraded (15.559%); 1.8 MiB/s, 61 objects/s recovering 2023-10-09T04:42:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:18 smithi029 ceph-mon[223858]: pgmap v1043: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 925 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 6916/44451 objects degraded (15.559%); 1.8 MiB/s, 61 objects/s recovering 2023-10-09T04:42:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:18 smithi029 ceph-mon[216903]: pgmap v1043: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 925 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 6916/44451 objects degraded (15.559%); 1.8 MiB/s, 61 objects/s recovering 2023-10-09T04:42:19.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:19.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:19.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:19.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:19.312 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:19.391 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:19.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:19.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:19.583 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:20.082 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1039,"stamp":"2023-10-09T04:42:09.164481+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1289099192,"num_objects":19686,"num_object_clones":0,"num_object_copies":59058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6916,"num_objects_misplaced":84,"num_objects_unfound":0,"num_objects_dirty":19686,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":181746,"num_write_kb":6437838,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12306,"num_bytes_recovered":611983512,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139464,"ondisk_log_size":139464,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10166340,"kb_used_data":4471876,"kb_used_omap":20,"kb_used_meta":5694251,"kb_avail":739794876,"statfs":{"total":767960285184,"available":757549953024,"internally_reserved":0,"allocated":4579201024,"data_stored":4572794577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5830913678},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,4,10,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-358416384,"num_objects":-5469,"num_object_clones":0,"num_object_copies":-16407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-463,"num_objects_unfound":0,"num_objects_dirty":-5469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5469,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":463,"num_bytes_recovered":30343168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.196981"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914530+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:41:54.914530+0000","last_peered":"2023-10-09T04:41:54.914530+0000","last_clean":"2023-10-09T04:41:54.914530+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:41:54.914530+0000","last_undegraded":"2023-10-09T04:41:54.914530+0000","last_fullsized":"2023-10-09T04:41:54.914530+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":225,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906575+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:41:54.906575+0000","last_peered":"2023-10-09T04:41:54.906575+0000","last_clean":"2023-10-09T04:41:54.906575+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:41:54.906575+0000","last_undegraded":"2023-10-09T04:41:54.906575+0000","last_fullsized":"2023-10-09T04:41:54.906575+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":258,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906449+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:41:54.906449+0000","last_peered":"2023-10-09T04:41:54.906449+0000","last_clean":"2023-10-09T04:41:54.906449+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:41:54.906449+0000","last_undegraded":"2023-10-09T04:41:54.906449+0000","last_fullsized":"2023-10-09T04:41:54.906449+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":186,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951639+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:41:54.951639+0000","last_peered":"2023-10-09T04:41:54.951639+0000","last_clean":"2023-10-09T04:41:54.951639+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:41:54.951639+0000","last_undegraded":"2023-10-09T04:41:54.951639+0000","last_fullsized":"2023-10-09T04:41:54.951639+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":261,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906178+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:41:54.906178+0000","last_peered":"2023-10-09T04:41:54.906178+0000","last_clean":"2023-10-09T04:41:54.906178+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:41:54.906178+0000","last_undegraded":"2023-10-09T04:41:54.906178+0000","last_fullsized":"2023-10-09T04:41:54.906178+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":230,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951286+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:41:54.951286+0000","last_peered":"2023-10-09T04:41:54.951286+0000","last_clean":"2023-10-09T04:41:54.951286+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:41:54.951286+0000","last_undegraded":"2023-10-09T04:41:54.951286+0000","last_fullsized":"2023-10-09T04:41:54.951286+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":185,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913625+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:41:54.913625+0000","last_peered":"2023-10-09T04:41:54.913625+0000","last_clean":"2023-10-09T04:41:54.913625+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:41:54.913625+0000","last_undegraded":"2023-10-09T04:41:54.913625+0000","last_fullsized":"2023-10-09T04:41:54.913625+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":226,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:57.632302+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:41:57.632302+0000","last_peered":"2023-10-09T04:41:57.632302+0000","last_clean":"2023-10-09T04:41:57.632302+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:41:57.632302+0000","last_undegraded":"2023-10-09T04:41:57.632302+0000","last_fullsized":"2023-10-09T04:41:57.632302+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":233,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913330+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:41:54.913330+0000","last_peered":"2023-10-09T04:41:54.913330+0000","last_clean":"2023-10-09T04:41:54.913330+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:41:54.913330+0000","last_undegraded":"2023-10-09T04:41:54.913330+0000","last_fullsized":"2023-10-09T04:41:54.913330+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":199,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:57.632326+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:41:57.632326+0000","last_peered":"2023-10-09T04:41:57.632326+0000","last_clean":"2023-10-09T04:41:57.632326+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:41:57.632326+0000","last_undegraded":"2023-10-09T04:41:57.632326+0000","last_fullsized":"2023-10-09T04:41:57.632326+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":267,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952594+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:41:54.952594+0000","last_peered":"2023-10-09T04:41:54.952594+0000","last_clean":"2023-10-09T04:41:54.952594+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:41:54.952594+0000","last_undegraded":"2023-10-09T04:41:54.952594+0000","last_fullsized":"2023-10-09T04:41:54.952594+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":227,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952139+0000","last_change":"2023-10-09T04:39:40.406537+0000","last_active":"2023-10-09T04:41:54.952139+0000","last_peered":"2023-10-09T04:41:54.952139+0000","last_clean":"2023-10-09T04:41:54.952139+0000","last_became_active":"2023-10-09T04:39:40.406281+0000","last_became_peered":"2023-10-09T04:39:40.406281+0000","last_unstale":"2023-10-09T04:41:54.952139+0000","last_undegraded":"2023-10-09T04:41:54.952139+0000","last_fullsized":"2023-10-09T04:41:54.952139+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:09.883302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":31,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914638+0000","last_change":"2023-10-09T04:41:02.634559+0000","last_active":"2023-10-09T04:41:54.914638+0000","last_peered":"2023-10-09T04:41:54.914638+0000","last_clean":"2023-10-09T04:41:54.914638+0000","last_became_active":"2023-10-09T04:41:02.624597+0000","last_became_peered":"2023-10-09T04:41:02.624597+0000","last_unstale":"2023-10-09T04:41:54.914638+0000","last_undegraded":"2023-10-09T04:41:54.914638+0000","last_fullsized":"2023-10-09T04:41:54.914638+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:57.939757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":233,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914850+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:41:54.914850+0000","last_peered":"2023-10-09T04:41:54.914850+0000","last_clean":"2023-10-09T04:41:54.914850+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:41:54.914850+0000","last_undegraded":"2023-10-09T04:41:54.914850+0000","last_fullsized":"2023-10-09T04:41:54.914850+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":283,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.953380+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:41:54.953380+0000","last_peered":"2023-10-09T04:41:54.953380+0000","last_clean":"2023-10-09T04:41:54.953380+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:41:54.953380+0000","last_undegraded":"2023-10-09T04:41:54.953380+0000","last_fullsized":"2023-10-09T04:41:54.953380+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":226,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:57.632383+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:41:57.632383+0000","last_peered":"2023-10-09T04:41:57.632383+0000","last_clean":"2023-10-09T04:41:57.632383+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:41:57.632383+0000","last_undegraded":"2023-10-09T04:41:57.632383+0000","last_fullsized":"2023-10-09T04:41:57.632383+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":185,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951363+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:41:54.951363+0000","last_peered":"2023-10-09T04:41:54.951363+0000","last_clean":"2023-10-09T04:41:54.951363+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:41:54.951363+0000","last_undegraded":"2023-10-09T04:41:54.951363+0000","last_fullsized":"2023-10-09T04:41:54.951363+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913373+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:41:54.913373+0000","last_peered":"2023-10-09T04:41:54.913373+0000","last_clean":"2023-10-09T04:41:54.913373+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:41:54.913373+0000","last_undegraded":"2023-10-09T04:41:54.913373+0000","last_fullsized":"2023-10-09T04:41:54.913373+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":215,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951187+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:41:54.951187+0000","last_peered":"2023-10-09T04:41:54.951187+0000","last_clean":"2023-10-09T04:41:54.951187+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:41:54.951187+0000","last_undegraded":"2023-10-09T04:41:54.951187+0000","last_fullsized":"2023-10-09T04:41:54.951187+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":187,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906860+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:41:54.906860+0000","last_peered":"2023-10-09T04:41:54.906860+0000","last_clean":"2023-10-09T04:41:54.906860+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:41:54.906860+0000","last_undegraded":"2023-10-09T04:41:54.906860+0000","last_fullsized":"2023-10-09T04:41:54.906860+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":279,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.950938+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:41:54.950938+0000","last_peered":"2023-10-09T04:41:54.950938+0000","last_clean":"2023-10-09T04:41:54.950938+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:41:54.950938+0000","last_undegraded":"2023-10-09T04:41:54.950938+0000","last_fullsized":"2023-10-09T04:41:54.950938+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":225,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.907185+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:41:54.907185+0000","last_peered":"2023-10-09T04:41:54.907185+0000","last_clean":"2023-10-09T04:41:54.907185+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:41:54.907185+0000","last_undegraded":"2023-10-09T04:41:54.907185+0000","last_fullsized":"2023-10-09T04:41:54.907185+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"273'11480","reported_seq":18799,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:05.387400+0000","last_change":"2023-10-09T04:41:54.945309+0000","last_active":"2023-10-09T04:42:05.387400+0000","last_peered":"2023-10-09T04:42:05.387400+0000","last_clean":"2023-10-09T04:42:05.387400+0000","last_became_active":"2023-10-09T04:41:54.915620+0000","last_became_peered":"2023-10-09T04:41:54.915620+0000","last_unstale":"2023-10-09T04:42:05.387400+0000","last_undegraded":"2023-10-09T04:42:05.387400+0000","last_fullsized":"2023-10-09T04:42:05.387400+0000","mapping_epoch":271,"log_start":"242'1400","ondisk_log_start":"242'1400","created":237,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:35.953123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80216064,"num_objects":1224,"num_object_clones":0,"num_object_copies":3672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11480,"num_write_kb":406528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3622,"num_bytes_recovered":43122688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":289,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951943+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:41:54.951943+0000","last_peered":"2023-10-09T04:41:54.951943+0000","last_clean":"2023-10-09T04:41:54.951943+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:41:54.951943+0000","last_undegraded":"2023-10-09T04:41:54.951943+0000","last_fullsized":"2023-10-09T04:41:54.951943+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":215,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913236+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:41:54.913236+0000","last_peered":"2023-10-09T04:41:54.913236+0000","last_clean":"2023-10-09T04:41:54.913236+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:41:54.913236+0000","last_undegraded":"2023-10-09T04:41:54.913236+0000","last_fullsized":"2023-10-09T04:41:54.913236+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"273'11272","reported_seq":11318,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:06.691415+0000","last_change":"2023-10-09T04:37:52.255599+0000","last_active":"2023-10-09T04:42:06.691415+0000","last_peered":"2023-10-09T04:42:06.691415+0000","last_clean":"2023-10-09T04:42:06.691415+0000","last_became_active":"2023-10-09T04:37:52.255291+0000","last_became_peered":"2023-10-09T04:37:52.255291+0000","last_unstale":"2023-10-09T04:42:06.691415+0000","last_undegraded":"2023-10-09T04:42:06.691415+0000","last_fullsized":"2023-10-09T04:42:06.691415+0000","mapping_epoch":237,"log_start":"242'1200","ondisk_log_start":"242'1200","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.990132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11272,"num_write_kb":399744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":262,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951568+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:41:54.951568+0000","last_peered":"2023-10-09T04:41:54.951568+0000","last_clean":"2023-10-09T04:41:54.951568+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:41:54.951568+0000","last_undegraded":"2023-10-09T04:41:54.951568+0000","last_fullsized":"2023-10-09T04:41:54.951568+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":250,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914590+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:41:54.914590+0000","last_peered":"2023-10-09T04:41:54.914590+0000","last_clean":"2023-10-09T04:41:54.914590+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:41:54.914590+0000","last_undegraded":"2023-10-09T04:41:54.914590+0000","last_fullsized":"2023-10-09T04:41:54.914590+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"273'11403","reported_seq":11447,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:05.124181+0000","last_change":"2023-10-09T04:37:52.013171+0000","last_active":"2023-10-09T04:42:05.124181+0000","last_peered":"2023-10-09T04:42:05.124181+0000","last_clean":"2023-10-09T04:42:05.124181+0000","last_became_active":"2023-10-09T04:37:52.012873+0000","last_became_peered":"2023-10-09T04:37:52.012873+0000","last_unstale":"2023-10-09T04:42:05.124181+0000","last_undegraded":"2023-10-09T04:42:05.124181+0000","last_fullsized":"2023-10-09T04:42:05.124181+0000","mapping_epoch":237,"log_start":"244'3200","ondisk_log_start":"244'3200","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8203,"log_dups_size":0,"ondisk_log_size":8203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:25.833222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82771968,"num_objects":1263,"num_object_clones":0,"num_object_copies":3789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11403,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":216,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952455+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:41:54.952455+0000","last_peered":"2023-10-09T04:41:54.952455+0000","last_clean":"2023-10-09T04:41:54.952455+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:41:54.952455+0000","last_undegraded":"2023-10-09T04:41:54.952455+0000","last_fullsized":"2023-10-09T04:41:54.952455+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":257,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951848+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:41:54.951848+0000","last_peered":"2023-10-09T04:41:54.951848+0000","last_clean":"2023-10-09T04:41:54.951848+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:41:54.951848+0000","last_undegraded":"2023-10-09T04:41:54.951848+0000","last_fullsized":"2023-10-09T04:41:54.951848+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"273'11326","reported_seq":11370,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:05.082750+0000","last_change":"2023-10-09T04:37:52.012019+0000","last_active":"2023-10-09T04:42:05.082750+0000","last_peered":"2023-10-09T04:42:05.082750+0000","last_clean":"2023-10-09T04:42:05.082750+0000","last_became_active":"2023-10-09T04:37:52.011714+0000","last_became_peered":"2023-10-09T04:37:52.011714+0000","last_unstale":"2023-10-09T04:42:05.082750+0000","last_undegraded":"2023-10-09T04:42:05.082750+0000","last_fullsized":"2023-10-09T04:42:05.082750+0000","mapping_epoch":237,"log_start":"244'3200","ondisk_log_start":"244'3200","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:55.324035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87031808,"num_objects":1328,"num_object_clones":0,"num_object_copies":3984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11326,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913292+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:41:54.913292+0000","last_peered":"2023-10-09T04:41:54.913292+0000","last_clean":"2023-10-09T04:41:54.913292+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:41:54.913292+0000","last_undegraded":"2023-10-09T04:41:54.913292+0000","last_fullsized":"2023-10-09T04:41:54.913292+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":130,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913133+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:41:54.913133+0000","last_peered":"2023-10-09T04:41:54.913133+0000","last_clean":"2023-10-09T04:41:54.913133+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:41:54.913133+0000","last_undegraded":"2023-10-09T04:41:54.913133+0000","last_fullsized":"2023-10-09T04:41:54.913133+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"273'11067","reported_seq":11111,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:04.941987+0000","last_change":"2023-10-09T04:37:52.013681+0000","last_active":"2023-10-09T04:42:04.941987+0000","last_peered":"2023-10-09T04:42:04.941987+0000","last_clean":"2023-10-09T04:42:04.941987+0000","last_became_active":"2023-10-09T04:37:52.013382+0000","last_became_peered":"2023-10-09T04:37:52.013382+0000","last_unstale":"2023-10-09T04:42:04.941987+0000","last_undegraded":"2023-10-09T04:42:04.941987+0000","last_fullsized":"2023-10-09T04:42:04.941987+0000","mapping_epoch":237,"log_start":"242'2400","ondisk_log_start":"242'2400","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:30.877521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11067,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":270,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.907456+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:41:54.907456+0000","last_peered":"2023-10-09T04:41:54.907456+0000","last_clean":"2023-10-09T04:41:54.907456+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:41:54.907456+0000","last_undegraded":"2023-10-09T04:41:54.907456+0000","last_fullsized":"2023-10-09T04:41:54.907456+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914813+0000","last_change":"2023-10-09T04:41:02.624962+0000","last_active":"2023-10-09T04:41:54.914813+0000","last_peered":"2023-10-09T04:41:54.914813+0000","last_clean":"2023-10-09T04:41:54.914813+0000","last_became_active":"2023-10-09T04:41:02.624555+0000","last_became_peered":"2023-10-09T04:41:02.624555+0000","last_unstale":"2023-10-09T04:41:54.914813+0000","last_undegraded":"2023-10-09T04:41:54.914813+0000","last_fullsized":"2023-10-09T04:41:54.914813+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:48.506794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"273'11291","reported_seq":11338,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:06.791199+0000","last_change":"2023-10-09T04:37:52.255728+0000","last_active":"2023-10-09T04:42:06.791199+0000","last_peered":"2023-10-09T04:42:06.791199+0000","last_clean":"2023-10-09T04:42:06.791199+0000","last_became_active":"2023-10-09T04:37:52.255283+0000","last_became_peered":"2023-10-09T04:37:52.255283+0000","last_unstale":"2023-10-09T04:42:06.791199+0000","last_undegraded":"2023-10-09T04:42:06.791199+0000","last_fullsized":"2023-10-09T04:42:06.791199+0000","mapping_epoch":237,"log_start":"242'1200","ondisk_log_start":"242'1200","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10091,"log_dups_size":0,"ondisk_log_size":10091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:18.900085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79626240,"num_objects":1215,"num_object_clones":0,"num_object_copies":3645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11291,"num_write_kb":400192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951964+0000","last_change":"2023-10-09T04:41:02.603773+0000","last_active":"2023-10-09T04:41:54.951964+0000","last_peered":"2023-10-09T04:41:54.951964+0000","last_clean":"2023-10-09T04:41:54.951964+0000","last_became_active":"2023-10-09T04:41:02.603407+0000","last_became_peered":"2023-10-09T04:41:02.603407+0000","last_unstale":"2023-10-09T04:41:54.951964+0000","last_undegraded":"2023-10-09T04:41:54.951964+0000","last_fullsized":"2023-10-09T04:41:54.951964+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:18.401503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914847+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:41:54.914847+0000","last_peered":"2023-10-09T04:41:54.914847+0000","last_clean":"2023-10-09T04:41:54.914847+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:41:54.914847+0000","last_undegraded":"2023-10-09T04:41:54.914847+0000","last_fullsized":"2023-10-09T04:41:54.914847+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"273'11195","reported_seq":11238,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:04.906406+0000","last_change":"2023-10-09T04:37:52.256288+0000","last_active":"2023-10-09T04:42:04.906406+0000","last_peered":"2023-10-09T04:42:04.906406+0000","last_clean":"2023-10-09T04:42:04.906406+0000","last_became_active":"2023-10-09T04:37:52.256008+0000","last_became_peered":"2023-10-09T04:37:52.256008+0000","last_unstale":"2023-10-09T04:42:04.906406+0000","last_undegraded":"2023-10-09T04:42:04.906406+0000","last_fullsized":"2023-10-09T04:42:04.906406+0000","mapping_epoch":237,"log_start":"243'2600","ondisk_log_start":"243'2600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8595,"log_dups_size":0,"ondisk_log_size":8595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:34.570497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79101952,"num_objects":1207,"num_object_clones":0,"num_object_copies":3621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11195,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914407+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:41:54.914407+0000","last_peered":"2023-10-09T04:41:54.914407+0000","last_clean":"2023-10-09T04:41:54.914407+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:41:54.914407+0000","last_undegraded":"2023-10-09T04:41:54.914407+0000","last_fullsized":"2023-10-09T04:41:54.914407+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914978+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:41:54.914978+0000","last_peered":"2023-10-09T04:41:54.914978+0000","last_clean":"2023-10-09T04:41:54.914978+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:41:54.914978+0000","last_undegraded":"2023-10-09T04:41:54.914978+0000","last_fullsized":"2023-10-09T04:41:54.914978+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"273'11620","reported_seq":11674,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:07.610209+0000","last_change":"2023-10-09T04:39:40.403627+0000","last_active":"2023-10-09T04:42:07.610209+0000","last_peered":"2023-10-09T04:42:07.610209+0000","last_clean":"2023-10-09T04:42:07.610209+0000","last_became_active":"2023-10-09T04:39:40.393228+0000","last_became_peered":"2023-10-09T04:39:40.393228+0000","last_unstale":"2023-10-09T04:42:07.610209+0000","last_undegraded":"2023-10-09T04:42:07.610209+0000","last_fullsized":"2023-10-09T04:42:07.610209+0000","mapping_epoch":251,"log_start":"244'3700","ondisk_log_start":"244'3700","created":237,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7920,"log_dups_size":0,"ondisk_log_size":7920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:22.432701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11620,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":227,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906007+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:41:54.906007+0000","last_peered":"2023-10-09T04:41:54.906007+0000","last_clean":"2023-10-09T04:41:54.906007+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:41:54.906007+0000","last_undegraded":"2023-10-09T04:41:54.906007+0000","last_fullsized":"2023-10-09T04:41:54.906007+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":257,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:57.632432+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:41:57.632432+0000","last_peered":"2023-10-09T04:41:57.632432+0000","last_clean":"2023-10-09T04:41:57.632432+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:41:57.632432+0000","last_undegraded":"2023-10-09T04:41:57.632432+0000","last_fullsized":"2023-10-09T04:41:57.632432+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"273'11377","reported_seq":17994,"reported_epoch":273,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:42:06.778529+0000","last_change":"2023-10-09T04:41:53.384634+0000","last_active":"2023-10-09T04:42:06.778529+0000","last_peered":"2023-10-09T04:42:06.778529+0000","last_clean":"2023-10-09T04:39:11.377114+0000","last_became_active":"2023-10-09T04:41:02.731395+0000","last_became_peered":"2023-10-09T04:41:02.731395+0000","last_unstale":"2023-10-09T04:42:06.778529+0000","last_undegraded":"2023-10-09T04:42:06.778529+0000","last_fullsized":"2023-10-09T04:41:02.586350+0000","mapping_epoch":261,"log_start":"242'1300","ondisk_log_start":"242'1300","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10077,"log_dups_size":0,"ondisk_log_size":10077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:11.256631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81526808,"num_objects":1245,"num_object_clones":0,"num_object_copies":3735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":84,"num_objects_unfound":0,"num_objects_dirty":1245,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11377,"num_write_kb":403841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3276,"num_bytes_recovered":214695936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,6","objects":84},{"shards":"2,6","objects":1161}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":150,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952196+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:41:54.952196+0000","last_peered":"2023-10-09T04:41:54.952196+0000","last_clean":"2023-10-09T04:41:54.952196+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:41:54.952196+0000","last_undegraded":"2023-10-09T04:41:54.952196+0000","last_fullsized":"2023-10-09T04:41:54.952196+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":267,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913396+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:41:54.913396+0000","last_peered":"2023-10-09T04:41:54.913396+0000","last_clean":"2023-10-09T04:41:54.913396+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:41:54.913396+0000","last_undegraded":"2023-10-09T04:41:54.913396+0000","last_fullsized":"2023-10-09T04:41:54.913396+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":244,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913176+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:41:54.913176+0000","last_peered":"2023-10-09T04:41:54.913176+0000","last_clean":"2023-10-09T04:41:54.913176+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:41:54.913176+0000","last_undegraded":"2023-10-09T04:41:54.913176+0000","last_fullsized":"2023-10-09T04:41:54.913176+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"273'11205","reported_seq":11251,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:07.537135+0000","last_change":"2023-10-09T04:37:52.255930+0000","last_active":"2023-10-09T04:42:07.537135+0000","last_peered":"2023-10-09T04:42:07.537135+0000","last_clean":"2023-10-09T04:42:07.537135+0000","last_became_active":"2023-10-09T04:37:52.255584+0000","last_became_peered":"2023-10-09T04:37:52.255584+0000","last_unstale":"2023-10-09T04:42:07.537135+0000","last_undegraded":"2023-10-09T04:42:07.537135+0000","last_fullsized":"2023-10-09T04:42:07.537135+0000","mapping_epoch":237,"log_start":"244'3300","ondisk_log_start":"244'3300","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7905,"log_dups_size":0,"ondisk_log_size":7905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:52.503096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78184448,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11205,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":237,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913229+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:41:54.913229+0000","last_peered":"2023-10-09T04:41:54.913229+0000","last_clean":"2023-10-09T04:41:54.913229+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:41:54.913229+0000","last_undegraded":"2023-10-09T04:41:54.913229+0000","last_fullsized":"2023-10-09T04:41:54.913229+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914686+0000","last_change":"2023-10-09T04:41:02.608773+0000","last_active":"2023-10-09T04:41:54.914686+0000","last_peered":"2023-10-09T04:41:54.914686+0000","last_clean":"2023-10-09T04:41:54.914686+0000","last_became_active":"2023-10-09T04:41:02.608455+0000","last_became_peered":"2023-10-09T04:41:02.608455+0000","last_unstale":"2023-10-09T04:41:54.914686+0000","last_undegraded":"2023-10-09T04:41:54.914686+0000","last_fullsized":"2023-10-09T04:41:54.914686+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:29.807000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.919242+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:41:54.919242+0000","last_peered":"2023-10-09T04:41:54.919242+0000","last_clean":"2023-10-09T04:41:54.919242+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:41:54.919242+0000","last_undegraded":"2023-10-09T04:41:54.919242+0000","last_fullsized":"2023-10-09T04:41:54.919242+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"273'11179","reported_seq":11225,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:07.625298+0000","last_change":"2023-10-09T04:37:52.012774+0000","last_active":"2023-10-09T04:42:07.625298+0000","last_peered":"2023-10-09T04:42:07.625298+0000","last_clean":"2023-10-09T04:42:07.625298+0000","last_became_active":"2023-10-09T04:37:52.012402+0000","last_became_peered":"2023-10-09T04:37:52.012402+0000","last_unstale":"2023-10-09T04:42:07.625298+0000","last_undegraded":"2023-10-09T04:42:07.625298+0000","last_fullsized":"2023-10-09T04:42:07.625298+0000","mapping_epoch":237,"log_start":"244'3200","ondisk_log_start":"244'3200","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7979,"log_dups_size":0,"ondisk_log_size":7979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:37.167042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75956224,"num_objects":1159,"num_object_clones":0,"num_object_copies":3477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11179,"num_write_kb":394816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":229,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.918929+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:41:54.918929+0000","last_peered":"2023-10-09T04:41:54.918929+0000","last_clean":"2023-10-09T04:41:54.918929+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:41:54.918929+0000","last_undegraded":"2023-10-09T04:41:54.918929+0000","last_fullsized":"2023-10-09T04:41:54.918929+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":228,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.918935+0000","last_change":"2023-10-09T04:39:40.406420+0000","last_active":"2023-10-09T04:41:54.918935+0000","last_peered":"2023-10-09T04:41:54.918935+0000","last_clean":"2023-10-09T04:41:54.918935+0000","last_became_active":"2023-10-09T04:39:40.406072+0000","last_became_peered":"2023-10-09T04:39:40.406072+0000","last_unstale":"2023-10-09T04:41:54.918935+0000","last_undegraded":"2023-10-09T04:41:54.918935+0000","last_fullsized":"2023-10-09T04:41:54.918935+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:08.385113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":145,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913598+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:41:54.913598+0000","last_peered":"2023-10-09T04:41:54.913598+0000","last_clean":"2023-10-09T04:41:54.913598+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:41:54.913598+0000","last_undegraded":"2023-10-09T04:41:54.913598+0000","last_fullsized":"2023-10-09T04:41:54.913598+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"273'11628","reported_seq":11674,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:07.625467+0000","last_change":"2023-10-09T04:37:52.013105+0000","last_active":"2023-10-09T04:42:07.625467+0000","last_peered":"2023-10-09T04:42:07.625467+0000","last_clean":"2023-10-09T04:42:07.625467+0000","last_became_active":"2023-10-09T04:37:52.012773+0000","last_became_peered":"2023-10-09T04:37:52.012773+0000","last_unstale":"2023-10-09T04:42:07.625467+0000","last_undegraded":"2023-10-09T04:42:07.625467+0000","last_fullsized":"2023-10-09T04:42:07.625467+0000","mapping_epoch":237,"log_start":"244'3700","ondisk_log_start":"244'3700","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.291873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79822848,"num_objects":1218,"num_object_clones":0,"num_object_copies":3654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11628,"num_write_kb":411072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":254,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913160+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:41:54.913160+0000","last_peered":"2023-10-09T04:41:54.913160+0000","last_clean":"2023-10-09T04:41:54.913160+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:41:54.913160+0000","last_undegraded":"2023-10-09T04:41:54.913160+0000","last_fullsized":"2023-10-09T04:41:54.913160+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":665,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952856+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:41:54.952856+0000","last_peered":"2023-10-09T04:41:54.952856+0000","last_clean":"2023-10-09T04:41:54.952856+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:41:54.952856+0000","last_undegraded":"2023-10-09T04:41:54.952856+0000","last_fullsized":"2023-10-09T04:41:54.952856+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":198,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906527+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:41:54.906527+0000","last_peered":"2023-10-09T04:41:54.906527+0000","last_clean":"2023-10-09T04:41:54.906527+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:41:54.906527+0000","last_undegraded":"2023-10-09T04:41:54.906527+0000","last_fullsized":"2023-10-09T04:41:54.906527+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":233,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951770+0000","last_change":"2023-10-09T04:41:02.612580+0000","last_active":"2023-10-09T04:41:54.951770+0000","last_peered":"2023-10-09T04:41:54.951770+0000","last_clean":"2023-10-09T04:41:54.951770+0000","last_became_active":"2023-10-09T04:41:02.612258+0000","last_became_peered":"2023-10-09T04:41:02.612258+0000","last_unstale":"2023-10-09T04:41:54.951770+0000","last_undegraded":"2023-10-09T04:41:54.951770+0000","last_fullsized":"2023-10-09T04:41:54.951770+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:53.656800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"273'11358","reported_seq":11404,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:07.655753+0000","last_change":"2023-10-09T04:37:52.257119+0000","last_active":"2023-10-09T04:42:07.655753+0000","last_peered":"2023-10-09T04:42:07.655753+0000","last_clean":"2023-10-09T04:42:07.655753+0000","last_became_active":"2023-10-09T04:37:52.256760+0000","last_became_peered":"2023-10-09T04:37:52.256760+0000","last_unstale":"2023-10-09T04:42:07.655753+0000","last_undegraded":"2023-10-09T04:42:07.655753+0000","last_fullsized":"2023-10-09T04:42:07.655753+0000","mapping_epoch":237,"log_start":"244'3400","ondisk_log_start":"244'3400","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:16.001253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78118912,"num_objects":1192,"num_object_clones":0,"num_object_copies":3576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11358,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":273,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952784+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:41:54.952784+0000","last_peered":"2023-10-09T04:41:54.952784+0000","last_clean":"2023-10-09T04:41:54.952784+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:41:54.952784+0000","last_undegraded":"2023-10-09T04:41:54.952784+0000","last_fullsized":"2023-10-09T04:41:54.952784+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914501+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:41:54.914501+0000","last_peered":"2023-10-09T04:41:54.914501+0000","last_clean":"2023-10-09T04:41:54.914501+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:41:54.914501+0000","last_undegraded":"2023-10-09T04:41:54.914501+0000","last_fullsized":"2023-10-09T04:41:54.914501+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":149,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951644+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:41:54.951644+0000","last_peered":"2023-10-09T04:41:54.951644+0000","last_clean":"2023-10-09T04:41:54.951644+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:41:54.951644+0000","last_undegraded":"2023-10-09T04:41:54.951644+0000","last_fullsized":"2023-10-09T04:41:54.951644+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"273'11253","reported_seq":16835,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:42:06.756530+0000","last_change":"2023-10-09T04:41:03.882872+0000","last_active":"2023-10-09T04:42:06.756530+0000","last_peered":"2023-10-09T04:42:06.756530+0000","last_clean":"2023-10-09T04:39:11.375310+0000","last_became_active":"2023-10-09T04:41:03.851493+0000","last_became_peered":"2023-10-09T04:41:03.851493+0000","last_unstale":"2023-10-09T04:42:06.756530+0000","last_undegraded":"2023-10-09T04:41:03.605177+0000","last_fullsized":"2023-10-09T04:41:03.588185+0000","mapping_epoch":262,"log_start":"242'1200","ondisk_log_start":"242'1200","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10053,"log_dups_size":0,"ondisk_log_size":10053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:51.273314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77660160,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3394,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11253,"num_write_kb":398016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2757,"num_bytes_recovered":180682752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":549},{"shards":"1,2,7","objects":635}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":276,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.952521+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:41:54.952521+0000","last_peered":"2023-10-09T04:41:54.952521+0000","last_clean":"2023-10-09T04:41:54.952521+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:41:54.952521+0000","last_undegraded":"2023-10-09T04:41:54.952521+0000","last_fullsized":"2023-10-09T04:41:54.952521+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":228,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.919164+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:41:54.919164+0000","last_peered":"2023-10-09T04:41:54.919164+0000","last_clean":"2023-10-09T04:41:54.919164+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:41:54.919164+0000","last_undegraded":"2023-10-09T04:41:54.919164+0000","last_fullsized":"2023-10-09T04:41:54.919164+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":117,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913214+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:41:54.913214+0000","last_peered":"2023-10-09T04:41:54.913214+0000","last_clean":"2023-10-09T04:41:54.913214+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:41:54.913214+0000","last_undegraded":"2023-10-09T04:41:54.913214+0000","last_fullsized":"2023-10-09T04:41:54.913214+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"273'11279","reported_seq":11323,"reported_epoch":273,"state":"active+clean","last_fresh":"2023-10-09T04:42:05.062888+0000","last_change":"2023-10-09T04:37:52.013669+0000","last_active":"2023-10-09T04:42:05.062888+0000","last_peered":"2023-10-09T04:42:05.062888+0000","last_clean":"2023-10-09T04:42:05.062888+0000","last_became_active":"2023-10-09T04:37:52.013386+0000","last_became_peered":"2023-10-09T04:37:52.013386+0000","last_unstale":"2023-10-09T04:42:05.062888+0000","last_undegraded":"2023-10-09T04:42:05.062888+0000","last_fullsized":"2023-10-09T04:42:05.062888+0000","mapping_epoch":237,"log_start":"243'3100","ondisk_log_start":"243'3100","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8179,"log_dups_size":0,"ondisk_log_size":8179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:57.288541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81985536,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11279,"num_write_kb":400960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":270,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.907106+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:41:54.907106+0000","last_peered":"2023-10-09T04:41:54.907106+0000","last_clean":"2023-10-09T04:41:54.907106+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:41:54.907106+0000","last_undegraded":"2023-10-09T04:41:54.907106+0000","last_fullsized":"2023-10-09T04:41:54.907106+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":243,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951182+0000","last_change":"2023-10-09T04:41:02.629667+0000","last_active":"2023-10-09T04:41:54.951182+0000","last_peered":"2023-10-09T04:41:54.951182+0000","last_clean":"2023-10-09T04:41:54.951182+0000","last_became_active":"2023-10-09T04:41:02.629305+0000","last_became_peered":"2023-10-09T04:41:02.629305+0000","last_unstale":"2023-10-09T04:41:54.951182+0000","last_undegraded":"2023-10-09T04:41:54.951182+0000","last_fullsized":"2023-10-09T04:41:54.951182+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:36.600453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":260,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:57.632264+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:41:57.632264+0000","last_peered":"2023-10-09T04:41:57.632264+0000","last_clean":"2023-10-09T04:41:57.632264+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:41:57.632264+0000","last_undegraded":"2023-10-09T04:41:57.632264+0000","last_fullsized":"2023-10-09T04:41:57.632264+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"273'11513","reported_seq":16785,"reported_epoch":273,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:42:07.189067+0000","last_change":"2023-10-09T04:41:03.880649+0000","last_active":"2023-10-09T04:42:07.189067+0000","last_peered":"2023-10-09T04:42:07.189067+0000","last_clean":"2023-10-09T04:39:11.359026+0000","last_became_active":"2023-10-09T04:41:03.853327+0000","last_became_peered":"2023-10-09T04:41:03.853327+0000","last_unstale":"2023-10-09T04:42:07.189067+0000","last_undegraded":"2023-10-09T04:41:03.604660+0000","last_fullsized":"2023-10-09T04:41:03.586462+0000","mapping_epoch":262,"log_start":"244'4098","ondisk_log_start":"244'4098","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7415,"log_dups_size":0,"ondisk_log_size":7415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:23:51.288086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80543744,"num_objects":1229,"num_object_clones":0,"num_object_copies":3687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3522,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11513,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2607,"num_bytes_recovered":170524672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,1,7","objects":680},{"shards":"1,7","objects":549}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":252,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.951236+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:41:54.951236+0000","last_peered":"2023-10-09T04:41:54.951236+0000","last_clean":"2023-10-09T04:41:54.951236+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:41:54.951236+0000","last_undegraded":"2023-10-09T04:41:54.951236+0000","last_fullsized":"2023-10-09T04:41:54.951236+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":45,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.915027+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:41:54.915027+0000","last_peered":"2023-10-09T04:41:54.915027+0000","last_clean":"2023-10-09T04:41:54.915027+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:41:54.915027+0000","last_undegraded":"2023-10-09T04:41:54.915027+0000","last_fullsized":"2023-10-09T04:41:54.915027+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914392+0000","last_change":"2023-10-09T04:41:02.614004+0000","last_active":"2023-10-09T04:41:54.914392+0000","last_peered":"2023-10-09T04:41:54.914392+0000","last_clean":"2023-10-09T04:41:54.914392+0000","last_became_active":"2023-10-09T04:41:02.612300+0000","last_became_peered":"2023-10-09T04:41:02.612300+0000","last_unstale":"2023-10-09T04:41:54.914392+0000","last_undegraded":"2023-10-09T04:41:54.914392+0000","last_fullsized":"2023-10-09T04:41:54.914392+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:04.375059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":113,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914763+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:41:54.914763+0000","last_peered":"2023-10-09T04:41:54.914763+0000","last_clean":"2023-10-09T04:41:54.914763+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:41:54.914763+0000","last_undegraded":"2023-10-09T04:41:54.914763+0000","last_fullsized":"2023-10-09T04:41:54.914763+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":222,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.919319+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:41:54.919319+0000","last_peered":"2023-10-09T04:41:54.919319+0000","last_clean":"2023-10-09T04:41:54.919319+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:41:54.919319+0000","last_undegraded":"2023-10-09T04:41:54.919319+0000","last_fullsized":"2023-10-09T04:41:54.919319+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":263,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913178+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:41:54.913178+0000","last_peered":"2023-10-09T04:41:54.913178+0000","last_clean":"2023-10-09T04:41:54.913178+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:41:54.913178+0000","last_undegraded":"2023-10-09T04:41:54.913178+0000","last_fullsized":"2023-10-09T04:41:54.913178+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914633+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:41:54.914633+0000","last_peered":"2023-10-09T04:41:54.914633+0000","last_clean":"2023-10-09T04:41:54.914633+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:41:54.914633+0000","last_undegraded":"2023-10-09T04:41:54.914633+0000","last_fullsized":"2023-10-09T04:41:54.914633+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":218,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913423+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:41:54.913423+0000","last_peered":"2023-10-09T04:41:54.913423+0000","last_clean":"2023-10-09T04:41:54.913423+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:41:54.913423+0000","last_undegraded":"2023-10-09T04:41:54.913423+0000","last_fullsized":"2023-10-09T04:41:54.913423+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914840+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:41:54.914840+0000","last_peered":"2023-10-09T04:41:54.914840+0000","last_clean":"2023-10-09T04:41:54.914840+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:41:54.914840+0000","last_undegraded":"2023-10-09T04:41:54.914840+0000","last_fullsized":"2023-10-09T04:41:54.914840+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":261,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:57.632407+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:41:57.632407+0000","last_peered":"2023-10-09T04:41:57.632407+0000","last_clean":"2023-10-09T04:41:57.632407+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:41:57.632407+0000","last_undegraded":"2023-10-09T04:41:57.632407+0000","last_fullsized":"2023-10-09T04:41:57.632407+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":292,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.906989+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:41:54.906989+0000","last_peered":"2023-10-09T04:41:54.906989+0000","last_clean":"2023-10-09T04:41:54.906989+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:41:54.906989+0000","last_undegraded":"2023-10-09T04:41:54.906989+0000","last_fullsized":"2023-10-09T04:41:54.906989+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":147,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.913721+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:41:54.913721+0000","last_peered":"2023-10-09T04:41:54.913721+0000","last_clean":"2023-10-09T04:41:54.913721+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:41:54.913721+0000","last_undegraded":"2023-10-09T04:41:54.913721+0000","last_fullsized":"2023-10-09T04:41:54.913721+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":272,"state":"active+clean","last_fresh":"2023-10-09T04:41:54.914383+0000","last_change":"2023-10-09T04:41:02.608725+0000","last_active":"2023-10-09T04:41:54.914383+0000","last_peered":"2023-10-09T04:41:54.914383+0000","last_clean":"2023-10-09T04:41:54.914383+0000","last_became_active":"2023-10-09T04:41:02.606509+0000","last_became_peered":"2023-10-09T04:41:02.606509+0000","last_unstale":"2023-10-09T04:41:54.914383+0000","last_undegraded":"2023-10-09T04:41:54.914383+0000","last_fullsized":"2023-10-09T04:41:54.914383+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:49.430101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1288503320,"num_objects":19662,"num_object_clones":0,"num_object_copies":58986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6916,"num_objects_misplaced":84,"num_objects_unfound":0,"num_objects_dirty":19662,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181446,"num_write_kb":6435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12262,"num_bytes_recovered":609026048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4484509696,"data_stored":4484497480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139248,"ondisk_log_size":139248,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":5624,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234422,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1450232,"kb_used_data":543344,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":92294920,"statfs":{"total":95995035648,"available":94509998080,"internally_reserved":0,"allocated":556384256,"data_stored":555796316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:35 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":2.6349999999999998,"15min":4.1559999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":14.571999999999999,"5min":44.122999999999998,"15min":186.446},"last":0.30099999999999999},{"interface":"front","average":{"1min":3.992,"5min":3.0579999999999998,"15min":4.3620000000000001},"min":{"1min":0.29299999999999998,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":14.170999999999999,"5min":43.313000000000002,"15min":186.429},"last":0.36099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:41:35 2023","interfaces":[{"interface":"back","average":{"1min":3.4449999999999998,"5min":6.1289999999999996,"15min":5.3879999999999999},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":19.298999999999999,"5min":264.62,"15min":264.62},"last":3.2850000000000001},{"interface":"front","average":{"1min":2.9220000000000002,"5min":6.4630000000000001,"15min":5.4969999999999999},"min":{"1min":0.29899999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":19.251999999999999,"5min":264.65899999999999,"15min":264.65899999999999},"last":0.33600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:41:54 2023","interfaces":[{"interface":"back","average":{"1min":11.278,"5min":7.556,"15min":3.7200000000000002},"min":{"1min":0.34899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":156.053,"5min":184.72,"15min":184.72},"last":0.379},{"interface":"front","average":{"1min":12.067,"5min":7.8570000000000002,"15min":4.1539999999999999},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":156.036,"5min":184.697,"15min":184.697},"last":0.44900000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:41:18 2023","interfaces":[{"interface":"back","average":{"1min":3.4220000000000002,"5min":7.0350000000000001,"15min":7.5499999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":20.469999999999999,"5min":182.839,"15min":249.989},"last":0.434},{"interface":"front","average":{"1min":4.8120000000000003,"5min":7.1369999999999996,"15min":7.7130000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":20.765000000000001,"5min":182.678,"15min":250.21700000000001},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:41:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5619999999999998,"5min":1.4810000000000001,"15min":1.4019999999999999},"min":{"1min":0.315,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.257999999999999,"5min":14.257999999999999,"15min":14.257999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.095,"15min":1.1599999999999999},"min":{"1min":0.29099999999999998,"5min":0.28399999999999997,"15min":0.21199999999999999},"max":{"1min":7.2290000000000001,"5min":7.2290000000000001,"15min":10.613},"last":0.32400000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:41:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9980000000000002,"5min":1.256,"15min":1.1140000000000001},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.27900000000000003},"max":{"1min":13.319000000000001,"5min":13.319000000000001,"15min":13.319000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":3.9470000000000001,"5min":1.6120000000000001,"15min":1.506},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":18.408000000000001,"5min":18.408000000000001,"15min":18.408000000000001},"last":0.62}]},{"osd":6,"last update":"Mon Oct 9 04:41:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":1.202,"15min":1.335},"min":{"1min":0.374,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":11.388,"5min":11.388,"15min":12.007},"last":0.46000000000000002},{"interface":"front","average":{"1min":4.0110000000000001,"5min":1.643,"15min":1.623},"min":{"1min":0.38100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":18.311,"5min":18.311,"15min":18.311},"last":0.58599999999999997}]}]},{"osd":6,"up_from":46,"seq":197568496056,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576364,"kb_used_data":678116,"kb_used_omap":0,"kb_used_meta":898176,"kb_avail":92168788,"statfs":{"total":95995035648,"available":94380838912,"internally_reserved":0,"allocated":694390784,"data_stored":693814687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919732224},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":4.5620000000000003,"15min":8.1639999999999997},"min":{"1min":0.23499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":5.383,"5min":113.86799999999999,"15min":1284.521},"last":1},{"interface":"front","average":{"1min":2.3540000000000001,"5min":5.008,"15min":8.2889999999999997},"min":{"1min":0.26600000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.0050000000000008,"5min":109.63500000000001,"15min":1284.4690000000001},"last":4.4690000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:41:26 2023","interfaces":[{"interface":"back","average":{"1min":2.9660000000000002,"5min":7.1600000000000001,"15min":5.3419999999999996},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":21.939,"5min":150.75,"15min":185.74000000000001},"last":3.2810000000000001},{"interface":"front","average":{"1min":3.5760000000000001,"5min":7.3040000000000003,"15min":5.4509999999999996},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.26100000000000001},"max":{"1min":21.751999999999999,"5min":150.81999999999999,"15min":189.03999999999999},"last":0.97099999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:41:57 2023","interfaces":[{"interface":"back","average":{"1min":23.649000000000001,"5min":8.6799999999999997,"15min":4.2919999999999998},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":209.44399999999999,"5min":209.44399999999999,"15min":209.44399999999999},"last":4.5419999999999998},{"interface":"front","average":{"1min":23.584,"5min":8.7539999999999996,"15min":4.3049999999999997},"min":{"1min":0.41199999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":209.428,"5min":209.428,"15min":209.428},"last":3.3330000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:41:26 2023","interfaces":[{"interface":"back","average":{"1min":16.07,"5min":10.82,"15min":5.8230000000000004},"min":{"1min":0.34499999999999997,"5min":0.253,"15min":0.253},"max":{"1min":157.93100000000001,"5min":167.56100000000001,"15min":167.56100000000001},"last":1.0209999999999999},{"interface":"front","average":{"1min":16.552,"5min":11.131,"15min":5.9630000000000001},"min":{"1min":0.38500000000000001,"5min":0.33000000000000002,"15min":0.313},"max":{"1min":157.95099999999999,"5min":167.577,"15min":167.577},"last":4.5149999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.169,"15min":1.3560000000000001},"min":{"1min":0.28899999999999998,"5min":0.27200000000000002,"15min":0.246},"max":{"1min":6.8380000000000001,"5min":14.430999999999999,"15min":14.430999999999999},"last":3.359},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.0389999999999999,"15min":1.1299999999999999},"min":{"1min":0.36499999999999999,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":3.5209999999999999,"5min":6.0430000000000001,"15min":12.028},"last":1.2390000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.1990000000000001,"15min":1.798},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.28100000000000003},"max":{"1min":3.8250000000000002,"5min":14.631,"15min":81.087000000000003},"last":3.4540000000000002},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.127,"15min":1.7470000000000001},"min":{"1min":0.39200000000000002,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":3.7890000000000001,"5min":8.6479999999999997,"15min":81.706999999999994},"last":6.3280000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:42:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":1.2709999999999999,"15min":1.6539999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":8.3350000000000009,"5min":16.512,"15min":81.674999999999997},"last":6.2370000000000001},{"interface":"front","average":{"1min":2.512,"5min":1.53,"15min":1.7989999999999999},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.308},"max":{"1min":7.8049999999999997,"5min":17.715,"15min":81.055000000000007},"last":3.4300000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141146,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1230532,"kb_used_data":674428,"kb_used_omap":4,"kb_used_meta":556091,"kb_avail":92514620,"statfs":{"total":95995035648,"available":94734970880,"internally_reserved":0,"allocated":690614272,"data_stored":689488737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":569438094},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9340000000000002,"5min":4.516,"15min":4.0359999999999996},"min":{"1min":0.14799999999999999,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":14.407999999999999,"5min":93.171999999999997,"15min":97.308000000000007},"last":0.502},{"interface":"front","average":{"1min":5.4749999999999996,"5min":4.1600000000000001,"15min":4.1479999999999997},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":27.321000000000002,"5min":93.147000000000006,"15min":97.055999999999997},"last":7.3579999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:41:52 2023","interfaces":[{"interface":"back","average":{"1min":10.272,"5min":4.2430000000000003,"15min":3.1720000000000002},"min":{"1min":0.23300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":115.70399999999999,"5min":115.70399999999999,"15min":115.70399999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":8.9060000000000006,"5min":4.9000000000000004,"15min":3.5990000000000002},"min":{"1min":0.308,"5min":0.254,"15min":0.254},"max":{"1min":115.34099999999999,"5min":115.34099999999999,"15min":115.34099999999999},"last":7.0590000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:41:20 2023","interfaces":[{"interface":"back","average":{"1min":14.061,"5min":8.8849999999999998,"15min":5.1139999999999999},"min":{"1min":0.38,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":118.837,"5min":142.34200000000001,"15min":195.035},"last":6.6079999999999997},{"interface":"front","average":{"1min":15.640000000000001,"5min":10.092000000000001,"15min":5.75},"min":{"1min":0.39300000000000002,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":117.88500000000001,"5min":141.75200000000001,"15min":195.465},"last":9.6050000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.383,"15min":2.4889999999999999},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.245},"max":{"1min":9.593,"5min":14.449,"15min":170.072},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.343,"5min":2.5150000000000001,"15min":2.2130000000000001},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.28799999999999998},"max":{"1min":7.8010000000000002,"5min":75.513999999999996,"15min":169.94499999999999},"last":0.54400000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1680000000000001,"5min":2.113,"15min":2.4399999999999999},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":11.499000000000001,"5min":11.499000000000001,"15min":171.036},"last":7.1239999999999997},{"interface":"front","average":{"1min":2.6469999999999998,"5min":2.375,"15min":2.6299999999999999},"min":{"1min":0.38400000000000001,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":9.4900000000000002,"5min":14.557,"15min":170.88},"last":0.61599999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":2.137,"15min":1.9850000000000001},"min":{"1min":0.34300000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.545,"5min":11.548,"15min":13.388},"last":7.1559999999999997},{"interface":"front","average":{"1min":2.6629999999999998,"5min":2.6760000000000002,"15min":2.3069999999999999},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":9.4459999999999997,"5min":14.500999999999999,"15min":14.500999999999999},"last":0.83199999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:41:41 2023","interfaces":[{"interface":"back","average":{"1min":3.1930000000000001,"5min":2.7719999999999998,"15min":2.8769999999999998},"min":{"1min":0.434,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":9.5180000000000007,"5min":14.528,"15min":174.75999999999999},"last":1.4430000000000001},{"interface":"front","average":{"1min":3.2149999999999999,"5min":2.9119999999999999,"15min":2.5449999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":8.4309999999999992,"5min":75.471999999999994,"15min":174.779},"last":1.413}]}]},{"osd":0,"up_from":91,"seq":390842024203,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1244376,"kb_used_data":435792,"kb_used_omap":5,"kb_used_meta":808570,"kb_avail":92500776,"statfs":{"total":95995035648,"available":94720794624,"internally_reserved":0,"allocated":446251008,"data_stored":445223756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":827976314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:41:31 2023","interfaces":[{"interface":"back","average":{"1min":7.4660000000000002,"5min":6.5350000000000001,"15min":5.0620000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":119.392,"5min":138.536,"15min":167.92699999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":9.0709999999999997,"5min":7.4199999999999999,"15min":5.4500000000000002},"min":{"1min":0.23200000000000001,"5min":0.23000000000000001,"15min":0.19500000000000001},"max":{"1min":119.34399999999999,"5min":138.89599999999999,"15min":169.78999999999999},"last":0.223}]},{"osd":2,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":3.0920000000000001,"5min":6.1040000000000001,"15min":8.3339999999999996},"min":{"1min":0.247,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.935,"5min":131.30699999999999,"15min":131.30699999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":3.3500000000000001,"5min":6.4409999999999998,"15min":8.7370000000000001},"min":{"1min":0.22800000000000001,"5min":0.219,"15min":0.219},"max":{"1min":10.208,"5min":129.922,"15min":129.922},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":2.919,"5min":4.9589999999999996,"15min":4.1219999999999999},"min":{"1min":0.27000000000000002,"5min":0.216,"15min":0.20399999999999999},"max":{"1min":11.949,"5min":91.578999999999994,"15min":213.28399999999999},"last":156.69800000000001},{"interface":"front","average":{"1min":4.6360000000000001,"5min":5.9779999999999998,"15min":4.5549999999999997},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":28.696999999999999,"5min":92.067999999999998,"15min":212.27000000000001},"last":161.40199999999999}]},{"osd":4,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":3.7509999999999999,"15min":2.419},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.25700000000000001},"max":{"1min":4.7670000000000003,"5min":161.67099999999999,"15min":161.67099999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.9990000000000001,"5min":4.1189999999999998,"15min":2.6299999999999999},"min":{"1min":0.39000000000000001,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":8.0589999999999993,"5min":160.74799999999999,"15min":160.74799999999999},"last":0.41899999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":8.8889999999999993,"5min":3.673,"15min":2.4809999999999999},"min":{"1min":0.36599999999999999,"5min":0.32100000000000001,"15min":0.27900000000000003},"max":{"1min":136.322,"5min":136.322,"15min":136.322},"last":0.34499999999999997},{"interface":"front","average":{"1min":8.7119999999999997,"5min":3.4220000000000002,"15min":2.2069999999999999},"min":{"1min":0.439,"5min":0.32700000000000001,"15min":0.29099999999999998},"max":{"1min":136.15799999999999,"5min":136.15799999999999,"15min":136.15799999999999},"last":0.28999999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":2.4060000000000001,"15min":2.4889999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":8.1039999999999992,"5min":14.162000000000001,"15min":108.63800000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.5910000000000002,"5min":1.8720000000000001,"15min":2.1829999999999998},"min":{"1min":0.40400000000000003,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":7.2300000000000004,"5min":9.9510000000000005,"15min":108.76300000000001},"last":0.318}]},{"osd":7,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.901,"15min":1.8240000000000001},"min":{"1min":0.50700000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.2960000000000003,"5min":9.9809999999999999,"15min":11.890000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.587,"5min":1.909,"15min":1.9139999999999999},"min":{"1min":0.40000000000000002,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":4.133,"5min":11.260999999999999,"15min":11.260999999999999},"last":0.373}]}]},{"osd":2,"up_from":221,"seq":949187772481,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":749412,"kb_used_data":481116,"kb_used_omap":7,"kb_used_meta":268280,"kb_avail":92995740,"statfs":{"total":95995035648,"available":95227637760,"internally_reserved":0,"allocated":492662784,"data_stored":491684340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":274718801},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":4.0620000000000003,"5min":4.4539999999999997,"15min":8.4399999999999995},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":10.663,"5min":72.608000000000004,"15min":72.608000000000004},"last":0.313},{"interface":"front","average":{"1min":2.548,"5min":4.0540000000000003,"15min":7.9720000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.526,"5min":72.632000000000005,"15min":72.632000000000005},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":8.0559999999999992,"5min":3.9260000000000002,"15min":4.8520000000000003},"min":{"1min":0.27000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":75.590999999999994,"5min":75.590999999999994,"15min":75.590999999999994},"last":0.28000000000000003},{"interface":"front","average":{"1min":8.3770000000000007,"5min":4.5910000000000002,"15min":5.431},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":74.673000000000002,"5min":74.673000000000002,"15min":74.673000000000002},"last":0.26600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":4.2770000000000001,"5min":7.4050000000000002,"15min":6.0720000000000001},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.702,"5min":335.928,"15min":335.928},"last":225.85499999999999},{"interface":"front","average":{"1min":5.1210000000000004,"5min":7.9059999999999997,"15min":6.1459999999999999},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":19.041,"5min":335.68599999999998,"15min":335.68599999999998},"last":233.53299999999999}]},{"osd":4,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":4.4139999999999997,"5min":3.1600000000000001,"15min":4.3920000000000003},"min":{"1min":0.35999999999999999,"5min":0.315,"15min":0.315},"max":{"1min":13.641,"5min":72.944999999999993,"15min":72.944999999999993},"last":0.34399999999999997},{"interface":"front","average":{"1min":4.6849999999999996,"5min":3.0779999999999998,"15min":4.5380000000000003},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.378},"max":{"1min":14.183,"5min":73.054000000000002,"15min":73.054000000000002},"last":0.42599999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":6.3630000000000004,"5min":3.3860000000000001,"15min":4.2140000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":65.388000000000005,"5min":72.986000000000004,"15min":72.986000000000004},"last":0.39800000000000002},{"interface":"front","average":{"1min":7.5010000000000003,"5min":3.7610000000000001,"15min":4.7089999999999996},"min":{"1min":0.372,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":67.799000000000007,"5min":72.963999999999999,"15min":72.963999999999999},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":5.9139999999999997,"5min":4.8620000000000001,"15min":9.9979999999999993},"min":{"1min":0.45800000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":34.753,"5min":135.614,"15min":135.614},"last":0.50700000000000001},{"interface":"front","average":{"1min":5.3209999999999997,"5min":4.6239999999999997,"15min":9.3079999999999998},"min":{"1min":0.51500000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":34.796999999999997,"5min":130.196,"15min":130.196},"last":0.61699999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":6.0499999999999998,"5min":3.403,"15min":4.7869999999999999},"min":{"1min":0.55000000000000004,"5min":0.378,"15min":0.378},"max":{"1min":28.126999999999999,"5min":73.079999999999998,"15min":73.079999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":4.9489999999999998,"5min":3.1080000000000001,"15min":4.1479999999999997},"min":{"1min":0.437,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":28.106000000000002,"5min":73.069000000000003,"15min":73.069000000000003},"last":0.58099999999999996}]}]},{"osd":3,"up_from":142,"seq":609885356204,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005968,"kb_used_data":548360,"kb_used_omap":2,"kb_used_meta":457533,"kb_avail":92739184,"statfs":{"total":95995035648,"available":94964924416,"internally_reserved":0,"allocated":561520640,"data_stored":560558057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":468513845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":1.734,"15min":2.327},"min":{"1min":0.24299999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":17.648,"5min":17.648,"15min":142.14500000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":4.1529999999999996,"5min":2.327,"15min":2.52},"min":{"1min":0.26800000000000002,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":14.951000000000001,"5min":14.951000000000001,"15min":138.14599999999999},"last":0.79600000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1059999999999999,"5min":2.4409999999999998,"15min":3.464},"min":{"1min":0.254,"5min":0.19800000000000001,"15min":0.192},"max":{"1min":21.879000000000001,"5min":21.879000000000001,"15min":180.02500000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":4.3979999999999997,"5min":2.0720000000000001,"15min":3.5},"min":{"1min":0.29299999999999998,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":21.911999999999999,"5min":21.911999999999999,"15min":184.00899999999999},"last":2.5219999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:41:57 2023","interfaces":[{"interface":"back","average":{"1min":3.9039999999999999,"5min":5.3010000000000002,"15min":2.8839999999999999},"min":{"1min":0.30199999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":12.808999999999999,"5min":177.28399999999999,"15min":177.28399999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":17.309999999999999,"5min":7.3410000000000002,"15min":3.2869999999999999},"min":{"1min":0.29899999999999999,"5min":0.254,"15min":0.254},"max":{"1min":255.05199999999999,"5min":255.05199999999999,"15min":255.05199999999999},"last":9.6530000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.9570000000000001,"15min":2.0179999999999998},"min":{"1min":0.373,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":12.144,"5min":12.144,"15min":103.208},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.4089999999999998,"5min":1.613,"15min":1.7470000000000001},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.307},"max":{"1min":17.082000000000001,"5min":17.082000000000001,"15min":103.218},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":2.488,"5min":1.8320000000000001,"15min":2.2850000000000001},"min":{"1min":0.315,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.045000000000002,"5min":24.675999999999998,"15min":90.972999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.056,"5min":2.2090000000000001,"15min":2.371},"min":{"1min":0.434,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":14.867000000000001,"5min":17.422999999999998,"15min":90.959000000000003},"last":0.77200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":3.077,"5min":2.3170000000000002,"15min":2.161},"min":{"1min":0.40000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.916,"5min":14.916,"15min":104.422},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.98,"5min":3.2890000000000001,"15min":2.577},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":12.177,"5min":123.505,"15min":123.505},"last":1.226}]},{"osd":7,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":2.1179999999999999,"15min":2.2770000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.32300000000000001},"max":{"1min":12.096,"5min":12.096,"15min":57.399000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.698,"5min":1.702,"15min":2.0139999999999998},"min":{"1min":0.45900000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":16.984999999999999,"5min":16.984999999999999,"15min":57.447000000000003},"last":0.436}]}]},{"osd":4,"up_from":30,"seq":128849019330,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1678896,"kb_used_data":778664,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92066256,"statfs":{"total":95995035648,"available":94275846144,"internally_reserved":0,"allocated":797351936,"data_stored":796771574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":3.9180000000000001,"5min":4.2530000000000001,"15min":5.3250000000000002},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":23.114999999999998,"5min":171.13499999999999,"15min":219.577},"last":6.0430000000000001},{"interface":"front","average":{"1min":3.0529999999999999,"5min":4.2450000000000001,"15min":5.1150000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.247},"max":{"1min":13.992000000000001,"5min":171.11799999999999,"15min":218.96000000000001},"last":1.3160000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:41:35 2023","interfaces":[{"interface":"back","average":{"1min":3.7040000000000002,"5min":3.3260000000000001,"15min":5.0640000000000001},"min":{"1min":0.316,"5min":0.25,"15min":0.23400000000000001},"max":{"1min":33.322000000000003,"5min":61.764000000000003,"15min":142.74799999999999},"last":3.2120000000000002},{"interface":"front","average":{"1min":3.1469999999999998,"5min":3.294,"15min":5.1100000000000003},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.26700000000000002},"max":{"1min":33.418999999999997,"5min":61.747999999999998,"15min":139.09700000000001},"last":3.964}]},{"osd":2,"last update":"Mon Oct 9 04:41:55 2023","interfaces":[{"interface":"back","average":{"1min":13.916,"5min":5.9690000000000003,"15min":4.4189999999999996},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.307},"max":{"1min":195.267,"5min":195.267,"15min":195.267},"last":7.5039999999999996},{"interface":"front","average":{"1min":12.153,"5min":5.593,"15min":3.8809999999999998},"min":{"1min":0.30599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":195.15799999999999,"5min":195.15799999999999,"15min":195.15799999999999},"last":0.626}]},{"osd":3,"last update":"Mon Oct 9 04:41:14 2023","interfaces":[{"interface":"back","average":{"1min":23.756,"5min":6.9429999999999996,"15min":6.8600000000000003},"min":{"1min":0.38900000000000001,"5min":0.32600000000000001,"15min":0.27900000000000003},"max":{"1min":357.125,"5min":357.125,"15min":357.125},"last":0.65100000000000002},{"interface":"front","average":{"1min":23.155000000000001,"5min":6.7270000000000003,"15min":7.0039999999999996},"min":{"1min":0.34300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":359.02800000000002,"5min":359.02800000000002,"15min":359.02800000000002},"last":3.169}]},{"osd":5,"last update":"Mon Oct 9 04:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.1819999999999999,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":9.5609999999999999,"5min":9.5609999999999999,"15min":107.476},"last":3.286},{"interface":"front","average":{"1min":1.137,"5min":1.286,"15min":2.089},"min":{"1min":0.311,"5min":0.26900000000000002,"15min":0.23300000000000001},"max":{"1min":6.694,"5min":9.0210000000000008,"15min":107.414},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:41:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.1479999999999999,"15min":1.869},"min":{"1min":0.32200000000000001,"5min":0.31,"15min":0.26800000000000002},"max":{"1min":9.6099999999999994,"5min":9.6099999999999994,"15min":65.75},"last":3.238},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.2270000000000001,"15min":1.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.1630000000000003,"5min":9.3320000000000007,"15min":65.849999999999994},"last":6.0010000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.2470000000000001,"15min":1.6120000000000001},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.28000000000000003},"max":{"1min":5.0590000000000002,"5min":6.8259999999999996,"15min":76.283000000000001},"last":1.008},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.1899999999999999,"15min":1.583},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.32100000000000001},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":76.259},"last":3.2610000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757694,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1230560,"kb_used_data":332056,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":92514592,"statfs":{"total":95995035648,"available":94734942208,"internally_reserved":0,"allocated":340025344,"data_stored":339457110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:41:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":3.992,"15min":5.117},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.22},"max":{"1min":7.7249999999999996,"5min":88.346000000000004,"15min":210.85499999999999},"last":1.8049999999999999},{"interface":"front","average":{"1min":3.278,"5min":4.5339999999999998,"15min":5.1429999999999998},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":18.606999999999999,"5min":88.375,"15min":210.23500000000001},"last":0.69399999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:41:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":4.0810000000000004,"15min":4.3440000000000003},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.255},"max":{"1min":11.345000000000001,"5min":145.577,"15min":145.577},"last":2},{"interface":"front","average":{"1min":2.2029999999999998,"5min":4.0250000000000004,"15min":5.1840000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":12.547000000000001,"5min":145.792,"15min":145.792},"last":4.0579999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:41:54 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":3.4830000000000001,"15min":6.0309999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.0990000000000002,"5min":90.763999999999996,"15min":90.763999999999996},"last":0.82499999999999996},{"interface":"front","average":{"1min":3.1080000000000001,"5min":3.8580000000000001,"15min":6.2560000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.8090000000000002,"5min":91.340000000000003,"15min":91.340000000000003},"last":0.872}]},{"osd":3,"last update":"Mon Oct 9 04:41:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":8.9920000000000009,"15min":6.6029999999999998},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28100000000000003},"max":{"1min":8.4100000000000001,"5min":268.63299999999998,"15min":268.63299999999998},"last":2.3559999999999999},{"interface":"front","average":{"1min":2.4620000000000002,"5min":8.8689999999999998,"15min":6.4640000000000004},"min":{"1min":0.46600000000000003,"5min":0.32200000000000001,"15min":0.31},"max":{"1min":9.0899999999999999,"5min":268.92500000000001,"15min":268.92500000000001},"last":0.91300000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.76400000000000001,"15min":2.448},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.248},"max":{"1min":2.077,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.37},{"interface":"front","average":{"1min":1.823,"5min":1.242,"15min":2.798},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.6280000000000001,"5min":8.7599999999999998,"15min":361.71300000000002},"last":0.38200000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.97299999999999998,"15min":1.2869999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.7640000000000002,"5min":5.7640000000000002,"15min":11.192},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.1819999999999999,"5min":1.409,"15min":1.4490000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.231999999999999,"5min":13.231999999999999,"15min":14.334},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.163,"15min":1.4039999999999999},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.5899999999999999,"5min":8.5589999999999993,"15min":11.093},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.0349999999999999,"15min":1.2989999999999999},"min":{"1min":0.38800000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.4429999999999996,"5min":6.5129999999999999,"15min":15.257999999999999},"last":0.54600000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":439549952,"data_stored":439549952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":679874560,"data_stored":679870488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":480186368,"data_stored":480182296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":542965760,"data_stored":542965760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":789250048,"data_stored":789250048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":328990720,"data_stored":328990720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":674041856,"data_stored":674037784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":549650432,"data_stored":549650432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:42:20.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:42:20.158 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:20.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:20.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:20.423 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:20.431 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:20.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:20 smithi029 ceph-mon[223858]: pgmap v1044: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 847 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6916/40683 objects degraded (17.000%); 442 KiB/s, 40 objects/s recovering 2023-10-09T04:42:20.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/40683 objects degraded (17.000%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:20.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:20 smithi029 ceph-mon[216903]: pgmap v1044: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 847 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6916/40683 objects degraded (17.000%); 442 KiB/s, 40 objects/s recovering 2023-10-09T04:42:20.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/40683 objects degraded (17.000%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:20.808 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:20.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:20.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:20.914 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:21.024 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:20 smithi177 ceph-mon[195797]: pgmap v1044: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 847 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6916/40683 objects degraded (17.000%); 442 KiB/s, 40 objects/s recovering 2023-10-09T04:42:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/40683 objects degraded (17.000%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:190590 11.f e273/237)", 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.448930+0000", 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0011942019999999999, 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012203660000000001, 2023-10-09T04:42:21.455 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.448930+0000", 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.448930+0000", 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.448929+0000", 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.448937+0000", 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7470000000000005e-06 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.448953+0000", 2023-10-09T04:42:21.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6334999999999999e-05 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.448961+0000", 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6909999999999997e-06 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:190591 11.4 e273/237 11:25b6300e:::benchmark_data_smithi029_392508_object90035:head v 273'11931, mlcod=273'11929)", 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.449904+0000", 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.000220521, 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00030561100000000001, 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449904+0000", 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449904+0000", 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449903+0000", 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449909+0000", 2023-10-09T04:42:21.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6230000000000002e-06 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449921+0000", 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2325e-05 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449929+0000", 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7880000000000007e-06 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449967+0000", 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8352000000000001e-05 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449987+0000", 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9426999999999999e-05 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.45869.0:190581 11.c e273/262 ondisk, result = 0)", 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.450165+0000", 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4294967295.9999595, 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011417, 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450165+0000", 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450165+0000", 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450164+0000", 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450169+0000", 2023-10-09T04:42:21.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2410000000000003e-06 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450181+0000", 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.183e-05 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450188+0000", 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0060000000000003e-06 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450251+0000", 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3137000000000002e-05 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:190575 11.6 11:683362fc:::benchmark_data_smithi029_392508_object90019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.433001+0000", 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.017123499, 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017343786999999999, 2023-10-09T04:42:21.466 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 190575 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433001+0000", 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433001+0000", 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.432999+0000", 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433007+0000", 2023-10-09T04:42:21.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2599999999999999e-06 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433026+0000", 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9769e-05 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433035+0000", 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9069999999999996e-06 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:42:21.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439076+0000", 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060405110000000001 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439215+0000", 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013905800000000001 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:42:21.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439365+0000", 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014980200000000001 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.444982+0000", 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056175979999999997 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.471 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:190581 11.c 11:3d3bc7fd:::benchmark_data_smithi029_392508_object90025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.433054+0000", 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.017070345000000001, 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017383306000000001, 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:21.472 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 190581 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433054+0000", 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433054+0000", 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433053+0000", 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433057+0000", 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4340000000000001e-06 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433070+0000", 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2858e-05 2023-10-09T04:42:21.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433076+0000", 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7159999999999997e-06 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433140+0000", 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3621000000000006e-05 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433336+0000", 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000196386 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.433552+0000", 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021585800000000001 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.443651+0000", 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010099163 2023-10-09T04:42:21.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.449111+0000", 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054593030000000004 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450373+0000", 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012620870000000001 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:42:21.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.450438+0000", 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5310999999999999e-05 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:190584 11.f e273/237)", 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.438792+0000", 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.011332099, 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011748003999999999, 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:42:21.478 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.438792+0000", 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.438792+0000", 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.438791+0000", 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.438798+0000", 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8050000000000001e-06 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.438813+0000", 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4584e-05 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.438821+0000", 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0350000000000001e-06 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:190585 11.d 11:b194eeee:::benchmark_data_smithi029_392508_object90029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:21.439110+0000", 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.011014655, 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01148655, 2023-10-09T04:42:21.481 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 190585 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439110+0000", 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439110+0000", 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439108+0000", 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439115+0000", 2023-10-09T04:42:21.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.737e-06 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439127+0000", 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1902999999999999e-05 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439135+0000", 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.32e-06 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439238+0000", 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102981 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439377+0000", 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013915600000000001 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.439524+0000", 2023-10-09T04:42:21.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014617699999999999 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:21.447860+0000", 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083362309999999995 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:42:21.486 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 7 2023-10-09T04:42:21.487 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:42:21.706 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:42:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/179425753' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/179425753' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/179425753' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:42:22.745 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:22.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:22.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[216903]: pgmap v1045: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 775 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 6916/37245 objects degraded (18.569%); 442 KiB/s, 40 objects/s recovering 2023-10-09T04:42:22.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:22.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:42:22.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[216903]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:42:22.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[216903]: failed to encode map e274 with expected crc 2023-10-09T04:42:22.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[216903]: failed to encode map e274 with expected crc 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[223858]: pgmap v1045: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 775 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 6916/37245 objects degraded (18.569%); 442 KiB/s, 40 objects/s recovering 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[223858]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[223858]: failed to encode map e274 with expected crc 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:22 smithi029 ceph-mon[223858]: failed to encode map e274 with expected crc 2023-10-09T04:42:22.819 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:42:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:42:22] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-10-09T04:42:22.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:22.949 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:22 smithi177 ceph-mon[195797]: pgmap v1045: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 775 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 6916/37245 objects degraded (18.569%); 442 KiB/s, 40 objects/s recovering 2023-10-09T04:42:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:22 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:22 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:42:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:22 smithi177 ceph-mon[195797]: osdmap e274: 8 total, 8 up, 8 in 2023-10-09T04:42:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:22 smithi177 ceph-mon[195797]: failed to encode map e274 with expected crc 2023-10-09T04:42:23.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:22 smithi177 ceph-mon[195797]: failed to encode map e274 with expected crc 2023-10-09T04:42:23.531 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:42:23 smithi177 conmon[174387]: logger=cleanup t=2023-10-09T04:42:23.249226355Z level=info msg="Completed cleanup jobs" duration=65.111686ms 2023-10-09T04:42:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:24 smithi029 ceph-mon[216903]: pgmap v1047: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 654 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 6916/31455 objects degraded (21.987%); 0 B/s, 51 objects/s recovering 2023-10-09T04:42:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:24 smithi029 ceph-mon[223858]: pgmap v1047: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 654 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 6916/31455 objects degraded (21.987%); 0 B/s, 51 objects/s recovering 2023-10-09T04:42:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:24 smithi177 ceph-mon[195797]: pgmap v1047: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 654 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 6916/31455 objects degraded (21.987%); 0 B/s, 51 objects/s recovering 2023-10-09T04:42:25.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/31455 objects degraded (21.987%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/31455 objects degraded (21.987%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/31455 objects degraded (21.987%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:26.057 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:26.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:26.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:26.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:26.220 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:26.312 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:26.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:26.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:26.458 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:26.557 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:26.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:26.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:26.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:26.695 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:26.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:26.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:26.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:26.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:26 smithi029 ceph-mon[223858]: pgmap v1048: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 632 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6916/30372 objects degraded (22.771%); 0 B/s, 51 objects/s recovering 2023-10-09T04:42:26.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:26 smithi029 ceph-mon[216903]: pgmap v1048: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 632 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6916/30372 objects degraded (22.771%); 0 B/s, 51 objects/s recovering 2023-10-09T04:42:26.881 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:26.987 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:26.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:26 smithi177 ceph-mon[195797]: pgmap v1048: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 632 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6916/30372 objects degraded (22.771%); 0 B/s, 51 objects/s recovering 2023-10-09T04:42:27.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:27.169 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:27.744 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:27.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:27.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:27.877 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:27.923 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:27.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:27.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:28.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.125 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:28.626 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:28.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:28.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:28.741 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:28.823 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:28.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:28 smithi029 ceph-mon[216903]: pgmap v1049: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 543 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 6916/26133 objects degraded (26.465%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:28.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:28.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:28 smithi029 ceph-mon[223858]: pgmap v1049: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 543 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 6916/26133 objects degraded (26.465%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:28.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:29.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:28 smithi177 ceph-mon[195797]: pgmap v1049: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 543 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 6916/26133 objects degraded (26.465%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:29.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:29.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:42:29.948 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:29.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:30.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:30.110 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:30.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:30.177 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:30.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:30.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:30.337 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:30.449 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:30.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:30.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:30.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:30.594 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:30.658 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:30.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:30.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:30.800 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:30.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:30.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:30.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:30 smithi029 ceph-mon[216903]: pgmap v1050: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 476 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 6916/22872 objects degraded (30.238%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:30.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/22872 objects degraded (30.238%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:30 smithi029 ceph-mon[223858]: pgmap v1050: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 476 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 6916/22872 objects degraded (30.238%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/22872 objects degraded (30.238%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:30.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:31.023 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:42:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:30 smithi177 ceph-mon[195797]: pgmap v1050: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 476 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 6916/22872 objects degraded (30.238%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/22872 objects degraded (30.238%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:31.078 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:31.086 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:42:31.136 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:42:31.179 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:42:31.222 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:42:31.234 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:31.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:31.266 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:42:31.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:31.370 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:31.415 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:31.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:31.517 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:42:31.517 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:42:31.517 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:42:31.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:31.551 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:31.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:31.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:31.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:31.801 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:31.802 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3696607033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:31.910 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:31.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:31.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:31.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:42:32.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:32.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3696607033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:32.061 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3696607033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:42:32.136 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:32 smithi177 ceph-mon[195797]: pgmap v1051: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 407 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 6916/19596 objects degraded (35.293%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:33.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:32 smithi029 ceph-mon[216903]: pgmap v1051: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 407 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 6916/19596 objects degraded (35.293%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:33.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:42:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:42:32] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:42:33.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:32 smithi029 ceph-mon[223858]: pgmap v1051: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 407 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 6916/19596 objects degraded (35.293%); 0 B/s, 62 objects/s recovering 2023-10-09T04:42:33.345 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:33.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:33.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:33.494 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:33.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:33.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:33.667 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:33.877 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:34 smithi029 ceph-mon[216903]: pgmap v1052: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 298 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 6916/14370 objects degraded (48.128%); 0 B/s, 81 objects/s recovering 2023-10-09T04:42:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:34 smithi029 ceph-mon[223858]: pgmap v1052: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 298 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 6916/14370 objects degraded (48.128%); 0 B/s, 81 objects/s recovering 2023-10-09T04:42:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:34 smithi177 ceph-mon[195797]: pgmap v1052: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 298 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 6916/14370 objects degraded (48.128%); 0 B/s, 81 objects/s recovering 2023-10-09T04:42:35.843 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:35.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:35.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:35.945 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:35.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:36.072 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/14370 objects degraded (48.128%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:36.075 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/14370 objects degraded (48.128%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:36.091 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:36.146 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/14370 objects degraded (48.128%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:36.364 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:36.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:36.476 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:36.519 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:36.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:36.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:36.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:36.692 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:36.795 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:36.963 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:36.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:36.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:37.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:37.150 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:36 smithi029 ceph-mon[223858]: pgmap v1053: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 279 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 6916/13440 objects degraded (51.458%); 0 B/s, 55 objects/s recovering 2023-10-09T04:42:37.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:36 smithi029 ceph-mon[216903]: pgmap v1053: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 279 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 6916/13440 objects degraded (51.458%); 0 B/s, 55 objects/s recovering 2023-10-09T04:42:37.163 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:36 smithi177 ceph-mon[195797]: pgmap v1053: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 279 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 500 op/s; 6916/13440 objects degraded (51.458%); 0 B/s, 55 objects/s recovering 2023-10-09T04:42:37.389 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:42:37.389 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6916/13440 objects degraded (51.458%), 2 pgs degraded, 3 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1494,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":274,"num_osds":8,"num_up_osds":8,"osd_up_since":1696826206,"num_in_osds":8,"osd_in_since":1696825790,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":4480,"data_bytes":292558776,"bytes_used":7393132544,"bytes_avail":760567152640,"bytes_total":767960285184,"degraded_objects":6916,"degraded_total":13440,"degraded_ratio":0.51458333333333328,"recovering_objects_per_sec":55,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":667,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":500},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T04:37:47.018583+0000","services":{}},"progress_events":{"207ae389-d6c7-4169-b122-a0b4dcfa7052":{"message":"Global Recovery Event (3m)\n [===========================.] (remaining: 6s)","progress":0.96629214286804199,"add_to_ceph_s":true},"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T04:42:37.946 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:42:38.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2803963446' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:42:38.147 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2803963446' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:42:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2803963446' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:42:39.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:39.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:38 smithi029 ceph-mon[216903]: pgmap v1054: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 190 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6916/9159 objects degraded (75.510%); 0 B/s, 81 objects/s recovering 2023-10-09T04:42:39.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:38 smithi029 ceph-mon[223858]: pgmap v1054: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 190 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6916/9159 objects degraded (75.510%); 0 B/s, 81 objects/s recovering 2023-10-09T04:42:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:39.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:39.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:39.262 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:38 smithi177 ceph-mon[195797]: pgmap v1054: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 190 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6916/9159 objects degraded (75.510%); 0 B/s, 81 objects/s recovering 2023-10-09T04:42:39.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 100553 objects 2023-10-09T04:42:39.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :186.763 2023-10-09T04:42:39.916 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:39.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:40.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:40.047 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:40 smithi029 ceph-mon[216903]: pgmap v1055: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 127 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 6916/6144 objects degraded (112.565%); 0 B/s, 52 objects/s recovering 2023-10-09T04:42:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/6144 objects degraded (112.565%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:40 smithi029 ceph-mon[223858]: pgmap v1055: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 127 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 6916/6144 objects degraded (112.565%); 0 B/s, 52 objects/s recovering 2023-10-09T04:42:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/6144 objects degraded (112.565%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:40 smithi177 ceph-mon[195797]: pgmap v1055: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 127 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 6916/6144 objects degraded (112.565%); 0 B/s, 52 objects/s recovering 2023-10-09T04:42:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/6144 objects degraded (112.565%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:41.513 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:41.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:41.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:41.644 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:42.487 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:42:42.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:42:42.493 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:42.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:42.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:42.631 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:42.674 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:42:42.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:42.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:42.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:42.696 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-10-09T04:42:42.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-10-09T04:42:42.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:42.824 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:43.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:42 smithi177 ceph-mon[195797]: pgmap v1056: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 61 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 6916/2967 objects degraded (233.097%); 0 B/s, 52 objects/s recovering 2023-10-09T04:42:43.147 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:42 smithi029 ceph-mon[216903]: pgmap v1056: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 61 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 6916/2967 objects degraded (233.097%); 0 B/s, 52 objects/s recovering 2023-10-09T04:42:43.147 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:42:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:42:42] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:42:43.147 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:42 smithi029 ceph-mon[223858]: pgmap v1056: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 61 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 6916/2967 objects degraded (233.097%); 0 B/s, 52 objects/s recovering 2023-10-09T04:42:43.886 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:42:44.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/385498136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:42:44.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:44.016 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/385498136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:42:44.016 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:44.137 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:42:44.138 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:42:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/385498136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:42:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: pgmap v1057: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 106 objects/s recovering 2023-10-09T04:42:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/385498136' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[223858]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: pgmap v1057: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 106 objects/s recovering 2023-10-09T04:42:45.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/385498136' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:44 smithi029 ceph-mon[216903]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: pgmap v1057: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 106 objects/s recovering 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/385498136' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: osdmap e275: 8 total, 8 up, 8 in 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:44 smithi177 ceph-mon[195797]: failed to encode map e275 with expected crc 2023-10-09T04:42:45.433 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:45.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:45.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:45.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:45.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:46.022 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:46.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:46.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: failed to encode map e275 with expected crc 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.149 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 6916/72 objects degraded (9605.556%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:46.151 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: failed to encode map e275 with expected crc 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 6916/72 objects degraded (9605.556%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:46.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: failed to encode map e275 with expected crc 2023-10-09T04:42:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: osdmap e276: 8 total, 8 up, 8 in 2023-10-09T04:42:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: failed to encode map e276 with expected crc 2023-10-09T04:42:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 6916/72 objects degraded (9605.556%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:46.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:46.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:46.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:46.328 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:46.928 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:46.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:47.031 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:47.058 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:47.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:47.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[223858]: pgmap v1060: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 80 objects/s recovering 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[223858]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[223858]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[223858]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[223858]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[223858]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[216903]: pgmap v1060: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 80 objects/s recovering 2023-10-09T04:42:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[216903]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[216903]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[216903]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:42:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[216903]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:46 smithi029 ceph-mon[216903]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:47.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:46 smithi177 ceph-mon[195797]: pgmap v1060: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 80 objects/s recovering 2023-10-09T04:42:47.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:46 smithi177 ceph-mon[195797]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:46 smithi177 ceph-mon[195797]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:46 smithi177 ceph-mon[195797]: osdmap e277: 8 total, 8 up, 8 in 2023-10-09T04:42:47.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:46 smithi177 ceph-mon[195797]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.228 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:46 smithi177 ceph-mon[195797]: failed to encode map e277 with expected crc 2023-10-09T04:42:47.230 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:47.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:47.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:47.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:47.409 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:47.455 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:47.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:47.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:47.586 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:47.634 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:47.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:47.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:47.765 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:47.814 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:47.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:47.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:47.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:47.944 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:47.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:47.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:47.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:48.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:48.121 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:48.169 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:48.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:48.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:48.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:48.299 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:48.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:48.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:48.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:48.476 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:48.522 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:48.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:48.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:48.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:48.655 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:48.701 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:48.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:48.805 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:48.832 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:48.879 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:48.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:48.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:49.010 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:49.055 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:49.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:49.159 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:49.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:48 smithi029 ceph-mon[216903]: pgmap v1062: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 205 objects/s recovering 2023-10-09T04:42:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:48 smithi029 ceph-mon[223858]: pgmap v1062: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 205 objects/s recovering 2023-10-09T04:42:49.187 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:49.233 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:49.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:49.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:48 smithi177 ceph-mon[195797]: pgmap v1062: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 160 op/s; 6916/72 objects degraded (9605.556%); 0 B/s, 205 objects/s recovering 2023-10-09T04:42:49.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:49.364 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:49.410 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:49.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:49.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:49.541 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:49.589 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:49.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:49.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:49.718 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:49.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:49.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:49.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:49.895 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:49.943 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:49.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:50.046 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:50.074 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:50.120 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:50.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:50.223 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:50.251 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:50.297 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:50.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:50.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:50.428 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:50.475 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:50.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:50.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:50.606 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:50.651 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:50.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:50.755 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:50.784 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:50.829 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:50.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:50.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:50.962 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:51.009 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:51.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:51.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:50 smithi177 ceph-mon[195797]: pgmap v1063: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 177 objects/s recovering 2023-10-09T04:42:51.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4074/72 objects degraded (5658.333%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:51.139 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:50 smithi029 ceph-mon[216903]: pgmap v1063: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 177 objects/s recovering 2023-10-09T04:42:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4074/72 objects degraded (5658.333%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:50 smithi029 ceph-mon[223858]: pgmap v1063: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 177 objects/s recovering 2023-10-09T04:42:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4074/72 objects degraded (5658.333%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:42:51.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:51.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:51.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:51.314 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:51.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:51.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:51.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:51.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:51.489 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:51.538 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:51.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:51.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:51.670 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:51.717 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:51.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:51.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:51.847 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:51.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:51.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:51.997 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:52.023 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:52.068 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:52.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:52.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:52.200 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:52.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:52.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:52.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:52.378 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:52.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:52.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:52.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:52.557 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:52.604 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:52.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:52.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:52.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:52.736 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:52.782 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:52.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:52.886 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:52.913 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:52.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:52.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:53.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:53.090 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:52 smithi029 ceph-mon[216903]: pgmap v1064: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 145 objects/s recovering 2023-10-09T04:42:53.090 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:42:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:42:52] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:42:53.090 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:52 smithi029 ceph-mon[223858]: pgmap v1064: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 145 objects/s recovering 2023-10-09T04:42:53.092 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:53.140 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:53.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:53.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:53.272 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:52 smithi177 ceph-mon[195797]: pgmap v1064: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 145 objects/s recovering 2023-10-09T04:42:53.318 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:53.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:53.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:53.449 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:53.496 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:53.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:53.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:53.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:53.627 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:53.675 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:53.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:53.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:53.806 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:53.854 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:53.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:53.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:53.985 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:54.033 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:54.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:54.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:54.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:54.164 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:54.214 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:54.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:54.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:54.351 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:54.425 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:54.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:54.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:54.560 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:54.604 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:54.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:54.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:54.737 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:54.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:54.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:54.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:54.923 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:54.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:54.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:54.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:54 smithi177 ceph-mon[195797]: pgmap v1065: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 228 objects/s recovering 2023-10-09T04:42:55.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:55.110 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:55.158 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:55.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:54 smithi029 ceph-mon[216903]: pgmap v1065: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 228 objects/s recovering 2023-10-09T04:42:55.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:54 smithi029 ceph-mon[223858]: pgmap v1065: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 228 objects/s recovering 2023-10-09T04:42:55.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:55.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:55.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:55.290 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:55.339 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:55.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:55.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:55.470 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:55.520 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:55.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:55.626 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:55.657 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:55.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:55.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:55.817 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:55.843 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:55.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:55.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:56.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:56.031 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:56.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:56.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:56.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:42:56.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:56.226 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:56.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:56.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:56.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:56.386 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:56.414 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:56.462 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:56.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:56.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:56.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:56.594 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:56 smithi029 ceph-mon[216903]: pgmap v1066: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 190 objects/s recovering 2023-10-09T04:42:57.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:56 smithi029 ceph-mon[223858]: pgmap v1066: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 190 objects/s recovering 2023-10-09T04:42:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:56 smithi177 ceph-mon[195797]: pgmap v1066: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 190 objects/s recovering 2023-10-09T04:42:57.297 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:57.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:57.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:57.435 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:57.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:57.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:57.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:57.619 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:57.666 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:57.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:57.769 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:42:57.798 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:57.843 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:42:57.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:57.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:42:57.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:57.976 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:58.024 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:58.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:58.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:58.154 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:58.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:58.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:58.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:42:58.330 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:58.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:42:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:58.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:42:58.487 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:42:58.515 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:58.563 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:42:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:58.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:42:58.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:58.693 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:58.739 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:58.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:58.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:58.871 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:58.918 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:58.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:59.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:42:59.051 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:59.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:42:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:59.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:42:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:58 smithi029 ceph-mon[216903]: pgmap v1067: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 168 objects/s recovering 2023-10-09T04:42:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:42:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:58 smithi029 ceph-mon[223858]: pgmap v1067: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 168 objects/s recovering 2023-10-09T04:42:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:42:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:59.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:42:59.222 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:58 smithi177 ceph-mon[195797]: pgmap v1067: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 168 objects/s recovering 2023-10-09T04:42:59.222 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:42:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:42:59.224 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:59.272 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:42:59.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:59.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:42:59.379 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:42:59.405 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:59.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:42:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:59.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:42:59.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:59.583 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:59.628 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:59.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:59.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:59.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:42:59.759 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:59.805 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:42:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:59.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:42:59.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:42:59.936 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:42:59.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:42:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:42:59.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:00.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:00.120 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:00.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:00.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:00.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:00.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:00.298 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:00.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:00.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:00.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:00.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:00.482 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:00.679 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:00.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:00.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:00.811 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:00.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:00.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:00.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:00.989 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:01.036 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:01.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:01.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:01.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:01.163 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:00 smithi177 ceph-mon[195797]: pgmap v1068: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 189 objects/s recovering 2023-10-09T04:43:01.166 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:00 smithi029 ceph-mon[216903]: pgmap v1068: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 189 objects/s recovering 2023-10-09T04:43:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:00 smithi029 ceph-mon[223858]: pgmap v1068: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 189 objects/s recovering 2023-10-09T04:43:01.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:01.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:01.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:01.345 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:01.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:01.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:01.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:01.534 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:02.045 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:02.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:02.131 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:43:02.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:02.156 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:02.175 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:02.184 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:02.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:02.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:02.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:02.360 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:02.410 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:02.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:02.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:02.516 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:02.544 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:02.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:02.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:02.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:02.725 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:02.770 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:02.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:02.818 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:43:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:43:02] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:43:02.867 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:43:02.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:02.892 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:43:02.915 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:43:02.919 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:02.944 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:43:02.966 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:43:02.997 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:43:03.017 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:43:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:02 smithi177 ceph-mon[195797]: pgmap v1069: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 149 objects/s recovering 2023-10-09T04:43:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:02 smithi177 ceph-mon[195797]: failed to encode map e278 with expected crc 2023-10-09T04:43:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:02 smithi177 ceph-mon[195797]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:43:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1844444039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:03.037 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:43:03.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.039 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.039 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.039 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:43:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[216903]: pgmap v1069: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 149 objects/s recovering 2023-10-09T04:43:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[216903]: failed to encode map e278 with expected crc 2023-10-09T04:43:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[216903]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:43:03.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1844444039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[223858]: pgmap v1069: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 4074/72 objects degraded (5658.333%); 0 B/s, 149 objects/s recovering 2023-10-09T04:43:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[223858]: failed to encode map e278 with expected crc 2023-10-09T04:43:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[223858]: osdmap e278: 8 total, 8 up, 8 in 2023-10-09T04:43:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1844444039' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:03.236 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:03.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:03.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:03.340 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:03.366 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:03.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:03.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:03.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:03.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:03.549 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:03.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:43:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[216903]: failed to encode map e278 with expected crc 2023-10-09T04:43:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[216903]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[216903]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[216903]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T04:43:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[216903]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[216903]: failed to encode map e278 with expected crc 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[223858]: failed to encode map e278 with expected crc 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[223858]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[223858]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[223858]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[223858]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:03 smithi029 ceph-mon[223858]: failed to encode map e278 with expected crc 2023-10-09T04:43:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:03 smithi177 ceph-mon[195797]: failed to encode map e278 with expected crc 2023-10-09T04:43:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:03 smithi177 ceph-mon[195797]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:03 smithi177 ceph-mon[195797]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:03 smithi177 ceph-mon[195797]: osdmap e279: 8 total, 8 up, 8 in 2023-10-09T04:43:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:03 smithi177 ceph-mon[195797]: failed to encode map e279 with expected crc 2023-10-09T04:43:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:03 smithi177 ceph-mon[195797]: failed to encode map e278 with expected crc 2023-10-09T04:43:04.357 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:04.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:04.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:04.488 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:04.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:04.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:04.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:04.669 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:04 smithi029 ceph-mon[216903]: pgmap v1072: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1316/72 objects degraded (1827.778%); 0 B/s, 127 objects/s recovering 2023-10-09T04:43:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:04 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1316/72 objects degraded (1827.778%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:04 smithi029 ceph-mon[223858]: pgmap v1072: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1316/72 objects degraded (1827.778%); 0 B/s, 127 objects/s recovering 2023-10-09T04:43:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:04 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1316/72 objects degraded (1827.778%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:04 smithi177 ceph-mon[195797]: pgmap v1072: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 1316/72 objects degraded (1827.778%); 0 B/s, 127 objects/s recovering 2023-10-09T04:43:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:04 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1316/72 objects degraded (1827.778%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:43:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:06 smithi029 ceph-mon[216903]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:06 smithi029 ceph-mon[223858]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:06 smithi177 ceph-mon[195797]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:08 smithi029 ceph-mon[216903]: pgmap v1074: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:09.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:08 smithi029 ceph-mon[223858]: pgmap v1074: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:08 smithi177 ceph-mon[195797]: pgmap v1074: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:10 smithi029 ceph-mon[216903]: pgmap v1075: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 636/72 objects degraded (883.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:43:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:10 smithi029 ceph-mon[223858]: pgmap v1075: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 636/72 objects degraded (883.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:43:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:10 smithi177 ceph-mon[195797]: pgmap v1075: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 283 objects/s recovering 2023-10-09T04:43:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 636/72 objects degraded (883.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:43:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:12 smithi029 ceph-mon[216903]: pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 244 objects/s recovering 2023-10-09T04:43:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:43:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:43:12] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:43:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:12 smithi029 ceph-mon[223858]: pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 244 objects/s recovering 2023-10-09T04:43:13.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:12 smithi177 ceph-mon[195797]: pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 244 objects/s recovering 2023-10-09T04:43:13.749 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:13.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:13.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:13.881 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:13.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:13.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:14.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:14.068 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:14.118 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:14.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:14.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:14.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:14.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:14.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:14.250 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:14.251 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:43:14.253 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:14.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:14.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:14.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:14.551 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:14 smithi029 ceph-mon[216903]: pgmap v1077: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 310 objects/s recovering 2023-10-09T04:43:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:14 smithi029 ceph-mon[223858]: pgmap v1077: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 310 objects/s recovering 2023-10-09T04:43:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:14 smithi177 ceph-mon[195797]: pgmap v1077: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 310 objects/s recovering 2023-10-09T04:43:17.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:16 smithi029 ceph-mon[216903]: pgmap v1078: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 266 objects/s recovering 2023-10-09T04:43:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:16 smithi029 ceph-mon[223858]: pgmap v1078: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 266 objects/s recovering 2023-10-09T04:43:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:16 smithi177 ceph-mon[195797]: pgmap v1078: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 636/72 objects degraded (883.333%); 0 B/s, 266 objects/s recovering 2023-10-09T04:43:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:18.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:18.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:18 smithi029 ceph-mon[216903]: pgmap v1079: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 253 objects/s recovering 2023-10-09T04:43:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:18.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:18 smithi029 ceph-mon[223858]: pgmap v1079: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 253 objects/s recovering 2023-10-09T04:43:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:18 smithi177 ceph-mon[195797]: pgmap v1079: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 253 objects/s recovering 2023-10-09T04:43:19.151 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:19.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:19.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:19.290 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:19.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:19.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 288/72 objects degraded (400.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:43:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T04:43:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[216903]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 288/72 objects degraded (400.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:19 smithi029 ceph-mon[223858]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:19.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 288/72 objects degraded (400.000%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:43:19.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: osdmap e280: 8 total, 8 up, 8 in 2023-10-09T04:43:19.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:19 smithi177 ceph-mon[195797]: failed to encode map e280 with expected crc 2023-10-09T04:43:19.467 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:19.510 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:19.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:19.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:19.649 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:19.706 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:19.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:19.811 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:19.838 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:19.846 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:19.878 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:19.887 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:19.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:19.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:19.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:20.026 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:20.357 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:20.358 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[216903]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T04:43:20.358 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[216903]: failed to encode map e281 with expected crc 2023-10-09T04:43:20.358 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[216903]: pgmap v1082: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 260 objects/s recovering 2023-10-09T04:43:20.358 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[216903]: failed to encode map e281 with expected crc 2023-10-09T04:43:20.359 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[223858]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T04:43:20.359 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[223858]: failed to encode map e281 with expected crc 2023-10-09T04:43:20.359 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[223858]: pgmap v1082: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 260 objects/s recovering 2023-10-09T04:43:20.359 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:20 smithi029 ceph-mon[223858]: failed to encode map e281 with expected crc 2023-10-09T04:43:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:20.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:20.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:20.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:20 smithi177 ceph-mon[195797]: osdmap e281: 8 total, 8 up, 8 in 2023-10-09T04:43:20.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:20 smithi177 ceph-mon[195797]: failed to encode map e281 with expected crc 2023-10-09T04:43:20.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:20 smithi177 ceph-mon[195797]: pgmap v1082: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 260 objects/s recovering 2023-10-09T04:43:20.489 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:20 smithi177 ceph-mon[195797]: failed to encode map e281 with expected crc 2023-10-09T04:43:20.491 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:20.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:20.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:20.559 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:43:20.561 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:43:20.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:20.674 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[216903]: failed to encode map e281 with expected crc 2023-10-09T04:43:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[216903]: failed to encode map e281 with expected crc 2023-10-09T04:43:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[216903]: from='client.36162 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/639039497' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:43:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[223858]: failed to encode map e281 with expected crc 2023-10-09T04:43:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[223858]: failed to encode map e281 with expected crc 2023-10-09T04:43:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[223858]: from='client.36162 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/639039497' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:43:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:21 smithi177 ceph-mon[195797]: failed to encode map e281 with expected crc 2023-10-09T04:43:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:21 smithi177 ceph-mon[195797]: failed to encode map e281 with expected crc 2023-10-09T04:43:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:21 smithi177 ceph-mon[195797]: from='client.36162 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/639039497' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:43:21.547 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_6' removed 2023-10-09T04:43:21.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:43:22.348 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:22.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:22.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:22.482 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:22.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:22.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:22.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: pgmap v1083: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 136 objects/s recovering 2023-10-09T04:43:22.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/639039497' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:43:22.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T04:43:22.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[216903]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: pgmap v1083: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 136 objects/s recovering 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/639039497' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:22 smithi029 ceph-mon[223858]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:22.663 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: pgmap v1083: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 288/72 objects degraded (400.000%); 0 B/s, 136 objects/s recovering 2023-10-09T04:43:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/639039497' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:43:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: osdmap e282: 8 total, 8 up, 8 in 2023-10-09T04:43:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:22 smithi177 ceph-mon[195797]: failed to encode map e282 with expected crc 2023-10-09T04:43:22.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:43:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:43:22] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-10-09T04:43:23.308 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1082,"stamp":"2023-10-09T04:43:19.187708+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":201408,"num_write_kb":6437838,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21906,"num_bytes_recovered":617423024,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139026,"ondisk_log_size":139026,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5848380,"kb_used_data":106428,"kb_used_omap":20,"kb_used_meta":5741867,"kb_avail":744112836,"statfs":{"total":767960285184,"available":761971544064,"internally_reserved":0,"allocated":108982272,"data_stored":102518586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5879672462},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-348,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2082,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002101"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882337+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:43:01.882337+0000","last_peered":"2023-10-09T04:43:01.882337+0000","last_clean":"2023-10-09T04:43:01.882337+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:43:01.882337+0000","last_undegraded":"2023-10-09T04:43:01.882337+0000","last_fullsized":"2023-10-09T04:43:01.882337+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":232,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725548+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:43:02.725548+0000","last_peered":"2023-10-09T04:43:02.725548+0000","last_clean":"2023-10-09T04:43:02.725548+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:43:02.725548+0000","last_undegraded":"2023-10-09T04:43:02.725548+0000","last_fullsized":"2023-10-09T04:43:02.725548+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":264,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.880119+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:43:01.880119+0000","last_peered":"2023-10-09T04:43:01.880119+0000","last_clean":"2023-10-09T04:43:01.880119+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:43:01.880119+0000","last_undegraded":"2023-10-09T04:43:01.880119+0000","last_fullsized":"2023-10-09T04:43:01.880119+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":192,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.885132+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:43:01.885132+0000","last_peered":"2023-10-09T04:43:01.885132+0000","last_clean":"2023-10-09T04:43:01.885132+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:43:01.885132+0000","last_undegraded":"2023-10-09T04:43:01.885132+0000","last_fullsized":"2023-10-09T04:43:01.885132+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":268,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725063+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:43:02.725063+0000","last_peered":"2023-10-09T04:43:02.725063+0000","last_clean":"2023-10-09T04:43:02.725063+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:43:02.725063+0000","last_undegraded":"2023-10-09T04:43:02.725063+0000","last_fullsized":"2023-10-09T04:43:02.725063+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":235,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866599+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:42:45.866599+0000","last_peered":"2023-10-09T04:42:45.866599+0000","last_clean":"2023-10-09T04:42:45.866599+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:42:45.866599+0000","last_undegraded":"2023-10-09T04:42:45.866599+0000","last_fullsized":"2023-10-09T04:42:45.866599+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":191,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.540706+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:43:02.540706+0000","last_peered":"2023-10-09T04:43:02.540706+0000","last_clean":"2023-10-09T04:43:02.540706+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:43:02.540706+0000","last_undegraded":"2023-10-09T04:43:02.540706+0000","last_fullsized":"2023-10-09T04:43:02.540706+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":232,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725143+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:43:02.725143+0000","last_peered":"2023-10-09T04:43:02.725143+0000","last_clean":"2023-10-09T04:43:02.725143+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:43:02.725143+0000","last_undegraded":"2023-10-09T04:43:02.725143+0000","last_fullsized":"2023-10-09T04:43:02.725143+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":239,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539901+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:43:02.539901+0000","last_peered":"2023-10-09T04:43:02.539901+0000","last_clean":"2023-10-09T04:43:02.539901+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:43:02.539901+0000","last_undegraded":"2023-10-09T04:43:02.539901+0000","last_fullsized":"2023-10-09T04:43:02.539901+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":205,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725227+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:43:02.725227+0000","last_peered":"2023-10-09T04:43:02.725227+0000","last_clean":"2023-10-09T04:43:02.725227+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:43:02.725227+0000","last_undegraded":"2023-10-09T04:43:02.725227+0000","last_fullsized":"2023-10-09T04:43:02.725227+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":273,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884990+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:43:01.884990+0000","last_peered":"2023-10-09T04:43:01.884990+0000","last_clean":"2023-10-09T04:43:01.884990+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:43:01.884990+0000","last_undegraded":"2023-10-09T04:43:01.884990+0000","last_fullsized":"2023-10-09T04:43:01.884990+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":232,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866769+0000","last_change":"2023-10-09T04:39:40.406537+0000","last_active":"2023-10-09T04:42:45.866769+0000","last_peered":"2023-10-09T04:42:45.866769+0000","last_clean":"2023-10-09T04:42:45.866769+0000","last_became_active":"2023-10-09T04:39:40.406281+0000","last_became_peered":"2023-10-09T04:39:40.406281+0000","last_unstale":"2023-10-09T04:42:45.866769+0000","last_undegraded":"2023-10-09T04:42:45.866769+0000","last_fullsized":"2023-10-09T04:42:45.866769+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:09.883302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":37,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882442+0000","last_change":"2023-10-09T04:41:02.634559+0000","last_active":"2023-10-09T04:43:01.882442+0000","last_peered":"2023-10-09T04:43:01.882442+0000","last_clean":"2023-10-09T04:43:01.882442+0000","last_became_active":"2023-10-09T04:41:02.624597+0000","last_became_peered":"2023-10-09T04:41:02.624597+0000","last_unstale":"2023-10-09T04:43:01.882442+0000","last_undegraded":"2023-10-09T04:43:01.882442+0000","last_fullsized":"2023-10-09T04:43:01.882442+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:57.939757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":239,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882555+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:43:01.882555+0000","last_peered":"2023-10-09T04:43:01.882555+0000","last_clean":"2023-10-09T04:43:01.882555+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:43:01.882555+0000","last_undegraded":"2023-10-09T04:43:01.882555+0000","last_fullsized":"2023-10-09T04:43:01.882555+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":289,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884792+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:43:01.884792+0000","last_peered":"2023-10-09T04:43:01.884792+0000","last_clean":"2023-10-09T04:43:01.884792+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:43:01.884792+0000","last_undegraded":"2023-10-09T04:43:01.884792+0000","last_fullsized":"2023-10-09T04:43:01.884792+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":232,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725425+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:43:02.725425+0000","last_peered":"2023-10-09T04:43:02.725425+0000","last_clean":"2023-10-09T04:43:02.725425+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:43:02.725425+0000","last_undegraded":"2023-10-09T04:43:02.725425+0000","last_fullsized":"2023-10-09T04:43:02.725425+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":191,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884512+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:43:01.884512+0000","last_peered":"2023-10-09T04:43:01.884512+0000","last_clean":"2023-10-09T04:43:01.884512+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:43:01.884512+0000","last_undegraded":"2023-10-09T04:43:01.884512+0000","last_fullsized":"2023-10-09T04:43:01.884512+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":238,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539173+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:43:02.539173+0000","last_peered":"2023-10-09T04:43:02.539173+0000","last_clean":"2023-10-09T04:43:02.539173+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:43:02.539173+0000","last_undegraded":"2023-10-09T04:43:02.539173+0000","last_fullsized":"2023-10-09T04:43:02.539173+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":221,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884397+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:43:01.884397+0000","last_peered":"2023-10-09T04:43:01.884397+0000","last_clean":"2023-10-09T04:43:01.884397+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:43:01.884397+0000","last_undegraded":"2023-10-09T04:43:01.884397+0000","last_fullsized":"2023-10-09T04:43:01.884397+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":193,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725596+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:43:02.725596+0000","last_peered":"2023-10-09T04:43:02.725596+0000","last_clean":"2023-10-09T04:43:02.725596+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:43:02.725596+0000","last_undegraded":"2023-10-09T04:43:02.725596+0000","last_fullsized":"2023-10-09T04:43:02.725596+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":284,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866376+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:42:45.866376+0000","last_peered":"2023-10-09T04:42:45.866376+0000","last_clean":"2023-10-09T04:42:45.866376+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:42:45.866376+0000","last_undegraded":"2023-10-09T04:42:45.866376+0000","last_fullsized":"2023-10-09T04:42:45.866376+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":232,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725714+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:43:02.725714+0000","last_peered":"2023-10-09T04:43:02.725714+0000","last_clean":"2023-10-09T04:43:02.725714+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:43:02.725714+0000","last_undegraded":"2023-10-09T04:43:02.725714+0000","last_fullsized":"2023-10-09T04:43:02.725714+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"274'12704","reported_seq":20029,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.540785+0000","last_change":"2023-10-09T04:41:54.945309+0000","last_active":"2023-10-09T04:43:02.540785+0000","last_peered":"2023-10-09T04:43:02.540785+0000","last_clean":"2023-10-09T04:43:02.540785+0000","last_became_active":"2023-10-09T04:41:54.915620+0000","last_became_peered":"2023-10-09T04:41:54.915620+0000","last_unstale":"2023-10-09T04:43:02.540785+0000","last_undegraded":"2023-10-09T04:43:02.540785+0000","last_fullsized":"2023-10-09T04:43:02.540785+0000","mapping_epoch":271,"log_start":"243'2700","ondisk_log_start":"243'2700","created":237,"last_epoch_clean":272,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10004,"log_dups_size":0,"ondisk_log_size":10004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:35.953123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12704,"num_write_kb":406528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3622,"num_bytes_recovered":43122688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":295,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884843+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:43:01.884843+0000","last_peered":"2023-10-09T04:43:01.884843+0000","last_clean":"2023-10-09T04:43:01.884843+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:43:01.884843+0000","last_undegraded":"2023-10-09T04:43:01.884843+0000","last_fullsized":"2023-10-09T04:43:01.884843+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":221,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539566+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:43:02.539566+0000","last_peered":"2023-10-09T04:43:02.539566+0000","last_clean":"2023-10-09T04:43:02.539566+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:43:02.539566+0000","last_undegraded":"2023-10-09T04:43:02.539566+0000","last_fullsized":"2023-10-09T04:43:02.539566+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"274'12492","reported_seq":12545,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.890776+0000","last_change":"2023-10-09T04:37:52.255599+0000","last_active":"2023-10-09T04:43:01.890776+0000","last_peered":"2023-10-09T04:43:01.890776+0000","last_clean":"2023-10-09T04:43:01.890776+0000","last_became_active":"2023-10-09T04:37:52.255291+0000","last_became_peered":"2023-10-09T04:37:52.255291+0000","last_unstale":"2023-10-09T04:43:01.890776+0000","last_undegraded":"2023-10-09T04:43:01.890776+0000","last_fullsized":"2023-10-09T04:43:01.890776+0000","mapping_epoch":237,"log_start":"242'2400","ondisk_log_start":"242'2400","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:06.990132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12492,"num_write_kb":399744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":268,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.885112+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:43:01.885112+0000","last_peered":"2023-10-09T04:43:01.885112+0000","last_clean":"2023-10-09T04:43:01.885112+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:43:01.885112+0000","last_undegraded":"2023-10-09T04:43:01.885112+0000","last_fullsized":"2023-10-09T04:43:01.885112+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":256,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882593+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:43:01.882593+0000","last_peered":"2023-10-09T04:43:01.882593+0000","last_clean":"2023-10-09T04:43:01.882593+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:43:01.882593+0000","last_undegraded":"2023-10-09T04:43:01.882593+0000","last_fullsized":"2023-10-09T04:43:01.882593+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"274'12666","reported_seq":12716,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884581+0000","last_change":"2023-10-09T04:37:52.013171+0000","last_active":"2023-10-09T04:43:01.884581+0000","last_peered":"2023-10-09T04:43:01.884581+0000","last_clean":"2023-10-09T04:43:01.884581+0000","last_became_active":"2023-10-09T04:37:52.012873+0000","last_became_peered":"2023-10-09T04:37:52.012873+0000","last_unstale":"2023-10-09T04:43:01.884581+0000","last_undegraded":"2023-10-09T04:43:01.884581+0000","last_fullsized":"2023-10-09T04:43:01.884581+0000","mapping_epoch":237,"log_start":"245'4500","ondisk_log_start":"245'4500","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:25.833222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12666,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":222,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884693+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:43:01.884693+0000","last_peered":"2023-10-09T04:43:01.884693+0000","last_clean":"2023-10-09T04:43:01.884693+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:43:01.884693+0000","last_undegraded":"2023-10-09T04:43:01.884693+0000","last_fullsized":"2023-10-09T04:43:01.884693+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":262,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866444+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:42:45.866444+0000","last_peered":"2023-10-09T04:42:45.866444+0000","last_clean":"2023-10-09T04:42:45.866444+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:42:45.866444+0000","last_undegraded":"2023-10-09T04:42:45.866444+0000","last_fullsized":"2023-10-09T04:42:45.866444+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"274'12654","reported_seq":12704,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884588+0000","last_change":"2023-10-09T04:37:52.012019+0000","last_active":"2023-10-09T04:43:01.884588+0000","last_peered":"2023-10-09T04:43:01.884588+0000","last_clean":"2023-10-09T04:43:01.884588+0000","last_became_active":"2023-10-09T04:37:52.011714+0000","last_became_peered":"2023-10-09T04:37:52.011714+0000","last_unstale":"2023-10-09T04:43:01.884588+0000","last_undegraded":"2023-10-09T04:43:01.884588+0000","last_fullsized":"2023-10-09T04:43:01.884588+0000","mapping_epoch":237,"log_start":"245'4500","ondisk_log_start":"245'4500","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8154,"log_dups_size":0,"ondisk_log_size":8154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:55.324035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12654,"num_write_kb":404928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":239,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539533+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:43:02.539533+0000","last_peered":"2023-10-09T04:43:02.539533+0000","last_clean":"2023-10-09T04:43:02.539533+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:43:02.539533+0000","last_undegraded":"2023-10-09T04:43:02.539533+0000","last_fullsized":"2023-10-09T04:43:02.539533+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":136,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539287+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:43:02.539287+0000","last_peered":"2023-10-09T04:43:02.539287+0000","last_clean":"2023-10-09T04:43:02.539287+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:43:02.539287+0000","last_undegraded":"2023-10-09T04:43:02.539287+0000","last_fullsized":"2023-10-09T04:43:02.539287+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"274'12380","reported_seq":12429,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.867086+0000","last_change":"2023-10-09T04:37:52.013681+0000","last_active":"2023-10-09T04:42:45.867086+0000","last_peered":"2023-10-09T04:42:45.867086+0000","last_clean":"2023-10-09T04:42:45.867086+0000","last_became_active":"2023-10-09T04:37:52.013382+0000","last_became_peered":"2023-10-09T04:37:52.013382+0000","last_unstale":"2023-10-09T04:42:45.867086+0000","last_undegraded":"2023-10-09T04:42:45.867086+0000","last_fullsized":"2023-10-09T04:42:45.867086+0000","mapping_epoch":237,"log_start":"244'3800","ondisk_log_start":"244'3800","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8580,"log_dups_size":0,"ondisk_log_size":8580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:38:30.877521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12380,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":276,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725384+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:43:02.725384+0000","last_peered":"2023-10-09T04:43:02.725384+0000","last_clean":"2023-10-09T04:43:02.725384+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:43:02.725384+0000","last_undegraded":"2023-10-09T04:43:02.725384+0000","last_fullsized":"2023-10-09T04:43:02.725384+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882645+0000","last_change":"2023-10-09T04:41:02.624962+0000","last_active":"2023-10-09T04:43:01.882645+0000","last_peered":"2023-10-09T04:43:01.882645+0000","last_clean":"2023-10-09T04:43:01.882645+0000","last_became_active":"2023-10-09T04:41:02.624555+0000","last_became_peered":"2023-10-09T04:41:02.624555+0000","last_unstale":"2023-10-09T04:43:01.882645+0000","last_undegraded":"2023-10-09T04:43:01.882645+0000","last_fullsized":"2023-10-09T04:43:01.882645+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:48.506794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"274'12506","reported_seq":12559,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.889005+0000","last_change":"2023-10-09T04:37:52.255728+0000","last_active":"2023-10-09T04:43:01.889005+0000","last_peered":"2023-10-09T04:43:01.889005+0000","last_clean":"2023-10-09T04:43:01.889005+0000","last_became_active":"2023-10-09T04:37:52.255283+0000","last_became_peered":"2023-10-09T04:37:52.255283+0000","last_unstale":"2023-10-09T04:43:01.889005+0000","last_undegraded":"2023-10-09T04:43:01.889005+0000","last_fullsized":"2023-10-09T04:43:01.889005+0000","mapping_epoch":237,"log_start":"242'2500","ondisk_log_start":"242'2500","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:41:18.900085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12506,"num_write_kb":400192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":256,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866611+0000","last_change":"2023-10-09T04:41:02.603773+0000","last_active":"2023-10-09T04:42:45.866611+0000","last_peered":"2023-10-09T04:42:45.866611+0000","last_clean":"2023-10-09T04:42:45.866611+0000","last_became_active":"2023-10-09T04:41:02.603407+0000","last_became_peered":"2023-10-09T04:41:02.603407+0000","last_unstale":"2023-10-09T04:42:45.866611+0000","last_undegraded":"2023-10-09T04:42:45.866611+0000","last_fullsized":"2023-10-09T04:42:45.866611+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:18.401503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882768+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:43:01.882768+0000","last_peered":"2023-10-09T04:43:01.882768+0000","last_clean":"2023-10-09T04:43:01.882768+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:43:01.882768+0000","last_undegraded":"2023-10-09T04:43:01.882768+0000","last_fullsized":"2023-10-09T04:43:01.882768+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"274'12402","reported_seq":12451,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866441+0000","last_change":"2023-10-09T04:37:52.256288+0000","last_active":"2023-10-09T04:42:45.866441+0000","last_peered":"2023-10-09T04:42:45.866441+0000","last_clean":"2023-10-09T04:42:45.866441+0000","last_became_active":"2023-10-09T04:37:52.256008+0000","last_became_peered":"2023-10-09T04:37:52.256008+0000","last_unstale":"2023-10-09T04:42:45.866441+0000","last_undegraded":"2023-10-09T04:42:45.866441+0000","last_fullsized":"2023-10-09T04:42:45.866441+0000","mapping_epoch":237,"log_start":"244'3800","ondisk_log_start":"244'3800","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8602,"log_dups_size":0,"ondisk_log_size":8602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:43:34.570497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12402,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882236+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:43:01.882236+0000","last_peered":"2023-10-09T04:43:01.882236+0000","last_clean":"2023-10-09T04:43:01.882236+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:43:01.882236+0000","last_undegraded":"2023-10-09T04:43:01.882236+0000","last_fullsized":"2023-10-09T04:43:01.882236+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":239,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882832+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:43:01.882832+0000","last_peered":"2023-10-09T04:43:01.882832+0000","last_clean":"2023-10-09T04:43:01.882832+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:43:01.882832+0000","last_undegraded":"2023-10-09T04:43:01.882832+0000","last_fullsized":"2023-10-09T04:43:01.882832+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"274'12840","reported_seq":12900,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.724951+0000","last_change":"2023-10-09T04:39:40.403627+0000","last_active":"2023-10-09T04:43:02.724951+0000","last_peered":"2023-10-09T04:43:02.724951+0000","last_clean":"2023-10-09T04:43:02.724951+0000","last_became_active":"2023-10-09T04:39:40.393228+0000","last_became_peered":"2023-10-09T04:39:40.393228+0000","last_unstale":"2023-10-09T04:43:02.724951+0000","last_undegraded":"2023-10-09T04:43:02.724951+0000","last_fullsized":"2023-10-09T04:43:02.724951+0000","mapping_epoch":251,"log_start":"245'4900","ondisk_log_start":"245'4900","created":237,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7940,"log_dups_size":0,"ondisk_log_size":7940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:21:22.432701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12840,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":234,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:43:03.268492+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:43:03.268492+0000","last_peered":"2023-10-09T04:43:03.268492+0000","last_clean":"2023-10-09T04:43:03.268492+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:43:03.268492+0000","last_undegraded":"2023-10-09T04:43:03.268492+0000","last_fullsized":"2023-10-09T04:43:03.268492+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":263,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.724684+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:43:02.724684+0000","last_peered":"2023-10-09T04:43:02.724684+0000","last_clean":"2023-10-09T04:43:02.724684+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:43:02.724684+0000","last_undegraded":"2023-10-09T04:43:02.724684+0000","last_fullsized":"2023-10-09T04:43:02.724684+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"274'12622","reported_seq":25200,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.889443+0000","last_change":"2023-10-09T04:42:45.899045+0000","last_active":"2023-10-09T04:43:01.889443+0000","last_peered":"2023-10-09T04:43:01.889443+0000","last_clean":"2023-10-09T04:43:01.889443+0000","last_became_active":"2023-10-09T04:42:45.867902+0000","last_became_peered":"2023-10-09T04:42:45.867902+0000","last_unstale":"2023-10-09T04:43:01.889443+0000","last_undegraded":"2023-10-09T04:43:01.889443+0000","last_fullsized":"2023-10-09T04:43:01.889443+0000","mapping_epoch":276,"log_start":"243'2600","ondisk_log_start":"243'2600","created":237,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:16:59.075390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12622,"num_write_kb":403841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6248,"num_bytes_recovered":220135448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":156,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884638+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:43:01.884638+0000","last_peered":"2023-10-09T04:43:01.884638+0000","last_clean":"2023-10-09T04:43:01.884638+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:43:01.884638+0000","last_undegraded":"2023-10-09T04:43:01.884638+0000","last_fullsized":"2023-10-09T04:43:01.884638+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":273,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539870+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:43:02.539870+0000","last_peered":"2023-10-09T04:43:02.539870+0000","last_clean":"2023-10-09T04:43:02.539870+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:43:02.539870+0000","last_undegraded":"2023-10-09T04:43:02.539870+0000","last_fullsized":"2023-10-09T04:43:02.539870+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":250,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539703+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:43:02.539703+0000","last_peered":"2023-10-09T04:43:02.539703+0000","last_clean":"2023-10-09T04:43:02.539703+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:43:02.539703+0000","last_undegraded":"2023-10-09T04:43:02.539703+0000","last_fullsized":"2023-10-09T04:43:02.539703+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"274'12398","reported_seq":12451,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725283+0000","last_change":"2023-10-09T04:37:52.255930+0000","last_active":"2023-10-09T04:43:02.725283+0000","last_peered":"2023-10-09T04:43:02.725283+0000","last_clean":"2023-10-09T04:43:02.725283+0000","last_became_active":"2023-10-09T04:37:52.255584+0000","last_became_peered":"2023-10-09T04:37:52.255584+0000","last_unstale":"2023-10-09T04:43:02.725283+0000","last_undegraded":"2023-10-09T04:43:02.725283+0000","last_fullsized":"2023-10-09T04:43:02.725283+0000","mapping_epoch":237,"log_start":"245'4500","ondisk_log_start":"245'4500","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7898,"log_dups_size":0,"ondisk_log_size":7898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:52.503096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12398,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":243,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539743+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:43:02.539743+0000","last_peered":"2023-10-09T04:43:02.539743+0000","last_clean":"2023-10-09T04:43:02.539743+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:43:02.539743+0000","last_undegraded":"2023-10-09T04:43:02.539743+0000","last_fullsized":"2023-10-09T04:43:02.539743+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882380+0000","last_change":"2023-10-09T04:41:02.608773+0000","last_active":"2023-10-09T04:43:01.882380+0000","last_peered":"2023-10-09T04:43:01.882380+0000","last_clean":"2023-10-09T04:43:01.882380+0000","last_became_active":"2023-10-09T04:41:02.608455+0000","last_became_peered":"2023-10-09T04:41:02.608455+0000","last_unstale":"2023-10-09T04:43:01.882380+0000","last_undegraded":"2023-10-09T04:43:01.882380+0000","last_fullsized":"2023-10-09T04:43:01.882380+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:29.807000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":249,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.889508+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:43:01.889508+0000","last_peered":"2023-10-09T04:43:01.889508+0000","last_clean":"2023-10-09T04:43:01.889508+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:43:01.889508+0000","last_undegraded":"2023-10-09T04:43:01.889508+0000","last_fullsized":"2023-10-09T04:43:01.889508+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"274'12338","reported_seq":12391,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725088+0000","last_change":"2023-10-09T04:37:52.012774+0000","last_active":"2023-10-09T04:43:02.725088+0000","last_peered":"2023-10-09T04:43:02.725088+0000","last_clean":"2023-10-09T04:43:02.725088+0000","last_became_active":"2023-10-09T04:37:52.012402+0000","last_became_peered":"2023-10-09T04:37:52.012402+0000","last_unstale":"2023-10-09T04:43:02.725088+0000","last_undegraded":"2023-10-09T04:43:02.725088+0000","last_fullsized":"2023-10-09T04:43:02.725088+0000","mapping_epoch":237,"log_start":"245'4400","ondisk_log_start":"245'4400","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7938,"log_dups_size":0,"ondisk_log_size":7938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:17:37.167042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12338,"num_write_kb":394816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":236,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.013491+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:43:02.013491+0000","last_peered":"2023-10-09T04:43:02.013491+0000","last_clean":"2023-10-09T04:43:02.013491+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:43:02.013491+0000","last_undegraded":"2023-10-09T04:43:02.013491+0000","last_fullsized":"2023-10-09T04:43:02.013491+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":234,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.888874+0000","last_change":"2023-10-09T04:39:40.406420+0000","last_active":"2023-10-09T04:43:01.888874+0000","last_peered":"2023-10-09T04:43:01.888874+0000","last_clean":"2023-10-09T04:43:01.888874+0000","last_became_active":"2023-10-09T04:39:40.406072+0000","last_became_peered":"2023-10-09T04:39:40.406072+0000","last_unstale":"2023-10-09T04:43:01.888874+0000","last_undegraded":"2023-10-09T04:43:01.888874+0000","last_fullsized":"2023-10-09T04:43:01.888874+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:08.385113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.540849+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:43:02.540849+0000","last_peered":"2023-10-09T04:43:02.540849+0000","last_clean":"2023-10-09T04:43:02.540849+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:43:02.540849+0000","last_undegraded":"2023-10-09T04:43:02.540849+0000","last_fullsized":"2023-10-09T04:43:02.540849+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"274'12846","reported_seq":12899,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725203+0000","last_change":"2023-10-09T04:37:52.013105+0000","last_active":"2023-10-09T04:43:02.725203+0000","last_peered":"2023-10-09T04:43:02.725203+0000","last_clean":"2023-10-09T04:43:02.725203+0000","last_became_active":"2023-10-09T04:37:52.012773+0000","last_became_peered":"2023-10-09T04:37:52.012773+0000","last_unstale":"2023-10-09T04:43:02.725203+0000","last_undegraded":"2023-10-09T04:43:02.725203+0000","last_fullsized":"2023-10-09T04:43:02.725203+0000","mapping_epoch":237,"log_start":"245'4900","ondisk_log_start":"245'4900","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:31:18.291873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12846,"num_write_kb":411072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":260,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539418+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:43:02.539418+0000","last_peered":"2023-10-09T04:43:02.539418+0000","last_clean":"2023-10-09T04:43:02.539418+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:43:02.539418+0000","last_undegraded":"2023-10-09T04:43:02.539418+0000","last_fullsized":"2023-10-09T04:43:02.539418+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":671,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884333+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:43:01.884333+0000","last_peered":"2023-10-09T04:43:01.884333+0000","last_clean":"2023-10-09T04:43:01.884333+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:43:01.884333+0000","last_undegraded":"2023-10-09T04:43:01.884333+0000","last_fullsized":"2023-10-09T04:43:01.884333+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":205,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725248+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:43:02.725248+0000","last_peered":"2023-10-09T04:43:02.725248+0000","last_clean":"2023-10-09T04:43:02.725248+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:43:02.725248+0000","last_undegraded":"2023-10-09T04:43:02.725248+0000","last_fullsized":"2023-10-09T04:43:02.725248+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":238,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866860+0000","last_change":"2023-10-09T04:41:02.612580+0000","last_active":"2023-10-09T04:42:45.866860+0000","last_peered":"2023-10-09T04:42:45.866860+0000","last_clean":"2023-10-09T04:42:45.866860+0000","last_became_active":"2023-10-09T04:41:02.612258+0000","last_became_peered":"2023-10-09T04:41:02.612258+0000","last_unstale":"2023-10-09T04:42:45.866860+0000","last_undegraded":"2023-10-09T04:42:45.866860+0000","last_fullsized":"2023-10-09T04:42:45.866860+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:53.656800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"274'12550","reported_seq":12603,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725325+0000","last_change":"2023-10-09T04:37:52.257119+0000","last_active":"2023-10-09T04:43:02.725325+0000","last_peered":"2023-10-09T04:43:02.725325+0000","last_clean":"2023-10-09T04:43:02.725325+0000","last_became_active":"2023-10-09T04:37:52.256760+0000","last_became_peered":"2023-10-09T04:37:52.256760+0000","last_unstale":"2023-10-09T04:43:02.725325+0000","last_undegraded":"2023-10-09T04:43:02.725325+0000","last_fullsized":"2023-10-09T04:43:02.725325+0000","mapping_epoch":237,"log_start":"245'4600","ondisk_log_start":"245'4600","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7950,"log_dups_size":0,"ondisk_log_size":7950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:40:16.001253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12550,"num_write_kb":401600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":279,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884321+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:43:01.884321+0000","last_peered":"2023-10-09T04:43:01.884321+0000","last_clean":"2023-10-09T04:43:01.884321+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:43:01.884321+0000","last_undegraded":"2023-10-09T04:43:01.884321+0000","last_fullsized":"2023-10-09T04:43:01.884321+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882341+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:43:01.882341+0000","last_peered":"2023-10-09T04:43:01.882341+0000","last_clean":"2023-10-09T04:43:01.882341+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:43:01.882341+0000","last_undegraded":"2023-10-09T04:43:01.882341+0000","last_fullsized":"2023-10-09T04:43:01.882341+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":156,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884990+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:43:01.884990+0000","last_peered":"2023-10-09T04:43:01.884990+0000","last_clean":"2023-10-09T04:43:01.884990+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:43:01.884990+0000","last_undegraded":"2023-10-09T04:43:01.884990+0000","last_fullsized":"2023-10-09T04:43:01.884990+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"274'12438","reported_seq":24239,"reported_epoch":279,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:43:17.017421+0000","last_change":"2023-10-09T04:43:01.517159+0000","last_active":"2023-10-09T04:43:17.017421+0000","last_peered":"2023-10-09T04:43:17.017421+0000","last_clean":"2023-10-09T04:39:11.375310+0000","last_became_active":"2023-10-09T04:41:03.851493+0000","last_became_peered":"2023-10-09T04:41:03.851493+0000","last_unstale":"2023-10-09T04:43:17.017421+0000","last_undegraded":"2023-10-09T04:41:03.605177+0000","last_fullsized":"2023-10-09T04:41:03.588185+0000","mapping_epoch":262,"log_start":"242'2400","ondisk_log_start":"242'2400","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":10038,"log_dups_size":0,"ondisk_log_size":10038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:51.273314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12438,"num_write_kb":398016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5863,"num_bytes_recovered":180682752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":-288}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":282,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884919+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:43:01.884919+0000","last_peered":"2023-10-09T04:43:01.884919+0000","last_clean":"2023-10-09T04:43:01.884919+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:43:01.884919+0000","last_undegraded":"2023-10-09T04:43:01.884919+0000","last_fullsized":"2023-10-09T04:43:01.884919+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":234,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.889427+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:43:01.889427+0000","last_peered":"2023-10-09T04:43:01.889427+0000","last_clean":"2023-10-09T04:43:01.889427+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:43:01.889427+0000","last_undegraded":"2023-10-09T04:43:01.889427+0000","last_fullsized":"2023-10-09T04:43:01.889427+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":123,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539173+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:43:02.539173+0000","last_peered":"2023-10-09T04:43:02.539173+0000","last_clean":"2023-10-09T04:43:02.539173+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:43:02.539173+0000","last_undegraded":"2023-10-09T04:43:02.539173+0000","last_fullsized":"2023-10-09T04:43:02.539173+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"274'12530","reported_seq":12580,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.884745+0000","last_change":"2023-10-09T04:37:52.013669+0000","last_active":"2023-10-09T04:43:01.884745+0000","last_peered":"2023-10-09T04:43:01.884745+0000","last_clean":"2023-10-09T04:43:01.884745+0000","last_became_active":"2023-10-09T04:37:52.013386+0000","last_became_peered":"2023-10-09T04:37:52.013386+0000","last_unstale":"2023-10-09T04:43:01.884745+0000","last_undegraded":"2023-10-09T04:43:01.884745+0000","last_fullsized":"2023-10-09T04:43:01.884745+0000","mapping_epoch":237,"log_start":"245'4400","ondisk_log_start":"245'4400","created":237,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":8130,"log_dups_size":0,"ondisk_log_size":8130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:57.288541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12530,"num_write_kb":400960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":277,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:43:03.269691+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:43:03.269691+0000","last_peered":"2023-10-09T04:43:03.269691+0000","last_clean":"2023-10-09T04:43:03.269691+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:43:03.269691+0000","last_undegraded":"2023-10-09T04:43:03.269691+0000","last_fullsized":"2023-10-09T04:43:03.269691+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":248,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866731+0000","last_change":"2023-10-09T04:41:02.629667+0000","last_active":"2023-10-09T04:42:45.866731+0000","last_peered":"2023-10-09T04:42:45.866731+0000","last_clean":"2023-10-09T04:42:45.866731+0000","last_became_active":"2023-10-09T04:41:02.629305+0000","last_became_peered":"2023-10-09T04:41:02.629305+0000","last_unstale":"2023-10-09T04:42:45.866731+0000","last_undegraded":"2023-10-09T04:42:45.866731+0000","last_fullsized":"2023-10-09T04:42:45.866731+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:36.600453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":266,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.724995+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:43:02.724995+0000","last_peered":"2023-10-09T04:43:02.724995+0000","last_clean":"2023-10-09T04:43:02.724995+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:43:02.724995+0000","last_undegraded":"2023-10-09T04:43:02.724995+0000","last_fullsized":"2023-10-09T04:43:02.724995+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"274'12742","reported_seq":18030,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.929993+0000","last_change":"2023-10-09T04:43:02.929993+0000","last_active":"2023-10-09T04:43:02.929993+0000","last_peered":"2023-10-09T04:43:02.929993+0000","last_clean":"2023-10-09T04:43:02.929993+0000","last_became_active":"2023-10-09T04:43:02.897434+0000","last_became_peered":"2023-10-09T04:43:02.897434+0000","last_unstale":"2023-10-09T04:43:02.929993+0000","last_undegraded":"2023-10-09T04:43:02.929993+0000","last_fullsized":"2023-10-09T04:43:02.929993+0000","mapping_epoch":278,"log_start":"246'5398","ondisk_log_start":"246'5398","created":237,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:37:50.995948+0000","last_clean_scrub_stamp":"2023-10-09T04:37:50.995948+0000","objects_scrubbed":0,"log_size":7344,"log_dups_size":0,"ondisk_log_size":7344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:22.766065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12742,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6129,"num_bytes_recovered":170524672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":257,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-10-09T04:42:45.866525+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:42:45.866525+0000","last_peered":"2023-10-09T04:42:45.866525+0000","last_clean":"2023-10-09T04:42:45.866525+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:42:45.866525+0000","last_undegraded":"2023-10-09T04:42:45.866525+0000","last_fullsized":"2023-10-09T04:42:45.866525+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":51,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882930+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:43:01.882930+0000","last_peered":"2023-10-09T04:43:01.882930+0000","last_clean":"2023-10-09T04:43:01.882930+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:43:01.882930+0000","last_undegraded":"2023-10-09T04:43:01.882930+0000","last_fullsized":"2023-10-09T04:43:01.882930+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882232+0000","last_change":"2023-10-09T04:41:02.614004+0000","last_active":"2023-10-09T04:43:01.882232+0000","last_peered":"2023-10-09T04:43:01.882232+0000","last_clean":"2023-10-09T04:43:01.882232+0000","last_became_active":"2023-10-09T04:41:02.612300+0000","last_became_peered":"2023-10-09T04:41:02.612300+0000","last_unstale":"2023-10-09T04:43:01.882232+0000","last_undegraded":"2023-10-09T04:43:01.882232+0000","last_fullsized":"2023-10-09T04:43:01.882232+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:04.375059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":119,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882516+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:43:01.882516+0000","last_peered":"2023-10-09T04:43:01.882516+0000","last_clean":"2023-10-09T04:43:01.882516+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:43:01.882516+0000","last_undegraded":"2023-10-09T04:43:01.882516+0000","last_fullsized":"2023-10-09T04:43:01.882516+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":228,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.889581+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:43:01.889581+0000","last_peered":"2023-10-09T04:43:01.889581+0000","last_clean":"2023-10-09T04:43:01.889581+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:43:01.889581+0000","last_undegraded":"2023-10-09T04:43:01.889581+0000","last_fullsized":"2023-10-09T04:43:01.889581+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":269,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539387+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:43:02.539387+0000","last_peered":"2023-10-09T04:43:02.539387+0000","last_clean":"2023-10-09T04:43:02.539387+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:43:02.539387+0000","last_undegraded":"2023-10-09T04:43:02.539387+0000","last_fullsized":"2023-10-09T04:43:02.539387+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882471+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:43:01.882471+0000","last_peered":"2023-10-09T04:43:01.882471+0000","last_clean":"2023-10-09T04:43:01.882471+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:43:01.882471+0000","last_undegraded":"2023-10-09T04:43:01.882471+0000","last_fullsized":"2023-10-09T04:43:01.882471+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":224,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.539895+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:43:02.539895+0000","last_peered":"2023-10-09T04:43:02.539895+0000","last_clean":"2023-10-09T04:43:02.539895+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:43:02.539895+0000","last_undegraded":"2023-10-09T04:43:02.539895+0000","last_fullsized":"2023-10-09T04:43:02.539895+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882425+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:43:01.882425+0000","last_peered":"2023-10-09T04:43:01.882425+0000","last_clean":"2023-10-09T04:43:01.882425+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:43:01.882425+0000","last_undegraded":"2023-10-09T04:43:01.882425+0000","last_fullsized":"2023-10-09T04:43:01.882425+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":267,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725572+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:43:02.725572+0000","last_peered":"2023-10-09T04:43:02.725572+0000","last_clean":"2023-10-09T04:43:02.725572+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:43:02.725572+0000","last_undegraded":"2023-10-09T04:43:02.725572+0000","last_fullsized":"2023-10-09T04:43:02.725572+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":299,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.725155+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:43:02.725155+0000","last_peered":"2023-10-09T04:43:02.725155+0000","last_clean":"2023-10-09T04:43:02.725155+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:43:02.725155+0000","last_undegraded":"2023-10-09T04:43:02.725155+0000","last_fullsized":"2023-10-09T04:43:02.725155+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":153,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:02.540931+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:43:02.540931+0000","last_peered":"2023-10-09T04:43:02.540931+0000","last_clean":"2023-10-09T04:43:02.540931+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:43:02.540931+0000","last_undegraded":"2023-10-09T04:43:02.540931+0000","last_fullsized":"2023-10-09T04:43:02.540931+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":27,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-10-09T04:43:01.882246+0000","last_change":"2023-10-09T04:41:02.608725+0000","last_active":"2023-10-09T04:43:01.882246+0000","last_peered":"2023-10-09T04:43:01.882246+0000","last_clean":"2023-10-09T04:43:01.882246+0000","last_became_active":"2023-10-09T04:41:02.606509+0000","last_became_peered":"2023-10-09T04:41:02.606509+0000","last_unstale":"2023-10-09T04:43:01.882246+0000","last_undegraded":"2023-10-09T04:43:01.882246+0000","last_fullsized":"2023-10-09T04:43:01.882246+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:49.430101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":201108,"num_write_kb":6435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21862,"num_bytes_recovered":614465560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":76349440,"data_stored":76349440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138810,"ondisk_log_size":138810,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":5624,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234436,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":946536,"kb_used_data":43744,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":92798616,"statfs":{"total":95995035648,"available":95025782784,"internally_reserved":0,"allocated":44793856,"data_stored":44205208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:36 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.0390000000000001,"15min":4.0579999999999998},"min":{"1min":0.247,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":11.144,"5min":44.122999999999998,"15min":186.446},"last":0.27200000000000002},{"interface":"front","average":{"1min":3.5510000000000002,"5min":2.7690000000000001,"15min":4.407},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":10.42,"5min":43.313000000000002,"15min":186.429},"last":0.38900000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:42:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":6.3140000000000001,"15min":5.3949999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":11.256,"5min":264.62,"15min":264.62},"last":0.374},{"interface":"front","average":{"1min":3.2759999999999998,"5min":6.5979999999999999,"15min":5.5439999999999996},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":14.73,"5min":264.65899999999999,"15min":264.65899999999999},"last":0.501}]},{"osd":2,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":7.5010000000000003,"15min":3.7010000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.9949999999999992,"5min":184.72,"15min":184.72},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.4769999999999999,"5min":7.8920000000000003,"15min":4.1660000000000004},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.438000000000001,"5min":184.697,"15min":184.697},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:42:22 2023","interfaces":[{"interface":"back","average":{"1min":3.2450000000000001,"5min":7.1470000000000002,"15min":7.7300000000000004},"min":{"1min":0.41999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":11.339,"5min":182.839,"15min":249.989},"last":0.42499999999999999},{"interface":"front","average":{"1min":5.2850000000000001,"5min":7.734,"15min":8.0269999999999992},"min":{"1min":0.42699999999999999,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":15.837999999999999,"5min":182.678,"15min":250.21700000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.2789999999999999,"15min":1.389},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.388,"5min":14.257999999999999,"15min":14.257999999999999},"last":0.307},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.86199999999999999,"15min":1.131},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.21199999999999999},"max":{"1min":4.923,"5min":7.2290000000000001,"15min":10.613},"last":0.48399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.2290000000000001,"15min":1.1040000000000001},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.27900000000000003},"max":{"1min":4.8099999999999996,"5min":13.319000000000001,"15min":13.319000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.3799999999999999,"5min":1.728,"15min":1.5449999999999999},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.779999999999999,"5min":18.408000000000001,"15min":18.408000000000001},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.1439999999999999,"15min":1.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":11.388,"15min":11.388},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.643,"15min":1.6299999999999999},"min":{"1min":0.373,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.532999999999999,"5min":18.311,"15min":18.311},"last":0.46200000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496070,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905368,"kb_used_data":3088,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":92839784,"statfs":{"total":95995035648,"available":95067938816,"internally_reserved":0,"allocated":3162112,"data_stored":2589379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.274,"15min":8.1029999999999998},"min":{"1min":0.24199999999999999,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":8.1760000000000002,"5min":75.308000000000007,"15min":1284.521},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.3929999999999998,"5min":2.7000000000000002,"15min":8.2710000000000008},"min":{"1min":0.29799999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4820000000000002,"5min":75.432000000000002,"15min":1284.4690000000001},"last":0.439}]},{"osd":1,"last update":"Mon Oct 9 04:42:30 2023","interfaces":[{"interface":"back","average":{"1min":6.0679999999999996,"5min":7.9489999999999998,"15min":5.6050000000000004},"min":{"1min":0.32200000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":61.042000000000002,"5min":150.75,"15min":185.74000000000001},"last":1.655},{"interface":"front","average":{"1min":6.0359999999999996,"5min":8.1059999999999999,"15min":5.7190000000000003},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.26100000000000001},"max":{"1min":61.055999999999997,"5min":150.81999999999999,"15min":189.03999999999999},"last":0.41699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:42:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":8.7400000000000002,"15min":4.3120000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.30199999999999999},"max":{"1min":11.225,"5min":209.44399999999999,"15min":209.44399999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.825,"5min":8.7029999999999994,"15min":4.2880000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":11.239000000000001,"5min":209.428,"15min":209.428},"last":1.6910000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:42:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8760000000000003,"5min":7.992,"15min":6.1719999999999997},"min":{"1min":0.32400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":34.414000000000001,"5min":157.93100000000001,"15min":167.56100000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":5.2569999999999997,"5min":8.0220000000000002,"15min":6.2750000000000004},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.313},"max":{"1min":34.167000000000002,"5min":157.95099999999999,"15min":167.577},"last":0.46400000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":0.996,"15min":1.3169999999999999},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.25600000000000001},"max":{"1min":6.5800000000000001,"5min":6.8380000000000001,"15min":14.430999999999999},"last":1.5249999999999999},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.046,"15min":1.109},"min":{"1min":0.33600000000000002,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":9.2240000000000002,"5min":9.2240000000000002,"15min":9.2240000000000002},"last":0.33400000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.0800000000000001,"15min":1.8029999999999999},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":6.6310000000000002,"5min":7.3010000000000002,"15min":81.087000000000003},"last":1.5700000000000001},{"interface":"front","average":{"1min":2.4510000000000001,"5min":1.1639999999999999,"15min":1.746},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":8.2739999999999991,"5min":8.6479999999999997,"15min":81.706999999999994},"last":0.52400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:43:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.238,"15min":1.5600000000000001},"min":{"1min":0.378,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":4.6029999999999998,"5min":16.512,"15min":81.674999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.385,"5min":1.389,"15min":1.762},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":7.5289999999999999,"5min":17.715,"15min":81.055000000000007},"last":1.6120000000000001}]}]},{"osd":1,"up_from":118,"seq":506806141160,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628040,"kb_used_data":17792,"kb_used_omap":4,"kb_used_meta":610235,"kb_avail":93117112,"statfs":{"total":95995035648,"available":95351922688,"internally_reserved":0,"allocated":18219008,"data_stored":17074321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":624881550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":3.8260000000000001,"15min":3.9990000000000001},"min":{"1min":0.23200000000000001,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":13.573,"5min":75.620000000000005,"15min":97.308000000000007},"last":0.36799999999999999},{"interface":"front","average":{"1min":4.3570000000000002,"5min":3.7690000000000001,"15min":4.218},"min":{"1min":0.20499999999999999,"5min":0.193,"15min":0.193},"max":{"1min":13.366,"5min":69.701999999999998,"15min":97.055999999999997},"last":0.35199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:42:52 2023","interfaces":[{"interface":"back","average":{"1min":12.409000000000001,"5min":6.1970000000000001,"15min":3.823},"min":{"1min":0.253,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":263.60500000000002,"5min":263.60500000000002,"15min":263.60500000000002},"last":0.30599999999999999},{"interface":"front","average":{"1min":13.936,"5min":7.0970000000000004,"15min":4.3310000000000004},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":263.65199999999999,"5min":263.65199999999999,"15min":263.65199999999999},"last":0.27800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:42:25 2023","interfaces":[{"interface":"back","average":{"1min":5.2999999999999998,"5min":8.0850000000000009,"15min":5.4400000000000004},"min":{"1min":0.29099999999999998,"5min":0.252,"15min":0.24399999999999999},"max":{"1min":14.930999999999999,"5min":118.837,"15min":195.035},"last":0.41199999999999998},{"interface":"front","average":{"1min":5.2510000000000003,"5min":9.0180000000000007,"15min":6.0730000000000004},"min":{"1min":0.316,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":12.634,"5min":117.88500000000001,"15min":195.465},"last":0.40000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":2.1480000000000001,"15min":2.4039999999999999},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.245},"max":{"1min":8.0489999999999995,"5min":14.449,"15min":170.072},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.738,"15min":2.21},"min":{"1min":0.308,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":12.297000000000001,"5min":75.513999999999996,"15min":169.94499999999999},"last":0.498}]},{"osd":5,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":3.54,"5min":2.5,"15min":2.4340000000000002},"min":{"1min":0.40000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.8940000000000001,"5min":11.499000000000001,"15min":171.036},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.6560000000000001,"5min":2.3889999999999998,"15min":2.5960000000000001},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":13.031000000000001,"5min":14.557,"15min":170.88},"last":0.38600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":9.4359999999999999,"5min":3.6659999999999999,"15min":2.4159999999999999},"min":{"1min":0.46600000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":132.36099999999999,"5min":132.36099999999999,"15min":132.36099999999999},"last":0.441},{"interface":"front","average":{"1min":8.0470000000000006,"5min":3.4950000000000001,"15min":2.593},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":132.17699999999999,"5min":132.17699999999999,"15min":132.17699999999999},"last":0.46400000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":8.5210000000000008,"5min":3.8079999999999998,"15min":3.149},"min":{"1min":0.42599999999999999,"5min":0.36599999999999999,"15min":0.34100000000000003},"max":{"1min":132.136,"5min":132.136,"15min":174.75999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":7.8659999999999997,"5min":4.133,"15min":2.8530000000000002},"min":{"1min":0.44700000000000001,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":132.12,"5min":132.12,"15min":174.779},"last":0.56100000000000005}]}]},{"osd":0,"up_from":91,"seq":390842024217,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":814804,"kb_used_data":18508,"kb_used_omap":5,"kb_used_meta":796282,"kb_avail":92930348,"statfs":{"total":95995035648,"available":95160676352,"internally_reserved":0,"allocated":18952192,"data_stored":17912974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":815393402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":5.7599999999999998,"15min":4.9790000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":20.994,"5min":138.536,"15min":167.92699999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.585,"5min":6.5449999999999999,"15min":5.367},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":8.1010000000000009,"5min":138.89599999999999,"15min":169.78999999999999},"last":0.28799999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:42:55 2023","interfaces":[{"interface":"back","average":{"1min":7.0999999999999996,"5min":5.6340000000000003,"15min":8.1769999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":98.938000000000002,"5min":122.983,"15min":131.30699999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":7.649,"5min":5.9930000000000003,"15min":8.5879999999999992},"min":{"1min":0.26400000000000001,"5min":0.219,"15min":0.219},"max":{"1min":98.975999999999999,"5min":122.715,"15min":129.922},"last":0.32400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:42:19 2023","interfaces":[{"interface":"back","average":{"1min":13.063000000000001,"5min":6.3949999999999996,"15min":4.968},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":156.69800000000001,"5min":156.69800000000001,"15min":213.28399999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":15.638,"5min":7.8710000000000004,"15min":5.5709999999999997},"min":{"1min":0.374,"5min":0.26600000000000001,"15min":0.222},"max":{"1min":161.40199999999999,"5min":161.40199999999999,"15min":212.27000000000001},"last":0.34799999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.4930000000000001,"15min":2.2970000000000002},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.25700000000000001},"max":{"1min":6.0979999999999999,"5min":11.300000000000001,"15min":161.67099999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.847,"15min":2.5710000000000002},"min":{"1min":0.35499999999999998,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":5.6890000000000001,"5min":14.109999999999999,"15min":160.74799999999999},"last":0.46600000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":3.222,"15min":2.399},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27900000000000003},"max":{"1min":5.657,"5min":136.322,"15min":136.322},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.2080000000000002,"5min":3.1469999999999998,"15min":2.181},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.718,"5min":136.15799999999999,"15min":136.15799999999999},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":6.0629999999999997,"5min":3.113,"15min":2.7210000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":78.438999999999993,"5min":78.438999999999993,"15min":108.63800000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":5.6829999999999998,"5min":2.7090000000000001,"15min":2.3170000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":78.617999999999995,"5min":78.617999999999995,"15min":108.76300000000001},"last":0.438}]},{"osd":7,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":6.0199999999999996,"5min":2.7679999999999998,"15min":2.004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":77.682000000000002,"5min":77.682000000000002,"15min":77.682000000000002},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.9429999999999996,"5min":2.516,"15min":2.0190000000000001},"min":{"1min":0.373,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":77.606999999999999,"5min":77.606999999999999,"15min":77.606999999999999},"last":0.57899999999999996}]}]},{"osd":2,"up_from":221,"seq":949187772495,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285216,"kb_used_data":2968,"kb_used_omap":7,"kb_used_meta":282232,"kb_avail":93459936,"statfs":{"total":95995035648,"available":95702974464,"internally_reserved":0,"allocated":3039232,"data_stored":2049129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":289005649},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8849999999999998,"15min":7.9169999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.6999999999999993,"5min":13.813000000000001,"15min":72.608000000000004},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.585,"5min":2.3849999999999998,"15min":7.4160000000000004},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.8170000000000002,"5min":14.377000000000001,"15min":72.632000000000005},"last":0.34499999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":11.619,"5min":5.1859999999999999,"15min":5.2720000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.848,"5min":121.848,"15min":121.848},"last":0.27800000000000002},{"interface":"front","average":{"1min":10.805,"5min":5.5819999999999999,"15min":5.7610000000000001},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":121.827,"5min":121.827,"15min":121.827},"last":0.308}]},{"osd":3,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":15.984,"5min":9.5199999999999996,"15min":6.7770000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":225.85499999999999,"5min":335.928,"15min":335.928},"last":0.36099999999999999},{"interface":"front","average":{"1min":16.954000000000001,"5min":10.244,"15min":6.9260000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":233.53299999999999,"5min":335.68599999999998,"15min":335.68599999999998},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":2.48,"15min":4.1660000000000004},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.315},"max":{"1min":8.8360000000000003,"5min":14.452999999999999,"15min":72.944999999999993},"last":0.44},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.4359999999999999,"15min":4.3239999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.6359999999999992,"5min":14.183,"15min":73.054000000000002},"last":0.41899999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.71,"15min":3.988},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.2649999999999997,"5min":65.388000000000005,"15min":72.986000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":3.0659999999999998,"15min":4.4779999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.6440000000000001,"5min":67.799000000000007,"15min":72.963999999999999},"last":0.40500000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.7770000000000001,"15min":9.3030000000000008},"min":{"1min":0.39600000000000002,"5min":0.39000000000000001,"15min":0.33600000000000002},"max":{"1min":10.263,"5min":34.753,"15min":135.614},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.4830000000000001,"5min":2.5910000000000002,"15min":8.6310000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.35099999999999998},"max":{"1min":7.423,"5min":34.796999999999997,"15min":130.196},"last":0.60099999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3149999999999999,"5min":2.77,"15min":4.5759999999999996},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.378},"max":{"1min":10.228,"5min":28.126999999999999,"15min":73.079999999999998},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.536,"15min":3.9580000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.41499999999999998},"max":{"1min":11.669,"5min":28.106000000000002,"15min":73.069000000000003},"last":0.63500000000000001}]}]},{"osd":3,"up_from":142,"seq":609885356218,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":456400,"kb_used_data":2952,"kb_used_omap":2,"kb_used_meta":453437,"kb_avail":93288752,"statfs":{"total":95995035648,"available":95527682048,"internally_reserved":0,"allocated":3022848,"data_stored":2044563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":464319541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.7789999999999999,"15min":2.4009999999999998},"min":{"1min":0.23799999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":7.7000000000000002,"5min":17.648,"15min":142.14500000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":15.590999999999999,"5min":5.0460000000000003,"15min":3.532},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":254.38499999999999,"5min":254.38499999999999,"15min":254.38499999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":2.0529999999999999,"5min":2.597,"15min":3.5030000000000001},"min":{"1min":0.29799999999999999,"5min":0.23200000000000001,"15min":0.192},"max":{"1min":8.1609999999999996,"5min":21.879000000000001,"15min":180.02500000000001},"last":0.31},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.1139999999999999,"15min":3.5659999999999998},"min":{"1min":0.28399999999999997,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":6.4900000000000002,"5min":21.911999999999999,"15min":184.00899999999999},"last":0.27100000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":6.04,"5min":6.173,"15min":3.1749999999999998},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":77.772000000000006,"5min":177.28399999999999,"15min":177.28399999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":7.5359999999999996,"5min":8.5960000000000001,"15min":3.7050000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":77.968999999999994,"5min":255.05199999999999,"15min":255.05199999999999},"last":0.34300000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":15.137,"5min":4.524,"15min":2.907},"min":{"1min":0.32300000000000001,"5min":0.28999999999999998,"15min":0.27900000000000003},"max":{"1min":254.072,"5min":254.072,"15min":254.072},"last":0.39500000000000002},{"interface":"front","average":{"1min":15.058999999999999,"5min":4.2640000000000002,"15min":2.6850000000000001},"min":{"1min":0.40400000000000003,"5min":0.32500000000000001,"15min":0.307},"max":{"1min":253.49700000000001,"5min":253.49700000000001,"15min":253.49700000000001},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":18.327000000000002,"5min":5.218,"15min":3.444},"min":{"1min":0.373,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":253.57900000000001,"5min":253.57900000000001,"15min":253.57900000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":17.856000000000002,"5min":5.3220000000000001,"15min":3.516},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":254.191,"5min":254.191,"15min":254.191},"last":0.45600000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":18.550999999999998,"5min":5.5640000000000001,"15min":3.3540000000000001},"min":{"1min":0.38700000000000001,"5min":0.375,"15min":0.31900000000000001},"max":{"1min":254.14699999999999,"5min":254.14699999999999,"15min":254.14699999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":5.8150000000000004,"5min":2.8359999999999999,"15min":2.8460000000000001},"min":{"1min":0.35499999999999998,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":50.308,"5min":50.308,"15min":123.505},"last":0.42199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:42:20 2023","interfaces":[{"interface":"back","average":{"1min":4.3559999999999999,"5min":2.5470000000000002,"15min":2.4500000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.32300000000000001},"max":{"1min":43.835000000000001,"5min":43.835000000000001,"15min":57.399000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.9540000000000002,"5min":2.1509999999999998,"15min":2.2109999999999999},"min":{"1min":0.436,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":43.781999999999996,"5min":43.781999999999996,"15min":57.447000000000003},"last":0.61799999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019344,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906912,"kb_used_data":14872,"kb_used_omap":0,"kb_used_meta":892032,"kb_avail":92838240,"statfs":{"total":95995035648,"available":95066357760,"internally_reserved":0,"allocated":15228928,"data_stored":14647858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913440768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":2.8010000000000002,"15min":5.3499999999999996},"min":{"1min":0.26600000000000001,"5min":0.25600000000000001,"15min":0.23000000000000001},"max":{"1min":11.221,"5min":73.471000000000004,"15min":219.577},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.93600000000000005,"5min":2.5979999999999999,"15min":5.0519999999999996},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.255},"max":{"1min":5.3630000000000004,"5min":73.364999999999995,"15min":218.96000000000001},"last":0.55900000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:42:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":3.0350000000000001,"15min":5.0640000000000001},"min":{"1min":0.26400000000000001,"5min":0.255,"15min":0.24099999999999999},"max":{"1min":7.8579999999999997,"5min":56.122,"15min":142.74799999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":3.6240000000000001,"5min":3.1619999999999999,"15min":5.218},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":13.840999999999999,"5min":56.043999999999997,"15min":139.09700000000001},"last":0.42599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2189999999999999,"5min":5.6829999999999998,"15min":4.3239999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.372,"5min":195.267,"15min":195.267},"last":0.505},{"interface":"front","average":{"1min":2.2250000000000001,"5min":5.4329999999999998,"15min":3.8279999999999998},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.3620000000000001,"5min":195.15799999999999,"15min":195.15799999999999},"last":0.65400000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0259999999999998,"5min":7.1280000000000001,"15min":7.0199999999999996},"min":{"1min":0.34399999999999997,"5min":0.32600000000000001,"15min":0.27900000000000003},"max":{"1min":8.7200000000000006,"5min":357.125,"15min":357.125},"last":0.625},{"interface":"front","average":{"1min":2.4089999999999998,"5min":6.7510000000000003,"15min":7.1219999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.312},"max":{"1min":6.7169999999999996,"5min":359.02800000000002,"15min":359.02800000000002},"last":0.438}]},{"osd":5,"last update":"Mon Oct 9 04:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.024,"15min":1.893},"min":{"1min":0.32400000000000001,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":5.0339999999999998,"5min":9.5609999999999999,"15min":107.476},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":1.2090000000000001,"15min":2.1400000000000001},"min":{"1min":0.312,"5min":0.26900000000000002,"15min":0.23300000000000001},"max":{"1min":15.394,"5min":15.394,"15min":107.414},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:42:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.145,"15min":1.875},"min":{"1min":0.34599999999999997,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":6.2300000000000004,"5min":9.6099999999999994,"15min":65.75},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.113,"5min":1.147,"15min":1.8620000000000001},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.782,"5min":9.782,"15min":65.849999999999994},"last":0.46700000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.0680000000000001,"15min":1.6180000000000001},"min":{"1min":0.37,"5min":0.34399999999999997,"15min":0.28000000000000003},"max":{"1min":1.8320000000000001,"5min":8.6120000000000001,"15min":76.283000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.3069999999999999,"15min":1.6599999999999999},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.32100000000000001},"max":{"1min":4.2469999999999999,"5min":9.6310000000000002,"15min":76.259},"last":0.40100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757708,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905104,"kb_used_data":2504,"kb_used_omap":0,"kb_used_meta":902592,"kb_avail":92840048,"statfs":{"total":95995035648,"available":95068209152,"internally_reserved":0,"allocated":2564096,"data_stored":1995154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924254208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:38 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":3.5630000000000002,"15min":5.2800000000000002},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.22},"max":{"1min":75.855999999999995,"5min":76.587999999999994,"15min":210.85499999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":5.8559999999999999,"5min":4.1299999999999999,"15min":5.3879999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":75.816000000000003,"5min":76.682000000000002,"15min":210.23500000000001},"last":0.35099999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:42:38 2023","interfaces":[{"interface":"back","average":{"1min":5.2569999999999997,"5min":4.8369999999999997,"15min":4.5529999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":73.296000000000006,"5min":145.577,"15min":145.577},"last":0.40799999999999997},{"interface":"front","average":{"1min":6.048,"5min":4.9340000000000002,"15min":5.4160000000000004},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":73.045000000000002,"5min":145.792,"15min":145.792},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:42:56 2023","interfaces":[{"interface":"back","average":{"1min":6.2050000000000001,"5min":3.2629999999999999,"15min":5.9580000000000002},"min":{"1min":0.35899999999999999,"5min":0.28199999999999997,"15min":0.26800000000000002},"max":{"1min":66.984999999999999,"5min":66.984999999999999,"15min":90.763999999999996},"last":0.378},{"interface":"front","average":{"1min":5.2599999999999998,"5min":3.419,"15min":6.1100000000000003},"min":{"1min":0.36599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":67.686000000000007,"5min":67.686000000000007,"15min":91.340000000000003},"last":0.47499999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:42:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":7.8120000000000003,"15min":6.7770000000000001},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":15.063000000000001,"5min":268.63299999999998,"15min":268.63299999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.1869999999999998,"5min":7.4580000000000002,"15min":6.5640000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":15.263,"5min":268.92500000000001,"15min":268.92500000000001},"last":0.52200000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:42:17 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.71999999999999997,"15min":2.379},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.248},"max":{"1min":4.2009999999999996,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.2969999999999999,"15min":2.847},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":7.4859999999999998,"5min":8.7599999999999998,"15min":361.71300000000002},"last":0.50600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:42:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":1.1839999999999999},"min":{"1min":0.312,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.1669999999999998,"5min":5.7640000000000002,"15min":11.192},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.103,"5min":1.2130000000000001,"15min":1.333},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.5800000000000001,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.45800000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.2769999999999999,"15min":1.3400000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":1.78,"5min":13.268000000000001,"15min":13.268000000000001},"last":0.439},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.998,"15min":1.2410000000000001},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.734,"5min":6.5129999999999999,"15min":15.257999999999999},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":15269888,"data_stored":15269888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":15007744,"data_stored":15007744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4456448,"data_stored":4456448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":41615360,"data_stored":41615360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:43:23.513 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:23.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:23.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:23.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:23 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 288/72 objects degraded (400.000%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:43:23.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:23 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 288/72 objects degraded (400.000%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:43:23.646 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:23 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 288/72 objects degraded (400.000%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:43:24.312 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:24.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:24.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:24.373 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:43:24.374 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:43:24.374 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-10-09T04:43:24.374 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-10-09T04:43:24.417 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:24.444 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:24.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:24 smithi029 ceph-mon[216903]: pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:24.571 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:24 smithi029 ceph-mon[223858]: pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:24.768 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:24.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:24.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:24.872 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:24.897 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183705 10.13 10.550def73 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570147+0000", 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.32460340799997, 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96301429199999999, 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.898 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183705 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570147+0000", 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570147+0000", 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570146+0000", 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570164+0000", 2023-10-09T04:43:24.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7838000000000001e-05 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570176+0000", 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2649999999999999e-05 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570186+0000", 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.026e-05 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627369+0000", 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057182589999999998 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627380+0000", 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.079e-05 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533151+0000", 2023-10-09T04:43:24.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90577155499999995 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533162+0000", 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0236e-05 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.903 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183707 10.23 10.be4d5fe3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570198+0000", 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.32455323400001, 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96303084900000002, 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.904 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183707 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570198+0000", 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570198+0000", 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570197+0000", 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570200+0000", 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5969999999999999e-06 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570205+0000", 2023-10-09T04:43:24.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.84e-06 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570211+0000", 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0610000000000003e-06 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627385+0000", 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057174306000000001 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627394+0000", 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3950000000000007e-06 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533219+0000", 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90582470800000003 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533228+0000", 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4620000000000002e-06 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183713 10.13 10.8cd385d3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.32453626699998, 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96298166600000001, 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.910 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183713 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570214+0000", 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570216+0000", 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9259999999999999e-06 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570222+0000", 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1240000000000002e-06 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570226+0000", 2023-10-09T04:43:24.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1760000000000003e-06 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627405+0000", 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057178527999999999 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.627412+0000", 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5000000000000002e-06 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533186+0000", 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90577387600000003 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.533196+0000", 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8530000000000004e-06 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.915 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183714 10.15 10.f5e10575 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.32452053899999, 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96215133799999997, 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.916 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183714 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570230+0000", 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570232+0000", 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0679999999999999e-06 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570237+0000", 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7389999999999999e-06 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570242+0000", 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0509999999999996e-06 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607094+0000", 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036852426000000001 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607105+0000", 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0959e-05 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532372+0000", 2023-10-09T04:43:24.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92526699999999995 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532382+0000", 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4830000000000005e-06 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183718 10.8 10.ac006e88 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.570247+0000", 2023-10-09T04:43:24.921 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.324504021, 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96047792499999995, 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183718 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570247+0000", 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570247+0000", 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570246+0000", 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:43:24.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570254+0000", 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6129999999999998e-06 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570259+0000", 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3469999999999998e-06 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.570264+0000", 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1960000000000001e-06 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593203+0000", 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022939365 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593219+0000", 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5820000000000001e-05 2023-10-09T04:43:24.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530714+0000", 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93749524500000003 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530725+0000", 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0659000000000001e-05 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.926 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183731 10.15 10.37a67415 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.583550+0000", 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.31120089000001, 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94886407800000006, 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183731 2023-10-09T04:43:24.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583550+0000", 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583550+0000", 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583549+0000", 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583554+0000", 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9749999999999998e-06 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583562+0000", 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0220000000000004e-06 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.583568+0000", 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7660000000000001e-06 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607153+0000", 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023584963 2023-10-09T04:43:24.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607159+0000", 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.117e-06 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532404+0000", 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92524475299999998 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532414+0000", 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7100000000000002e-06 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183735 10.5 10.e68c7665 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.593341+0000", 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.30140966200003, 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93897671800000004, 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.932 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183735 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593341+0000", 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593341+0000", 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593339+0000", 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593350+0000", 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0436e-05 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593363+0000", 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3105000000000001e-05 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.593375+0000", 2023-10-09T04:43:24.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2364999999999999e-05 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607125+0000", 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013750149 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607132+0000", 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.224e-06 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532305+0000", 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92517361099999995 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532318+0000", 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2707999999999999e-05 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.937 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183739 10.15 10.23ae88b5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.600199+0000", 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.29455174999998, 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93224512900000001, 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.938 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183739 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600199+0000", 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600199+0000", 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600198+0000", 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600205+0000", 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2069999999999998e-06 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600220+0000", 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5398999999999999e-05 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.600230+0000", 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0006e-05 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607186+0000", 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006955653 2023-10-09T04:43:24.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.607193+0000", 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.973e-06 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532435+0000", 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.925241804 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532444+0000", 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5999999999999996e-06 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183746 10.5 10.14db3525 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.611980+0000", 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.28277095499999, 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92037076200000001, 2023-10-09T04:43:24.943 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183746 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611980+0000", 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611980+0000", 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611978+0000", 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611986+0000", 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2899999999999997e-06 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.611998+0000", 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2118e-05 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.612012+0000", 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4317e-05 2023-10-09T04:43:24.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.612243+0000", 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023038900000000001 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.612250+0000", 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6009999999999997e-06 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532341+0000", 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92009069700000001 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.532351+0000", 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6870000000000006e-06 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.948 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:183750 10.18 10.1e4f0c38 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:46.624114+0000", 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 398.27063690199998, 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90657211299999996, 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 183750 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624114+0000", 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624114+0000", 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624112+0000", 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624121+0000", 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3180000000000007e-06 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624140+0000", 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9738e-05 2023-10-09T04:43:24.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624149+0000", 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3850000000000002e-06 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624201+0000", 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2445999999999999e-05 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:46.624211+0000", 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3680000000000008e-06 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530672+0000", 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.90646186799999995 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:47.530686+0000", 2023-10-09T04:43:24.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3596e-05 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89779 11.d 11:b6440cbe:::benchmark_data_smithi029_392508_object89778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.422302+0000", 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 253.47244913200001, 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2239466000000001, 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:24.954 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89779 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422302+0000", 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422302+0000", 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422300+0000", 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422328+0000", 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7678000000000002e-05 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422345+0000", 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.751e-05 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422354+0000", 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7600000000000008e-06 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422420+0000", 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6419000000000002e-05 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422433+0000", 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2203e-05 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394893+0000", 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.972460455 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394908+0000", 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4713e-05 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.633240+0000", 2023-10-09T04:43:24.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.238332558 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.633379+0000", 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013892099999999999 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.633492+0000", 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011317899999999999 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641640+0000", 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081479450000000002 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641688+0000", 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8102999999999997e-05 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641701+0000", 2023-10-09T04:43:24.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3006999999999999e-05 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641763+0000", 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1185000000000005e-05 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646248+0000", 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044856119999999999 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.962 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89791 11.d 11:b817d833:::benchmark_data_smithi029_392508_object89790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.438144+0000", 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 253.45660726200001, 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.239980504, 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:24.963 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89791 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438144+0000", 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438144+0000", 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438142+0000", 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438168+0000", 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6075999999999999e-05 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438178+0000", 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6059999999999992e-06 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438197+0000", 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8683000000000001e-05 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438265+0000", 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8207999999999995e-05 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438279+0000", 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3917000000000001e-05 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394933+0000", 2023-10-09T04:43:24.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95665392599999999 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394942+0000", 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4579999999999999e-06 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646427+0000", 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2514851170000001 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646596+0000", 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000169013 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646783+0000", 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000186331 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.660928+0000", 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014145061 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.660997+0000", 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9444999999999997e-05 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.673775+0000", 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012778273 2023-10-09T04:43:24.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.673841+0000", 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5617999999999994e-05 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.678124+0000", 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042831040000000003 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.971 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89794 11.d 11:b962c3ba:::benchmark_data_smithi029_392508_object89793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.440680+0000", 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 253.45407058500001, 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2416845639999998, 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:24.972 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89794 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440680+0000", 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440680+0000", 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440679+0000", 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440701+0000", 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1917999999999999e-05 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440716+0000", 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5790999999999998e-05 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440723+0000", 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7170000000000002e-06 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440772+0000", 2023-10-09T04:43:24.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8900999999999998e-05 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440779+0000", 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1999999999999997e-06 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394965+0000", 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95418614000000002 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394974+0000", 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2199999999999992e-06 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.649824+0000", 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.254850244 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:24.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.649952+0000", 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012858499999999999 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.650115+0000", 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016252899999999999 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.660967+0000", 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010852185 2023-10-09T04:43:24.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.661004+0000", 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7236000000000001e-05 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.678184+0000", 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017179507 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.678245+0000", 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1424e-05 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.682365+0000", 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041193979999999998 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:140707 11.c 11:397d142f:::benchmark_data_smithi029_392508_object40151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:40:50.998126+0000", 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 153.896625228, 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.49069680300000001, 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:24.981 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 140707 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998126+0000", 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998126+0000", 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998124+0000", 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998130+0000", 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.122e-06 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998144+0000", 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.344e-05 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998152+0000", 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9969999999999998e-06 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:50.998219+0000", 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7512e-05 2023-10-09T04:43:24.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.002915+0000", 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046961529999999998 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.010801+0000", 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078856029999999997 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.010892+0000", 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1167000000000003e-05 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.011023+0000", 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013088100000000001 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.018791+0000", 2023-10-09T04:43:24.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007768067 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.019360+0000", 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00056920799999999998 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.486691+0000", 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.467330462 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.486738+0000", 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7299000000000002e-05 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.488822+0000", 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020844129999999998 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:140708 11.c 11:3710fa76:::benchmark_data_smithi029_392508_object40152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:40:51.007108+0000", 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 153.88764295199999, 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50857920899999998, 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 140708 2023-10-09T04:43:24.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.007108+0000", 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.007108+0000", 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.007106+0000", 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.007112+0000", 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7640000000000002e-06 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:24.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.007126+0000", 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3621000000000001e-05 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.007134+0000", 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1489999999999994e-06 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.017642+0000", 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010507829999999999 2023-10-09T04:43:24.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.017979+0000", 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000337134 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.496053+0000", 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47807455399999998 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.496160+0000", 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010681 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.496291+0000", 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013074899999999999 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.502536+0000", 2023-10-09T04:43:24.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062450120000000003 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.503494+0000", 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00095830700000000004 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.513188+0000", 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096935690000000008 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.513235+0000", 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7329e-05 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:40:51.515687+0000", 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024518539999999998 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:24.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145927 11.d 11:be473cf7:::benchmark_data_smithi029_392508_object45371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.631539+0000", 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 143.263211603, 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2090651059999999, 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145927 2023-10-09T04:43:24.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631539+0000", 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631539+0000", 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631538+0000", 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631545+0000", 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.306e-06 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:24.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631562+0000", 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7039999999999999e-05 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631571+0000", 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6710000000000005e-06 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631637+0000", 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5897999999999998e-05 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631651+0000", 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4246e-05 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.757167+0000", 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1255164500000001 2023-10-09T04:43:25.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.757305+0000", 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137119 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.757422+0000", 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117538 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.768164+0000", 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010741516 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.780860+0000", 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012696146 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.839627+0000", 2023-10-09T04:43:25.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.058767698 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.839695+0000", 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7335999999999994e-05 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.840604+0000", 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00090948899999999998 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145928 11.d 11:b60f9d16:::benchmark_data_smithi029_392508_object45372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.631947+0000", 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 143.26280410800001, 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.209480036, 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:25.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145928 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631947+0000", 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631947+0000", 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631946+0000", 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631951+0000", 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8459999999999999e-06 2023-10-09T04:43:25.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631963+0000", 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2666e-05 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631970+0000", 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6510000000000004e-06 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.632035+0000", 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4949000000000004e-05 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.632045+0000", 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6190000000000006e-06 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.762300+0000", 2023-10-09T04:43:25.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.130255303 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.762418+0000", 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117657 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.762510+0000", 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2128999999999998e-05 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771289+0000", 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087791169999999995 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.793784+0000", 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022494858 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.840683+0000", 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.046899376999999999 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.840747+0000", 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3975999999999999e-05 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.841427+0000", 2023-10-09T04:43:25.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00067968599999999998 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145934 11.d 11:b5fb235c:::benchmark_data_smithi029_392508_object45378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.638048+0000", 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 143.25670231699999, 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.204085243, 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:25.013 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145934 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638048+0000", 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:25.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638048+0000", 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638047+0000", 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638053+0000", 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638068+0000", 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4608e-05 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638076+0000", 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1300000000000001e-06 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638154+0000", 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8053999999999995e-05 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638166+0000", 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2418999999999999e-05 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.765706+0000", 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.127539327 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.765786+0000", 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0094999999999998e-05 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.765863+0000", 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7152999999999998e-05 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771298+0000", 2023-10-09T04:43:25.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054349400000000001 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.793840+0000", 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022542736000000001 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.841500+0000", 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.047659941999999997 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.841556+0000", 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5578999999999998e-05 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842134+0000", 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057772800000000001 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145964 11.d 11:b6b54d43:::benchmark_data_smithi029_392508_object45408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.911106+0000", 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 142.983644365, 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.931445309, 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145964 2023-10-09T04:43:25.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911106+0000", 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911106+0000", 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911105+0000", 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911111+0000", 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7610000000000004e-06 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911125+0000", 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3961000000000001e-05 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911136+0000", 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0842000000000001e-05 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911185+0000", 2023-10-09T04:43:25.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8922999999999999e-05 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.911192+0000", 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.571e-06 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.768213+0000", 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85702041200000001 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.768356+0000", 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000143213 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.768476+0000", 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000119734 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.774511+0000", 2023-10-09T04:43:25.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060349749999999997 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.793874+0000", 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019363279000000001 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842184+0000", 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048310522000000002 2023-10-09T04:43:25.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842236+0000", 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1539999999999998e-05 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842552+0000", 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031579999999999998 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:25.028 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145970 11.d 11:b13f26c3:::benchmark_data_smithi029_392508_object45414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.924004+0000", 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 142.97074714300001, 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.91921223500000004, 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145970 2023-10-09T04:43:25.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924004+0000", 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924004+0000", 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:43:25.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924002+0000", 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924018+0000", 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6311e-05 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:43:25.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924035+0000", 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6951000000000001e-05 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924044+0000", 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8780000000000004e-06 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924100+0000", 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5250000000000001e-05 2023-10-09T04:43:25.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.924110+0000", 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0331e-05 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771323+0000", 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84721304500000005 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771417+0000", 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4334999999999997e-05 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771503+0000", 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5620999999999999e-05 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.774604+0000", 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031007399999999998 2023-10-09T04:43:25.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.793897+0000", 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019293675999999999 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842749+0000", 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048852238999999999 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842800+0000", 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0569000000000003e-05 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.843216+0000", 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041579099999999999 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:43:25.036 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:43:25.037 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:24 smithi177 ceph-mon[195797]: pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:25.038 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:25.039 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:25.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:25.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:25.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:25.169 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:25.566 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:43:25.684 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:25.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:25.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:25.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:25.813 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2610452216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:25.813 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:25.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2610452216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:25.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:25.816 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:25.872 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:43:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:25.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:25.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:43:25.977 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:26.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2610452216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:26.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:43:26.009 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:26.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:26.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:26.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:26.195 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:26.316 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:43:26.734 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:26.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:26.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:26.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:26.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:26.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:43:26.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:43:26.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:26.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T04:43:26.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:26 smithi177 ceph-mon[195797]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.870 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:26.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[223858]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: osdmap e283: 8 total, 8 up, 8 in 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:26 smithi029 ceph-mon[216903]: failed to encode map e283 with expected crc 2023-10-09T04:43:26.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:26.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:27.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:27.050 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:27.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:27.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:27.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:27.229 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:27.239 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:27.609 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:27.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:27.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:27.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:27.742 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:27.784 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:27.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:27.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:27.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:27 smithi029 ceph-mon[216903]: failed to encode map e283 with expected crc 2023-10-09T04:43:27.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:27 smithi029 ceph-mon[223858]: failed to encode map e283 with expected crc 2023-10-09T04:43:27.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:27.915 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:27 smithi177 ceph-mon[195797]: failed to encode map e283 with expected crc 2023-10-09T04:43:27.917 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:27.959 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:27.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:28.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:28.102 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:28.303 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:28.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:28.309 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:43:28.328 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:43:28.350 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:43:28.370 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:43:28.389 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:43:28.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:28.410 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:43:28.436 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:43:28.441 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:28.458 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.462 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:43:28.744 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[216903]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/364695354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[223858]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/364695354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:28.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:28.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:43:28.985 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:28 smithi177 ceph-mon[195797]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/364695354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:29.947 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:29.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:29.988 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:30.051 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:30.086 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:30.212 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:30.212 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:30.218 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:43:30.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:30.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:30.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:30.356 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:30.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:30.604 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:43:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:30.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:30.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:30.740 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:30.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:30.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:30.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:30.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:30.926 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:31.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:31 smithi029 ceph-mon[216903]: pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:31.268 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:31 smithi029 ceph-mon[223858]: pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:31.359 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:31.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:31.470 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:31.503 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:31 smithi177 ceph-mon[195797]: pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:32.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:43:32.405 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:32.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:32.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:32.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:32.541 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:32.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:32.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:32.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:32.726 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:32.816 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:43:32.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:32.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:32.922 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:32.953 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:33.001 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:33.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:33.084 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:43:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:43:32] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2023-10-09T04:43:33.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:33.138 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:33.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:33.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:33.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:33.325 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:33.552 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_7' created 2023-10-09T04:43:33.720 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:43:33.722 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:43:33.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:33 smithi029 ceph-mon[216903]: pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:33.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:33 smithi029 ceph-mon[223858]: pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:33 smithi177 ceph-mon[195797]: pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:34.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4263012461' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: failed to encode map e284 with expected crc 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4263012461' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: failed to encode map e284 with expected crc 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: from='client.46079 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4263012461' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T04:43:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: failed to encode map e284 with expected crc 2023-10-09T04:43:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4263012461' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: failed to encode map e284 with expected crc 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: from='client.46079 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4263012461' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: failed to encode map e284 with expected crc 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4263012461' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-09T04:43:34.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: osdmap e284: 8 total, 8 up, 8 in 2023-10-09T04:43:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: failed to encode map e284 with expected crc 2023-10-09T04:43:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: from='client.46079 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:43:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:34.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:35.223 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1092,"stamp":"2023-10-09T04:43:33.534963+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2957464,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":220,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5813972,"kb_used_data":45780,"kb_used_omap":20,"kb_used_meta":5768107,"kb_avail":744147244,"statfs":{"total":767960285184,"available":762006777856,"internally_reserved":0,"allocated":46878720,"data_stored":40321155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5906542222},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.345338"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012257+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:43:22.012257+0000","last_peered":"2023-10-09T04:43:22.012257+0000","last_clean":"2023-10-09T04:43:22.012257+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:43:22.012257+0000","last_undegraded":"2023-10-09T04:43:22.012257+0000","last_fullsized":"2023-10-09T04:43:22.012257+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.552171+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:43:21.552171+0000","last_peered":"2023-10-09T04:43:21.552171+0000","last_clean":"2023-10-09T04:43:21.552171+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:43:21.552171+0000","last_undegraded":"2023-10-09T04:43:21.552171+0000","last_fullsized":"2023-10-09T04:43:21.552171+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":269,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-10-09T04:43:25.560841+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:43:25.560841+0000","last_peered":"2023-10-09T04:43:25.560841+0000","last_clean":"2023-10-09T04:43:25.560841+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:43:25.560841+0000","last_undegraded":"2023-10-09T04:43:25.560841+0000","last_fullsized":"2023-10-09T04:43:25.560841+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":196,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.020987+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:43:22.020987+0000","last_peered":"2023-10-09T04:43:22.020987+0000","last_clean":"2023-10-09T04:43:22.020987+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:43:22.020987+0000","last_undegraded":"2023-10-09T04:43:22.020987+0000","last_fullsized":"2023-10-09T04:43:22.020987+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551155+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:43:21.551155+0000","last_peered":"2023-10-09T04:43:21.551155+0000","last_clean":"2023-10-09T04:43:21.551155+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:43:21.551155+0000","last_undegraded":"2023-10-09T04:43:21.551155+0000","last_fullsized":"2023-10-09T04:43:21.551155+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":239,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.013196+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:43:22.013196+0000","last_peered":"2023-10-09T04:43:22.013196+0000","last_clean":"2023-10-09T04:43:22.013196+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:43:22.013196+0000","last_undegraded":"2023-10-09T04:43:22.013196+0000","last_fullsized":"2023-10-09T04:43:22.013196+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":195,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011453+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:43:22.011453+0000","last_peered":"2023-10-09T04:43:22.011453+0000","last_clean":"2023-10-09T04:43:22.011453+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:43:22.011453+0000","last_undegraded":"2023-10-09T04:43:22.011453+0000","last_fullsized":"2023-10-09T04:43:22.011453+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.554649+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:43:21.554649+0000","last_peered":"2023-10-09T04:43:21.554649+0000","last_clean":"2023-10-09T04:43:21.554649+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:43:21.554649+0000","last_undegraded":"2023-10-09T04:43:21.554649+0000","last_fullsized":"2023-10-09T04:43:21.554649+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011358+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:43:22.011358+0000","last_peered":"2023-10-09T04:43:22.011358+0000","last_clean":"2023-10-09T04:43:22.011358+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:43:22.011358+0000","last_undegraded":"2023-10-09T04:43:22.011358+0000","last_fullsized":"2023-10-09T04:43:22.011358+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":209,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.553344+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:43:21.553344+0000","last_peered":"2023-10-09T04:43:21.553344+0000","last_clean":"2023-10-09T04:43:21.553344+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:43:21.553344+0000","last_undegraded":"2023-10-09T04:43:21.553344+0000","last_fullsized":"2023-10-09T04:43:21.553344+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":277,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.019493+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:43:22.019493+0000","last_peered":"2023-10-09T04:43:22.019493+0000","last_clean":"2023-10-09T04:43:22.019493+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:43:22.019493+0000","last_undegraded":"2023-10-09T04:43:22.019493+0000","last_fullsized":"2023-10-09T04:43:22.019493+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.014729+0000","last_change":"2023-10-09T04:39:40.406537+0000","last_active":"2023-10-09T04:43:22.014729+0000","last_peered":"2023-10-09T04:43:22.014729+0000","last_clean":"2023-10-09T04:43:22.014729+0000","last_became_active":"2023-10-09T04:39:40.406281+0000","last_became_peered":"2023-10-09T04:39:40.406281+0000","last_unstale":"2023-10-09T04:43:22.014729+0000","last_undegraded":"2023-10-09T04:43:22.014729+0000","last_fullsized":"2023-10-09T04:43:22.014729+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:09.883302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":41,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.013138+0000","last_change":"2023-10-09T04:41:02.634559+0000","last_active":"2023-10-09T04:43:22.013138+0000","last_peered":"2023-10-09T04:43:22.013138+0000","last_clean":"2023-10-09T04:43:22.013138+0000","last_became_active":"2023-10-09T04:41:02.624597+0000","last_became_peered":"2023-10-09T04:41:02.624597+0000","last_unstale":"2023-10-09T04:43:22.013138+0000","last_undegraded":"2023-10-09T04:43:22.013138+0000","last_fullsized":"2023-10-09T04:43:22.013138+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:57.939757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012868+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:43:22.012868+0000","last_peered":"2023-10-09T04:43:22.012868+0000","last_clean":"2023-10-09T04:43:22.012868+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:43:22.012868+0000","last_undegraded":"2023-10-09T04:43:22.012868+0000","last_fullsized":"2023-10-09T04:43:22.012868+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":293,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.017871+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:43:22.017871+0000","last_peered":"2023-10-09T04:43:22.017871+0000","last_clean":"2023-10-09T04:43:22.017871+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:43:22.017871+0000","last_undegraded":"2023-10-09T04:43:22.017871+0000","last_fullsized":"2023-10-09T04:43:22.017871+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551253+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:43:21.551253+0000","last_peered":"2023-10-09T04:43:21.551253+0000","last_clean":"2023-10-09T04:43:21.551253+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:43:21.551253+0000","last_undegraded":"2023-10-09T04:43:21.551253+0000","last_fullsized":"2023-10-09T04:43:21.551253+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":195,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.017166+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:43:22.017166+0000","last_peered":"2023-10-09T04:43:22.017166+0000","last_clean":"2023-10-09T04:43:22.017166+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:43:22.017166+0000","last_undegraded":"2023-10-09T04:43:22.017166+0000","last_fullsized":"2023-10-09T04:43:22.017166+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":242,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011834+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:43:22.011834+0000","last_peered":"2023-10-09T04:43:22.011834+0000","last_clean":"2023-10-09T04:43:22.011834+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:43:22.011834+0000","last_undegraded":"2023-10-09T04:43:22.011834+0000","last_fullsized":"2023-10-09T04:43:22.011834+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.017784+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:43:22.017784+0000","last_peered":"2023-10-09T04:43:22.017784+0000","last_clean":"2023-10-09T04:43:22.017784+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:43:22.017784+0000","last_undegraded":"2023-10-09T04:43:22.017784+0000","last_fullsized":"2023-10-09T04:43:22.017784+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":197,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551810+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:43:21.551810+0000","last_peered":"2023-10-09T04:43:21.551810+0000","last_clean":"2023-10-09T04:43:21.551810+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:43:21.551810+0000","last_undegraded":"2023-10-09T04:43:21.551810+0000","last_fullsized":"2023-10-09T04:43:21.551810+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":288,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012183+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:43:22.012183+0000","last_peered":"2023-10-09T04:43:22.012183+0000","last_clean":"2023-10-09T04:43:22.012183+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:43:22.012183+0000","last_undegraded":"2023-10-09T04:43:22.012183+0000","last_fullsized":"2023-10-09T04:43:22.012183+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":236,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551500+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:43:21.551500+0000","last_peered":"2023-10-09T04:43:21.551500+0000","last_clean":"2023-10-09T04:43:21.551500+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:43:21.551500+0000","last_undegraded":"2023-10-09T04:43:21.551500+0000","last_fullsized":"2023-10-09T04:43:21.551500+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":299,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.018941+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:43:22.018941+0000","last_peered":"2023-10-09T04:43:22.018941+0000","last_clean":"2023-10-09T04:43:22.018941+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:43:22.018941+0000","last_undegraded":"2023-10-09T04:43:22.018941+0000","last_fullsized":"2023-10-09T04:43:22.018941+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":225,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.010802+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:43:22.010802+0000","last_peered":"2023-10-09T04:43:22.010802+0000","last_clean":"2023-10-09T04:43:22.010802+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:43:22.010802+0000","last_undegraded":"2023-10-09T04:43:22.010802+0000","last_fullsized":"2023-10-09T04:43:22.010802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":272,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.020906+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:43:22.020906+0000","last_peered":"2023-10-09T04:43:22.020906+0000","last_clean":"2023-10-09T04:43:22.020906+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:43:22.020906+0000","last_undegraded":"2023-10-09T04:43:22.020906+0000","last_fullsized":"2023-10-09T04:43:22.020906+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012447+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:43:22.012447+0000","last_peered":"2023-10-09T04:43:22.012447+0000","last_clean":"2023-10-09T04:43:22.012447+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:43:22.012447+0000","last_undegraded":"2023-10-09T04:43:22.012447+0000","last_fullsized":"2023-10-09T04:43:22.012447+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":226,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.019305+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:43:22.019305+0000","last_peered":"2023-10-09T04:43:22.019305+0000","last_clean":"2023-10-09T04:43:22.019305+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:43:22.019305+0000","last_undegraded":"2023-10-09T04:43:22.019305+0000","last_fullsized":"2023-10-09T04:43:22.019305+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":266,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012876+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:43:22.012876+0000","last_peered":"2023-10-09T04:43:22.012876+0000","last_clean":"2023-10-09T04:43:22.012876+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:43:22.012876+0000","last_undegraded":"2023-10-09T04:43:22.012876+0000","last_fullsized":"2023-10-09T04:43:22.012876+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011259+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:43:22.011259+0000","last_peered":"2023-10-09T04:43:22.011259+0000","last_clean":"2023-10-09T04:43:22.011259+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:43:22.011259+0000","last_undegraded":"2023-10-09T04:43:22.011259+0000","last_fullsized":"2023-10-09T04:43:22.011259+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.010984+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:43:22.010984+0000","last_peered":"2023-10-09T04:43:22.010984+0000","last_clean":"2023-10-09T04:43:22.010984+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:43:22.010984+0000","last_undegraded":"2023-10-09T04:43:22.010984+0000","last_fullsized":"2023-10-09T04:43:22.010984+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":280,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.553172+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:43:21.553172+0000","last_peered":"2023-10-09T04:43:21.553172+0000","last_clean":"2023-10-09T04:43:21.553172+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:43:21.553172+0000","last_undegraded":"2023-10-09T04:43:21.553172+0000","last_fullsized":"2023-10-09T04:43:21.553172+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.013221+0000","last_change":"2023-10-09T04:41:02.624962+0000","last_active":"2023-10-09T04:43:22.013221+0000","last_peered":"2023-10-09T04:43:22.013221+0000","last_clean":"2023-10-09T04:43:22.013221+0000","last_became_active":"2023-10-09T04:41:02.624555+0000","last_became_peered":"2023-10-09T04:41:02.624555+0000","last_unstale":"2023-10-09T04:43:22.013221+0000","last_undegraded":"2023-10-09T04:43:22.013221+0000","last_fullsized":"2023-10-09T04:43:22.013221+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:48.506794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.013016+0000","last_change":"2023-10-09T04:41:02.603773+0000","last_active":"2023-10-09T04:43:22.013016+0000","last_peered":"2023-10-09T04:43:22.013016+0000","last_clean":"2023-10-09T04:43:22.013016+0000","last_became_active":"2023-10-09T04:41:02.603407+0000","last_became_peered":"2023-10-09T04:41:02.603407+0000","last_unstale":"2023-10-09T04:43:22.013016+0000","last_undegraded":"2023-10-09T04:43:22.013016+0000","last_fullsized":"2023-10-09T04:43:22.013016+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:18.401503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012734+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:43:22.012734+0000","last_peered":"2023-10-09T04:43:22.012734+0000","last_clean":"2023-10-09T04:43:22.012734+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:43:22.012734+0000","last_undegraded":"2023-10-09T04:43:22.012734+0000","last_fullsized":"2023-10-09T04:43:22.012734+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012267+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:43:22.012267+0000","last_peered":"2023-10-09T04:43:22.012267+0000","last_clean":"2023-10-09T04:43:22.012267+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:43:22.012267+0000","last_undegraded":"2023-10-09T04:43:22.012267+0000","last_fullsized":"2023-10-09T04:43:22.012267+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.013861+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:43:22.013861+0000","last_peered":"2023-10-09T04:43:22.013861+0000","last_clean":"2023-10-09T04:43:22.013861+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:43:22.013861+0000","last_undegraded":"2023-10-09T04:43:22.013861+0000","last_fullsized":"2023-10-09T04:43:22.013861+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":238,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.550454+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:43:21.550454+0000","last_peered":"2023-10-09T04:43:21.550454+0000","last_clean":"2023-10-09T04:43:21.550454+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:43:21.550454+0000","last_undegraded":"2023-10-09T04:43:21.550454+0000","last_fullsized":"2023-10-09T04:43:21.550454+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":267,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551415+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:43:21.551415+0000","last_peered":"2023-10-09T04:43:21.551415+0000","last_clean":"2023-10-09T04:43:21.551415+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:43:21.551415+0000","last_undegraded":"2023-10-09T04:43:21.551415+0000","last_fullsized":"2023-10-09T04:43:21.551415+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":160,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.021149+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:43:22.021149+0000","last_peered":"2023-10-09T04:43:22.021149+0000","last_clean":"2023-10-09T04:43:22.021149+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:43:22.021149+0000","last_undegraded":"2023-10-09T04:43:22.021149+0000","last_fullsized":"2023-10-09T04:43:22.021149+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":277,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011885+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:43:22.011885+0000","last_peered":"2023-10-09T04:43:22.011885+0000","last_clean":"2023-10-09T04:43:22.011885+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:43:22.011885+0000","last_undegraded":"2023-10-09T04:43:22.011885+0000","last_fullsized":"2023-10-09T04:43:22.011885+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":254,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011641+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:43:22.011641+0000","last_peered":"2023-10-09T04:43:22.011641+0000","last_clean":"2023-10-09T04:43:22.011641+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:43:22.011641+0000","last_undegraded":"2023-10-09T04:43:22.011641+0000","last_fullsized":"2023-10-09T04:43:22.011641+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":247,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011570+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:43:22.011570+0000","last_peered":"2023-10-09T04:43:22.011570+0000","last_clean":"2023-10-09T04:43:22.011570+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:43:22.011570+0000","last_undegraded":"2023-10-09T04:43:22.011570+0000","last_fullsized":"2023-10-09T04:43:22.011570+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012553+0000","last_change":"2023-10-09T04:41:02.608773+0000","last_active":"2023-10-09T04:43:22.012553+0000","last_peered":"2023-10-09T04:43:22.012553+0000","last_clean":"2023-10-09T04:43:22.012553+0000","last_became_active":"2023-10-09T04:41:02.608455+0000","last_became_peered":"2023-10-09T04:41:02.608455+0000","last_unstale":"2023-10-09T04:43:22.012553+0000","last_undegraded":"2023-10-09T04:43:22.012553+0000","last_fullsized":"2023-10-09T04:43:22.012553+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:29.807000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":253,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.503573+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:43:21.503573+0000","last_peered":"2023-10-09T04:43:21.503573+0000","last_clean":"2023-10-09T04:43:21.503573+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:43:21.503573+0000","last_undegraded":"2023-10-09T04:43:21.503573+0000","last_fullsized":"2023-10-09T04:43:21.503573+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":240,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.505282+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:43:21.505282+0000","last_peered":"2023-10-09T04:43:21.505282+0000","last_clean":"2023-10-09T04:43:21.505282+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:43:21.505282+0000","last_undegraded":"2023-10-09T04:43:21.505282+0000","last_fullsized":"2023-10-09T04:43:21.505282+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":238,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.502266+0000","last_change":"2023-10-09T04:39:40.406420+0000","last_active":"2023-10-09T04:43:21.502266+0000","last_peered":"2023-10-09T04:43:21.502266+0000","last_clean":"2023-10-09T04:43:21.502266+0000","last_became_active":"2023-10-09T04:39:40.406072+0000","last_became_peered":"2023-10-09T04:39:40.406072+0000","last_unstale":"2023-10-09T04:43:21.502266+0000","last_undegraded":"2023-10-09T04:43:21.502266+0000","last_fullsized":"2023-10-09T04:43:21.502266+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:08.385113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":155,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011526+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:43:22.011526+0000","last_peered":"2023-10-09T04:43:22.011526+0000","last_clean":"2023-10-09T04:43:22.011526+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:43:22.011526+0000","last_undegraded":"2023-10-09T04:43:22.011526+0000","last_fullsized":"2023-10-09T04:43:22.011526+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":264,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011068+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:43:22.011068+0000","last_peered":"2023-10-09T04:43:22.011068+0000","last_clean":"2023-10-09T04:43:22.011068+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:43:22.011068+0000","last_undegraded":"2023-10-09T04:43:22.011068+0000","last_fullsized":"2023-10-09T04:43:22.011068+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":675,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.017001+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:43:22.017001+0000","last_peered":"2023-10-09T04:43:22.017001+0000","last_clean":"2023-10-09T04:43:22.017001+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:43:22.017001+0000","last_undegraded":"2023-10-09T04:43:22.017001+0000","last_fullsized":"2023-10-09T04:43:22.017001+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":209,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551215+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:43:21.551215+0000","last_peered":"2023-10-09T04:43:21.551215+0000","last_clean":"2023-10-09T04:43:21.551215+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:43:21.551215+0000","last_undegraded":"2023-10-09T04:43:21.551215+0000","last_fullsized":"2023-10-09T04:43:21.551215+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.015854+0000","last_change":"2023-10-09T04:41:02.612580+0000","last_active":"2023-10-09T04:43:22.015854+0000","last_peered":"2023-10-09T04:43:22.015854+0000","last_clean":"2023-10-09T04:43:22.015854+0000","last_became_active":"2023-10-09T04:41:02.612258+0000","last_became_peered":"2023-10-09T04:41:02.612258+0000","last_unstale":"2023-10-09T04:43:22.015854+0000","last_undegraded":"2023-10-09T04:43:22.015854+0000","last_fullsized":"2023-10-09T04:43:22.015854+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:53.656800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":283,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.016924+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:43:22.016924+0000","last_peered":"2023-10-09T04:43:22.016924+0000","last_clean":"2023-10-09T04:43:22.016924+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:43:22.016924+0000","last_undegraded":"2023-10-09T04:43:22.016924+0000","last_fullsized":"2023-10-09T04:43:22.016924+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012385+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:43:22.012385+0000","last_peered":"2023-10-09T04:43:22.012385+0000","last_clean":"2023-10-09T04:43:22.012385+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:43:22.012385+0000","last_undegraded":"2023-10-09T04:43:22.012385+0000","last_fullsized":"2023-10-09T04:43:22.012385+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":160,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.010165+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:43:22.010165+0000","last_peered":"2023-10-09T04:43:22.010165+0000","last_clean":"2023-10-09T04:43:22.010165+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:43:22.010165+0000","last_undegraded":"2023-10-09T04:43:22.010165+0000","last_fullsized":"2023-10-09T04:43:22.010165+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":286,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.019397+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:43:22.019397+0000","last_peered":"2023-10-09T04:43:22.019397+0000","last_clean":"2023-10-09T04:43:22.019397+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:43:22.019397+0000","last_undegraded":"2023-10-09T04:43:22.019397+0000","last_fullsized":"2023-10-09T04:43:22.019397+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":238,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.503493+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:43:21.503493+0000","last_peered":"2023-10-09T04:43:21.503493+0000","last_clean":"2023-10-09T04:43:21.503493+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:43:21.503493+0000","last_undegraded":"2023-10-09T04:43:21.503493+0000","last_fullsized":"2023-10-09T04:43:21.503493+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":127,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011135+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:43:22.011135+0000","last_peered":"2023-10-09T04:43:22.011135+0000","last_clean":"2023-10-09T04:43:22.011135+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:43:22.011135+0000","last_undegraded":"2023-10-09T04:43:22.011135+0000","last_fullsized":"2023-10-09T04:43:22.011135+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":281,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.551504+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:43:21.551504+0000","last_peered":"2023-10-09T04:43:21.551504+0000","last_clean":"2023-10-09T04:43:21.551504+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:43:21.551504+0000","last_undegraded":"2023-10-09T04:43:21.551504+0000","last_fullsized":"2023-10-09T04:43:21.551504+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":252,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012187+0000","last_change":"2023-10-09T04:41:02.629667+0000","last_active":"2023-10-09T04:43:22.012187+0000","last_peered":"2023-10-09T04:43:22.012187+0000","last_clean":"2023-10-09T04:43:22.012187+0000","last_became_active":"2023-10-09T04:41:02.629305+0000","last_became_peered":"2023-10-09T04:41:02.629305+0000","last_unstale":"2023-10-09T04:43:22.012187+0000","last_undegraded":"2023-10-09T04:43:22.012187+0000","last_fullsized":"2023-10-09T04:43:22.012187+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:36.600453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":270,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.550553+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:43:21.550553+0000","last_peered":"2023-10-09T04:43:21.550553+0000","last_clean":"2023-10-09T04:43:21.550553+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:43:21.550553+0000","last_undegraded":"2023-10-09T04:43:21.550553+0000","last_fullsized":"2023-10-09T04:43:21.550553+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-09T04:43:33.532834+0000","last_change":"2023-10-09T04:43:33.532834+0000","last_active":"2023-10-09T04:43:33.532834+0000","last_peered":"2023-10-09T04:43:33.532834+0000","last_clean":"2023-10-09T04:43:33.532834+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-09T04:43:33.532834+0000","last_undegraded":"2023-10-09T04:43:33.532834+0000","last_fullsized":"2023-10-09T04:43:33.532834+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":261,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.013113+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:43:22.013113+0000","last_peered":"2023-10-09T04:43:22.013113+0000","last_clean":"2023-10-09T04:43:22.013113+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:43:22.013113+0000","last_undegraded":"2023-10-09T04:43:22.013113+0000","last_fullsized":"2023-10-09T04:43:22.013113+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":55,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.014536+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:43:22.014536+0000","last_peered":"2023-10-09T04:43:22.014536+0000","last_clean":"2023-10-09T04:43:22.014536+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:43:22.014536+0000","last_undegraded":"2023-10-09T04:43:22.014536+0000","last_fullsized":"2023-10-09T04:43:22.014536+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012194+0000","last_change":"2023-10-09T04:41:02.614004+0000","last_active":"2023-10-09T04:43:22.012194+0000","last_peered":"2023-10-09T04:43:22.012194+0000","last_clean":"2023-10-09T04:43:22.012194+0000","last_became_active":"2023-10-09T04:41:02.612300+0000","last_became_peered":"2023-10-09T04:41:02.612300+0000","last_unstale":"2023-10-09T04:43:22.012194+0000","last_undegraded":"2023-10-09T04:43:22.012194+0000","last_fullsized":"2023-10-09T04:43:22.012194+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:04.375059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":123,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012787+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:43:22.012787+0000","last_peered":"2023-10-09T04:43:22.012787+0000","last_clean":"2023-10-09T04:43:22.012787+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:43:22.012787+0000","last_undegraded":"2023-10-09T04:43:22.012787+0000","last_fullsized":"2023-10-09T04:43:22.012787+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":232,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.503649+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:43:21.503649+0000","last_peered":"2023-10-09T04:43:21.503649+0000","last_clean":"2023-10-09T04:43:21.503649+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:43:21.503649+0000","last_undegraded":"2023-10-09T04:43:21.503649+0000","last_fullsized":"2023-10-09T04:43:21.503649+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":273,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011157+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:43:22.011157+0000","last_peered":"2023-10-09T04:43:22.011157+0000","last_clean":"2023-10-09T04:43:22.011157+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:43:22.011157+0000","last_undegraded":"2023-10-09T04:43:22.011157+0000","last_fullsized":"2023-10-09T04:43:22.011157+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012646+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:43:22.012646+0000","last_peered":"2023-10-09T04:43:22.012646+0000","last_clean":"2023-10-09T04:43:22.012646+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:43:22.012646+0000","last_undegraded":"2023-10-09T04:43:22.012646+0000","last_fullsized":"2023-10-09T04:43:22.012646+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":228,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011926+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:43:22.011926+0000","last_peered":"2023-10-09T04:43:22.011926+0000","last_clean":"2023-10-09T04:43:22.011926+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:43:22.011926+0000","last_undegraded":"2023-10-09T04:43:22.011926+0000","last_fullsized":"2023-10-09T04:43:22.011926+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012396+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:43:22.012396+0000","last_peered":"2023-10-09T04:43:22.012396+0000","last_clean":"2023-10-09T04:43:22.012396+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:43:22.012396+0000","last_undegraded":"2023-10-09T04:43:22.012396+0000","last_fullsized":"2023-10-09T04:43:22.012396+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":271,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.552332+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:43:21.552332+0000","last_peered":"2023-10-09T04:43:21.552332+0000","last_clean":"2023-10-09T04:43:21.552332+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:43:21.552332+0000","last_undegraded":"2023-10-09T04:43:21.552332+0000","last_fullsized":"2023-10-09T04:43:21.552332+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":303,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:21.553268+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:43:21.553268+0000","last_peered":"2023-10-09T04:43:21.553268+0000","last_clean":"2023-10-09T04:43:21.553268+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:43:21.553268+0000","last_undegraded":"2023-10-09T04:43:21.553268+0000","last_fullsized":"2023-10-09T04:43:21.553268+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":157,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.011677+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:43:22.011677+0000","last_peered":"2023-10-09T04:43:22.011677+0000","last_clean":"2023-10-09T04:43:22.011677+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:43:22.011677+0000","last_undegraded":"2023-10-09T04:43:22.011677+0000","last_fullsized":"2023-10-09T04:43:22.011677+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-10-09T04:43:22.012082+0000","last_change":"2023-10-09T04:41:02.608725+0000","last_active":"2023-10-09T04:43:22.012082+0000","last_peered":"2023-10-09T04:43:22.012082+0000","last_clean":"2023-10-09T04:43:22.012082+0000","last_became_active":"2023-10-09T04:41:02.606509+0000","last_became_peered":"2023-10-09T04:41:02.606509+0000","last_unstale":"2023-10-09T04:43:22.012082+0000","last_undegraded":"2023-10-09T04:43:22.012082+0000","last_fullsized":"2023-10-09T04:43:22.012082+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:49.430101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":5624,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234439,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":927500,"kb_used_data":24708,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":92817652,"statfs":{"total":95995035648,"available":95045275648,"internally_reserved":0,"allocated":25300992,"data_stored":24707179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:36 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.0390000000000001,"15min":4.0579999999999998},"min":{"1min":0.247,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":11.144,"5min":44.122999999999998,"15min":186.446},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.5510000000000002,"5min":2.7690000000000001,"15min":4.407},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":10.42,"5min":43.313000000000002,"15min":186.429},"last":0.58699999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:42:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":6.3140000000000001,"15min":5.3949999999999996},"min":{"1min":0.253,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":11.256,"5min":264.62,"15min":264.62},"last":0.433},{"interface":"front","average":{"1min":3.2759999999999998,"5min":6.5979999999999999,"15min":5.5439999999999996},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":14.73,"5min":264.65899999999999,"15min":264.65899999999999},"last":0.53800000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":7.5010000000000003,"15min":3.7010000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.9949999999999992,"5min":184.72,"15min":184.72},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.4769999999999999,"5min":7.8920000000000003,"15min":4.1660000000000004},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.438000000000001,"5min":184.697,"15min":184.697},"last":0.45000000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":5.3259999999999996,"15min":7.7530000000000001},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.3819999999999997,"5min":182.839,"15min":249.989},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.2330000000000001,"5min":5.8860000000000001,"15min":8.0700000000000003},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":10.278,"5min":182.678,"15min":250.21700000000001},"last":0.48599999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.2789999999999999,"15min":1.389},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.388,"5min":14.257999999999999,"15min":14.257999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.86199999999999999,"15min":1.131},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.21199999999999999},"max":{"1min":4.923,"5min":7.2290000000000001,"15min":10.613},"last":0.41699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.2290000000000001,"15min":1.1040000000000001},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.27900000000000003},"max":{"1min":4.8099999999999996,"5min":13.319000000000001,"15min":13.319000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.3799999999999999,"5min":1.728,"15min":1.5449999999999999},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.779999999999999,"5min":18.408000000000001,"15min":18.408000000000001},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.1439999999999999,"15min":1.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":11.388,"15min":11.388},"last":0.504},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.643,"15min":1.6299999999999999},"min":{"1min":0.373,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.532999999999999,"5min":18.311,"15min":18.311},"last":0.55400000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496073,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905404,"kb_used_data":3124,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":92839748,"statfs":{"total":95995035648,"available":95067901952,"internally_reserved":0,"allocated":3198976,"data_stored":2621078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.274,"15min":8.1029999999999998},"min":{"1min":0.24199999999999999,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":8.1760000000000002,"5min":75.308000000000007,"15min":1284.521},"last":0.443},{"interface":"front","average":{"1min":2.3929999999999998,"5min":2.7000000000000002,"15min":8.2710000000000008},"min":{"1min":0.29799999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4820000000000002,"5min":75.432000000000002,"15min":1284.4690000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:42:30 2023","interfaces":[{"interface":"back","average":{"1min":6.0679999999999996,"5min":7.9489999999999998,"15min":5.6050000000000004},"min":{"1min":0.32200000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":61.042000000000002,"5min":150.75,"15min":185.74000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":6.0359999999999996,"5min":8.1059999999999999,"15min":5.7190000000000003},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.26100000000000001},"max":{"1min":61.055999999999997,"5min":150.81999999999999,"15min":189.03999999999999},"last":0.57799999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:42:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":8.7400000000000002,"15min":4.3120000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.30199999999999999},"max":{"1min":11.225,"5min":209.44399999999999,"15min":209.44399999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.825,"5min":8.7029999999999994,"15min":4.2880000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":11.239000000000001,"5min":209.428,"15min":209.428},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:42:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8760000000000003,"5min":7.992,"15min":6.1719999999999997},"min":{"1min":0.32400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":34.414000000000001,"5min":157.93100000000001,"15min":167.56100000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":5.2569999999999997,"5min":8.0220000000000002,"15min":6.2750000000000004},"min":{"1min":0.39800000000000002,"5min":0.33000000000000002,"15min":0.313},"max":{"1min":34.167000000000002,"5min":157.95099999999999,"15min":167.577},"last":0.71399999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":0.996,"15min":1.3169999999999999},"min":{"1min":0.33300000000000002,"5min":0.27200000000000002,"15min":0.25600000000000001},"max":{"1min":6.5800000000000001,"5min":6.8380000000000001,"15min":14.430999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.046,"15min":1.109},"min":{"1min":0.33600000000000002,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":9.2240000000000002,"5min":9.2240000000000002,"15min":9.2240000000000002},"last":0.73999999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:42:35 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.0800000000000001,"15min":1.8029999999999999},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":6.6310000000000002,"5min":7.3010000000000002,"15min":81.087000000000003},"last":0.75900000000000001},{"interface":"front","average":{"1min":2.4510000000000001,"5min":1.1639999999999999,"15min":1.746},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":8.2739999999999991,"5min":8.6479999999999997,"15min":81.706999999999994},"last":0.77700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:43:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.238,"15min":1.5600000000000001},"min":{"1min":0.378,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":4.6029999999999998,"5min":16.512,"15min":81.674999999999997},"last":0.80100000000000005},{"interface":"front","average":{"1min":1.385,"5min":1.389,"15min":1.762},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":7.5289999999999999,"5min":17.715,"15min":81.055000000000007},"last":0.82999999999999996}]}]},{"osd":1,"up_from":118,"seq":506806141163,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":631484,"kb_used_data":3188,"kb_used_omap":4,"kb_used_meta":628283,"kb_avail":93113668,"statfs":{"total":95995035648,"available":95348396032,"internally_reserved":0,"allocated":3264512,"data_stored":2099645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":643362702},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":3.8260000000000001,"15min":3.9990000000000001},"min":{"1min":0.23200000000000001,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":13.573,"5min":75.620000000000005,"15min":97.308000000000007},"last":0.873},{"interface":"front","average":{"1min":4.3570000000000002,"5min":3.7690000000000001,"15min":4.218},"min":{"1min":0.20499999999999999,"5min":0.193,"15min":0.193},"max":{"1min":13.366,"5min":69.701999999999998,"15min":97.055999999999997},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:42:52 2023","interfaces":[{"interface":"back","average":{"1min":12.409000000000001,"5min":6.1970000000000001,"15min":3.823},"min":{"1min":0.253,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":263.60500000000002,"5min":263.60500000000002,"15min":263.60500000000002},"last":0.41499999999999998},{"interface":"front","average":{"1min":13.936,"5min":7.0970000000000004,"15min":4.3310000000000004},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":263.65199999999999,"5min":263.65199999999999,"15min":263.65199999999999},"last":0.434}]},{"osd":3,"last update":"Mon Oct 9 04:43:27 2023","interfaces":[{"interface":"back","average":{"1min":15.497999999999999,"5min":10.351000000000001,"15min":6.4470000000000001},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":379.80900000000003,"5min":379.80900000000003,"15min":379.80900000000003},"last":2.2389999999999999},{"interface":"front","average":{"1min":15.800000000000001,"5min":11.353999999999999,"15min":7.0990000000000002},"min":{"1min":0.29399999999999998,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":379.95800000000003,"5min":379.95800000000003,"15min":379.95800000000003},"last":0.36499999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":2.1480000000000001,"15min":2.4039999999999999},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.245},"max":{"1min":8.0489999999999995,"5min":14.449,"15min":170.072},"last":0.995},{"interface":"front","average":{"1min":2.6499999999999999,"5min":2.738,"15min":2.21},"min":{"1min":0.308,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":12.297000000000001,"5min":75.513999999999996,"15min":169.94499999999999},"last":0.90200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":3.54,"5min":2.5,"15min":2.4340000000000002},"min":{"1min":0.40000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.8940000000000001,"5min":11.499000000000001,"15min":171.036},"last":0.77600000000000002},{"interface":"front","average":{"1min":2.6560000000000001,"5min":2.3889999999999998,"15min":2.5960000000000001},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":13.031000000000001,"5min":14.557,"15min":170.88},"last":0.94999999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":9.4359999999999999,"5min":3.6659999999999999,"15min":2.4159999999999999},"min":{"1min":0.46600000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":132.36099999999999,"5min":132.36099999999999,"15min":132.36099999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":8.0470000000000006,"5min":3.4950000000000001,"15min":2.593},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":132.17699999999999,"5min":132.17699999999999,"15min":132.17699999999999},"last":0.92400000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:42:45 2023","interfaces":[{"interface":"back","average":{"1min":8.5210000000000008,"5min":3.8079999999999998,"15min":3.149},"min":{"1min":0.42599999999999999,"5min":0.36599999999999999,"15min":0.34100000000000003},"max":{"1min":132.136,"5min":132.136,"15min":174.75999999999999},"last":1.028},{"interface":"front","average":{"1min":7.8659999999999997,"5min":4.133,"15min":2.8530000000000002},"min":{"1min":0.44700000000000001,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":132.12,"5min":132.12,"15min":174.779},"last":0.83799999999999997}]}]},{"osd":0,"up_from":91,"seq":390842024220,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":799940,"kb_used_data":3644,"kb_used_omap":5,"kb_used_meta":796282,"kb_avail":92945212,"statfs":{"total":95995035648,"available":95175897088,"internally_reserved":0,"allocated":3731456,"data_stored":2675794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":815393402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":5.7599999999999998,"15min":4.9790000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":20.994,"5min":138.536,"15min":167.92699999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.585,"5min":6.5449999999999999,"15min":5.367},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":8.1010000000000009,"5min":138.89599999999999,"15min":169.78999999999999},"last":0.443}]},{"osd":2,"last update":"Mon Oct 9 04:42:55 2023","interfaces":[{"interface":"back","average":{"1min":7.0999999999999996,"5min":5.6340000000000003,"15min":8.1769999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":98.938000000000002,"5min":122.983,"15min":131.30699999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":7.649,"5min":5.9930000000000003,"15min":8.5879999999999992},"min":{"1min":0.26400000000000001,"5min":0.219,"15min":0.219},"max":{"1min":98.975999999999999,"5min":122.715,"15min":129.922},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:43:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":6.4050000000000002,"15min":5.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":6.0220000000000002,"5min":156.69800000000001,"15min":213.28399999999999},"last":0.435},{"interface":"front","average":{"1min":1.9470000000000001,"5min":7.7539999999999996,"15min":5.6740000000000004},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.222},"max":{"1min":13.529,"5min":161.40199999999999,"15min":212.27000000000001},"last":0.42299999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.4930000000000001,"15min":2.2970000000000002},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.25700000000000001},"max":{"1min":6.0979999999999999,"5min":11.300000000000001,"15min":161.67099999999999},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.847,"15min":2.5710000000000002},"min":{"1min":0.35499999999999998,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":5.6890000000000001,"5min":14.109999999999999,"15min":160.74799999999999},"last":0.71399999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":3.222,"15min":2.399},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27900000000000003},"max":{"1min":5.657,"5min":136.322,"15min":136.322},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.2080000000000002,"5min":3.1469999999999998,"15min":2.181},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.718,"5min":136.15799999999999,"15min":136.15799999999999},"last":0.68400000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":6.0629999999999997,"5min":3.113,"15min":2.7210000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":78.438999999999993,"5min":78.438999999999993,"15min":108.63800000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":5.6829999999999998,"5min":2.7090000000000001,"15min":2.3170000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":78.617999999999995,"5min":78.617999999999995,"15min":108.76300000000001},"last":0.755}]},{"osd":7,"last update":"Mon Oct 9 04:42:48 2023","interfaces":[{"interface":"back","average":{"1min":6.0199999999999996,"5min":2.7679999999999998,"15min":2.004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":77.682000000000002,"5min":77.682000000000002,"15min":77.682000000000002},"last":0.90100000000000002},{"interface":"front","average":{"1min":4.9429999999999996,"5min":2.516,"15min":2.0190000000000001},"min":{"1min":0.373,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":77.606999999999999,"5min":77.606999999999999,"15min":77.606999999999999},"last":0.89000000000000001}]}]},{"osd":2,"up_from":221,"seq":949187772498,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285264,"kb_used_data":3016,"kb_used_omap":7,"kb_used_meta":282232,"kb_avail":93459888,"statfs":{"total":95995035648,"available":95702925312,"internally_reserved":0,"allocated":3088384,"data_stored":2081866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":289005649},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8849999999999998,"15min":7.9169999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.6999999999999993,"5min":13.813000000000001,"15min":72.608000000000004},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.585,"5min":2.3849999999999998,"15min":7.4160000000000004},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.8170000000000002,"5min":14.377000000000001,"15min":72.632000000000005},"last":0.308}]},{"osd":1,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":11.619,"5min":5.1859999999999999,"15min":5.2720000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.848,"5min":121.848,"15min":121.848},"last":0.23000000000000001},{"interface":"front","average":{"1min":10.805,"5min":5.5819999999999999,"15min":5.7610000000000001},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":121.827,"5min":121.827,"15min":121.827},"last":0.25}]},{"osd":3,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":15.984,"5min":9.5199999999999996,"15min":6.7770000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":225.85499999999999,"5min":335.928,"15min":335.928},"last":0.32400000000000001},{"interface":"front","average":{"1min":16.954000000000001,"5min":10.244,"15min":6.9260000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":233.53299999999999,"5min":335.68599999999998,"15min":335.68599999999998},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":2.48,"15min":4.1660000000000004},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.315},"max":{"1min":8.8360000000000003,"5min":14.452999999999999,"15min":72.944999999999993},"last":0.495},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.4359999999999999,"15min":4.3239999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.6359999999999992,"5min":14.183,"15min":73.054000000000002},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.71,"15min":3.988},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.2649999999999997,"5min":65.388000000000005,"15min":72.986000000000004},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":3.0659999999999998,"15min":4.4779999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.6440000000000001,"5min":67.799000000000007,"15min":72.963999999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.7770000000000001,"15min":9.3030000000000008},"min":{"1min":0.39600000000000002,"5min":0.39000000000000001,"15min":0.33600000000000002},"max":{"1min":10.263,"5min":34.753,"15min":135.614},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.4830000000000001,"5min":2.5910000000000002,"15min":8.6310000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.35099999999999998},"max":{"1min":7.423,"5min":34.796999999999997,"15min":130.196},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3149999999999999,"5min":2.77,"15min":4.5759999999999996},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.378},"max":{"1min":10.228,"5min":28.126999999999999,"15min":73.079999999999998},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.536,"15min":3.9580000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.41499999999999998},"max":{"1min":11.669,"5min":28.106000000000002,"15min":73.069000000000003},"last":0.626}]}]},{"osd":3,"up_from":142,"seq":609885356221,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":456452,"kb_used_data":3004,"kb_used_omap":2,"kb_used_meta":453437,"kb_avail":93288700,"statfs":{"total":95995035648,"available":95527628800,"internally_reserved":0,"allocated":3076096,"data_stored":2077631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":464319541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":1.9399999999999999,"15min":2.4820000000000002},"min":{"1min":0.23300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.7690000000000001,"5min":17.648,"15min":142.14500000000001},"last":0.311},{"interface":"front","average":{"1min":2.1539999999999999,"5min":5.2519999999999998,"15min":3.649},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":10.725,"5min":254.38499999999999,"15min":254.38499999999999},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":2.7909999999999999,"15min":3.54},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.192},"max":{"1min":9.8450000000000006,"5min":21.879000000000001,"15min":180.02500000000001},"last":0.434},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3069999999999999,"15min":3.6389999999999998},"min":{"1min":0.251,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":9.7319999999999993,"5min":21.911999999999999,"15min":184.00899999999999},"last":0.379}]},{"osd":2,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":6.04,"5min":6.173,"15min":3.1749999999999998},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":77.772000000000006,"5min":177.28399999999999,"15min":177.28399999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":7.5359999999999996,"5min":8.5960000000000001,"15min":3.7050000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":77.968999999999994,"5min":255.05199999999999,"15min":255.05199999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.6559999999999997,"15min":2.9220000000000002},"min":{"1min":0.316,"5min":0.28999999999999998,"15min":0.27900000000000003},"max":{"1min":9.5109999999999992,"5min":254.072,"15min":254.072},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.9550000000000001,"5min":4.3970000000000002,"15min":2.75},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.307},"max":{"1min":8.2360000000000007,"5min":253.49700000000001,"15min":253.49700000000001},"last":0.63300000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":5.2439999999999998,"15min":3.4969999999999999},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.8719999999999999,"5min":253.57900000000001,"15min":253.57900000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.5449999999999999,"5min":5.3040000000000003,"15min":3.573},"min":{"1min":0.40100000000000002,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":8.1039999999999992,"5min":254.191,"15min":254.191},"last":0.57599999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":5.532,"15min":3.4039999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.31900000000000001},"max":{"1min":8.1419999999999995,"5min":254.14699999999999,"15min":254.14699999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.819,"5min":2.786,"15min":2.8490000000000002},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":9.5380000000000003,"5min":50.308,"15min":123.505},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":2.653,"15min":2.456},"min":{"1min":0.41199999999999998,"5min":0.378,"15min":0.32300000000000001},"max":{"1min":9.4789999999999992,"5min":43.835000000000001,"15min":57.399000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.627,"5min":2.222,"15min":2.254},"min":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":7.907,"5min":43.781999999999996,"15min":57.447000000000003},"last":0.56599999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019347,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902788,"kb_used_data":2556,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":92842364,"statfs":{"total":95995035648,"available":95070580736,"internally_reserved":0,"allocated":2617344,"data_stored":2031109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":2.8010000000000002,"15min":5.3499999999999996},"min":{"1min":0.26600000000000001,"5min":0.25600000000000001,"15min":0.23000000000000001},"max":{"1min":11.221,"5min":73.471000000000004,"15min":219.577},"last":0.504},{"interface":"front","average":{"1min":0.93600000000000005,"5min":2.5979999999999999,"15min":5.0519999999999996},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.255},"max":{"1min":5.3630000000000004,"5min":73.364999999999995,"15min":218.96000000000001},"last":0.65300000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:42:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":3.0350000000000001,"15min":5.0640000000000001},"min":{"1min":0.26400000000000001,"5min":0.255,"15min":0.24099999999999999},"max":{"1min":7.8579999999999997,"5min":56.122,"15min":142.74799999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":3.6240000000000001,"5min":3.1619999999999999,"15min":5.218},"min":{"1min":0.29599999999999999,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":13.840999999999999,"5min":56.043999999999997,"15min":139.09700000000001},"last":0.57599999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2189999999999999,"5min":5.6829999999999998,"15min":4.3239999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.372,"5min":195.267,"15min":195.267},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.2250000000000001,"5min":5.4329999999999998,"15min":3.8279999999999998},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.3620000000000001,"5min":195.15799999999999,"15min":195.15799999999999},"last":0.70699999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":7.6150000000000002,"5min":8.3160000000000007,"15min":7.4859999999999998},"min":{"1min":0.38400000000000001,"5min":0.32600000000000001,"15min":0.27900000000000003},"max":{"1min":146.02199999999999,"5min":357.125,"15min":357.125},"last":0.60299999999999998},{"interface":"front","average":{"1min":7.9699999999999998,"5min":8.1500000000000004,"15min":7.6120000000000001},"min":{"1min":0.38500000000000001,"5min":0.314,"15min":0.312},"max":{"1min":147.75700000000001,"5min":359.02800000000002,"15min":359.02800000000002},"last":0.67300000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.95499999999999996,"15min":1.8580000000000001},"min":{"1min":0.312,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":5.8540000000000001,"5min":9.5609999999999999,"15min":107.476},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0680000000000001,"15min":2.0779999999999998},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.23300000000000001},"max":{"1min":1.6259999999999999,"5min":15.394,"15min":107.414},"last":0.44600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:42:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.145,"15min":1.875},"min":{"1min":0.34599999999999997,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":6.2300000000000004,"5min":9.6099999999999994,"15min":65.75},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.113,"5min":1.147,"15min":1.8620000000000001},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.782,"5min":9.782,"15min":65.849999999999994},"last":0.68700000000000006}]},{"osd":7,"last update":"Mon Oct 9 04:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.0680000000000001,"15min":1.6180000000000001},"min":{"1min":0.37,"5min":0.34399999999999997,"15min":0.28000000000000003},"max":{"1min":1.8320000000000001,"5min":8.6120000000000001,"15min":76.283000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.3069999999999999,"15min":1.6599999999999999},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.32100000000000001},"max":{"1min":4.2469999999999999,"5min":9.6310000000000002,"15min":76.259},"last":0.53900000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757711,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905140,"kb_used_data":2540,"kb_used_omap":0,"kb_used_meta":902592,"kb_avail":92840012,"statfs":{"total":95995035648,"available":95068172288,"internally_reserved":0,"allocated":2600960,"data_stored":2026853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924254208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:38 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":3.5630000000000002,"15min":5.2800000000000002},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.22},"max":{"1min":75.855999999999995,"5min":76.587999999999994,"15min":210.85499999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":5.8559999999999999,"5min":4.1299999999999999,"15min":5.3879999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":75.816000000000003,"5min":76.682000000000002,"15min":210.23500000000001},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:42:38 2023","interfaces":[{"interface":"back","average":{"1min":5.2569999999999997,"5min":4.8369999999999997,"15min":4.5529999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":73.296000000000006,"5min":145.577,"15min":145.577},"last":0.54000000000000004},{"interface":"front","average":{"1min":6.048,"5min":4.9340000000000002,"15min":5.4160000000000004},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":73.045000000000002,"5min":145.792,"15min":145.792},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:42:56 2023","interfaces":[{"interface":"back","average":{"1min":6.2050000000000001,"5min":3.2629999999999999,"15min":5.9580000000000002},"min":{"1min":0.35899999999999999,"5min":0.28199999999999997,"15min":0.26800000000000002},"max":{"1min":66.984999999999999,"5min":66.984999999999999,"15min":90.763999999999996},"last":0.36399999999999999},{"interface":"front","average":{"1min":5.2599999999999998,"5min":3.419,"15min":6.1100000000000003},"min":{"1min":0.36599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":67.686000000000007,"5min":67.686000000000007,"15min":91.340000000000003},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:43:19 2023","interfaces":[{"interface":"back","average":{"1min":5.1710000000000003,"5min":8.5860000000000003,"15min":7.0789999999999997},"min":{"1min":0.31,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":67.129999999999995,"5min":268.63299999999998,"15min":268.63299999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":5.4299999999999997,"5min":8.3550000000000004,"15min":6.8810000000000002},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":67.189999999999998,"5min":268.92500000000001,"15min":268.92500000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:43:19 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.64100000000000001,"15min":2.3370000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.248},"max":{"1min":0.70799999999999996,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.206,"15min":2.7799999999999998},"min":{"1min":0.26900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":3.1819999999999999,"5min":7.4859999999999998,"15min":361.71300000000002},"last":0.56799999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:42:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":1.1839999999999999},"min":{"1min":0.312,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.1669999999999998,"5min":5.7640000000000002,"15min":11.192},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.103,"5min":1.2130000000000001,"15min":1.333},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.5800000000000001,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.627}]},{"osd":7,"last update":"Mon Oct 9 04:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.2769999999999999,"15min":1.3400000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":1.78,"5min":13.268000000000001,"15min":13.268000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.998,"15min":1.2410000000000001},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.734,"5min":6.5129999999999999,"15min":15.257999999999999},"last":0.66900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:43:35.229 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:43:35.285 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:43:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:35.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:35.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:35.392 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:43:35.392 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:43:35.422 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:35.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T04:43:35.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: failed to encode map e285 with expected crc 2023-10-09T04:43:35.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:35.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:35.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:35.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: osdmap e285: 8 total, 8 up, 8 in 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: failed to encode map e285 with expected crc 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: failed to encode map e285 with expected crc 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: failed to encode map e285 with expected crc 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: failed to encode map e285 with expected crc 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:43:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:36.403 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:36.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:36.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:36.538 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:36.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:36.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:36.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[216903]: pgmap v1094: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[216903]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T04:43:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[216903]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[216903]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[216903]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[216903]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[223858]: pgmap v1094: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[223858]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T04:43:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[223858]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[223858]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[223858]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:36 smithi029 ceph-mon[223858]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.692 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:36.727 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:36.728 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:36.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:43:36.802 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:36.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:36.882 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:36.911 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:36.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:36.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:36 smithi177 ceph-mon[195797]: pgmap v1094: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:36.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:36 smithi177 ceph-mon[195797]: osdmap e286: 8 total, 8 up, 8 in 2023-10-09T04:43:36.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:36 smithi177 ceph-mon[195797]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:36 smithi177 ceph-mon[195797]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:36 smithi177 ceph-mon[195797]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:36 smithi177 ceph-mon[195797]: failed to encode map e286 with expected crc 2023-10-09T04:43:36.944 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:36.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:37.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:37.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:37.129 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:37.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:37.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:37.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:37.279 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:37.305 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:37.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:37.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:37.450 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:37.476 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:37.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:37.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:37.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:37.648 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:37.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:37.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:37.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:37.822 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:37.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:37.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:37.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:37.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:37.997 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:38.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:38.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:38.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:38.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.174 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:38.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:38.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:38.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:38.345 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:38.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:38.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:38.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:38.495 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.521 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:38.558 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:43:38.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:38.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:38.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:38 smithi029 ceph-mon[216903]: pgmap v1096: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:38.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3144241016' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:38.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:38 smithi029 ceph-mon[223858]: pgmap v1096: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:38.580 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3144241016' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:38.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:38 smithi177 ceph-mon[195797]: pgmap v1096: 89 pgs: 3 unknown, 86 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:38.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3144241016' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:43:38.694 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:38.735 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:38.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:38.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:38.873 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:38.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:38.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:39.020 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:39.053 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:39.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[216903]: failed to encode map e287 with expected crc 2023-10-09T04:43:39.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3144241016' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:43:39.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[216903]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T04:43:39.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[216903]: failed to encode map e287 with expected crc 2023-10-09T04:43:39.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[216903]: failed to encode map e287 with expected crc 2023-10-09T04:43:39.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:39.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[223858]: failed to encode map e287 with expected crc 2023-10-09T04:43:39.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3144241016' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:43:39.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[223858]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T04:43:39.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[223858]: failed to encode map e287 with expected crc 2023-10-09T04:43:39.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:39 smithi029 ceph-mon[223858]: failed to encode map e287 with expected crc 2023-10-09T04:43:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:39 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:43:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:39 smithi177 ceph-mon[195797]: failed to encode map e287 with expected crc 2023-10-09T04:43:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3144241016' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:43:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:39 smithi177 ceph-mon[195797]: osdmap e287: 8 total, 8 up, 8 in 2023-10-09T04:43:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:39 smithi177 ceph-mon[195797]: failed to encode map e287 with expected crc 2023-10-09T04:43:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:39 smithi177 ceph-mon[195797]: failed to encode map e287 with expected crc 2023-10-09T04:43:40.218 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:40.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:40.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:40.355 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:40 smithi029 ceph-mon[216903]: pgmap v1098: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:40 smithi029 ceph-mon[216903]: failed to encode map e287 with expected crc 2023-10-09T04:43:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:40 smithi029 ceph-mon[223858]: pgmap v1098: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:40 smithi029 ceph-mon[223858]: failed to encode map e287 with expected crc 2023-10-09T04:43:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:40 smithi177 ceph-mon[195797]: pgmap v1098: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:40 smithi177 ceph-mon[195797]: failed to encode map e287 with expected crc 2023-10-09T04:43:41.129 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:41.162 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:41.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:41.222 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:43:41.242 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:43:41.262 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:43:41.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:43:41.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:43:41.298 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:41.307 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:43:41.330 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:43:41.349 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:43:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:41.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:43:41.354 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:43:41.375 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.377 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:43:41.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:41.487 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:41.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:41.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:41.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:43:41.669 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:41.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:43:41.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:41.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:43:41.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1026766462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:41.821 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:41.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1026766462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:41.849 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1026766462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:43:41.851 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:42.573 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-10-09T04:43:42.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:43:42.604 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:42.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:42.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:42.739 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:42.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:42.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:42.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:42 smithi029 ceph-mon[216903]: pgmap v1099: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3231680769' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:43:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:42 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:43:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:42 smithi029 ceph-mon[223858]: pgmap v1099: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3231680769' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:43:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:42 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:43:42.894 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:42.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:42 smithi177 ceph-mon[195797]: pgmap v1099: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:42.920 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3231680769' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:43:42.921 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:42 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:43:42.922 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:42.965 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:42.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:43.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:43:43.099 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:43.122 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:43:43.123 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1559,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":288,"num_osds":8,"num_up_osds":8,"osd_up_since":1696826206,"num_in_osds":8,"osd_in_since":1696825790,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":24,"data_bytes":595872,"bytes_used":5938425856,"bytes_avail":762021859328,"bytes_total":767960285184},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":26,"modified":"2023-10-09T04:37:47.018583+0000","services":{}},"progress_events":{"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T04:43:43.141 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:43:43.145 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:43:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:43:42] "GET /metrics HTTP/1.1" 200 34125 "" "Prometheus/2.43.0" 2023-10-09T04:43:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:43.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:43:43.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:43.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:43.718 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:43:43.718 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T04:43:43.718 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.718 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.718 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.718 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3622263745' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: failed to encode map e288 with expected crc 2023-10-09T04:43:43.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3622263745' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:43:43.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:43.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: osdmap e288: 8 total, 8 up, 8 in 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: failed to encode map e288 with expected crc 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: failed to encode map e288 with expected crc 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: failed to encode map e288 with expected crc 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: failed to encode map e288 with expected crc 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: failed to encode map e288 with expected crc 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3622263745' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:44.405 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T04:43:44.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:43:44.408 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:44.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:44.515 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:44.543 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:44.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:43:44.690 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:44.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:44.794 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:44.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:45.032 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-10-09T04:43:45.034 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:43:45.034 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:43:45.105 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:43:45.105 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:43:45.105 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_442458 2023-10-09T04:43:45.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:43:45.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:43:45.174 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:45.251 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:45.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:45.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:45.363 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:45.415 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:45 smithi029 ceph-mon[223858]: pgmap v1101: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:45 smithi029 ceph-mon[216903]: pgmap v1101: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:45.429 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:45.461 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:43:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:45 smithi177 ceph-mon[195797]: pgmap v1101: 89 pgs: 89 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail 2023-10-09T04:43:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:43:45.628 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:45.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:45.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:43:45.773 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:45.850 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:43:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:45.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:43:45.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:46.024 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:46.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 2862 2846 177.866 177.875 0.00584466 0.00559909 2023-10-09T04:43:47.035 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:43:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:43:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:43:47.036 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:43:47.036 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:43:47.075 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:47 smithi029 ceph-mon[216903]: pgmap v1102: 89 pgs: 89 active+clean; 4.4 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 488 KiB/s wr, 7 op/s 2023-10-09T04:43:47.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6304 6288 196.485 215.125 0.00351761 0.00507767 2023-10-09T04:43:47.307 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:47.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:47.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:47 smithi029 ceph-mon[223858]: pgmap v1102: 89 pgs: 89 active+clean; 4.4 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 488 KiB/s wr, 7 op/s 2023-10-09T04:43:47.415 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:47.479 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:47.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:47 smithi177 ceph-mon[195797]: pgmap v1102: 89 pgs: 89 active+clean; 4.4 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 488 KiB/s wr, 7 op/s 2023-10-09T04:43:48.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 8966 8950 186.443 166.375 0.00641694 0.0053548 2023-10-09T04:43:49.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 12079 12063 188.469 194.562 0.0031118 0.0052987 2023-10-09T04:43:49.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:49 smithi029 ceph-mon[216903]: pgmap v1103: 89 pgs: 89 active+clean; 59 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 6.7 MiB/s wr, 107 op/s 2023-10-09T04:43:49.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:49 smithi029 ceph-mon[223858]: pgmap v1103: 89 pgs: 89 active+clean; 59 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 6.7 MiB/s wr, 107 op/s 2023-10-09T04:43:49.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:49 smithi177 ceph-mon[195797]: pgmap v1103: 89 pgs: 89 active+clean; 59 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 6.7 MiB/s wr, 107 op/s 2023-10-09T04:43:49.578 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:49.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:49.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:49.742 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:50.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 14857 14841 185.493 173.625 0.0052295 0.00538381 2023-10-09T04:43:50.858 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:43:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:50.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:43:50.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:51.017 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:51.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 17498 17482 182.085 165.062 0.00802182 0.00548401 2023-10-09T04:43:51.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:51 smithi029 ceph-mon[216903]: pgmap v1104: 89 pgs: 89 active+clean; 149 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 15 MiB/s wr, 237 op/s 2023-10-09T04:43:51.127 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:51 smithi029 ceph-mon[223858]: pgmap v1104: 89 pgs: 89 active+clean; 149 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 15 MiB/s wr, 237 op/s 2023-10-09T04:43:51.397 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:51.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:51.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:51.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:43:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:51 smithi177 ceph-mon[195797]: pgmap v1104: 89 pgs: 89 active+clean; 149 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 15 MiB/s wr, 237 op/s 2023-10-09T04:43:51.574 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:52.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 19881 19865 177.348 148.938 0.00751793 0.00563423 2023-10-09T04:43:52.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:52 smithi029 ceph-mon[216903]: pgmap v1105: 89 pgs: 89 active+clean; 773 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:43:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:52 smithi029 ceph-mon[223858]: pgmap v1105: 89 pgs: 89 active+clean; 773 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:43:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:52 smithi177 ceph-mon[195797]: pgmap v1105: 89 pgs: 89 active+clean; 773 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-09T04:43:53.106 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 22343 22327 174.412 153.875 0.00905143 0.0057268 2023-10-09T04:43:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:43:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:43:52] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-10-09T04:43:53.500 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:43:53.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:53.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:43:53.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:53.662 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:54.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 24189 24173 167.846 115.375 0.006061 0.00595108 2023-10-09T04:43:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:54 smithi177 ceph-mon[195797]: pgmap v1106: 89 pgs: 89 active+clean; 1.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:43:54.552 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:54.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:54.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:54 smithi029 ceph-mon[216903]: pgmap v1106: 89 pgs: 89 active+clean; 1.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:43:54.560 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:54 smithi029 ceph-mon[223858]: pgmap v1106: 89 pgs: 89 active+clean; 1.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-09T04:43:54.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:43:54.733 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:55.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 26108 26092 163.054 119.938 0.0074599 0.00612842 2023-10-09T04:43:56.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 28273 28257 160.531 135.312 0.00768096 0.00622484 2023-10-09T04:43:56.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:56 smithi177 ceph-mon[195797]: pgmap v1107: 89 pgs: 89 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:43:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:56 smithi029 ceph-mon[216903]: pgmap v1107: 89 pgs: 89 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:43:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:56 smithi029 ceph-mon[223858]: pgmap v1107: 89 pgs: 89 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-09T04:43:57.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 29964 29948 155.96 105.688 0.00191389 0.00637467 2023-10-09T04:43:58.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 31717 31701 152.39 109.562 0.00637077 0.00655625 2023-10-09T04:43:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:58 smithi177 ceph-mon[195797]: pgmap v1108: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:43:58.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:58 smithi029 ceph-mon[216903]: pgmap v1108: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:43:58.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:58 smithi029 ceph-mon[223858]: pgmap v1108: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-10-09T04:43:58.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:43:59.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 32668 32652 145.75 59.4375 0.00180573 0.00683848 2023-10-09T04:43:59.373 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:43:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:59.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:43:59.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:43:59.521 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:59.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:43:59.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:43:59.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:43:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:43:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:43:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:43:59.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:43:59.770 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:43:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:43:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:00.039 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:00.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:00.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:00.107 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 34048 34032 141.783 86.25 0.0129363 0.00704474 2023-10-09T04:44:00.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:00.236 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:00.633 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:00.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:00.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:00.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:00.789 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:00 smithi177 ceph-mon[195797]: pgmap v1109: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:44:00.790 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:00.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:00 smithi029 ceph-mon[223858]: pgmap v1109: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:44:00.852 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:00.854 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:00 smithi029 ceph-mon[216903]: pgmap v1109: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-09T04:44:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:00.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:00.956 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:00.976 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:44:00.998 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:01.012 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:44:01.049 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:44:01.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:01.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:01.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:01.084 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:44:01.108 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 35536 35520 138.729 93 0.0169067 0.00720127 2023-10-09T04:44:01.124 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:44:01.154 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:44:01.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:01.174 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:44:01.193 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:44:01.194 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.194 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.194 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.194 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.194 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.195 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.195 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.195 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:44:01.196 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:01.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:01.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:01.354 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:01.404 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:01.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:01.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:01.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3328470557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:01.552 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3434463951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:01.553 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3328470557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:01.553 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3434463951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:01.553 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:01 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:01.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:44:01.591 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:44:01.645 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3328470557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3434463951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:01.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:01 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:01.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:01 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:02.108 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 36759 36743 135.065 76.4375 0.00340438 0.00739945 2023-10-09T04:44:02.323 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:44:02.332 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: pgmap v1110: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:02.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[223858]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: pgmap v1110: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:02.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T04:44:02.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: failed to encode map e289 with expected crc 2023-10-09T04:44:02.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:02 smithi029 ceph-mon[216903]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: pgmap v1110: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2023-10-09T04:44:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: failed to encode map e289 with expected crc 2023-10-09T04:44:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: failed to encode map e289 with expected crc 2023-10-09T04:44:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: failed to encode map e289 with expected crc 2023-10-09T04:44:03.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:03.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: osdmap e289: 8 total, 8 up, 8 in 2023-10-09T04:44:03.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: failed to encode map e289 with expected crc 2023-10-09T04:44:03.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: failed to encode map e289 with expected crc 2023-10-09T04:44:03.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:02 smithi177 ceph-mon[195797]: from='client.46121 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:03.108 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 38490 38474 133.571 108.188 0.00207924 0.0074708 2023-10-09T04:44:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:44:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:44:02] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.43.0" 2023-10-09T04:44:04.109 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 39479 39463 129.785 61.8125 0.0104016 0.00769932 2023-10-09T04:44:04.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:04 smithi029 ceph-mon[216903]: pgmap v1112: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:44:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:04 smithi029 ceph-mon[223858]: pgmap v1112: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:44:04.969 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:44:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:04.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:44:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:04 smithi177 ceph-mon[195797]: pgmap v1112: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-10-09T04:44:05.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:05.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:44:05.110429+0000 min lat: 0.00124565 max lat: 0.309603 avg lat: 0.00790193 2023-10-09T04:44:05.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:44:05.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 40496 40480 126.472 63.5625 0.00948361 0.00790193 2023-10-09T04:44:05.137 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:06.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 41769 41753 124.238 79.5625 0.00763414 0.00804315 2023-10-09T04:44:06.169 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:06.172 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1111,"stamp":"2023-10-09T04:44:01.550654+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2099713952,"num_objects":32054,"num_object_clones":0,"num_object_copies":96162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32054,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":32330,"num_write_kb":2052365,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2957464,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32246,"ondisk_log_size":32246,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11811228,"kb_used_data":5975644,"kb_used_omap":20,"kb_used_meta":5835435,"kb_avail":738149988,"statfs":{"total":767960285184,"available":755865587712,"internally_reserved":0,"allocated":6119059456,"data_stored":6112430421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5975486094},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,2,3,4,1,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1289355264,"num_objects":19674,"num_object_clones":0,"num_object_copies":59022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19674,"num_write_kb":1259136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.354419"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416504+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:43:39.416504+0000","last_peered":"2023-10-09T04:43:39.416504+0000","last_clean":"2023-10-09T04:43:39.416504+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:43:39.416504+0000","last_undegraded":"2023-10-09T04:43:39.416504+0000","last_fullsized":"2023-10-09T04:43:39.416504+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":241,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414611+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:43:39.414611+0000","last_peered":"2023-10-09T04:43:39.414611+0000","last_clean":"2023-10-09T04:43:39.414611+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:43:39.414611+0000","last_undegraded":"2023-10-09T04:43:39.414611+0000","last_fullsized":"2023-10-09T04:43:39.414611+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":274,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.551673+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:43:38.551673+0000","last_peered":"2023-10-09T04:43:38.551673+0000","last_clean":"2023-10-09T04:43:38.551673+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:43:38.551673+0000","last_undegraded":"2023-10-09T04:43:38.551673+0000","last_fullsized":"2023-10-09T04:43:38.551673+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":201,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416326+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:43:39.416326+0000","last_peered":"2023-10-09T04:43:39.416326+0000","last_clean":"2023-10-09T04:43:39.416326+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:43:39.416326+0000","last_undegraded":"2023-10-09T04:43:39.416326+0000","last_fullsized":"2023-10-09T04:43:39.416326+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.413883+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:43:39.413883+0000","last_peered":"2023-10-09T04:43:39.413883+0000","last_clean":"2023-10-09T04:43:39.413883+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:43:39.413883+0000","last_undegraded":"2023-10-09T04:43:39.413883+0000","last_fullsized":"2023-10-09T04:43:39.413883+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":244,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415952+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:43:39.415952+0000","last_peered":"2023-10-09T04:43:39.415952+0000","last_clean":"2023-10-09T04:43:39.415952+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:43:39.415952+0000","last_undegraded":"2023-10-09T04:43:39.415952+0000","last_fullsized":"2023-10-09T04:43:39.415952+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":200,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555273+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:43:38.555273+0000","last_peered":"2023-10-09T04:43:38.555273+0000","last_clean":"2023-10-09T04:43:38.555273+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:43:38.555273+0000","last_undegraded":"2023-10-09T04:43:38.555273+0000","last_fullsized":"2023-10-09T04:43:38.555273+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":241,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.413927+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:43:39.413927+0000","last_peered":"2023-10-09T04:43:39.413927+0000","last_clean":"2023-10-09T04:43:39.413927+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:43:39.413927+0000","last_undegraded":"2023-10-09T04:43:39.413927+0000","last_fullsized":"2023-10-09T04:43:39.413927+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":248,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555155+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:43:38.555155+0000","last_peered":"2023-10-09T04:43:38.555155+0000","last_clean":"2023-10-09T04:43:38.555155+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:43:38.555155+0000","last_undegraded":"2023-10-09T04:43:38.555155+0000","last_fullsized":"2023-10-09T04:43:38.555155+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":214,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414102+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:43:39.414102+0000","last_peered":"2023-10-09T04:43:39.414102+0000","last_clean":"2023-10-09T04:43:39.414102+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:43:39.414102+0000","last_undegraded":"2023-10-09T04:43:39.414102+0000","last_fullsized":"2023-10-09T04:43:39.414102+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":282,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416344+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:43:39.416344+0000","last_peered":"2023-10-09T04:43:39.416344+0000","last_clean":"2023-10-09T04:43:39.416344+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:43:39.416344+0000","last_undegraded":"2023-10-09T04:43:39.416344+0000","last_fullsized":"2023-10-09T04:43:39.416344+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":241,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416719+0000","last_change":"2023-10-09T04:39:40.406537+0000","last_active":"2023-10-09T04:43:39.416719+0000","last_peered":"2023-10-09T04:43:39.416719+0000","last_clean":"2023-10-09T04:43:39.416719+0000","last_became_active":"2023-10-09T04:39:40.406281+0000","last_became_peered":"2023-10-09T04:39:40.406281+0000","last_unstale":"2023-10-09T04:43:39.416719+0000","last_undegraded":"2023-10-09T04:43:39.416719+0000","last_fullsized":"2023-10-09T04:43:39.416719+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:09.883302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":46,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416019+0000","last_change":"2023-10-09T04:41:02.634559+0000","last_active":"2023-10-09T04:43:39.416019+0000","last_peered":"2023-10-09T04:43:39.416019+0000","last_clean":"2023-10-09T04:43:39.416019+0000","last_became_active":"2023-10-09T04:41:02.624597+0000","last_became_peered":"2023-10-09T04:41:02.624597+0000","last_unstale":"2023-10-09T04:43:39.416019+0000","last_undegraded":"2023-10-09T04:43:39.416019+0000","last_fullsized":"2023-10-09T04:43:39.416019+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:57.939757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":248,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415980+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:43:39.415980+0000","last_peered":"2023-10-09T04:43:39.415980+0000","last_clean":"2023-10-09T04:43:39.415980+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:43:39.415980+0000","last_undegraded":"2023-10-09T04:43:39.415980+0000","last_fullsized":"2023-10-09T04:43:39.415980+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":298,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415970+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:43:39.415970+0000","last_peered":"2023-10-09T04:43:39.415970+0000","last_clean":"2023-10-09T04:43:39.415970+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:43:39.415970+0000","last_undegraded":"2023-10-09T04:43:39.415970+0000","last_fullsized":"2023-10-09T04:43:39.415970+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.413982+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:43:39.413982+0000","last_peered":"2023-10-09T04:43:39.413982+0000","last_clean":"2023-10-09T04:43:39.413982+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:43:39.413982+0000","last_undegraded":"2023-10-09T04:43:39.413982+0000","last_fullsized":"2023-10-09T04:43:39.413982+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":200,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415670+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:43:39.415670+0000","last_peered":"2023-10-09T04:43:39.415670+0000","last_clean":"2023-10-09T04:43:39.415670+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:43:39.415670+0000","last_undegraded":"2023-10-09T04:43:39.415670+0000","last_fullsized":"2023-10-09T04:43:39.415670+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":247,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555241+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:43:38.555241+0000","last_peered":"2023-10-09T04:43:38.555241+0000","last_clean":"2023-10-09T04:43:38.555241+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:43:38.555241+0000","last_undegraded":"2023-10-09T04:43:38.555241+0000","last_fullsized":"2023-10-09T04:43:38.555241+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":230,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416822+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:43:39.416822+0000","last_peered":"2023-10-09T04:43:39.416822+0000","last_clean":"2023-10-09T04:43:39.416822+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:43:39.416822+0000","last_undegraded":"2023-10-09T04:43:39.416822+0000","last_fullsized":"2023-10-09T04:43:39.416822+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":202,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414548+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:43:39.414548+0000","last_peered":"2023-10-09T04:43:39.414548+0000","last_clean":"2023-10-09T04:43:39.414548+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:43:39.414548+0000","last_undegraded":"2023-10-09T04:43:39.414548+0000","last_fullsized":"2023-10-09T04:43:39.414548+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":293,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415329+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:43:39.415329+0000","last_peered":"2023-10-09T04:43:39.415329+0000","last_clean":"2023-10-09T04:43:39.415329+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:43:39.415329+0000","last_undegraded":"2023-10-09T04:43:39.415329+0000","last_fullsized":"2023-10-09T04:43:39.415329+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":241,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414640+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:43:39.414640+0000","last_peered":"2023-10-09T04:43:39.414640+0000","last_clean":"2023-10-09T04:43:39.414640+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:43:39.414640+0000","last_undegraded":"2023-10-09T04:43:39.414640+0000","last_fullsized":"2023-10-09T04:43:39.414640+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"288'2092","reported_seq":2104,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.357042+0000","last_change":"2023-10-09T04:43:34.548310+0000","last_active":"2023-10-09T04:43:59.357042+0000","last_peered":"2023-10-09T04:43:59.357042+0000","last_clean":"2023-10-09T04:43:59.357042+0000","last_became_active":"2023-10-09T04:43:34.548012+0000","last_became_peered":"2023-10-09T04:43:34.548012+0000","last_unstale":"2023-10-09T04:43:59.357042+0000","last_undegraded":"2023-10-09T04:43:59.357042+0000","last_fullsized":"2023-10-09T04:43:59.357042+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2092,"log_dups_size":0,"ondisk_log_size":2092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:10.801596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2092,"num_write_kb":133888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":304,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415999+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:43:39.415999+0000","last_peered":"2023-10-09T04:43:39.415999+0000","last_clean":"2023-10-09T04:43:39.415999+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:43:39.415999+0000","last_undegraded":"2023-10-09T04:43:39.415999+0000","last_fullsized":"2023-10-09T04:43:39.415999+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":230,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555042+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:43:38.555042+0000","last_peered":"2023-10-09T04:43:38.555042+0000","last_clean":"2023-10-09T04:43:38.555042+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:43:38.555042+0000","last_undegraded":"2023-10-09T04:43:38.555042+0000","last_fullsized":"2023-10-09T04:43:38.555042+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"288'2024","reported_seq":2036,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.374247+0000","last_change":"2023-10-09T04:43:34.548528+0000","last_active":"2023-10-09T04:43:59.374247+0000","last_peered":"2023-10-09T04:43:59.374247+0000","last_clean":"2023-10-09T04:43:59.374247+0000","last_became_active":"2023-10-09T04:43:34.548236+0000","last_became_peered":"2023-10-09T04:43:34.548236+0000","last_unstale":"2023-10-09T04:43:59.374247+0000","last_undegraded":"2023-10-09T04:43:59.374247+0000","last_fullsized":"2023-10-09T04:43:59.374247+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2024,"log_dups_size":0,"ondisk_log_size":2024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:24.666864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132644864,"num_objects":2024,"num_object_clones":0,"num_object_copies":6072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2024,"num_write_kb":129536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":277,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416244+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:43:39.416244+0000","last_peered":"2023-10-09T04:43:39.416244+0000","last_clean":"2023-10-09T04:43:39.416244+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:43:39.416244+0000","last_undegraded":"2023-10-09T04:43:39.416244+0000","last_fullsized":"2023-10-09T04:43:39.416244+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416349+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:43:39.416349+0000","last_peered":"2023-10-09T04:43:39.416349+0000","last_clean":"2023-10-09T04:43:39.416349+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:43:39.416349+0000","last_undegraded":"2023-10-09T04:43:39.416349+0000","last_fullsized":"2023-10-09T04:43:39.416349+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"288'2061","reported_seq":2073,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.370228+0000","last_change":"2023-10-09T04:43:34.547460+0000","last_active":"2023-10-09T04:43:59.370228+0000","last_peered":"2023-10-09T04:43:59.370228+0000","last_clean":"2023-10-09T04:43:59.370228+0000","last_became_active":"2023-10-09T04:43:34.547150+0000","last_became_peered":"2023-10-09T04:43:34.547150+0000","last_unstale":"2023-10-09T04:43:59.370228+0000","last_undegraded":"2023-10-09T04:43:59.370228+0000","last_fullsized":"2023-10-09T04:43:59.370228+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2061,"log_dups_size":0,"ondisk_log_size":2061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:43.085276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135069696,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2061,"num_write_kb":131904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":231,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416165+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:43:39.416165+0000","last_peered":"2023-10-09T04:43:39.416165+0000","last_clean":"2023-10-09T04:43:39.416165+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:43:39.416165+0000","last_undegraded":"2023-10-09T04:43:39.416165+0000","last_fullsized":"2023-10-09T04:43:39.416165+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":271,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416505+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:43:39.416505+0000","last_peered":"2023-10-09T04:43:39.416505+0000","last_clean":"2023-10-09T04:43:39.416505+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:43:39.416505+0000","last_undegraded":"2023-10-09T04:43:39.416505+0000","last_fullsized":"2023-10-09T04:43:39.416505+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"288'2060","reported_seq":2072,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.358141+0000","last_change":"2023-10-09T04:43:34.881088+0000","last_active":"2023-10-09T04:43:59.358141+0000","last_peered":"2023-10-09T04:43:59.358141+0000","last_clean":"2023-10-09T04:43:59.358141+0000","last_became_active":"2023-10-09T04:43:34.880601+0000","last_became_peered":"2023-10-09T04:43:34.880601+0000","last_unstale":"2023-10-09T04:43:59.358141+0000","last_undegraded":"2023-10-09T04:43:59.358141+0000","last_fullsized":"2023-10-09T04:43:59.358141+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2060,"log_dups_size":0,"ondisk_log_size":2060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:27.274950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135004160,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2060,"num_write_kb":131840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":248,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555372+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:43:38.555372+0000","last_peered":"2023-10-09T04:43:38.555372+0000","last_clean":"2023-10-09T04:43:38.555372+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:43:38.555372+0000","last_undegraded":"2023-10-09T04:43:38.555372+0000","last_fullsized":"2023-10-09T04:43:38.555372+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":145,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555422+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:43:38.555422+0000","last_peered":"2023-10-09T04:43:38.555422+0000","last_clean":"2023-10-09T04:43:38.555422+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:43:38.555422+0000","last_undegraded":"2023-10-09T04:43:38.555422+0000","last_fullsized":"2023-10-09T04:43:38.555422+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.4","version":"288'2054","reported_seq":2066,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.360351+0000","last_change":"2023-10-09T04:43:34.548768+0000","last_active":"2023-10-09T04:43:59.360351+0000","last_peered":"2023-10-09T04:43:59.360351+0000","last_clean":"2023-10-09T04:43:59.360351+0000","last_became_active":"2023-10-09T04:43:34.548440+0000","last_became_peered":"2023-10-09T04:43:34.548440+0000","last_unstale":"2023-10-09T04:43:59.360351+0000","last_undegraded":"2023-10-09T04:43:59.360351+0000","last_fullsized":"2023-10-09T04:43:59.360351+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2054,"log_dups_size":0,"ondisk_log_size":2054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:54.014901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134610944,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2054,"num_write_kb":131456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":286,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:42.692714+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:43:42.692714+0000","last_peered":"2023-10-09T04:43:42.692714+0000","last_clean":"2023-10-09T04:43:42.692714+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:43:42.692714+0000","last_undegraded":"2023-10-09T04:43:42.692714+0000","last_fullsized":"2023-10-09T04:43:42.692714+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416072+0000","last_change":"2023-10-09T04:41:02.624962+0000","last_active":"2023-10-09T04:43:39.416072+0000","last_peered":"2023-10-09T04:43:39.416072+0000","last_clean":"2023-10-09T04:43:39.416072+0000","last_became_active":"2023-10-09T04:41:02.624555+0000","last_became_peered":"2023-10-09T04:41:02.624555+0000","last_unstale":"2023-10-09T04:43:39.416072+0000","last_undegraded":"2023-10-09T04:43:39.416072+0000","last_fullsized":"2023-10-09T04:43:39.416072+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:48.506794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"288'2051","reported_seq":2063,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.355763+0000","last_change":"2023-10-09T04:43:34.548194+0000","last_active":"2023-10-09T04:43:59.355763+0000","last_peered":"2023-10-09T04:43:59.355763+0000","last_clean":"2023-10-09T04:43:59.355763+0000","last_became_active":"2023-10-09T04:43:34.547902+0000","last_became_peered":"2023-10-09T04:43:34.547902+0000","last_unstale":"2023-10-09T04:43:59.355763+0000","last_undegraded":"2023-10-09T04:43:59.355763+0000","last_fullsized":"2023-10-09T04:43:59.355763+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2051,"log_dups_size":0,"ondisk_log_size":2051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:14.652242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134414336,"num_objects":2051,"num_object_clones":0,"num_object_copies":6153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2051,"num_write_kb":131264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":265,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416643+0000","last_change":"2023-10-09T04:41:02.603773+0000","last_active":"2023-10-09T04:43:39.416643+0000","last_peered":"2023-10-09T04:43:39.416643+0000","last_clean":"2023-10-09T04:43:39.416643+0000","last_became_active":"2023-10-09T04:41:02.603407+0000","last_became_peered":"2023-10-09T04:41:02.603407+0000","last_unstale":"2023-10-09T04:43:39.416643+0000","last_undegraded":"2023-10-09T04:43:39.416643+0000","last_fullsized":"2023-10-09T04:43:39.416643+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:18.401503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415735+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:43:39.415735+0000","last_peered":"2023-10-09T04:43:39.415735+0000","last_clean":"2023-10-09T04:43:39.415735+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:43:39.415735+0000","last_undegraded":"2023-10-09T04:43:39.415735+0000","last_fullsized":"2023-10-09T04:43:39.415735+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"288'1871","reported_seq":1884,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:57.004197+0000","last_change":"2023-10-09T04:43:34.881151+0000","last_active":"2023-10-09T04:43:57.004197+0000","last_peered":"2023-10-09T04:43:57.004197+0000","last_clean":"2023-10-09T04:43:57.004197+0000","last_became_active":"2023-10-09T04:43:34.880832+0000","last_became_peered":"2023-10-09T04:43:34.880832+0000","last_unstale":"2023-10-09T04:43:57.004197+0000","last_undegraded":"2023-10-09T04:43:57.004197+0000","last_fullsized":"2023-10-09T04:43:57.004197+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1871,"log_dups_size":0,"ondisk_log_size":1871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:48.504273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122617856,"num_objects":1871,"num_object_clones":0,"num_object_copies":5613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1871,"num_write_kb":119744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416240+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:43:39.416240+0000","last_peered":"2023-10-09T04:43:39.416240+0000","last_clean":"2023-10-09T04:43:39.416240+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:43:39.416240+0000","last_undegraded":"2023-10-09T04:43:39.416240+0000","last_fullsized":"2023-10-09T04:43:39.416240+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":248,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416809+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:43:39.416809+0000","last_peered":"2023-10-09T04:43:39.416809+0000","last_clean":"2023-10-09T04:43:39.416809+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:43:39.416809+0000","last_undegraded":"2023-10-09T04:43:39.416809+0000","last_fullsized":"2023-10-09T04:43:39.416809+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"288'2033","reported_seq":2045,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.365684+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:43:59.365684+0000","last_peered":"2023-10-09T04:43:59.365684+0000","last_clean":"2023-10-09T04:43:59.365684+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:43:59.365684+0000","last_undegraded":"2023-10-09T04:43:59.365684+0000","last_fullsized":"2023-10-09T04:43:59.365684+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2033,"log_dups_size":0,"ondisk_log_size":2033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133234688,"num_objects":2033,"num_object_clones":0,"num_object_copies":6099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2033,"num_write_kb":130112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":243,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.413790+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:43:39.413790+0000","last_peered":"2023-10-09T04:43:39.413790+0000","last_clean":"2023-10-09T04:43:39.413790+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:43:39.413790+0000","last_undegraded":"2023-10-09T04:43:39.413790+0000","last_fullsized":"2023-10-09T04:43:39.413790+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":272,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414590+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:43:39.414590+0000","last_peered":"2023-10-09T04:43:39.414590+0000","last_clean":"2023-10-09T04:43:39.414590+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:43:39.414590+0000","last_undegraded":"2023-10-09T04:43:39.414590+0000","last_fullsized":"2023-10-09T04:43:39.414590+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"288'1969","reported_seq":1982,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:57.014485+0000","last_change":"2023-10-09T04:43:34.883699+0000","last_active":"2023-10-09T04:43:57.014485+0000","last_peered":"2023-10-09T04:43:57.014485+0000","last_clean":"2023-10-09T04:43:57.014485+0000","last_became_active":"2023-10-09T04:43:34.882682+0000","last_became_peered":"2023-10-09T04:43:34.882682+0000","last_unstale":"2023-10-09T04:43:57.014485+0000","last_undegraded":"2023-10-09T04:43:57.014485+0000","last_fullsized":"2023-10-09T04:43:57.014485+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1969,"log_dups_size":0,"ondisk_log_size":1969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.341363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129040384,"num_objects":1969,"num_object_clones":0,"num_object_copies":5907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1969,"num_write_kb":126016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":165,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416146+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:43:39.416146+0000","last_peered":"2023-10-09T04:43:39.416146+0000","last_clean":"2023-10-09T04:43:39.416146+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:43:39.416146+0000","last_undegraded":"2023-10-09T04:43:39.416146+0000","last_fullsized":"2023-10-09T04:43:39.416146+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":282,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555325+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:43:38.555325+0000","last_peered":"2023-10-09T04:43:38.555325+0000","last_clean":"2023-10-09T04:43:38.555325+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:43:38.555325+0000","last_undegraded":"2023-10-09T04:43:38.555325+0000","last_fullsized":"2023-10-09T04:43:38.555325+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":259,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555155+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:43:38.555155+0000","last_peered":"2023-10-09T04:43:38.555155+0000","last_clean":"2023-10-09T04:43:38.555155+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:43:38.555155+0000","last_undegraded":"2023-10-09T04:43:38.555155+0000","last_fullsized":"2023-10-09T04:43:38.555155+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"288'2078","reported_seq":2090,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.366909+0000","last_change":"2023-10-09T04:43:34.881122+0000","last_active":"2023-10-09T04:43:59.366909+0000","last_peered":"2023-10-09T04:43:59.366909+0000","last_clean":"2023-10-09T04:43:59.366909+0000","last_became_active":"2023-10-09T04:43:34.880680+0000","last_became_peered":"2023-10-09T04:43:34.880680+0000","last_unstale":"2023-10-09T04:43:59.366909+0000","last_undegraded":"2023-10-09T04:43:59.366909+0000","last_fullsized":"2023-10-09T04:43:59.366909+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":2078,"log_dups_size":0,"ondisk_log_size":2078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:08.954210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136183808,"num_objects":2078,"num_object_clones":0,"num_object_copies":6234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2078,"num_write_kb":132992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":252,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555168+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:43:38.555168+0000","last_peered":"2023-10-09T04:43:38.555168+0000","last_clean":"2023-10-09T04:43:38.555168+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:43:38.555168+0000","last_undegraded":"2023-10-09T04:43:38.555168+0000","last_fullsized":"2023-10-09T04:43:38.555168+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415687+0000","last_change":"2023-10-09T04:41:02.608773+0000","last_active":"2023-10-09T04:43:39.415687+0000","last_peered":"2023-10-09T04:43:39.415687+0000","last_clean":"2023-10-09T04:43:39.415687+0000","last_became_active":"2023-10-09T04:41:02.608455+0000","last_became_peered":"2023-10-09T04:41:02.608455+0000","last_unstale":"2023-10-09T04:43:39.415687+0000","last_undegraded":"2023-10-09T04:43:39.415687+0000","last_fullsized":"2023-10-09T04:43:39.415687+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:29.807000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":259,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:42.567300+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:43:42.567300+0000","last_peered":"2023-10-09T04:43:42.567300+0000","last_clean":"2023-10-09T04:43:42.567300+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:43:42.567300+0000","last_undegraded":"2023-10-09T04:43:42.567300+0000","last_fullsized":"2023-10-09T04:43:42.567300+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"288'1961","reported_seq":1972,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.355611+0000","last_change":"2023-10-09T04:43:34.547468+0000","last_active":"2023-10-09T04:43:59.355611+0000","last_peered":"2023-10-09T04:43:59.355611+0000","last_clean":"2023-10-09T04:43:59.355611+0000","last_became_active":"2023-10-09T04:43:34.547129+0000","last_became_peered":"2023-10-09T04:43:34.547129+0000","last_unstale":"2023-10-09T04:43:59.355611+0000","last_undegraded":"2023-10-09T04:43:59.355611+0000","last_fullsized":"2023-10-09T04:43:59.355611+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1961,"log_dups_size":0,"ondisk_log_size":1961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:58.616531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1961,"num_write_kb":125504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":245,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414402+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:43:39.414402+0000","last_peered":"2023-10-09T04:43:39.414402+0000","last_clean":"2023-10-09T04:43:39.414402+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:43:39.414402+0000","last_undegraded":"2023-10-09T04:43:39.414402+0000","last_fullsized":"2023-10-09T04:43:39.414402+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":244,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:42.020646+0000","last_change":"2023-10-09T04:39:40.406420+0000","last_active":"2023-10-09T04:43:42.020646+0000","last_peered":"2023-10-09T04:43:42.020646+0000","last_clean":"2023-10-09T04:43:42.020646+0000","last_became_active":"2023-10-09T04:39:40.406072+0000","last_became_peered":"2023-10-09T04:39:40.406072+0000","last_unstale":"2023-10-09T04:43:42.020646+0000","last_undegraded":"2023-10-09T04:43:42.020646+0000","last_fullsized":"2023-10-09T04:43:42.020646+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:08.385113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555259+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:43:38.555259+0000","last_peered":"2023-10-09T04:43:38.555259+0000","last_clean":"2023-10-09T04:43:38.555259+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:43:38.555259+0000","last_undegraded":"2023-10-09T04:43:38.555259+0000","last_fullsized":"2023-10-09T04:43:38.555259+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"288'1983","reported_seq":1995,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.368491+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:43:59.368491+0000","last_peered":"2023-10-09T04:43:59.368491+0000","last_clean":"2023-10-09T04:43:59.368491+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:43:59.368491+0000","last_undegraded":"2023-10-09T04:43:59.368491+0000","last_fullsized":"2023-10-09T04:43:59.368491+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1983,"log_dups_size":0,"ondisk_log_size":1983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129957888,"num_objects":1983,"num_object_clones":0,"num_object_copies":5949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1983,"num_write_kb":126912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":269,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555462+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:43:38.555462+0000","last_peered":"2023-10-09T04:43:38.555462+0000","last_clean":"2023-10-09T04:43:38.555462+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:43:38.555462+0000","last_undegraded":"2023-10-09T04:43:38.555462+0000","last_fullsized":"2023-10-09T04:43:38.555462+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":680,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415423+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:43:39.415423+0000","last_peered":"2023-10-09T04:43:39.415423+0000","last_clean":"2023-10-09T04:43:39.415423+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:43:39.415423+0000","last_undegraded":"2023-10-09T04:43:39.415423+0000","last_fullsized":"2023-10-09T04:43:39.415423+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":214,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414237+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:43:39.414237+0000","last_peered":"2023-10-09T04:43:39.414237+0000","last_clean":"2023-10-09T04:43:39.414237+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:43:39.414237+0000","last_undegraded":"2023-10-09T04:43:39.414237+0000","last_fullsized":"2023-10-09T04:43:39.414237+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":247,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.417156+0000","last_change":"2023-10-09T04:41:02.612580+0000","last_active":"2023-10-09T04:43:39.417156+0000","last_peered":"2023-10-09T04:43:39.417156+0000","last_clean":"2023-10-09T04:43:39.417156+0000","last_became_active":"2023-10-09T04:41:02.612258+0000","last_became_peered":"2023-10-09T04:41:02.612258+0000","last_unstale":"2023-10-09T04:43:39.417156+0000","last_undegraded":"2023-10-09T04:43:39.417156+0000","last_fullsized":"2023-10-09T04:43:39.417156+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:53.656800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"288'1992","reported_seq":2004,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:59.350511+0000","last_change":"2023-10-09T04:43:34.880958+0000","last_active":"2023-10-09T04:43:59.350511+0000","last_peered":"2023-10-09T04:43:59.350511+0000","last_clean":"2023-10-09T04:43:59.350511+0000","last_became_active":"2023-10-09T04:43:34.880522+0000","last_became_peered":"2023-10-09T04:43:34.880522+0000","last_unstale":"2023-10-09T04:43:59.350511+0000","last_undegraded":"2023-10-09T04:43:59.350511+0000","last_fullsized":"2023-10-09T04:43:59.350511+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1992,"log_dups_size":0,"ondisk_log_size":1992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:04.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130547712,"num_objects":1992,"num_object_clones":0,"num_object_copies":5976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1992,"num_write_kb":127488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":288,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415326+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:43:39.415326+0000","last_peered":"2023-10-09T04:43:39.415326+0000","last_clean":"2023-10-09T04:43:39.415326+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:43:39.415326+0000","last_undegraded":"2023-10-09T04:43:39.415326+0000","last_fullsized":"2023-10-09T04:43:39.415326+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416310+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:43:39.416310+0000","last_peered":"2023-10-09T04:43:39.416310+0000","last_clean":"2023-10-09T04:43:39.416310+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:43:39.416310+0000","last_undegraded":"2023-10-09T04:43:39.416310+0000","last_fullsized":"2023-10-09T04:43:39.416310+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":165,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.792844+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:43:38.792844+0000","last_peered":"2023-10-09T04:43:38.792844+0000","last_clean":"2023-10-09T04:43:38.792844+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:43:38.792844+0000","last_undegraded":"2023-10-09T04:43:38.792844+0000","last_fullsized":"2023-10-09T04:43:38.792844+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"288'1979","reported_seq":1992,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:57.690343+0000","last_change":"2023-10-09T04:43:34.547726+0000","last_active":"2023-10-09T04:43:57.690343+0000","last_peered":"2023-10-09T04:43:57.690343+0000","last_clean":"2023-10-09T04:43:57.690343+0000","last_became_active":"2023-10-09T04:43:34.547321+0000","last_became_peered":"2023-10-09T04:43:34.547321+0000","last_unstale":"2023-10-09T04:43:57.690343+0000","last_undegraded":"2023-10-09T04:43:57.690343+0000","last_fullsized":"2023-10-09T04:43:57.690343+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1979,"log_dups_size":0,"ondisk_log_size":1979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:56.973914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1979,"num_write_kb":126656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":291,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416252+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:43:39.416252+0000","last_peered":"2023-10-09T04:43:39.416252+0000","last_clean":"2023-10-09T04:43:39.416252+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:43:39.416252+0000","last_undegraded":"2023-10-09T04:43:39.416252+0000","last_fullsized":"2023-10-09T04:43:39.416252+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":244,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:42.567225+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:43:42.567225+0000","last_peered":"2023-10-09T04:43:42.567225+0000","last_clean":"2023-10-09T04:43:42.567225+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:43:42.567225+0000","last_undegraded":"2023-10-09T04:43:42.567225+0000","last_fullsized":"2023-10-09T04:43:42.567225+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555500+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:43:38.555500+0000","last_peered":"2023-10-09T04:43:38.555500+0000","last_clean":"2023-10-09T04:43:38.555500+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:43:38.555500+0000","last_undegraded":"2023-10-09T04:43:38.555500+0000","last_fullsized":"2023-10-09T04:43:38.555500+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"288'1896","reported_seq":1909,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:57.692670+0000","last_change":"2023-10-09T04:43:34.880794+0000","last_active":"2023-10-09T04:43:57.692670+0000","last_peered":"2023-10-09T04:43:57.692670+0000","last_clean":"2023-10-09T04:43:57.692670+0000","last_became_active":"2023-10-09T04:43:34.880481+0000","last_became_peered":"2023-10-09T04:43:34.880481+0000","last_unstale":"2023-10-09T04:43:57.692670+0000","last_undegraded":"2023-10-09T04:43:57.692670+0000","last_fullsized":"2023-10-09T04:43:57.692670+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1896,"log_dups_size":0,"ondisk_log_size":1896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:13.398120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124256256,"num_objects":1896,"num_object_clones":0,"num_object_copies":5688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1896,"num_write_kb":121344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":286,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414618+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:43:39.414618+0000","last_peered":"2023-10-09T04:43:39.414618+0000","last_clean":"2023-10-09T04:43:39.414618+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:43:39.414618+0000","last_undegraded":"2023-10-09T04:43:39.414618+0000","last_fullsized":"2023-10-09T04:43:39.414618+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":257,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416154+0000","last_change":"2023-10-09T04:41:02.629667+0000","last_active":"2023-10-09T04:43:39.416154+0000","last_peered":"2023-10-09T04:43:39.416154+0000","last_clean":"2023-10-09T04:43:39.416154+0000","last_became_active":"2023-10-09T04:41:02.629305+0000","last_became_peered":"2023-10-09T04:41:02.629305+0000","last_unstale":"2023-10-09T04:43:39.416154+0000","last_undegraded":"2023-10-09T04:43:39.416154+0000","last_fullsized":"2023-10-09T04:43:39.416154+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:36.600453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":276,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:42.867919+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:43:42.867919+0000","last_peered":"2023-10-09T04:43:42.867919+0000","last_clean":"2023-10-09T04:43:42.867919+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:43:42.867919+0000","last_undegraded":"2023-10-09T04:43:42.867919+0000","last_fullsized":"2023-10-09T04:43:42.867919+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"288'1926","reported_seq":1939,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:57.015836+0000","last_change":"2023-10-09T04:43:34.548840+0000","last_active":"2023-10-09T04:43:57.015836+0000","last_peered":"2023-10-09T04:43:57.015836+0000","last_clean":"2023-10-09T04:43:57.015836+0000","last_became_active":"2023-10-09T04:43:34.548573+0000","last_became_peered":"2023-10-09T04:43:34.548573+0000","last_unstale":"2023-10-09T04:43:57.015836+0000","last_undegraded":"2023-10-09T04:43:57.015836+0000","last_fullsized":"2023-10-09T04:43:57.015836+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":1926,"log_dups_size":0,"ondisk_log_size":1926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:49.901133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126222336,"num_objects":1926,"num_object_clones":0,"num_object_copies":5778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1926,"num_write_kb":123264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":266,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415867+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:43:39.415867+0000","last_peered":"2023-10-09T04:43:39.415867+0000","last_clean":"2023-10-09T04:43:39.415867+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:43:39.415867+0000","last_undegraded":"2023-10-09T04:43:39.415867+0000","last_fullsized":"2023-10-09T04:43:39.415867+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":60,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416899+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:43:39.416899+0000","last_peered":"2023-10-09T04:43:39.416899+0000","last_clean":"2023-10-09T04:43:39.416899+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:43:39.416899+0000","last_undegraded":"2023-10-09T04:43:39.416899+0000","last_fullsized":"2023-10-09T04:43:39.416899+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416198+0000","last_change":"2023-10-09T04:41:02.614004+0000","last_active":"2023-10-09T04:43:39.416198+0000","last_peered":"2023-10-09T04:43:39.416198+0000","last_clean":"2023-10-09T04:43:39.416198+0000","last_became_active":"2023-10-09T04:41:02.612300+0000","last_became_peered":"2023-10-09T04:41:02.612300+0000","last_unstale":"2023-10-09T04:43:39.416198+0000","last_undegraded":"2023-10-09T04:43:39.416198+0000","last_fullsized":"2023-10-09T04:43:39.416198+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:04.375059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":128,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415903+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:43:39.415903+0000","last_peered":"2023-10-09T04:43:39.415903+0000","last_clean":"2023-10-09T04:43:39.415903+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:43:39.415903+0000","last_undegraded":"2023-10-09T04:43:39.415903+0000","last_fullsized":"2023-10-09T04:43:39.415903+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":237,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414986+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:43:39.414986+0000","last_peered":"2023-10-09T04:43:39.414986+0000","last_clean":"2023-10-09T04:43:39.414986+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:43:39.414986+0000","last_undegraded":"2023-10-09T04:43:39.414986+0000","last_fullsized":"2023-10-09T04:43:39.414986+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":278,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555161+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:43:38.555161+0000","last_peered":"2023-10-09T04:43:38.555161+0000","last_clean":"2023-10-09T04:43:38.555161+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:43:38.555161+0000","last_undegraded":"2023-10-09T04:43:38.555161+0000","last_fullsized":"2023-10-09T04:43:38.555161+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415764+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:43:39.415764+0000","last_peered":"2023-10-09T04:43:39.415764+0000","last_clean":"2023-10-09T04:43:39.415764+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:43:39.415764+0000","last_undegraded":"2023-10-09T04:43:39.415764+0000","last_fullsized":"2023-10-09T04:43:39.415764+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":233,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555339+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:43:38.555339+0000","last_peered":"2023-10-09T04:43:38.555339+0000","last_clean":"2023-10-09T04:43:38.555339+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:43:38.555339+0000","last_undegraded":"2023-10-09T04:43:38.555339+0000","last_fullsized":"2023-10-09T04:43:38.555339+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.416584+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:43:39.416584+0000","last_peered":"2023-10-09T04:43:39.416584+0000","last_clean":"2023-10-09T04:43:39.416584+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:43:39.416584+0000","last_undegraded":"2023-10-09T04:43:39.416584+0000","last_fullsized":"2023-10-09T04:43:39.416584+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":277,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-10-09T04:43:42.869546+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:43:42.869546+0000","last_peered":"2023-10-09T04:43:42.869546+0000","last_clean":"2023-10-09T04:43:42.869546+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:43:42.869546+0000","last_undegraded":"2023-10-09T04:43:42.869546+0000","last_fullsized":"2023-10-09T04:43:42.869546+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":308,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.414015+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:43:39.414015+0000","last_peered":"2023-10-09T04:43:39.414015+0000","last_clean":"2023-10-09T04:43:39.414015+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:43:39.414015+0000","last_undegraded":"2023-10-09T04:43:39.414015+0000","last_fullsized":"2023-10-09T04:43:39.414015+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":162,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:38.555375+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:43:38.555375+0000","last_peered":"2023-10-09T04:43:38.555375+0000","last_clean":"2023-10-09T04:43:38.555375+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:43:38.555375+0000","last_undegraded":"2023-10-09T04:43:38.555375+0000","last_fullsized":"2023-10-09T04:43:38.555375+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":36,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-10-09T04:43:39.415511+0000","last_change":"2023-10-09T04:41:02.608725+0000","last_active":"2023-10-09T04:43:39.415511+0000","last_peered":"2023-10-09T04:43:39.415511+0000","last_clean":"2023-10-09T04:43:39.415511+0000","last_became_active":"2023-10-09T04:41:02.606509+0000","last_became_peered":"2023-10-09T04:41:02.606509+0000","last_unstale":"2023-10-09T04:43:39.415511+0000","last_undegraded":"2023-10-09T04:43:39.415511+0000","last_fullsized":"2023-10-09T04:43:39.415511+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:49.430101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2099118080,"num_objects":32030,"num_object_clones":0,"num_object_copies":96090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":32030,"num_write_kb":2049920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6190071808,"data_stored":6190071808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32030,"ondisk_log_size":32030,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":5624,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234445,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1812404,"kb_used_data":909612,"kb_used_omap":0,"kb_used_meta":902784,"kb_avail":91932748,"statfs":{"total":95995035648,"available":94139133952,"internally_reserved":0,"allocated":931442688,"data_stored":930847963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924450816},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.972,"15min":4.0300000000000002},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.4449999999999998,"5min":44.122999999999998,"15min":186.446},"last":1.028},{"interface":"front","average":{"1min":0.47699999999999998,"5min":2.722,"15min":4.3769999999999998},"min":{"1min":0.32700000000000001,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":0.67400000000000004,"5min":43.313000000000002,"15min":186.429},"last":2.8959999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":5.8159999999999998,"15min":5.3529999999999998},"min":{"1min":0.34000000000000002,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":0.63400000000000001,"5min":264.62,"15min":264.62},"last":1.825},{"interface":"front","average":{"1min":0.47699999999999998,"5min":6.0869999999999997,"15min":5.5190000000000001},"min":{"1min":0.31,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67800000000000005,"5min":264.65899999999999,"15min":264.65899999999999},"last":1.0960000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":7.5010000000000003,"15min":3.7010000000000001},"min":{"1min":0.379,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.9949999999999992,"5min":184.72,"15min":184.72},"last":1.2969999999999999},{"interface":"front","average":{"1min":2.4769999999999999,"5min":7.8920000000000003,"15min":4.1660000000000004},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.438000000000001,"5min":184.697,"15min":184.697},"last":2.8519999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":5.3259999999999996,"15min":7.7530000000000001},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.3819999999999997,"5min":182.839,"15min":249.989},"last":1.3280000000000001},{"interface":"front","average":{"1min":1.2330000000000001,"5min":5.8860000000000001,"15min":8.0700000000000003},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":10.278,"5min":182.678,"15min":250.21700000000001},"last":2.9780000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.2789999999999999,"15min":1.389},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.388,"5min":14.257999999999999,"15min":14.257999999999999},"last":3.0449999999999999},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.86199999999999999,"15min":1.131},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.21199999999999999},"max":{"1min":4.923,"5min":7.2290000000000001,"15min":10.613},"last":0.64000000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.2290000000000001,"15min":1.1040000000000001},"min":{"1min":0.371,"5min":0.29399999999999998,"15min":0.27900000000000003},"max":{"1min":4.8099999999999996,"5min":13.319000000000001,"15min":13.319000000000001},"last":2.4049999999999998},{"interface":"front","average":{"1min":2.3799999999999999,"5min":1.728,"15min":1.5449999999999999},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.779999999999999,"5min":18.408000000000001,"15min":18.408000000000001},"last":2.9460000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.1439999999999999,"15min":1.2410000000000001},"min":{"1min":0.38200000000000001,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.9489999999999998,"5min":11.388,"15min":11.388},"last":1.069},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.643,"15min":1.6299999999999999},"min":{"1min":0.373,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.532999999999999,"5min":18.311,"15min":18.311},"last":3.0190000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496079,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1822436,"kb_used_data":920156,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":91922716,"statfs":{"total":95995035648,"available":94128861184,"internally_reserved":0,"allocated":942239744,"data_stored":941660934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":2.2559999999999998,"15min":8.0470000000000006},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":0.66700000000000004,"5min":75.308000000000007,"15min":1284.521},"last":4.1379999999999999},{"interface":"front","average":{"1min":0.81100000000000005,"5min":2.7450000000000001,"15min":8.2680000000000007},"min":{"1min":0.29099999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.8209999999999997,"5min":75.432000000000002,"15min":1284.4690000000001},"last":1.6339999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:43:32 2023","interfaces":[{"interface":"back","average":{"1min":3.7490000000000001,"5min":7.4779999999999998,"15min":5.7670000000000003},"min":{"1min":0.27300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":66.510999999999996,"5min":150.75,"15min":185.74000000000001},"last":1.679},{"interface":"front","average":{"1min":4.2110000000000003,"5min":7.7809999999999997,"15min":5.8250000000000002},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.26300000000000001},"max":{"1min":66.418000000000006,"5min":150.81999999999999,"15min":189.03999999999999},"last":2.4500000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:42:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":8.7400000000000002,"15min":4.3120000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.30199999999999999},"max":{"1min":11.225,"5min":209.44399999999999,"15min":209.44399999999999},"last":1.4770000000000001},{"interface":"front","average":{"1min":1.825,"5min":8.7029999999999994,"15min":4.2880000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":11.239000000000001,"5min":209.428,"15min":209.428},"last":3.3839999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:43:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":6.694,"15min":6.1669999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.253},"max":{"1min":1.3620000000000001,"5min":157.93100000000001,"15min":167.56100000000001},"last":2.032},{"interface":"front","average":{"1min":1.121,"5min":6.7960000000000003,"15min":6.3120000000000003},"min":{"1min":0.38700000000000001,"5min":0.33000000000000002,"15min":0.313},"max":{"1min":7.7779999999999996,"5min":157.95099999999999,"15min":167.577},"last":1.9179999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.98299999999999998,"15min":1.286},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.25600000000000001},"max":{"1min":2.6709999999999998,"5min":6.8380000000000001,"15min":14.430999999999999},"last":1.6160000000000001},{"interface":"front","average":{"1min":0.45400000000000001,"5min":1.0129999999999999,"15min":1.04},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":0.80000000000000004,"5min":9.2240000000000002,"15min":9.2240000000000002},"last":2.0710000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.0860000000000001,"15min":1.78},"min":{"1min":0.32100000000000001,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":4.0890000000000004,"5min":7.3010000000000002,"15min":81.087000000000003},"last":1.655},{"interface":"front","average":{"1min":0.49399999999999999,"5min":1.1579999999999999,"15min":1.6719999999999999},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":0.83699999999999997,"5min":8.6479999999999997,"15min":81.706999999999994},"last":2.3839999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:43:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.238,"15min":1.5600000000000001},"min":{"1min":0.378,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":4.6029999999999998,"5min":16.512,"15min":81.674999999999997},"last":1.996},{"interface":"front","average":{"1min":1.385,"5min":1.389,"15min":1.762},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":7.5289999999999999,"5min":17.715,"15min":81.055000000000007},"last":1.8180000000000001}]}]},{"osd":1,"up_from":118,"seq":506806141168,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215224,"kb_used_data":591024,"kb_used_omap":4,"kb_used_meta":624187,"kb_avail":92529928,"statfs":{"total":95995035648,"available":94750646272,"internally_reserved":0,"allocated":605208576,"data_stored":604025047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":639168398},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":3.6469999999999998,"15min":3.9359999999999999},"min":{"1min":0.23599999999999999,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":0.873,"5min":75.620000000000005,"15min":97.308000000000007},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.502,"5min":3.7189999999999999,"15min":4.1740000000000004},"min":{"1min":0.252,"5min":0.193,"15min":0.193},"max":{"1min":3.367,"5min":69.701999999999998,"15min":97.055999999999997},"last":0.58499999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:42:52 2023","interfaces":[{"interface":"back","average":{"1min":12.409000000000001,"5min":6.1970000000000001,"15min":3.823},"min":{"1min":0.253,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":263.60500000000002,"5min":263.60500000000002,"15min":263.60500000000002},"last":1.3520000000000001},{"interface":"front","average":{"1min":13.936,"5min":7.0970000000000004,"15min":4.3310000000000004},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":263.65199999999999,"5min":263.65199999999999,"15min":263.65199999999999},"last":0.69399999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:43:27 2023","interfaces":[{"interface":"back","average":{"1min":15.497999999999999,"5min":10.351000000000001,"15min":6.4470000000000001},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":379.80900000000003,"5min":379.80900000000003,"15min":379.80900000000003},"last":1.3899999999999999},{"interface":"front","average":{"1min":15.800000000000001,"5min":11.353999999999999,"15min":7.0990000000000002},"min":{"1min":0.29399999999999998,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":379.95800000000003,"5min":379.95800000000003,"15min":379.95800000000003},"last":0.64600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":2.173,"15min":2.3860000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.245},"max":{"1min":11.978999999999999,"5min":14.449,"15min":170.072},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.056,"5min":2.7149999999999999,"15min":2.2370000000000001},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":11.935,"5min":75.513999999999996,"15min":169.94499999999999},"last":0.73499999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.4460000000000002,"15min":2.4220000000000002},"min":{"1min":0.42299999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.4889999999999999,"5min":11.499000000000001,"15min":171.036},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.78400000000000003,"5min":2.3079999999999998,"15min":2.5489999999999999},"min":{"1min":0.38600000000000001,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":6.2329999999999997,"5min":14.557,"15min":170.88},"last":0.82499999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":3.5950000000000002,"15min":2.4169999999999998},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.4049999999999998,"5min":132.36099999999999,"15min":132.36099999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.82799999999999996,"5min":3.4350000000000001,"15min":2.548},"min":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":6.1929999999999996,"5min":132.17699999999999,"15min":132.17699999999999},"last":0.71899999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":3.7269999999999999,"15min":3.0990000000000002},"min":{"1min":0.42799999999999999,"5min":0.40400000000000003,"15min":0.34100000000000003},"max":{"1min":6.4189999999999996,"5min":132.136,"15min":174.75999999999999},"last":0.75},{"interface":"front","average":{"1min":0.60199999999999998,"5min":3.9980000000000002,"15min":2.8460000000000001},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":0.85699999999999998,"5min":132.12,"15min":174.779},"last":0.77000000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024225,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1579712,"kb_used_data":761272,"kb_used_omap":5,"kb_used_meta":818426,"kb_avail":92165440,"statfs":{"total":95995035648,"available":94377410560,"internally_reserved":0,"allocated":779542528,"data_stored":778468204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":838068858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:43:35 2023","interfaces":[{"interface":"back","average":{"1min":3.4820000000000002,"5min":6.3010000000000002,"15min":5.1509999999999998},"min":{"1min":0.20899999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":55.008000000000003,"5min":138.536,"15min":167.92699999999999},"last":1.849},{"interface":"front","average":{"1min":3.4209999999999998,"5min":7.0170000000000003,"15min":5.5549999999999997},"min":{"1min":0.25900000000000001,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":55.027999999999999,"5min":138.89599999999999,"15min":169.78999999999999},"last":1.0880000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":4.1550000000000002,"15min":7.5880000000000001},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.0190000000000001,"5min":98.938000000000002,"15min":131.30699999999999},"last":1.74},{"interface":"front","average":{"1min":0.61699999999999999,"5min":4.4009999999999998,"15min":7.9699999999999998},"min":{"1min":0.255,"5min":0.219,"15min":0.219},"max":{"1min":1.8819999999999999,"5min":98.975999999999999,"15min":129.922},"last":1.034}]},{"osd":3,"last update":"Mon Oct 9 04:43:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":6.4050000000000002,"15min":5.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":6.0220000000000002,"5min":156.69800000000001,"15min":213.28399999999999},"last":1.784},{"interface":"front","average":{"1min":1.9470000000000001,"5min":7.7539999999999996,"15min":5.6740000000000004},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.222},"max":{"1min":13.529,"5min":161.40199999999999,"15min":212.27000000000001},"last":1.0640000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.4810000000000001,"15min":2.2970000000000002},"min":{"1min":0.38400000000000001,"5min":0.30199999999999999,"15min":0.25700000000000001},"max":{"1min":1.915,"5min":11.300000000000001,"15min":161.67099999999999},"last":1.8240000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.8460000000000001,"15min":2.5680000000000001},"min":{"1min":0.379,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":1.7090000000000001,"5min":14.109999999999999,"15min":160.74799999999999},"last":3.0739999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":3.2069999999999999,"15min":2.3980000000000001},"min":{"1min":0.376,"5min":0.29199999999999998,"15min":0.27900000000000003},"max":{"1min":1.7949999999999999,"5min":136.322,"15min":136.322},"last":3.0379999999999998},{"interface":"front","average":{"1min":0.63200000000000001,"5min":3.1200000000000001,"15min":2.1789999999999998},"min":{"1min":0.39700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0880000000000001,"5min":136.15799999999999,"15min":136.15799999999999},"last":1.591}]},{"osd":6,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.109,"15min":2.6899999999999999},"min":{"1min":0.39500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":1.76,"5min":78.438999999999993,"15min":108.63800000000001},"last":3.004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":2.661,"15min":2.3119999999999998},"min":{"1min":0.41899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.99399999999999999,"5min":78.617999999999995,"15min":108.76300000000001},"last":3.3180000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":2.7549999999999999,"15min":1.994},"min":{"1min":0.35199999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.0960000000000001,"5min":77.682000000000002,"15min":77.682000000000002},"last":1.869},{"interface":"front","average":{"1min":0.72099999999999997,"5min":2.5179999999999998,"15min":2.0099999999999998},"min":{"1min":0.42099999999999999,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":1.9730000000000001,"5min":77.606999999999999,"15min":77.606999999999999},"last":3.7349999999999999}]}]},{"osd":2,"up_from":221,"seq":949187772503,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1454728,"kb_used_data":1158464,"kb_used_omap":7,"kb_used_meta":296184,"kb_avail":92290424,"statfs":{"total":95995035648,"available":94505394176,"internally_reserved":0,"allocated":1186267136,"data_stored":1185245369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":303292497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.8849999999999998,"15min":7.9169999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.6999999999999993,"5min":13.813000000000001,"15min":72.608000000000004},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.585,"5min":2.3849999999999998,"15min":7.4160000000000004},"min":{"1min":0.251,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.8170000000000002,"5min":14.377000000000001,"15min":72.632000000000005},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":11.619,"5min":5.1859999999999999,"15min":5.2720000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":121.848,"5min":121.848,"15min":121.848},"last":0.41699999999999998},{"interface":"front","average":{"1min":10.805,"5min":5.5819999999999999,"15min":5.7610000000000001},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":121.827,"5min":121.827,"15min":121.827},"last":0.35299999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":15.984,"5min":9.5199999999999996,"15min":6.7770000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":225.85499999999999,"5min":335.928,"15min":335.928},"last":84.840999999999994},{"interface":"front","average":{"1min":16.954000000000001,"5min":10.244,"15min":6.9260000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":233.53299999999999,"5min":335.68599999999998,"15min":335.68599999999998},"last":85.147999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":2.48,"15min":4.1660000000000004},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.315},"max":{"1min":8.8360000000000003,"5min":14.452999999999999,"15min":72.944999999999993},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.4359999999999999,"15min":4.3239999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.6359999999999992,"5min":14.183,"15min":73.054000000000002},"last":0.52800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.71,"15min":3.988},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.2649999999999997,"5min":65.388000000000005,"15min":72.986000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.7090000000000001,"5min":3.0659999999999998,"15min":4.4779999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.6440000000000001,"5min":67.799000000000007,"15min":72.963999999999999},"last":0.54200000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.7770000000000001,"15min":9.3030000000000008},"min":{"1min":0.39600000000000002,"5min":0.39000000000000001,"15min":0.33600000000000002},"max":{"1min":10.263,"5min":34.753,"15min":135.614},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.4830000000000001,"5min":2.5910000000000002,"15min":8.6310000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.35099999999999998},"max":{"1min":7.423,"5min":34.796999999999997,"15min":130.196},"last":0.55400000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3149999999999999,"5min":2.77,"15min":4.5759999999999996},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.378},"max":{"1min":10.228,"5min":28.126999999999999,"15min":73.079999999999998},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.536,"15min":3.9580000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.41499999999999998},"max":{"1min":11.669,"5min":28.106000000000002,"15min":73.069000000000003},"last":0.61499999999999999}]}]},{"osd":3,"up_from":142,"seq":609885356226,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484476,"kb_used_data":979316,"kb_used_omap":2,"kb_used_meta":505149,"kb_avail":92260676,"statfs":{"total":95995035648,"available":94474932224,"internally_reserved":0,"allocated":1002819584,"data_stored":1001805870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":517272629},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":1.9399999999999999,"15min":2.4820000000000002},"min":{"1min":0.23300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.7690000000000001,"5min":17.648,"15min":142.14500000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.1539999999999999,"5min":5.2519999999999998,"15min":3.649},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":10.725,"5min":254.38499999999999,"15min":254.38499999999999},"last":0.27200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":2.7909999999999999,"15min":3.54},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.192},"max":{"1min":9.8450000000000006,"5min":21.879000000000001,"15min":180.02500000000001},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3069999999999999,"15min":3.6389999999999998},"min":{"1min":0.251,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":9.7319999999999993,"5min":21.911999999999999,"15min":184.00899999999999},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:42:57 2023","interfaces":[{"interface":"back","average":{"1min":6.04,"5min":6.173,"15min":3.1749999999999998},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":77.772000000000006,"5min":177.28399999999999,"15min":177.28399999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":7.5359999999999996,"5min":8.5960000000000001,"15min":3.7050000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":77.968999999999994,"5min":255.05199999999999,"15min":255.05199999999999},"last":0.57599999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.6559999999999997,"15min":2.9220000000000002},"min":{"1min":0.316,"5min":0.28999999999999998,"15min":0.27900000000000003},"max":{"1min":9.5109999999999992,"5min":254.072,"15min":254.072},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.9550000000000001,"5min":4.3970000000000002,"15min":2.75},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.307},"max":{"1min":8.2360000000000007,"5min":253.49700000000001,"15min":253.49700000000001},"last":1.754}]},{"osd":5,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":5.2439999999999998,"15min":3.4969999999999999},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.8719999999999999,"5min":253.57900000000001,"15min":253.57900000000001},"last":1.6879999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":5.3040000000000003,"15min":3.573},"min":{"1min":0.40100000000000002,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":8.1039999999999992,"5min":254.191,"15min":254.191},"last":0.51600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":5.532,"15min":3.4039999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.31900000000000001},"max":{"1min":8.1419999999999995,"5min":254.14699999999999,"15min":254.14699999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.819,"5min":2.786,"15min":2.8490000000000002},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":9.5380000000000003,"5min":50.308,"15min":123.505},"last":1.2090000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":2.653,"15min":2.456},"min":{"1min":0.41199999999999998,"5min":0.378,"15min":0.32300000000000001},"max":{"1min":9.4789999999999992,"5min":43.835000000000001,"15min":57.399000000000001},"last":1.2350000000000001},{"interface":"front","average":{"1min":1.627,"5min":2.222,"15min":2.254},"min":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":7.907,"5min":43.781999999999996,"15min":57.447000000000003},"last":1.7310000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019353,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1283244,"kb_used_data":395300,"kb_used_omap":0,"kb_used_meta":887936,"kb_avail":92461908,"statfs":{"total":95995035648,"available":94680993792,"internally_reserved":0,"allocated":404787200,"data_stored":404200053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909246464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":2.0659999999999998,"15min":5.351},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.23000000000000001},"max":{"1min":1.405,"5min":23.114999999999998,"15min":219.577},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.8169999999999999,"15min":5.0419999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.255},"max":{"1min":1.4950000000000001,"5min":13.992000000000001,"15min":218.96000000000001},"last":0.43099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":3.3690000000000002,"15min":5.1820000000000004},"min":{"1min":0.28599999999999998,"5min":0.255,"15min":0.24099999999999999},"max":{"1min":37.356000000000002,"5min":56.122,"15min":142.74799999999999},"last":2.5019999999999998},{"interface":"front","average":{"1min":2.7429999999999999,"5min":3.5030000000000001,"15min":5.3289999999999997},"min":{"1min":0.36199999999999999,"5min":0.28100000000000003,"15min":0.27300000000000002},"max":{"1min":37.311999999999998,"5min":56.043999999999997,"15min":139.09700000000001},"last":0.34699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:43:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2189999999999999,"5min":5.6829999999999998,"15min":4.3239999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.372,"5min":195.267,"15min":195.267},"last":1.3540000000000001},{"interface":"front","average":{"1min":2.2250000000000001,"5min":5.4329999999999998,"15min":3.8279999999999998},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.3620000000000001,"5min":195.15799999999999,"15min":195.15799999999999},"last":0.46400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":7.6150000000000002,"5min":8.3160000000000007,"15min":7.4859999999999998},"min":{"1min":0.38400000000000001,"5min":0.32600000000000001,"15min":0.27900000000000003},"max":{"1min":146.02199999999999,"5min":357.125,"15min":357.125},"last":0.48599999999999999},{"interface":"front","average":{"1min":7.9699999999999998,"5min":8.1500000000000004,"15min":7.6120000000000001},"min":{"1min":0.38500000000000001,"5min":0.314,"15min":0.312},"max":{"1min":147.75700000000001,"5min":359.02800000000002,"15min":359.02800000000002},"last":1.2150000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.95499999999999996,"15min":1.8580000000000001},"min":{"1min":0.312,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":5.8540000000000001,"5min":9.5609999999999999,"15min":107.476},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0680000000000001,"15min":2.0779999999999998},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.23300000000000001},"max":{"1min":1.6259999999999999,"5min":15.394,"15min":107.414},"last":0.39300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.123,"15min":1.8859999999999999},"min":{"1min":0.311,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":5.8109999999999999,"5min":9.6099999999999994,"15min":65.75},"last":1.2689999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.133,"15min":1.861},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":2.2309999999999999,"5min":9.782,"15min":65.849999999999994},"last":0.495}]},{"osd":7,"last update":"Mon Oct 9 04:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.0680000000000001,"15min":1.6180000000000001},"min":{"1min":0.37,"5min":0.34399999999999997,"15min":0.28000000000000003},"max":{"1min":1.8320000000000001,"5min":8.6120000000000001,"15min":76.283000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.3069999999999999,"15min":1.6599999999999999},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.32100000000000001},"max":{"1min":4.2469999999999999,"5min":9.6310000000000002,"15min":76.259},"last":1.2509999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757717,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1159004,"kb_used_data":260500,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":92586148,"statfs":{"total":95995035648,"available":94808215552,"internally_reserved":0,"allocated":266752000,"data_stored":266176981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":2.8769999999999998,"15min":5.2359999999999998},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.22},"max":{"1min":0.79300000000000004,"5min":75.855999999999995,"15min":210.85499999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":3.448,"15min":5.359},"min":{"1min":0.30399999999999999,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":0.88,"5min":75.816000000000003,"15min":210.23500000000001},"last":0.45000000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":3.516,"15min":4.5439999999999996},"min":{"1min":0.33500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.413,"5min":117.84099999999999,"15min":145.577},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.6139999999999999,"15min":4.9249999999999998},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.83599999999999997,"5min":117.90600000000001,"15min":145.792},"last":0.44}]},{"osd":2,"last update":"Mon Oct 9 04:43:58 2023","interfaces":[{"interface":"back","average":{"1min":4.7560000000000002,"5min":3.8159999999999998,"15min":5.7880000000000003},"min":{"1min":0.36399999999999999,"5min":0.29099999999999998,"15min":0.26800000000000002},"max":{"1min":66.930000000000007,"5min":66.984999999999999,"15min":90.763999999999996},"last":66.930000000000007},{"interface":"front","average":{"1min":4.8879999999999999,"5min":4.0389999999999997,"15min":5.9390000000000001},"min":{"1min":0.39200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":67.085999999999999,"5min":67.686000000000007,"15min":91.340000000000003},"last":67.085999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:43:19 2023","interfaces":[{"interface":"back","average":{"1min":5.1710000000000003,"5min":8.5860000000000003,"15min":7.0789999999999997},"min":{"1min":0.31,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":67.129999999999995,"5min":268.63299999999998,"15min":268.63299999999998},"last":0.433},{"interface":"front","average":{"1min":5.4299999999999997,"5min":8.3550000000000004,"15min":6.8810000000000002},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":67.189999999999998,"5min":268.92500000000001,"15min":268.92500000000001},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:43:19 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.64100000000000001,"15min":2.3370000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.248},"max":{"1min":0.70799999999999996,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.206,"15min":2.7799999999999998},"min":{"1min":0.26900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":3.1819999999999999,"5min":7.4859999999999998,"15min":361.71300000000002},"last":0.40300000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.65900000000000003,"15min":1.1759999999999999},"min":{"1min":0.316,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.71299999999999997,"5min":5.7640000000000002,"15min":11.192},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.1799999999999999,"15min":1.327},"min":{"1min":0.40400000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6619999999999999,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.35299999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.2769999999999999,"15min":1.3400000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":1.78,"5min":13.268000000000001,"15min":13.268000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.998,"15min":1.2410000000000001},"min":{"1min":0.36799999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.734,"5min":6.5129999999999999,"15min":15.257999999999999},"last":0.52400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":787152896,"data_stored":787152896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":608829440,"data_stored":608829440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1230110720,"data_stored":1230110720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1022820352,"data_stored":1022820352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":402718720,"data_stored":402718720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":264699904,"data_stored":264699904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":943063040,"data_stored":943063040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":930676736,"data_stored":930676736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:44:06.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:06.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:06.233 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:06.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:06.380 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:06.492 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:06.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:06.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:06.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:06.653 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:06 smithi029 ceph-mon[216903]: pgmap v1113: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:44:06.656 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:06 smithi029 ceph-mon[223858]: pgmap v1113: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:44:06.688 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:06.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:06.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:06.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:07.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:06 smithi177 ceph-mon[195797]: pgmap v1113: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-10-09T04:44:07.031 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:07.097 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:07.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:07.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 43260 43244 122.827 93.1875 0.0108327 0.00813515 2023-10-09T04:44:07.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:07.253 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:07.329 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:07.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:07.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:07.477 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:07.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:07.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:07.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:07.748 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:07.861 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:07.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:07.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:08.026 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:08.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:08.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:08.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 44887 44871 121.907 101.688 0.0129363 0.00819884 2023-10-09T04:44:08.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:44:08.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:08.234 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:08.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:44:08.279 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:08.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:08.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:08.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:08.410 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:08.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:08.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:08.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:08.594 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:08.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:08 smithi177 ceph-mon[195797]: pgmap v1114: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:44:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:08 smithi029 ceph-mon[216903]: pgmap v1114: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:44:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:08 smithi029 ceph-mon[223858]: pgmap v1114: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-09T04:44:09.038 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:09.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:09.110 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 45773 45757 119.135 55.375 0.00424045 0.00838925 2023-10-09T04:44:09.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:09.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:44:09.218 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:09.420 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:09.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:09.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:09.523 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:09.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:09.551 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:09.649 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:10.113 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 47005 46989 117.441 77 0.0099223 0.00850947 2023-10-09T04:44:10.150 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:10.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:10.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:10.290 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:10.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:10.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:10.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:10.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:10.522 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:10.938 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:44:10.938 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:44:10.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:10 smithi029 ceph-mon[216903]: pgmap v1115: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:44:10.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:10 smithi029 ceph-mon[223858]: pgmap v1115: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:44:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:10.940 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:10.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:10 smithi177 ceph-mon[195797]: pgmap v1115: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-09T04:44:11.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:11.080 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:11.115 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 48286 48270 116.003 80.0625 0.00277481 0.00858095 2023-10-09T04:44:12.112 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 49719 49703 115.024 89.5625 0.00677365 0.00868786 2023-10-09T04:44:12.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:12 smithi029 ceph-mon[216903]: pgmap v1116: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:44:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:12 smithi029 ceph-mon[223858]: pgmap v1116: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:44:12.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:44:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:44:12] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.43.0" 2023-10-09T04:44:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:12 smithi177 ceph-mon[195797]: pgmap v1116: 89 pgs: 89 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-10-09T04:44:13.113 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 51107 51091 114.014 86.75 0.0163627 0.00876624 2023-10-09T04:44:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:13.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:14.113 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 52088 52072 112.196 61.3125 0.0115091 0.00890784 2023-10-09T04:44:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:14 smithi029 ceph-mon[216903]: pgmap v1117: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T04:44:14.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:14 smithi029 ceph-mon[223858]: pgmap v1117: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T04:44:14.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:44:15.006 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:15.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:14 smithi177 ceph-mon[195797]: pgmap v1117: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-09T04:44:15.113 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 53300 53284 110.982 75.75 0.0121937 0.00900569 2023-10-09T04:44:15.122 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:15.155 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:15.209 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:15.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:15.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:15.284 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:15.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:44:15.411 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:15.667 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:44:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:15.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:44:15.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:15.813 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:15.827 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:44:15.862 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:44:15.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:15.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:15.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:15.913 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:44:15.981 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:44:15.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:16.014 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:16.018 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:44:16.056 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:44:16.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:16.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:16.102 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:44:16.113 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 54350 54334 109.518 65.625 0.00982318 0.00912534 2023-10-09T04:44:16.159 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:44:16.162 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.162 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.162 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.162 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.163 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.163 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.163 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.163 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:44:16.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:16.214 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:16.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:16.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:16.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:16.421 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:16.489 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:16.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:16.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:16.646 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:16.652 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:44:16.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:16 smithi029 ceph-mon[223858]: pgmap v1118: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T04:44:16.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2025256388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:16.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1749287626' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:16.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:16 smithi029 ceph-mon[216903]: pgmap v1118: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T04:44:16.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2025256388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:16.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1749287626' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:16.709 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:16.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:16.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:16 smithi177 ceph-mon[195797]: pgmap v1118: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-09T04:44:16.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2025256388' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:16.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1749287626' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:44:16.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:16.847 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:16.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:16.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:17.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:17.068 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:17.079 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:17.115 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 55514 55498 108.363 72.75 0.0118541 0.00922113 2023-10-09T04:44:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1749287626' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:44:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[216903]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1749287626' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: failed to encode map e290 with expected crc 2023-10-09T04:44:17.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:17 smithi029 ceph-mon[223858]: failed to encode map e290 with expected crc 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1749287626' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: osdmap e290: 8 total, 8 up, 8 in 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: failed to encode map e290 with expected crc 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: failed to encode map e290 with expected crc 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: failed to encode map e290 with expected crc 2023-10-09T04:44:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:17 smithi177 ceph-mon[195797]: failed to encode map e290 with expected crc 2023-10-09T04:44:18.115 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 56703 56687 107.331 74.3125 0.00438898 0.00927375 2023-10-09T04:44:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:18 smithi029 ceph-mon[216903]: pgmap v1120: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:44:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:18 smithi029 ceph-mon[223858]: pgmap v1120: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:44:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:18 smithi177 ceph-mon[195797]: pgmap v1120: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-09T04:44:19.118 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 57199 57183 105.077 31 0.0129309 0.00951153 2023-10-09T04:44:19.383 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:19.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:19.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:19.558 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:19.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:19.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:19.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:19.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:19.819 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:20.118 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 58431 58415 104.275 77 0.0053507 0.00958405 2023-10-09T04:44:20.686 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:20.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:20.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:20.834 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:20.905 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:20.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:20 smithi177 ceph-mon[195797]: pgmap v1121: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:44:20.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:20.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:20 smithi029 ceph-mon[216903]: pgmap v1121: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:44:20.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:20 smithi029 ceph-mon[223858]: pgmap v1121: 89 pgs: 89 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:44:21.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:21.056 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:21.121 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 59759 59743 103.684 83 0.00619134 0.00964052 2023-10-09T04:44:21.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:21.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:21.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:21.283 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:21.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:21.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:21.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:21.471 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:21.514 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:21.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:21.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:21.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:21.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:44:21.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:21.779 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:21.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:44:22.118 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 60861 60845 102.743 68.875 0.00482106 0.00971042 2023-10-09T04:44:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:22 smithi029 ceph-mon[216903]: pgmap v1122: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:44:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:22 smithi029 ceph-mon[223858]: pgmap v1122: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:44:22.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:44:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:44:22] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2023-10-09T04:44:22.927 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:22.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:22 smithi177 ceph-mon[195797]: pgmap v1122: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-09T04:44:23.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:23.109 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:23.118 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 61860 61844 101.683 62.4375 0.0124533 0.00982608 2023-10-09T04:44:23.343 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:23.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:23.373 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:23.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:23.492 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:23.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:23.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:23.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:23.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:23.698 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:23.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:23.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:23.824 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:44:23.841 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:44:23.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:23.873 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:62629 12.6 12:68018ea0:::benchmark_data_smithi029_442458_object62628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:44:23.891702+0000", 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.010513878000000001, 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010549227, 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:44:23.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 62629 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891702+0000", 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891702+0000", 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891700+0000", 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999971 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:23.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891737+0000", 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7255000000000002e-05 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891777+0000", 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0212000000000001e-05 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891791+0000", 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.362e-05 2023-10-09T04:44:23.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.891983+0000", 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019207800000000001 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.892173+0000", 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000190514 2023-10-09T04:44:23.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.892328+0000", 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015509000000000001 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.898258+0000", 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059297450000000002 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.914 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:62627 12.5 e290/284 12:a2e0d5ea:::benchmark_data_smithi029_442458_object62626:head v 290'3942, mlcod=290'3940)", 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:44:23.893052+0000", 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0091638810000000005, 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092954179999999997, 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:23.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893052+0000", 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893052+0000", 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893051+0000", 2023-10-09T04:44:23.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893090+0000", 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9190000000000003e-05 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893107+0000", 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6637e-05 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893115+0000", 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.559e-06 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893848+0000", 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00073314400000000003 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.893872+0000", 2023-10-09T04:44:23.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.427e-05 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:62631 12.5 e290/284 12:a5175439:::benchmark_data_smithi029_442458_object62630:head v 290'3943, mlcod=290'3940)", 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:44:23.894585+0000", 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.007631054, 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078293779999999997, 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:44:23.919 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.894585+0000", 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.894585+0000", 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:23.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.894585+0000", 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.894618+0000", 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3732999999999997e-05 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.894631+0000", 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2431e-05 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.894639+0000", 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8679999999999999e-06 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.896061+0000", 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014221749999999999 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:23.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:23.896082+0000", 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0812999999999999e-05 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 3 2023-10-09T04:44:23.923 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:44:23.956 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:24.119 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 62850 62834 100.659 61.875 0.0167587 0.00992975 2023-10-09T04:44:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:24 smithi029 ceph-mon[216903]: pgmap v1123: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:44:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:24 smithi029 ceph-mon[216903]: from='client.46133 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:24.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:24 smithi029 ceph-mon[223858]: pgmap v1123: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:44:24.956 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:24 smithi029 ceph-mon[223858]: from='client.46133 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:24 smithi177 ceph-mon[195797]: pgmap v1123: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-09T04:44:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:24 smithi177 ceph-mon[195797]: from='client.46133 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:44:25.120 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:44:25.120128+0000 min lat: 0.00124565 max lat: 0.333044 avg lat: 0.00998114 2023-10-09T04:44:25.120 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:44:25.120 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 64122 64106 100.13 79.5 0.00409954 0.00998114 2023-10-09T04:44:25.878 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:25.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:25.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:26.030 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:26.104 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:26.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:26.119 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 65023 65007 99.0617 56.3125 0.0210433 0.0100901 2023-10-09T04:44:26.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:26.254 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:26.321 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:26.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:26.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:44:26.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:26.460 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:26.512 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:26.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:26.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:26.656 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:26.719 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:26.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:26.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:26.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:26.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:26.905 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:26 smithi177 ceph-mon[195797]: pgmap v1124: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:44:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:26.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:27.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:27.045 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:27.114 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:27.122 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 65788 65772 97.8374 47.8125 0.0130252 0.0102159 2023-10-09T04:44:27.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:27.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:26 smithi029 ceph-mon[223858]: pgmap v1124: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:44:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:26 smithi029 ceph-mon[216903]: pgmap v1124: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:44:27.223 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:27.268 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:27.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:27.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:27.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:27.490 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:27.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:27.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:27.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:27.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:27.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:27.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:27.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:27.857 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:27.891 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:27.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:27.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:28.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:28.122 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 66656 66640 96.824 54.25 0.00688167 0.0103221 2023-10-09T04:44:28.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:28.344 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1123,"stamp":"2023-10-09T04:44:23.209978+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3844020128,"num_objects":58670,"num_object_clones":0,"num_object_copies":176010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58670,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":58946,"num_write_kb":3755789,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2957464,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58862,"ondisk_log_size":58862,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17240884,"kb_used_data":11329076,"kb_used_omap":20,"kb_used_meta":5911723,"kb_avail":732720332,"statfs":{"total":767960285184,"available":750305619968,"internally_reserved":0,"allocated":11600973824,"data_stored":11594319383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":6053605006},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,7,10,4,1,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":743571456,"num_objects":11346,"num_object_clones":0,"num_object_copies":34038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11346,"num_write_kb":726144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006082"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563473+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:44:01.563473+0000","last_peered":"2023-10-09T04:44:01.563473+0000","last_clean":"2023-10-09T04:44:01.563473+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:44:01.563473+0000","last_undegraded":"2023-10-09T04:44:01.563473+0000","last_fullsized":"2023-10-09T04:44:01.563473+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":243,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566433+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:44:01.566433+0000","last_peered":"2023-10-09T04:44:01.566433+0000","last_clean":"2023-10-09T04:44:01.566433+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:44:01.566433+0000","last_undegraded":"2023-10-09T04:44:01.566433+0000","last_fullsized":"2023-10-09T04:44:01.566433+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":276,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.553376+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:44:01.553376+0000","last_peered":"2023-10-09T04:44:01.553376+0000","last_clean":"2023-10-09T04:44:01.553376+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:44:01.553376+0000","last_undegraded":"2023-10-09T04:44:01.553376+0000","last_fullsized":"2023-10-09T04:44:01.553376+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":203,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560655+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:44:01.560655+0000","last_peered":"2023-10-09T04:44:01.560655+0000","last_clean":"2023-10-09T04:44:01.560655+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:44:01.560655+0000","last_undegraded":"2023-10-09T04:44:01.560655+0000","last_fullsized":"2023-10-09T04:44:01.560655+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":279,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.564664+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:44:01.564664+0000","last_peered":"2023-10-09T04:44:01.564664+0000","last_clean":"2023-10-09T04:44:01.564664+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:44:01.564664+0000","last_undegraded":"2023-10-09T04:44:01.564664+0000","last_fullsized":"2023-10-09T04:44:01.564664+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":246,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566200+0000","last_change":"2023-10-09T04:24:07.959053+0000","last_active":"2023-10-09T04:44:01.566200+0000","last_peered":"2023-10-09T04:44:01.566200+0000","last_clean":"2023-10-09T04:44:01.566200+0000","last_became_active":"2023-10-09T04:24:07.958670+0000","last_became_peered":"2023-10-09T04:24:07.958670+0000","last_unstale":"2023-10-09T04:44:01.566200+0000","last_undegraded":"2023-10-09T04:44:01.566200+0000","last_fullsized":"2023-10-09T04:44:01.566200+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:02.572060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":202,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570269+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:44:01.570269+0000","last_peered":"2023-10-09T04:44:01.570269+0000","last_clean":"2023-10-09T04:44:01.570269+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:44:01.570269+0000","last_undegraded":"2023-10-09T04:44:01.570269+0000","last_fullsized":"2023-10-09T04:44:01.570269+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":244,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:16.647416+0000","last_change":"2023-10-09T04:27:57.987735+0000","last_active":"2023-10-09T04:44:16.647416+0000","last_peered":"2023-10-09T04:44:16.647416+0000","last_clean":"2023-10-09T04:44:16.647416+0000","last_became_active":"2023-10-09T04:27:57.987437+0000","last_became_peered":"2023-10-09T04:27:57.987437+0000","last_unstale":"2023-10-09T04:44:16.647416+0000","last_undegraded":"2023-10-09T04:44:16.647416+0000","last_fullsized":"2023-10-09T04:44:16.647416+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:43.040965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570150+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:44:01.570150+0000","last_peered":"2023-10-09T04:44:01.570150+0000","last_clean":"2023-10-09T04:44:01.570150+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:44:01.570150+0000","last_undegraded":"2023-10-09T04:44:01.570150+0000","last_fullsized":"2023-10-09T04:44:01.570150+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":217,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:16.648803+0000","last_change":"2023-10-09T04:27:57.990152+0000","last_active":"2023-10-09T04:44:16.648803+0000","last_peered":"2023-10-09T04:44:16.648803+0000","last_clean":"2023-10-09T04:44:16.648803+0000","last_became_active":"2023-10-09T04:27:57.989825+0000","last_became_peered":"2023-10-09T04:27:57.989825+0000","last_unstale":"2023-10-09T04:44:16.648803+0000","last_undegraded":"2023-10-09T04:44:16.648803+0000","last_fullsized":"2023-10-09T04:44:16.648803+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:14.414974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":284,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560522+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:44:01.560522+0000","last_peered":"2023-10-09T04:44:01.560522+0000","last_clean":"2023-10-09T04:44:01.560522+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:44:01.560522+0000","last_undegraded":"2023-10-09T04:44:01.560522+0000","last_fullsized":"2023-10-09T04:44:01.560522+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":243,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565851+0000","last_change":"2023-10-09T04:39:40.406537+0000","last_active":"2023-10-09T04:44:01.565851+0000","last_peered":"2023-10-09T04:44:01.565851+0000","last_clean":"2023-10-09T04:44:01.565851+0000","last_became_active":"2023-10-09T04:39:40.406281+0000","last_became_peered":"2023-10-09T04:39:40.406281+0000","last_unstale":"2023-10-09T04:44:01.565851+0000","last_undegraded":"2023-10-09T04:44:01.565851+0000","last_fullsized":"2023-10-09T04:44:01.565851+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:09.883302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":48,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563368+0000","last_change":"2023-10-09T04:41:02.634559+0000","last_active":"2023-10-09T04:44:01.563368+0000","last_peered":"2023-10-09T04:44:01.563368+0000","last_clean":"2023-10-09T04:44:01.563368+0000","last_became_active":"2023-10-09T04:41:02.624597+0000","last_became_peered":"2023-10-09T04:41:02.624597+0000","last_unstale":"2023-10-09T04:44:01.563368+0000","last_undegraded":"2023-10-09T04:44:01.563368+0000","last_fullsized":"2023-10-09T04:44:01.563368+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:57.939757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563659+0000","last_change":"2023-10-09T04:24:07.915134+0000","last_active":"2023-10-09T04:44:01.563659+0000","last_peered":"2023-10-09T04:44:01.563659+0000","last_clean":"2023-10-09T04:44:01.563659+0000","last_became_active":"2023-10-09T04:24:07.914755+0000","last_became_peered":"2023-10-09T04:24:07.914755+0000","last_unstale":"2023-10-09T04:44:01.563659+0000","last_undegraded":"2023-10-09T04:44:01.563659+0000","last_fullsized":"2023-10-09T04:44:01.563659+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:54.303815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":300,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560788+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:44:01.560788+0000","last_peered":"2023-10-09T04:44:01.560788+0000","last_clean":"2023-10-09T04:44:01.560788+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:44:01.560788+0000","last_undegraded":"2023-10-09T04:44:01.560788+0000","last_fullsized":"2023-10-09T04:44:01.560788+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":244,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:16.648959+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:44:16.648959+0000","last_peered":"2023-10-09T04:44:16.648959+0000","last_clean":"2023-10-09T04:44:16.648959+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:44:16.648959+0000","last_undegraded":"2023-10-09T04:44:16.648959+0000","last_fullsized":"2023-10-09T04:44:16.648959+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":202,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560261+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:44:01.560261+0000","last_peered":"2023-10-09T04:44:01.560261+0000","last_clean":"2023-10-09T04:44:01.560261+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:44:01.560261+0000","last_undegraded":"2023-10-09T04:44:01.560261+0000","last_fullsized":"2023-10-09T04:44:01.560261+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570113+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:44:01.570113+0000","last_peered":"2023-10-09T04:44:01.570113+0000","last_clean":"2023-10-09T04:44:01.570113+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:44:01.570113+0000","last_undegraded":"2023-10-09T04:44:01.570113+0000","last_fullsized":"2023-10-09T04:44:01.570113+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":232,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560154+0000","last_change":"2023-10-09T04:07:34.630710+0000","last_active":"2023-10-09T04:44:01.560154+0000","last_peered":"2023-10-09T04:44:01.560154+0000","last_clean":"2023-10-09T04:44:01.560154+0000","last_became_active":"2023-10-09T04:07:34.630426+0000","last_became_peered":"2023-10-09T04:07:34.630426+0000","last_unstale":"2023-10-09T04:44:01.560154+0000","last_undegraded":"2023-10-09T04:44:01.560154+0000","last_fullsized":"2023-10-09T04:44:01.560154+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:09.124035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":205,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:02.703537+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:44:02.703537+0000","last_peered":"2023-10-09T04:44:02.703537+0000","last_clean":"2023-10-09T04:44:02.703537+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:44:02.703537+0000","last_undegraded":"2023-10-09T04:44:02.703537+0000","last_fullsized":"2023-10-09T04:44:02.703537+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":295,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565528+0000","last_change":"2023-10-09T04:29:51.068634+0000","last_active":"2023-10-09T04:44:01.565528+0000","last_peered":"2023-10-09T04:44:01.565528+0000","last_clean":"2023-10-09T04:44:01.565528+0000","last_became_active":"2023-10-09T04:29:51.059294+0000","last_became_peered":"2023-10-09T04:29:51.059294+0000","last_unstale":"2023-10-09T04:44:01.565528+0000","last_undegraded":"2023-10-09T04:44:01.565528+0000","last_fullsized":"2023-10-09T04:44:01.565528+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:11.642296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":243,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566921+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:44:01.566921+0000","last_peered":"2023-10-09T04:44:01.566921+0000","last_clean":"2023-10-09T04:44:01.566921+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:44:01.566921+0000","last_undegraded":"2023-10-09T04:44:01.566921+0000","last_fullsized":"2023-10-09T04:44:01.566921+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"290'3686","reported_seq":3700,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.374087+0000","last_change":"2023-10-09T04:43:34.548310+0000","last_active":"2023-10-09T04:44:19.374087+0000","last_peered":"2023-10-09T04:44:19.374087+0000","last_clean":"2023-10-09T04:44:19.374087+0000","last_became_active":"2023-10-09T04:43:34.548012+0000","last_became_peered":"2023-10-09T04:43:34.548012+0000","last_unstale":"2023-10-09T04:44:19.374087+0000","last_undegraded":"2023-10-09T04:44:19.374087+0000","last_fullsized":"2023-10-09T04:44:19.374087+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3686,"log_dups_size":0,"ondisk_log_size":3686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:10.801596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241565696,"num_objects":3686,"num_object_clones":0,"num_object_copies":11058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3686,"num_write_kb":235904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":306,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560574+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:44:01.560574+0000","last_peered":"2023-10-09T04:44:01.560574+0000","last_clean":"2023-10-09T04:44:01.560574+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:44:01.560574+0000","last_undegraded":"2023-10-09T04:44:01.560574+0000","last_fullsized":"2023-10-09T04:44:01.560574+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":232,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.569680+0000","last_change":"2023-10-09T04:07:36.029518+0000","last_active":"2023-10-09T04:44:01.569680+0000","last_peered":"2023-10-09T04:44:01.569680+0000","last_clean":"2023-10-09T04:44:01.569680+0000","last_became_active":"2023-10-09T04:07:36.029128+0000","last_became_peered":"2023-10-09T04:07:36.029128+0000","last_unstale":"2023-10-09T04:44:01.569680+0000","last_undegraded":"2023-10-09T04:44:01.569680+0000","last_fullsized":"2023-10-09T04:44:01.569680+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:44.820476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"290'3619","reported_seq":3633,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.382410+0000","last_change":"2023-10-09T04:43:34.548528+0000","last_active":"2023-10-09T04:44:19.382410+0000","last_peered":"2023-10-09T04:44:19.382410+0000","last_clean":"2023-10-09T04:44:19.382410+0000","last_became_active":"2023-10-09T04:43:34.548236+0000","last_became_peered":"2023-10-09T04:43:34.548236+0000","last_unstale":"2023-10-09T04:44:19.382410+0000","last_undegraded":"2023-10-09T04:44:19.382410+0000","last_fullsized":"2023-10-09T04:44:19.382410+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3619,"log_dups_size":0,"ondisk_log_size":3619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:29:24.666864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237174784,"num_objects":3619,"num_object_clones":0,"num_object_copies":10857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3619,"num_write_kb":231616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":279,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560583+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:44:01.560583+0000","last_peered":"2023-10-09T04:44:01.560583+0000","last_clean":"2023-10-09T04:44:01.560583+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:44:01.560583+0000","last_undegraded":"2023-10-09T04:44:01.560583+0000","last_fullsized":"2023-10-09T04:44:01.560583+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":267,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563244+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:44:01.563244+0000","last_peered":"2023-10-09T04:44:01.563244+0000","last_clean":"2023-10-09T04:44:01.563244+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:44:01.563244+0000","last_undegraded":"2023-10-09T04:44:01.563244+0000","last_fullsized":"2023-10-09T04:44:01.563244+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"290'3597","reported_seq":3611,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.366755+0000","last_change":"2023-10-09T04:43:34.547460+0000","last_active":"2023-10-09T04:44:19.366755+0000","last_peered":"2023-10-09T04:44:19.366755+0000","last_clean":"2023-10-09T04:44:19.366755+0000","last_became_active":"2023-10-09T04:43:34.547150+0000","last_became_peered":"2023-10-09T04:43:34.547150+0000","last_unstale":"2023-10-09T04:44:19.366755+0000","last_undegraded":"2023-10-09T04:44:19.366755+0000","last_fullsized":"2023-10-09T04:44:19.366755+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3597,"log_dups_size":0,"ondisk_log_size":3597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:43.085276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235732992,"num_objects":3597,"num_object_clones":0,"num_object_copies":10791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3597,"num_write_kb":230208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":233,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560361+0000","last_change":"2023-10-09T04:07:34.634923+0000","last_active":"2023-10-09T04:44:01.560361+0000","last_peered":"2023-10-09T04:44:01.560361+0000","last_clean":"2023-10-09T04:44:01.560361+0000","last_became_active":"2023-10-09T04:07:34.634666+0000","last_became_peered":"2023-10-09T04:07:34.634666+0000","last_unstale":"2023-10-09T04:44:01.560361+0000","last_undegraded":"2023-10-09T04:44:01.560361+0000","last_fullsized":"2023-10-09T04:44:01.560361+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:48.458239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":273,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565568+0000","last_change":"2023-10-09T04:36:47.545147+0000","last_active":"2023-10-09T04:44:01.565568+0000","last_peered":"2023-10-09T04:44:01.565568+0000","last_clean":"2023-10-09T04:44:01.565568+0000","last_became_active":"2023-10-09T04:36:47.544827+0000","last_became_peered":"2023-10-09T04:36:47.544827+0000","last_unstale":"2023-10-09T04:44:01.565568+0000","last_undegraded":"2023-10-09T04:44:01.565568+0000","last_fullsized":"2023-10-09T04:44:01.565568+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:24.974414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"290'3634","reported_seq":3648,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.375306+0000","last_change":"2023-10-09T04:43:34.881088+0000","last_active":"2023-10-09T04:44:19.375306+0000","last_peered":"2023-10-09T04:44:19.375306+0000","last_clean":"2023-10-09T04:44:19.375306+0000","last_became_active":"2023-10-09T04:43:34.880601+0000","last_became_peered":"2023-10-09T04:43:34.880601+0000","last_unstale":"2023-10-09T04:44:19.375306+0000","last_undegraded":"2023-10-09T04:44:19.375306+0000","last_fullsized":"2023-10-09T04:44:19.375306+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3634,"log_dups_size":0,"ondisk_log_size":3634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:27.274950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238157824,"num_objects":3634,"num_object_clones":0,"num_object_copies":10902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3634,"num_write_kb":232576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570172+0000","last_change":"2023-10-09T04:27:57.992844+0000","last_active":"2023-10-09T04:44:01.570172+0000","last_peered":"2023-10-09T04:44:01.570172+0000","last_clean":"2023-10-09T04:44:01.570172+0000","last_became_active":"2023-10-09T04:27:57.992312+0000","last_became_peered":"2023-10-09T04:27:57.992312+0000","last_unstale":"2023-10-09T04:44:01.570172+0000","last_undegraded":"2023-10-09T04:44:01.570172+0000","last_fullsized":"2023-10-09T04:44:01.570172+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:41.347283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":147,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.569740+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:44:01.569740+0000","last_peered":"2023-10-09T04:44:01.569740+0000","last_clean":"2023-10-09T04:44:01.569740+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:44:01.569740+0000","last_undegraded":"2023-10-09T04:44:01.569740+0000","last_fullsized":"2023-10-09T04:44:01.569740+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.4","version":"290'3582","reported_seq":3596,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.358344+0000","last_change":"2023-10-09T04:43:34.548768+0000","last_active":"2023-10-09T04:44:19.358344+0000","last_peered":"2023-10-09T04:44:19.358344+0000","last_clean":"2023-10-09T04:44:19.358344+0000","last_became_active":"2023-10-09T04:43:34.548440+0000","last_became_peered":"2023-10-09T04:43:34.548440+0000","last_unstale":"2023-10-09T04:44:19.358344+0000","last_undegraded":"2023-10-09T04:44:19.358344+0000","last_fullsized":"2023-10-09T04:44:19.358344+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3582,"log_dups_size":0,"ondisk_log_size":3582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:54.014901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234749952,"num_objects":3582,"num_object_clones":0,"num_object_copies":10746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3582,"num_write_kb":229248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":288,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.564803+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:44:01.564803+0000","last_peered":"2023-10-09T04:44:01.564803+0000","last_clean":"2023-10-09T04:44:01.564803+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:44:01.564803+0000","last_undegraded":"2023-10-09T04:44:01.564803+0000","last_fullsized":"2023-10-09T04:44:01.564803+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563453+0000","last_change":"2023-10-09T04:41:02.624962+0000","last_active":"2023-10-09T04:44:01.563453+0000","last_peered":"2023-10-09T04:44:01.563453+0000","last_clean":"2023-10-09T04:44:01.563453+0000","last_became_active":"2023-10-09T04:41:02.624555+0000","last_became_peered":"2023-10-09T04:41:02.624555+0000","last_unstale":"2023-10-09T04:44:01.563453+0000","last_undegraded":"2023-10-09T04:44:01.563453+0000","last_fullsized":"2023-10-09T04:44:01.563453+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:48.506794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"290'3598","reported_seq":3612,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.381171+0000","last_change":"2023-10-09T04:43:34.548194+0000","last_active":"2023-10-09T04:44:19.381171+0000","last_peered":"2023-10-09T04:44:19.381171+0000","last_clean":"2023-10-09T04:44:19.381171+0000","last_became_active":"2023-10-09T04:43:34.547902+0000","last_became_peered":"2023-10-09T04:43:34.547902+0000","last_unstale":"2023-10-09T04:44:19.381171+0000","last_undegraded":"2023-10-09T04:44:19.381171+0000","last_fullsized":"2023-10-09T04:44:19.381171+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3598,"log_dups_size":0,"ondisk_log_size":3598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:14.652242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235798528,"num_objects":3598,"num_object_clones":0,"num_object_copies":10794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3598,"num_write_kb":230272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":267,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565720+0000","last_change":"2023-10-09T04:41:02.603773+0000","last_active":"2023-10-09T04:44:01.565720+0000","last_peered":"2023-10-09T04:44:01.565720+0000","last_clean":"2023-10-09T04:44:01.565720+0000","last_became_active":"2023-10-09T04:41:02.603407+0000","last_became_peered":"2023-10-09T04:41:02.603407+0000","last_unstale":"2023-10-09T04:44:01.565720+0000","last_undegraded":"2023-10-09T04:44:01.565720+0000","last_fullsized":"2023-10-09T04:44:01.565720+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:18.401503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563402+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:44:01.563402+0000","last_peered":"2023-10-09T04:44:01.563402+0000","last_clean":"2023-10-09T04:44:01.563402+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:44:01.563402+0000","last_undegraded":"2023-10-09T04:44:01.563402+0000","last_fullsized":"2023-10-09T04:44:01.563402+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"290'3763","reported_seq":3777,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:22.049355+0000","last_change":"2023-10-09T04:43:34.881151+0000","last_active":"2023-10-09T04:44:22.049355+0000","last_peered":"2023-10-09T04:44:22.049355+0000","last_clean":"2023-10-09T04:44:22.049355+0000","last_became_active":"2023-10-09T04:43:34.880832+0000","last_became_peered":"2023-10-09T04:43:34.880832+0000","last_unstale":"2023-10-09T04:44:22.049355+0000","last_undegraded":"2023-10-09T04:44:22.049355+0000","last_fullsized":"2023-10-09T04:44:22.049355+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3763,"log_dups_size":0,"ondisk_log_size":3763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:48.504273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246611968,"num_objects":3763,"num_object_clones":0,"num_object_copies":11289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3763,"num_write_kb":240832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563060+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:44:01.563060+0000","last_peered":"2023-10-09T04:44:01.563060+0000","last_clean":"2023-10-09T04:44:01.563060+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:44:01.563060+0000","last_undegraded":"2023-10-09T04:44:01.563060+0000","last_fullsized":"2023-10-09T04:44:01.563060+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563599+0000","last_change":"2023-10-09T04:24:07.922357+0000","last_active":"2023-10-09T04:44:01.563599+0000","last_peered":"2023-10-09T04:44:01.563599+0000","last_clean":"2023-10-09T04:44:01.563599+0000","last_became_active":"2023-10-09T04:24:07.921983+0000","last_became_peered":"2023-10-09T04:24:07.921983+0000","last_unstale":"2023-10-09T04:44:01.563599+0000","last_undegraded":"2023-10-09T04:44:01.563599+0000","last_fullsized":"2023-10-09T04:44:01.563599+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:27:13.800110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"290'3574","reported_seq":3588,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.328954+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:44:19.328954+0000","last_peered":"2023-10-09T04:44:19.328954+0000","last_clean":"2023-10-09T04:44:19.328954+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:44:19.328954+0000","last_undegraded":"2023-10-09T04:44:19.328954+0000","last_fullsized":"2023-10-09T04:44:19.328954+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3574,"log_dups_size":0,"ondisk_log_size":3574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234225664,"num_objects":3574,"num_object_clones":0,"num_object_copies":10722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3574,"num_write_kb":228736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":245,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.564342+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:44:01.564342+0000","last_peered":"2023-10-09T04:44:01.564342+0000","last_clean":"2023-10-09T04:44:01.564342+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:44:01.564342+0000","last_undegraded":"2023-10-09T04:44:01.564342+0000","last_fullsized":"2023-10-09T04:44:01.564342+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":275,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:12.715504+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:44:12.715504+0000","last_peered":"2023-10-09T04:44:12.715504+0000","last_clean":"2023-10-09T04:44:12.715504+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:44:12.715504+0000","last_undegraded":"2023-10-09T04:44:12.715504+0000","last_fullsized":"2023-10-09T04:44:12.715504+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"290'3856","reported_seq":3871,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:22.036162+0000","last_change":"2023-10-09T04:43:34.883699+0000","last_active":"2023-10-09T04:44:22.036162+0000","last_peered":"2023-10-09T04:44:22.036162+0000","last_clean":"2023-10-09T04:44:22.036162+0000","last_became_active":"2023-10-09T04:43:34.882682+0000","last_became_peered":"2023-10-09T04:43:34.882682+0000","last_unstale":"2023-10-09T04:44:22.036162+0000","last_undegraded":"2023-10-09T04:44:22.036162+0000","last_fullsized":"2023-10-09T04:44:22.036162+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3856,"log_dups_size":0,"ondisk_log_size":3856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.341363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252706816,"num_objects":3856,"num_object_clones":0,"num_object_copies":11568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3856,"num_write_kb":246784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":167,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560348+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:44:01.560348+0000","last_peered":"2023-10-09T04:44:01.560348+0000","last_clean":"2023-10-09T04:44:01.560348+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:44:01.560348+0000","last_undegraded":"2023-10-09T04:44:01.560348+0000","last_fullsized":"2023-10-09T04:44:01.560348+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570182+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:44:01.570182+0000","last_peered":"2023-10-09T04:44:01.570182+0000","last_clean":"2023-10-09T04:44:01.570182+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:44:01.570182+0000","last_undegraded":"2023-10-09T04:44:01.570182+0000","last_fullsized":"2023-10-09T04:44:01.570182+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":261,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.569771+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:44:01.569771+0000","last_peered":"2023-10-09T04:44:01.569771+0000","last_clean":"2023-10-09T04:44:01.569771+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:44:01.569771+0000","last_undegraded":"2023-10-09T04:44:01.569771+0000","last_fullsized":"2023-10-09T04:44:01.569771+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"290'3661","reported_seq":3675,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.352837+0000","last_change":"2023-10-09T04:43:34.881122+0000","last_active":"2023-10-09T04:44:19.352837+0000","last_peered":"2023-10-09T04:44:19.352837+0000","last_clean":"2023-10-09T04:44:19.352837+0000","last_became_active":"2023-10-09T04:43:34.880680+0000","last_became_peered":"2023-10-09T04:43:34.880680+0000","last_unstale":"2023-10-09T04:44:19.352837+0000","last_undegraded":"2023-10-09T04:44:19.352837+0000","last_fullsized":"2023-10-09T04:44:19.352837+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3661,"log_dups_size":0,"ondisk_log_size":3661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:08.954210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239927296,"num_objects":3661,"num_object_clones":0,"num_object_copies":10983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3661,"num_write_kb":234304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":254,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.569849+0000","last_change":"2023-10-09T04:24:07.944599+0000","last_active":"2023-10-09T04:44:01.569849+0000","last_peered":"2023-10-09T04:44:01.569849+0000","last_clean":"2023-10-09T04:44:01.569849+0000","last_became_active":"2023-10-09T04:24:07.944188+0000","last_became_peered":"2023-10-09T04:24:07.944188+0000","last_unstale":"2023-10-09T04:44:01.569849+0000","last_undegraded":"2023-10-09T04:44:01.569849+0000","last_fullsized":"2023-10-09T04:44:01.569849+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:11:00.663617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563282+0000","last_change":"2023-10-09T04:41:02.608773+0000","last_active":"2023-10-09T04:44:01.563282+0000","last_peered":"2023-10-09T04:44:01.563282+0000","last_clean":"2023-10-09T04:44:01.563282+0000","last_became_active":"2023-10-09T04:41:02.608455+0000","last_became_peered":"2023-10-09T04:41:02.608455+0000","last_unstale":"2023-10-09T04:44:01.563282+0000","last_undegraded":"2023-10-09T04:44:01.563282+0000","last_fullsized":"2023-10-09T04:44:01.563282+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:29.807000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":261,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565695+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:44:01.565695+0000","last_peered":"2023-10-09T04:44:01.565695+0000","last_clean":"2023-10-09T04:44:01.565695+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:44:01.565695+0000","last_undegraded":"2023-10-09T04:44:01.565695+0000","last_fullsized":"2023-10-09T04:44:01.565695+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"290'3403","reported_seq":3417,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.374231+0000","last_change":"2023-10-09T04:43:34.547468+0000","last_active":"2023-10-09T04:44:19.374231+0000","last_peered":"2023-10-09T04:44:19.374231+0000","last_clean":"2023-10-09T04:44:19.374231+0000","last_became_active":"2023-10-09T04:43:34.547129+0000","last_became_peered":"2023-10-09T04:43:34.547129+0000","last_unstale":"2023-10-09T04:44:19.374231+0000","last_undegraded":"2023-10-09T04:44:19.374231+0000","last_fullsized":"2023-10-09T04:44:19.374231+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3403,"log_dups_size":0,"ondisk_log_size":3403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:58.616531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223019008,"num_objects":3403,"num_object_clones":0,"num_object_copies":10209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3403,"num_write_kb":217792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":247,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565091+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:44:01.565091+0000","last_peered":"2023-10-09T04:44:01.565091+0000","last_clean":"2023-10-09T04:44:01.565091+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:44:01.565091+0000","last_undegraded":"2023-10-09T04:44:01.565091+0000","last_fullsized":"2023-10-09T04:44:01.565091+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":246,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565382+0000","last_change":"2023-10-09T04:39:40.406420+0000","last_active":"2023-10-09T04:44:01.565382+0000","last_peered":"2023-10-09T04:44:01.565382+0000","last_clean":"2023-10-09T04:44:01.565382+0000","last_became_active":"2023-10-09T04:39:40.406072+0000","last_became_peered":"2023-10-09T04:39:40.406072+0000","last_unstale":"2023-10-09T04:44:01.565382+0000","last_undegraded":"2023-10-09T04:44:01.565382+0000","last_fullsized":"2023-10-09T04:44:01.565382+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:08.385113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":162,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570345+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:44:01.570345+0000","last_peered":"2023-10-09T04:44:01.570345+0000","last_clean":"2023-10-09T04:44:01.570345+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:44:01.570345+0000","last_undegraded":"2023-10-09T04:44:01.570345+0000","last_fullsized":"2023-10-09T04:44:01.570345+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"290'3541","reported_seq":3555,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.377641+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:44:19.377641+0000","last_peered":"2023-10-09T04:44:19.377641+0000","last_clean":"2023-10-09T04:44:19.377641+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:44:19.377641+0000","last_undegraded":"2023-10-09T04:44:19.377641+0000","last_fullsized":"2023-10-09T04:44:19.377641+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3541,"log_dups_size":0,"ondisk_log_size":3541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232062976,"num_objects":3541,"num_object_clones":0,"num_object_copies":10623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3541,"num_write_kb":226624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":271,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.569996+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:44:01.569996+0000","last_peered":"2023-10-09T04:44:01.569996+0000","last_clean":"2023-10-09T04:44:01.569996+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:44:01.569996+0000","last_undegraded":"2023-10-09T04:44:01.569996+0000","last_fullsized":"2023-10-09T04:44:01.569996+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":682,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560063+0000","last_change":"2023-10-09T04:29:54.098901+0000","last_active":"2023-10-09T04:44:01.560063+0000","last_peered":"2023-10-09T04:44:01.560063+0000","last_clean":"2023-10-09T04:44:01.560063+0000","last_became_active":"2023-10-09T04:29:54.098459+0000","last_became_peered":"2023-10-09T04:29:54.098459+0000","last_unstale":"2023-10-09T04:44:01.560063+0000","last_undegraded":"2023-10-09T04:44:01.560063+0000","last_fullsized":"2023-10-09T04:44:01.560063+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:19.984620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":216,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.564570+0000","last_change":"2023-10-09T04:27:57.988547+0000","last_active":"2023-10-09T04:44:01.564570+0000","last_peered":"2023-10-09T04:44:01.564570+0000","last_clean":"2023-10-09T04:44:01.564570+0000","last_became_active":"2023-10-09T04:27:57.988239+0000","last_became_peered":"2023-10-09T04:27:57.988239+0000","last_unstale":"2023-10-09T04:44:01.564570+0000","last_undegraded":"2023-10-09T04:44:01.564570+0000","last_fullsized":"2023-10-09T04:44:01.564570+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:09:48.719914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.567797+0000","last_change":"2023-10-09T04:41:02.612580+0000","last_active":"2023-10-09T04:44:01.567797+0000","last_peered":"2023-10-09T04:44:01.567797+0000","last_clean":"2023-10-09T04:44:01.567797+0000","last_became_active":"2023-10-09T04:41:02.612258+0000","last_became_peered":"2023-10-09T04:41:02.612258+0000","last_unstale":"2023-10-09T04:44:01.567797+0000","last_undegraded":"2023-10-09T04:44:01.567797+0000","last_fullsized":"2023-10-09T04:44:01.567797+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:53.656800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"290'3561","reported_seq":3575,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:19.335790+0000","last_change":"2023-10-09T04:43:34.880958+0000","last_active":"2023-10-09T04:44:19.335790+0000","last_peered":"2023-10-09T04:44:19.335790+0000","last_clean":"2023-10-09T04:44:19.335790+0000","last_became_active":"2023-10-09T04:43:34.880522+0000","last_became_peered":"2023-10-09T04:43:34.880522+0000","last_unstale":"2023-10-09T04:44:19.335790+0000","last_undegraded":"2023-10-09T04:44:19.335790+0000","last_fullsized":"2023-10-09T04:44:19.335790+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3561,"log_dups_size":0,"ondisk_log_size":3561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:04.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233373696,"num_objects":3561,"num_object_clones":0,"num_object_copies":10683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3561,"num_write_kb":227904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":290,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560063+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:44:01.560063+0000","last_peered":"2023-10-09T04:44:01.560063+0000","last_clean":"2023-10-09T04:44:01.560063+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:44:01.560063+0000","last_undegraded":"2023-10-09T04:44:01.560063+0000","last_fullsized":"2023-10-09T04:44:01.560063+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563174+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:44:01.563174+0000","last_peered":"2023-10-09T04:44:01.563174+0000","last_clean":"2023-10-09T04:44:01.563174+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:44:01.563174+0000","last_undegraded":"2023-10-09T04:44:01.563174+0000","last_fullsized":"2023-10-09T04:44:01.563174+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":167,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.561133+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:44:01.561133+0000","last_peered":"2023-10-09T04:44:01.561133+0000","last_clean":"2023-10-09T04:44:01.561133+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:44:01.561133+0000","last_undegraded":"2023-10-09T04:44:01.561133+0000","last_fullsized":"2023-10-09T04:44:01.561133+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"290'3922","reported_seq":3937,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:22.685145+0000","last_change":"2023-10-09T04:43:34.547726+0000","last_active":"2023-10-09T04:44:22.685145+0000","last_peered":"2023-10-09T04:44:22.685145+0000","last_clean":"2023-10-09T04:44:22.685145+0000","last_became_active":"2023-10-09T04:43:34.547321+0000","last_became_peered":"2023-10-09T04:43:34.547321+0000","last_unstale":"2023-10-09T04:44:22.685145+0000","last_undegraded":"2023-10-09T04:44:22.685145+0000","last_fullsized":"2023-10-09T04:44:22.685145+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3922,"log_dups_size":0,"ondisk_log_size":3922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:11:56.973914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257032192,"num_objects":3922,"num_object_clones":0,"num_object_copies":11766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3922,"num_write_kb":251008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":293,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.560437+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:44:01.560437+0000","last_peered":"2023-10-09T04:44:01.560437+0000","last_clean":"2023-10-09T04:44:01.560437+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:44:01.560437+0000","last_undegraded":"2023-10-09T04:44:01.560437+0000","last_fullsized":"2023-10-09T04:44:01.560437+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":246,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.565612+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:44:01.565612+0000","last_peered":"2023-10-09T04:44:01.565612+0000","last_clean":"2023-10-09T04:44:01.565612+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:44:01.565612+0000","last_undegraded":"2023-10-09T04:44:01.565612+0000","last_fullsized":"2023-10-09T04:44:01.565612+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":134,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.569992+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:44:01.569992+0000","last_peered":"2023-10-09T04:44:01.569992+0000","last_clean":"2023-10-09T04:44:01.569992+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:44:01.569992+0000","last_undegraded":"2023-10-09T04:44:01.569992+0000","last_fullsized":"2023-10-09T04:44:01.569992+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"290'3842","reported_seq":3857,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:22.696244+0000","last_change":"2023-10-09T04:43:34.880794+0000","last_active":"2023-10-09T04:44:22.696244+0000","last_peered":"2023-10-09T04:44:22.696244+0000","last_clean":"2023-10-09T04:44:22.696244+0000","last_became_active":"2023-10-09T04:43:34.880481+0000","last_became_peered":"2023-10-09T04:43:34.880481+0000","last_unstale":"2023-10-09T04:44:22.696244+0000","last_undegraded":"2023-10-09T04:44:22.696244+0000","last_fullsized":"2023-10-09T04:44:22.696244+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3842,"log_dups_size":0,"ondisk_log_size":3842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:13.398120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251789312,"num_objects":3842,"num_object_clones":0,"num_object_copies":11526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3842,"num_write_kb":245888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":288,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566853+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:44:01.566853+0000","last_peered":"2023-10-09T04:44:01.566853+0000","last_clean":"2023-10-09T04:44:01.566853+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:44:01.566853+0000","last_undegraded":"2023-10-09T04:44:01.566853+0000","last_fullsized":"2023-10-09T04:44:01.566853+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":259,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566094+0000","last_change":"2023-10-09T04:41:02.629667+0000","last_active":"2023-10-09T04:44:01.566094+0000","last_peered":"2023-10-09T04:44:01.566094+0000","last_clean":"2023-10-09T04:44:01.566094+0000","last_became_active":"2023-10-09T04:41:02.629305+0000","last_became_peered":"2023-10-09T04:41:02.629305+0000","last_unstale":"2023-10-09T04:44:01.566094+0000","last_undegraded":"2023-10-09T04:44:01.566094+0000","last_fullsized":"2023-10-09T04:44:01.566094+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:36.600453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":278,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.564500+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:44:01.564500+0000","last_peered":"2023-10-09T04:44:01.564500+0000","last_clean":"2023-10-09T04:44:01.564500+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:44:01.564500+0000","last_undegraded":"2023-10-09T04:44:01.564500+0000","last_fullsized":"2023-10-09T04:44:01.564500+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"290'3807","reported_seq":3822,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:21.946929+0000","last_change":"2023-10-09T04:43:34.548840+0000","last_active":"2023-10-09T04:44:21.946929+0000","last_peered":"2023-10-09T04:44:21.946929+0000","last_clean":"2023-10-09T04:44:21.946929+0000","last_became_active":"2023-10-09T04:43:34.548573+0000","last_became_peered":"2023-10-09T04:43:34.548573+0000","last_unstale":"2023-10-09T04:44:21.946929+0000","last_undegraded":"2023-10-09T04:44:21.946929+0000","last_fullsized":"2023-10-09T04:44:21.946929+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":3807,"log_dups_size":0,"ondisk_log_size":3807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:49.901133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249495552,"num_objects":3807,"num_object_clones":0,"num_object_copies":11421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3807,"num_write_kb":243648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":268,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566166+0000","last_change":"2023-10-09T04:36:47.576051+0000","last_active":"2023-10-09T04:44:01.566166+0000","last_peered":"2023-10-09T04:44:01.566166+0000","last_clean":"2023-10-09T04:44:01.566166+0000","last_became_active":"2023-10-09T04:36:47.575707+0000","last_became_peered":"2023-10-09T04:36:47.575707+0000","last_unstale":"2023-10-09T04:44:01.566166+0000","last_undegraded":"2023-10-09T04:44:01.566166+0000","last_fullsized":"2023-10-09T04:44:01.566166+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:58.041056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":62,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563703+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:44:01.563703+0000","last_peered":"2023-10-09T04:44:01.563703+0000","last_clean":"2023-10-09T04:44:01.563703+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:44:01.563703+0000","last_undegraded":"2023-10-09T04:44:01.563703+0000","last_fullsized":"2023-10-09T04:44:01.563703+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563005+0000","last_change":"2023-10-09T04:41:02.614004+0000","last_active":"2023-10-09T04:44:01.563005+0000","last_peered":"2023-10-09T04:44:01.563005+0000","last_clean":"2023-10-09T04:44:01.563005+0000","last_became_active":"2023-10-09T04:41:02.612300+0000","last_became_peered":"2023-10-09T04:41:02.612300+0000","last_unstale":"2023-10-09T04:44:01.563005+0000","last_undegraded":"2023-10-09T04:44:01.563005+0000","last_fullsized":"2023-10-09T04:44:01.563005+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:04.375059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":130,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563568+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:44:01.563568+0000","last_peered":"2023-10-09T04:44:01.563568+0000","last_clean":"2023-10-09T04:44:01.563568+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:44:01.563568+0000","last_undegraded":"2023-10-09T04:44:01.563568+0000","last_fullsized":"2023-10-09T04:44:01.563568+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":240,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-10-09T04:44:16.650656+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:44:16.650656+0000","last_peered":"2023-10-09T04:44:16.650656+0000","last_clean":"2023-10-09T04:44:16.650656+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:44:16.650656+0000","last_undegraded":"2023-10-09T04:44:16.650656+0000","last_fullsized":"2023-10-09T04:44:16.650656+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":280,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570029+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:44:01.570029+0000","last_peered":"2023-10-09T04:44:01.570029+0000","last_clean":"2023-10-09T04:44:01.570029+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:44:01.570029+0000","last_undegraded":"2023-10-09T04:44:01.570029+0000","last_fullsized":"2023-10-09T04:44:01.570029+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563783+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:44:01.563783+0000","last_peered":"2023-10-09T04:44:01.563783+0000","last_clean":"2023-10-09T04:44:01.563783+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:44:01.563783+0000","last_undegraded":"2023-10-09T04:44:01.563783+0000","last_fullsized":"2023-10-09T04:44:01.563783+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":235,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570198+0000","last_change":"2023-10-09T04:07:34.635686+0000","last_active":"2023-10-09T04:44:01.570198+0000","last_peered":"2023-10-09T04:44:01.570198+0000","last_clean":"2023-10-09T04:44:01.570198+0000","last_became_active":"2023-10-09T04:07:34.635389+0000","last_became_peered":"2023-10-09T04:07:34.635389+0000","last_unstale":"2023-10-09T04:44:01.570198+0000","last_undegraded":"2023-10-09T04:44:01.570198+0000","last_fullsized":"2023-10-09T04:44:01.570198+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:03.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.563275+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:44:01.563275+0000","last_peered":"2023-10-09T04:44:01.563275+0000","last_clean":"2023-10-09T04:44:01.563275+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:44:01.563275+0000","last_undegraded":"2023-10-09T04:44:01.563275+0000","last_fullsized":"2023-10-09T04:44:01.563275+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":279,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.566378+0000","last_change":"2023-10-09T04:36:47.563433+0000","last_active":"2023-10-09T04:44:01.566378+0000","last_peered":"2023-10-09T04:44:01.566378+0000","last_clean":"2023-10-09T04:44:01.566378+0000","last_became_active":"2023-10-09T04:36:47.563053+0000","last_became_peered":"2023-10-09T04:36:47.563053+0000","last_unstale":"2023-10-09T04:44:01.566378+0000","last_undegraded":"2023-10-09T04:44:01.566378+0000","last_fullsized":"2023-10-09T04:44:01.566378+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:38:07.498991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":310,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.564738+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:44:01.564738+0000","last_peered":"2023-10-09T04:44:01.564738+0000","last_clean":"2023-10-09T04:44:01.564738+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:44:01.564738+0000","last_undegraded":"2023-10-09T04:44:01.564738+0000","last_fullsized":"2023-10-09T04:44:01.564738+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":164,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.570466+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:44:01.570466+0000","last_peered":"2023-10-09T04:44:01.570466+0000","last_clean":"2023-10-09T04:44:01.570466+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:44:01.570466+0000","last_undegraded":"2023-10-09T04:44:01.570466+0000","last_fullsized":"2023-10-09T04:44:01.570466+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-10-09T04:44:01.562996+0000","last_change":"2023-10-09T04:41:02.608725+0000","last_active":"2023-10-09T04:44:01.562996+0000","last_peered":"2023-10-09T04:44:01.562996+0000","last_clean":"2023-10-09T04:44:01.562996+0000","last_became_active":"2023-10-09T04:41:02.606509+0000","last_became_peered":"2023-10-09T04:41:02.606509+0000","last_unstale":"2023-10-09T04:44:01.562996+0000","last_undegraded":"2023-10-09T04:44:01.562996+0000","last_fullsized":"2023-10-09T04:44:01.562996+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:51:49.430101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3843424256,"num_objects":58646,"num_object_clones":0,"num_object_copies":175938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":58646,"num_write_kb":3753344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11630280704,"data_stored":11630280704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58646,"ondisk_log_size":58646,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":5624,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234449,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2507140,"kb_used_data":1600252,"kb_used_omap":0,"kb_used_meta":906880,"kb_avail":91238012,"statfs":{"total":95995035648,"available":93427724288,"internally_reserved":0,"allocated":1638658048,"data_stored":1638062971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928645120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.972,"15min":4.0300000000000002},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":2.4449999999999998,"5min":44.122999999999998,"15min":186.446},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":2.722,"15min":4.3769999999999998},"min":{"1min":0.32700000000000001,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":0.67400000000000004,"5min":43.313000000000002,"15min":186.429},"last":0.378}]},{"osd":1,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":5.8159999999999998,"15min":5.3529999999999998},"min":{"1min":0.34000000000000002,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":0.63400000000000001,"5min":264.62,"15min":264.62},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":6.0869999999999997,"15min":5.5190000000000001},"min":{"1min":0.31,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67800000000000005,"5min":264.65899999999999,"15min":264.65899999999999},"last":3.7010000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":7.4429999999999996,"15min":3.6349999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.294,"5min":184.72,"15min":184.72},"last":1.46},{"interface":"front","average":{"1min":0.90600000000000003,"5min":7.8259999999999996,"15min":4.0720000000000001},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6070000000000002,"5min":184.697,"15min":184.697},"last":1.502}]},{"osd":3,"last update":"Mon Oct 9 04:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":5.3259999999999996,"15min":7.7530000000000001},"min":{"1min":0.35499999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.3819999999999997,"5min":182.839,"15min":249.989},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.2330000000000001,"5min":5.8860000000000001,"15min":8.0700000000000003},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":10.278,"5min":182.678,"15min":250.21700000000001},"last":1.8520000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.23,"15min":1.3999999999999999},"min":{"1min":0.27200000000000002,"5min":0.26900000000000002,"15min":0.23300000000000001},"max":{"1min":3.0449999999999999,"5min":14.257999999999999,"15min":14.257999999999999},"last":1.018},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.77900000000000003,"15min":1.1319999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.21199999999999999},"max":{"1min":2.0129999999999999,"5min":7.2290000000000001,"15min":10.613},"last":0.35899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.2230000000000001,"15min":1.109},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27900000000000003},"max":{"1min":2.4049999999999998,"5min":13.319000000000001,"15min":13.319000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.641,"15min":1.5549999999999999},"min":{"1min":0.309,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.9460000000000002,"5min":18.408000000000001,"15min":18.408000000000001},"last":0.51500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.0700000000000001,"15min":1.2390000000000001},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":1.069,"5min":11.388,"15min":11.388},"last":1.2649999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.5900000000000001,"15min":1.635},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":3.0190000000000001,"5min":18.311,"15min":18.311},"last":1.6779999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496083,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2508404,"kb_used_data":1602028,"kb_used_omap":0,"kb_used_meta":906368,"kb_avail":91236748,"statfs":{"total":95995035648,"available":93426429952,"internally_reserved":0,"allocated":1640476672,"data_stored":1639897510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928120832},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":2.2559999999999998,"15min":8.0470000000000006},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.223},"max":{"1min":0.66700000000000004,"5min":75.308000000000007,"15min":1284.521},"last":0.623},{"interface":"front","average":{"1min":0.81100000000000005,"5min":2.7450000000000001,"15min":8.2680000000000007},"min":{"1min":0.29099999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.8209999999999997,"5min":75.432000000000002,"15min":1284.4690000000001},"last":1.6160000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:43:32 2023","interfaces":[{"interface":"back","average":{"1min":3.7490000000000001,"5min":7.4779999999999998,"15min":5.7670000000000003},"min":{"1min":0.27300000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":66.510999999999996,"5min":150.75,"15min":185.74000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":4.2110000000000003,"5min":7.7809999999999997,"15min":5.8250000000000002},"min":{"1min":0.30299999999999999,"5min":0.28899999999999998,"15min":0.26300000000000001},"max":{"1min":66.418000000000006,"5min":150.81999999999999,"15min":189.03999999999999},"last":0.84899999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":7.4900000000000002,"15min":4.2130000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.4770000000000001,"5min":209.44399999999999,"15min":209.44399999999999},"last":2.0600000000000001},{"interface":"front","average":{"1min":0.872,"5min":7.5250000000000004,"15min":4.2069999999999999},"min":{"1min":0.41699999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.3839999999999999,"5min":209.428,"15min":209.428},"last":2.0750000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:43:32 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":6.694,"15min":6.1669999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.253},"max":{"1min":1.3620000000000001,"5min":157.93100000000001,"15min":167.56100000000001},"last":4.1799999999999997},{"interface":"front","average":{"1min":1.121,"5min":6.7960000000000003,"15min":6.3120000000000003},"min":{"1min":0.38700000000000001,"5min":0.33000000000000002,"15min":0.313},"max":{"1min":7.7779999999999996,"5min":157.95099999999999,"15min":167.577},"last":0.71199999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.98299999999999998,"15min":1.286},"min":{"1min":0.27300000000000002,"5min":0.27200000000000002,"15min":0.25600000000000001},"max":{"1min":2.6709999999999998,"5min":6.8380000000000001,"15min":14.430999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.45400000000000001,"5min":1.0129999999999999,"15min":1.04},"min":{"1min":0.315,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":0.80000000000000004,"5min":9.2240000000000002,"15min":9.2240000000000002},"last":0.50800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.0860000000000001,"15min":1.78},"min":{"1min":0.32100000000000001,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":4.0890000000000004,"5min":7.3010000000000002,"15min":81.087000000000003},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":1.1579999999999999,"15min":1.6719999999999999},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":0.83699999999999997,"5min":8.6479999999999997,"15min":81.706999999999994},"last":0.38}]},{"osd":7,"last update":"Mon Oct 9 04:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.099,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32000000000000001},"max":{"1min":1.996,"5min":8.3350000000000009,"15min":81.674999999999997},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.2190000000000001,"15min":1.764},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":1.8180000000000001,"5min":7.8049999999999997,"15min":81.055000000000007},"last":0.65600000000000003}]}]},{"osd":1,"up_from":118,"seq":506806141173,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847376,"kb_used_data":1205128,"kb_used_omap":4,"kb_used_meta":642235,"kb_avail":91897776,"statfs":{"total":95995035648,"available":94103322624,"internally_reserved":0,"allocated":1234051072,"data_stored":1232859661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":657649550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":3.6469999999999998,"15min":3.9359999999999999},"min":{"1min":0.23599999999999999,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":0.873,"5min":75.620000000000005,"15min":97.308000000000007},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.502,"5min":3.7189999999999999,"15min":4.1740000000000004},"min":{"1min":0.252,"5min":0.193,"15min":0.193},"max":{"1min":3.367,"5min":69.701999999999998,"15min":97.055999999999997},"last":0.38600000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":5.9089999999999998,"15min":3.6890000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.5529999999999999,"5min":263.60500000000002,"15min":263.60500000000002},"last":92.253},{"interface":"front","average":{"1min":0.66100000000000003,"5min":7.008,"15min":4.1790000000000003},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":3.3170000000000002,"5min":263.65199999999999,"15min":263.65199999999999},"last":90.391999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:43:27 2023","interfaces":[{"interface":"back","average":{"1min":15.497999999999999,"5min":10.351000000000001,"15min":6.4470000000000001},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":379.80900000000003,"5min":379.80900000000003,"15min":379.80900000000003},"last":0.372},{"interface":"front","average":{"1min":15.800000000000001,"5min":11.353999999999999,"15min":7.0990000000000002},"min":{"1min":0.29399999999999998,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":379.95800000000003,"5min":379.95800000000003,"15min":379.95800000000003},"last":0.35399999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":2.173,"15min":2.3860000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.245},"max":{"1min":11.978999999999999,"5min":14.449,"15min":170.072},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.056,"5min":2.7149999999999999,"15min":2.2370000000000001},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":11.935,"5min":75.513999999999996,"15min":169.94499999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.4460000000000002,"15min":2.4220000000000002},"min":{"1min":0.42299999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.4889999999999999,"5min":11.499000000000001,"15min":171.036},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.78400000000000003,"5min":2.3079999999999998,"15min":2.5489999999999999},"min":{"1min":0.38600000000000001,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":6.2329999999999997,"5min":14.557,"15min":170.88},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":3.5950000000000002,"15min":2.4169999999999998},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.4049999999999998,"5min":132.36099999999999,"15min":132.36099999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.82799999999999996,"5min":3.4350000000000001,"15min":2.548},"min":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":6.1929999999999996,"5min":132.17699999999999,"15min":132.17699999999999},"last":0.64600000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":3.7269999999999999,"15min":3.0990000000000002},"min":{"1min":0.42799999999999999,"5min":0.40400000000000003,"15min":0.34100000000000003},"max":{"1min":6.4189999999999996,"5min":132.136,"15min":174.75999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":3.9980000000000002,"15min":2.8460000000000001},"min":{"1min":0.44400000000000001,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":0.85699999999999998,"5min":132.12,"15min":174.779},"last":0.63300000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024229,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2207064,"kb_used_data":1370576,"kb_used_omap":5,"kb_used_meta":836474,"kb_avail":91538088,"statfs":{"total":95995035648,"available":93735002112,"internally_reserved":0,"allocated":1403469824,"data_stored":1402387618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":856550010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:43:35 2023","interfaces":[{"interface":"back","average":{"1min":3.4820000000000002,"5min":6.3010000000000002,"15min":5.1509999999999998},"min":{"1min":0.20899999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":55.008000000000003,"5min":138.536,"15min":167.92699999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.4209999999999998,"5min":7.0170000000000003,"15min":5.5549999999999997},"min":{"1min":0.25900000000000001,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":55.027999999999999,"5min":138.89599999999999,"15min":169.78999999999999},"last":1.756}]},{"osd":2,"last update":"Mon Oct 9 04:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":4.1550000000000002,"15min":7.5880000000000001},"min":{"1min":0.26700000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.0190000000000001,"5min":98.938000000000002,"15min":131.30699999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":4.4009999999999998,"15min":7.9699999999999998},"min":{"1min":0.255,"5min":0.219,"15min":0.219},"max":{"1min":1.8819999999999999,"5min":98.975999999999999,"15min":129.922},"last":1.7909999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:43:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":6.4050000000000002,"15min":5.0179999999999998},"min":{"1min":0.28999999999999998,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":6.0220000000000002,"5min":156.69800000000001,"15min":213.28399999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.9470000000000001,"5min":7.7539999999999996,"15min":5.6740000000000004},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.222},"max":{"1min":13.529,"5min":161.40199999999999,"15min":212.27000000000001},"last":2.0790000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.4810000000000001,"15min":2.2970000000000002},"min":{"1min":0.38400000000000001,"5min":0.30199999999999999,"15min":0.25700000000000001},"max":{"1min":1.915,"5min":11.300000000000001,"15min":161.67099999999999},"last":1.3300000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.8460000000000001,"15min":2.5680000000000001},"min":{"1min":0.379,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":1.7090000000000001,"5min":14.109999999999999,"15min":160.74799999999999},"last":1.8600000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":3.2069999999999999,"15min":2.3980000000000001},"min":{"1min":0.376,"5min":0.29199999999999998,"15min":0.27900000000000003},"max":{"1min":1.7949999999999999,"5min":136.322,"15min":136.322},"last":2.681},{"interface":"front","average":{"1min":0.63200000000000001,"5min":3.1200000000000001,"15min":2.1789999999999998},"min":{"1min":0.39700000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0880000000000001,"5min":136.15799999999999,"15min":136.15799999999999},"last":4.8849999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.109,"15min":2.6899999999999999},"min":{"1min":0.39500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":1.76,"5min":78.438999999999993,"15min":108.63800000000001},"last":3.4420000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":2.661,"15min":2.3119999999999998},"min":{"1min":0.41899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.99399999999999999,"5min":78.617999999999995,"15min":108.76300000000001},"last":4.0410000000000004}]},{"osd":7,"last update":"Mon Oct 9 04:43:49 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":2.7549999999999999,"15min":1.994},"min":{"1min":0.35199999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.0960000000000001,"5min":77.682000000000002,"15min":77.682000000000002},"last":1.4199999999999999},{"interface":"front","average":{"1min":0.72099999999999997,"5min":2.5179999999999998,"15min":2.0099999999999998},"min":{"1min":0.42099999999999999,"5min":0.36699999999999999,"15min":0.36399999999999999},"max":{"1min":1.9730000000000001,"5min":77.606999999999999,"15min":77.606999999999999},"last":1.3819999999999999}]}]},{"osd":2,"up_from":221,"seq":949187772508,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2752860,"kb_used_data":2434516,"kb_used_omap":7,"kb_used_meta":318328,"kb_avail":90992292,"statfs":{"total":95995035648,"available":93176107008,"internally_reserved":0,"allocated":2492944384,"data_stored":2491918500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":325967953},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":2.5710000000000002,"15min":7.2469999999999999},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":1.1459999999999999,"5min":13.813000000000001,"15min":72.608000000000004},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":2.2610000000000001,"15min":6.7880000000000003},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.5920000000000001,"5min":14.377000000000001,"15min":72.632000000000005},"last":0.55400000000000005}]},{"osd":1,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":5.1379999999999999,"15min":4.9630000000000001},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.3639999999999999,"5min":121.848,"15min":121.848},"last":2.6589999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":5.3479999999999999,"15min":5.399},"min":{"1min":0.25,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.337,"5min":121.827,"15min":121.827},"last":3.9289999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":5.1299999999999999,"5min":10.228,"15min":6.7590000000000003},"min":{"1min":0.316,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":84.840999999999994,"5min":335.928,"15min":335.928},"last":7.2770000000000001},{"interface":"front","average":{"1min":5.1109999999999998,"5min":10.731,"15min":6.915},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":85.147999999999996,"5min":335.68599999999998,"15min":335.68599999999998},"last":0.66400000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":2.343,"15min":3.8839999999999999},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.315},"max":{"1min":4.6980000000000004,"5min":14.452999999999999,"15min":72.944999999999993},"last":3.887},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.3149999999999999,"15min":4.0099999999999998},"min":{"1min":0.41799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0389999999999999,"5min":14.183,"15min":73.054000000000002},"last":0.71099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":2.6139999999999999,"15min":3.738},"min":{"1min":0.41399999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9199999999999999,"5min":65.388000000000005,"15min":72.986000000000004},"last":2.5990000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":2.8460000000000001,"15min":4.1710000000000003},"min":{"1min":0.39100000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.524,"5min":67.799000000000007,"15min":72.963999999999999},"last":1.0649999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":2.661,"15min":8.5060000000000002},"min":{"1min":0.40300000000000002,"5min":0.39000000000000001,"15min":0.33600000000000002},"max":{"1min":1.264,"5min":34.753,"15min":135.614},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.79100000000000004,"5min":2.5190000000000001,"15min":7.907},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.9020000000000001,"5min":34.796999999999997,"15min":130.196},"last":2.6339999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":2.6779999999999999,"15min":4.2590000000000003},"min":{"1min":0.44500000000000001,"5min":0.38500000000000001,"15min":0.378},"max":{"1min":2.7639999999999998,"5min":28.126999999999999,"15min":73.079999999999998},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.4670000000000001,"15min":3.702},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.8639999999999999,"5min":28.106000000000002,"15min":73.069000000000003},"last":2.6890000000000001}]}]},{"osd":3,"up_from":142,"seq":609885356231,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2486480,"kb_used_data":1967368,"kb_used_omap":2,"kb_used_meta":519101,"kb_avail":91258672,"statfs":{"total":95995035648,"available":93448880128,"internally_reserved":0,"allocated":2014584832,"data_stored":2013567001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":531559477},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":1.9399999999999999,"15min":2.4820000000000002},"min":{"1min":0.23300000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.7690000000000001,"5min":17.648,"15min":142.14500000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.1539999999999999,"5min":5.2519999999999998,"15min":3.649},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":10.725,"5min":254.38499999999999,"15min":254.38499999999999},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":2.7909999999999999,"15min":3.54},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.192},"max":{"1min":9.8450000000000006,"5min":21.879000000000001,"15min":180.02500000000001},"last":2.0859999999999999},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.3069999999999999,"15min":3.6389999999999998},"min":{"1min":0.251,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":9.7319999999999993,"5min":21.911999999999999,"15min":184.00899999999999},"last":0.41899999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":4.4690000000000003,"15min":3.125},"min":{"1min":0.29399999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.7370000000000001,"5min":112.613,"15min":177.28399999999999},"last":3.234},{"interface":"front","average":{"1min":0.47399999999999998,"5min":6.7460000000000004,"15min":3.653},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.179,"5min":255.05199999999999,"15min":255.05199999999999},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":4.6559999999999997,"15min":2.9220000000000002},"min":{"1min":0.316,"5min":0.28999999999999998,"15min":0.27900000000000003},"max":{"1min":9.5109999999999992,"5min":254.072,"15min":254.072},"last":2.1699999999999999},{"interface":"front","average":{"1min":1.9550000000000001,"5min":4.3970000000000002,"15min":2.75},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.307},"max":{"1min":8.2360000000000007,"5min":253.49700000000001,"15min":253.49700000000001},"last":0.38900000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":5.2439999999999998,"15min":3.4969999999999999},"min":{"1min":0.34499999999999997,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":7.8719999999999999,"5min":253.57900000000001,"15min":253.57900000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.5449999999999999,"5min":5.3040000000000003,"15min":3.573},"min":{"1min":0.40100000000000002,"5min":0.35599999999999998,"15min":0.29799999999999999},"max":{"1min":8.1039999999999992,"5min":254.191,"15min":254.191},"last":0.433}]},{"osd":6,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":5.532,"15min":3.4039999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.31900000000000001},"max":{"1min":8.1419999999999995,"5min":254.14699999999999,"15min":254.14699999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.819,"5min":2.786,"15min":2.8490000000000002},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":9.5380000000000003,"5min":50.308,"15min":123.505},"last":2.2210000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:43:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":2.653,"15min":2.456},"min":{"1min":0.41199999999999998,"5min":0.378,"15min":0.32300000000000001},"max":{"1min":9.4789999999999992,"5min":43.835000000000001,"15min":57.399000000000001},"last":2.198},{"interface":"front","average":{"1min":1.627,"5min":2.222,"15min":2.254},"min":{"1min":0.45700000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":7.907,"5min":43.781999999999996,"15min":57.447000000000003},"last":0.48599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019357,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1582972,"kb_used_data":695028,"kb_used_omap":0,"kb_used_meta":887936,"kb_avail":92162180,"statfs":{"total":95995035648,"available":94374072320,"internally_reserved":0,"allocated":711708672,"data_stored":711121173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":909246464},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":2.0659999999999998,"15min":5.351},"min":{"1min":0.28399999999999997,"5min":0.25600000000000001,"15min":0.23000000000000001},"max":{"1min":1.405,"5min":23.114999999999998,"15min":219.577},"last":2.863},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.8169999999999999,"15min":5.0419999999999998},"min":{"1min":0.314,"5min":0.255,"15min":0.255},"max":{"1min":1.4950000000000001,"5min":13.992000000000001,"15min":218.96000000000001},"last":2.4780000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:43:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4780000000000002,"5min":3.3690000000000002,"15min":5.1820000000000004},"min":{"1min":0.28599999999999998,"5min":0.255,"15min":0.24099999999999999},"max":{"1min":37.356000000000002,"5min":56.122,"15min":142.74799999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.7429999999999999,"5min":3.5030000000000001,"15min":5.3289999999999997},"min":{"1min":0.36199999999999999,"5min":0.28100000000000003,"15min":0.27300000000000002},"max":{"1min":37.311999999999998,"5min":56.043999999999997,"15min":139.09700000000001},"last":3.0939999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:44:04 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":5.6699999999999999,"15min":4.1399999999999997},"min":{"1min":0.41399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.149,"5min":195.267,"15min":195.267},"last":2.726},{"interface":"front","average":{"1min":0.63700000000000001,"5min":5.298,"15min":3.669},"min":{"1min":0.33800000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":2.173,"5min":195.15799999999999,"15min":195.15799999999999},"last":1.8100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":7.6150000000000002,"5min":8.3160000000000007,"15min":7.4859999999999998},"min":{"1min":0.38400000000000001,"5min":0.32600000000000001,"15min":0.27900000000000003},"max":{"1min":146.02199999999999,"5min":357.125,"15min":357.125},"last":2.637},{"interface":"front","average":{"1min":7.9699999999999998,"5min":8.1500000000000004,"15min":7.6120000000000001},"min":{"1min":0.38500000000000001,"5min":0.314,"15min":0.312},"max":{"1min":147.75700000000001,"5min":359.02800000000002,"15min":359.02800000000002},"last":3.6269999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:43:22 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.95499999999999996,"15min":1.8580000000000001},"min":{"1min":0.312,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":5.8540000000000001,"5min":9.5609999999999999,"15min":107.476},"last":1.093},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.0680000000000001,"15min":2.0779999999999998},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.23300000000000001},"max":{"1min":1.6259999999999999,"5min":15.394,"15min":107.414},"last":1.274}]},{"osd":6,"last update":"Mon Oct 9 04:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.123,"15min":1.8859999999999999},"min":{"1min":0.311,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":5.8109999999999999,"5min":9.6099999999999994,"15min":65.75},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.133,"15min":1.861},"min":{"1min":0.34599999999999997,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":2.2309999999999999,"5min":9.782,"15min":65.849999999999994},"last":1.5669999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.0169999999999999,"15min":1.6200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":2.024,"5min":8.6120000000000001,"15min":76.283000000000001},"last":1.3100000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":1.288,"15min":1.6739999999999999},"min":{"1min":0.40400000000000003,"5min":0.379,"15min":0.32100000000000001},"max":{"1min":4.2850000000000001,"5min":9.6310000000000002,"15min":76.259},"last":0.48899999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757721,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1348588,"kb_used_data":454180,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":92396564,"statfs":{"total":95995035648,"available":94614081536,"internally_reserved":0,"allocated":465080320,"data_stored":464504949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":2.8769999999999998,"15min":5.2359999999999998},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.22},"max":{"1min":0.79300000000000004,"5min":75.855999999999995,"15min":210.85499999999999},"last":0.315},{"interface":"front","average":{"1min":0.52500000000000002,"5min":3.448,"15min":5.359},"min":{"1min":0.30399999999999999,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":0.88,"5min":75.816000000000003,"15min":210.23500000000001},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":3.516,"15min":4.5439999999999996},"min":{"1min":0.33500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.413,"5min":117.84099999999999,"15min":145.577},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.6139999999999999,"15min":4.9249999999999998},"min":{"1min":0.26600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.83599999999999997,"5min":117.90600000000001,"15min":145.792},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:43:58 2023","interfaces":[{"interface":"back","average":{"1min":4.7560000000000002,"5min":3.8159999999999998,"15min":5.7880000000000003},"min":{"1min":0.36399999999999999,"5min":0.29099999999999998,"15min":0.26800000000000002},"max":{"1min":66.930000000000007,"5min":66.984999999999999,"15min":90.763999999999996},"last":40.686},{"interface":"front","average":{"1min":4.8879999999999999,"5min":4.0389999999999997,"15min":5.9390000000000001},"min":{"1min":0.39200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":67.085999999999999,"5min":67.686000000000007,"15min":91.340000000000003},"last":40.539000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:43:19 2023","interfaces":[{"interface":"back","average":{"1min":5.1710000000000003,"5min":8.5860000000000003,"15min":7.0789999999999997},"min":{"1min":0.31,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":67.129999999999995,"5min":268.63299999999998,"15min":268.63299999999998},"last":0.30599999999999999},{"interface":"front","average":{"1min":5.4299999999999997,"5min":8.3550000000000004,"15min":6.8810000000000002},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":67.189999999999998,"5min":268.92500000000001,"15min":268.92500000000001},"last":0.45600000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:43:19 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.64100000000000001,"15min":2.3370000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.248},"max":{"1min":0.70799999999999996,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.438},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.206,"15min":2.7799999999999998},"min":{"1min":0.26900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":3.1819999999999999,"5min":7.4859999999999998,"15min":361.71300000000002},"last":0.28299999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.65900000000000003,"15min":1.1759999999999999},"min":{"1min":0.316,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.71299999999999997,"5min":5.7640000000000002,"15min":11.192},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.1799999999999999,"15min":1.327},"min":{"1min":0.40400000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.6619999999999999,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.38900000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:44:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.2589999999999999,"15min":1.3600000000000001},"min":{"1min":0.39200000000000002,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":2.403,"5min":13.268000000000001,"15min":13.268000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.94499999999999995,"15min":1.242},"min":{"1min":0.437,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.351,"5min":6.5129999999999999,"15min":15.257999999999999},"last":0.47399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1402601472,"data_stored":1402601472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1239023616,"data_stored":1239023616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2497445888,"data_stored":2497445888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2024079360,"data_stored":2024079360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":710737920,"data_stored":710737920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":462880768,"data_stored":462880768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1650130944,"data_stored":1650130944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1643380736,"data_stored":1643380736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:44:28.351 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:44:28.351 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:44:28.351 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2023-10-09T04:44:28.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T04:44:28.472 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:28.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:28.582 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:28.617 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:28.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:28.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:28.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:28.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:44:28.876 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:28 smithi177 ceph-mon[195797]: pgmap v1125: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:44:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:29.122 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 67530 67514 95.8651 54.625 0.00332121 0.0103992 2023-10-09T04:44:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:28 smithi029 ceph-mon[216903]: pgmap v1125: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:44:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:28 smithi029 ceph-mon[223858]: pgmap v1125: 89 pgs: 89 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-09T04:44:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:29.803 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:44:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:29.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:44:29.913 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:29.965 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:29.998 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:30.024 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:30.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:30.122 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 68403 68387 94.9469 54.5625 0.0152009 0.0105267 2023-10-09T04:44:30.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:30.233 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:30.512 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:30.731 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:30.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:30.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:30.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:44:30.912 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:30 smithi029 ceph-mon[216903]: pgmap v1126: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T04:44:30.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:30 smithi029 ceph-mon[223858]: pgmap v1126: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T04:44:30.920 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:30 smithi177 ceph-mon[195797]: pgmap v1126: 89 pgs: 89 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-10-09T04:44:31.079 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:44:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:31.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:44:31.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 69332 69316 94.1434 58.0625 0.0116664 0.0106172 2023-10-09T04:44:31.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:31.230 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:31.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:31.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:31.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:31.494 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:31.577 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:44:31.611 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:44:31.649 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:44:31.703 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:44:31.731 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:44:31.737 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:44:31.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1945205707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:31.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1761926084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:31.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1945205707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:31.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1761926084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:31.779 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:44:31.874 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:44:31.874 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:44:31.874 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.874 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.874 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.874 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.875 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.875 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.875 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:44:31.875 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:44:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1945205707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:44:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1761926084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:32.123 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 70148 70132 93.2256 51 0.0201971 0.0107217 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: pgmap v1127: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1945205707' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:32 smithi177 ceph-mon[195797]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.124 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 70751 70735 92.0672 37.6875 0.0135333 0.0108566 2023-10-09T04:44:33.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: pgmap v1127: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:44:33.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:44:33.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:44:33.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1945205707' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:33.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T04:44:33.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[216903]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:44:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:44:32] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: pgmap v1127: 89 pgs: 89 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:44:33.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1945205707' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:44:33.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: osdmap e291: 8 total, 8 up, 8 in 2023-10-09T04:44:33.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: failed to encode map e291 with expected crc 2023-10-09T04:44:33.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:32 smithi029 ceph-mon[223858]: failed to encode map e291 with expected crc 2023-10-09T04:44:34.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 71612 71596 91.2829 53.8125 0.0149742 0.0109498 2023-10-09T04:44:34.245 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:34.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:34.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:34.386 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:34 smithi177 ceph-mon[195797]: pgmap v1129: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:44:35.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 72408 72392 90.4524 49.75 0.0197024 0.0110511 2023-10-09T04:44:35.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:34 smithi029 ceph-mon[216903]: pgmap v1129: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:44:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:34 smithi029 ceph-mon[223858]: pgmap v1129: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-10-09T04:44:35.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:35.995 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:44:36.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 73302 73286 89.7745 55.875 0.00531641 0.0111323 2023-10-09T04:44:36.239 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:36.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:36.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:36.396 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:36.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:36.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:36.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:36.725 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:36.735 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:36 smithi029 ceph-mon[223858]: pgmap v1130: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T04:44:36.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:36.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:36.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:44:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:36 smithi177 ceph-mon[195797]: pgmap v1130: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T04:44:37.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:36 smithi029 ceph-mon[216903]: pgmap v1130: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-09T04:44:37.045 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:37.072 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:44:37.072 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:44:37.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 74210 74194 89.1309 56.75 0.0179066 0.0112132 2023-10-09T04:44:37.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:37 smithi029 conmon[216878]: 2023-10-09T04:44:37.308+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e291 definitely_dead 0 2023-10-09T04:44:37.323 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:44:37.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:37.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:44:37.435 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:44:37.486 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:37.785 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.6. 2023-10-09T04:44:37.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:44:38.023 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4211867979' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:44:38.023 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:37 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:44:38.023 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:37 smithi029 ceph-mon[216903]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:44:38.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4211867979' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:44:38.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:37 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:44:38.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:37 smithi029 ceph-mon[223858]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:44:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4211867979' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:44:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:37 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:44:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:37 smithi177 ceph-mon[195797]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:44:38.131 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 74742 74726 88.0769 33.25 0.00532781 0.0112747 2023-10-09T04:44:38.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:44:38.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: pgmap v1131: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-10-09T04:44:38.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:44:38.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T04:44:38.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[216903]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: pgmap v1131: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: failed to encode map e292 with expected crc 2023-10-09T04:44:38.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:38 smithi029 ceph-mon[223858]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: pgmap v1131: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-10-09T04:44:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: osdmap e292: 8 total, 8 up, 7 in 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:38 smithi177 ceph-mon[195797]: failed to encode map e292 with expected crc 2023-10-09T04:44:39.132 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 74742 74726 86.4464 0 - 0.0112747 2023-10-09T04:44:39.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:44:39.603 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:44:39.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:39.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:44:39.637 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:39.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:39.737 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.769 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:39.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: failed to encode map e293 with expected crc 2023-10-09T04:44:39.771 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:39.772 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:39.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:39.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:39.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:39.927 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:40.010 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:40.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: osdmap e293: 8 total, 8 up, 7 in 2023-10-09T04:44:40.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: failed to encode map e293 with expected crc 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:40.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:40.132 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 74798 74782 84.9389 1.75 0.0173004 0.0117665 2023-10-09T04:44:40.138 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:40.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:40.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:40.389 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:40.866 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:40.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: pgmap v1134: 89 pgs: 2 unknown, 5 peering, 82 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[216903]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:44:40.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:40.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: pgmap v1134: 89 pgs: 2 unknown, 5 peering, 82 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T04:44:40.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T04:44:40.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: failed to encode map e294 with expected crc 2023-10-09T04:44:40.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:40 smithi029 ceph-mon[223858]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:44:40.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:44:41.014 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:41.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: pgmap v1134: 89 pgs: 2 unknown, 5 peering, 82 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 789 op/s 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: osdmap e294: 8 total, 8 up, 7 in 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: failed to encode map e294 with expected crc 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: failed to encode map e294 with expected crc 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: failed to encode map e294 with expected crc 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: failed to encode map e294 with expected crc 2023-10-09T04:44:41.097 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:40 smithi177 ceph-mon[195797]: osdmap e295: 8 total, 8 up, 7 in 2023-10-09T04:44:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:41.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:44:41.132 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 75425 75409 84.1222 39.1875 0.0149642 0.0118829 2023-10-09T04:44:41.198 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:44:41.274 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:42.132 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 75962 75946 83.2354 33.5625 0.0182983 0.0119792 2023-10-09T04:44:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[216903]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[216903]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[216903]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[216903]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:44:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[223858]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[223858]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[223858]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:41 smithi029 ceph-mon[223858]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:44:42.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:41 smithi177 ceph-mon[195797]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:41 smithi177 ceph-mon[195797]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:41 smithi177 ceph-mon[195797]: failed to encode map e295 with expected crc 2023-10-09T04:44:42.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:41 smithi177 ceph-mon[195797]: osdmap e296: 8 total, 8 up, 7 in 2023-10-09T04:44:43.106 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:44:43.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[216903]: pgmap v1137: 89 pgs: 3 remapped+peering, 2 unknown, 5 peering, 79 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 1.2 KiB/s, 0 objects/s recovering 2023-10-09T04:44:43.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[216903]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[216903]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[216903]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[216903]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[223858]: pgmap v1137: 89 pgs: 3 remapped+peering, 2 unknown, 5 peering, 79 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 1.2 KiB/s, 0 objects/s recovering 2023-10-09T04:44:43.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[223858]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[223858]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[223858]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:42 smithi029 ceph-mon[223858]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.118 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:44:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:44:42] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-10-09T04:44:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:43.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:44:43.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 76574 76558 82.4594 38.25 0.0326883 0.012121 2023-10-09T04:44:43.221 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:42 smithi177 ceph-mon[195797]: pgmap v1137: 89 pgs: 3 remapped+peering, 2 unknown, 5 peering, 79 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 1.2 KiB/s, 0 objects/s recovering 2023-10-09T04:44:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:42 smithi177 ceph-mon[195797]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:42 smithi177 ceph-mon[195797]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:42 smithi177 ceph-mon[195797]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:42 smithi177 ceph-mon[195797]: failed to encode map e296 with expected crc 2023-10-09T04:44:43.318 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:44.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 77239 77223 81.7665 41.5625 0.0142798 0.0122237 2023-10-09T04:44:44.527 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:44:44.528 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:44:44.528 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T04:44:44.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:44:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[216903]: pgmap v1139: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 548 op/s; 18862/226449 objects degraded (8.329%); 4550/226449 objects misplaced (2.009%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 18862/226449 objects degraded (8.329%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[223858]: pgmap v1139: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 548 op/s; 18862/226449 objects degraded (8.329%); 4550/226449 objects misplaced (2.009%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:44 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 18862/226449 objects degraded (8.329%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:44 smithi177 ceph-mon[195797]: pgmap v1139: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 548 op/s; 18862/226449 objects degraded (8.329%); 4550/226449 objects misplaced (2.009%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T04:44:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:44 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 18862/226449 objects degraded (8.329%), 4 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:45.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:44:45.133270+0000 min lat: 0.00124565 max lat: 2.32654 avg lat: 0.0123496 2023-10-09T04:44:45.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:44:45.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 77745 77729 80.931 31.625 0.0164819 0.0123496 2023-10-09T04:44:45.232 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:45.505 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:45.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:45.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:45.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:44:45.654 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:45.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:44:45.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:45.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:44:45.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:45.929 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:46.122 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:46.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:46.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 78337 78321 80.211 37 0.00958103 0.0124593 2023-10-09T04:44:46.234 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:44:46.270 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:46.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:44:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:46.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:44:46.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:46.541 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:44:46.547 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:46.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:44:46.664 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:44:46.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:46 smithi029 ceph-mon[223858]: pgmap v1140: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s; 23613/230349 objects degraded (10.251%); 8806/230349 objects misplaced (3.823%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:44:46.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3310349231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:46.704 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:46 smithi029 ceph-mon[216903]: pgmap v1140: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s; 23613/230349 objects degraded (10.251%); 8806/230349 objects misplaced (3.823%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:44:46.704 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3310349231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:46.714 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:44:46.758 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:44:46.803 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:44:46.850 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:44:46.898 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:44:46.911 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:44:46.912 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:44:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:46 smithi177 ceph-mon[195797]: pgmap v1140: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 625 op/s; 23613/230349 objects degraded (10.251%); 8806/230349 objects misplaced (3.823%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:44:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3310349231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:44:47.141 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 78957 78941 79.5369 38.75 0.0172962 0.0125672 2023-10-09T04:44:48.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 79644 79628 78.9561 42.9375 0.0409822 0.0126595 2023-10-09T04:44:48.609 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:44:48.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:48 smithi029 ceph-mon[216903]: pgmap v1141: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s; 23613/232098 objects degraded (10.174%); 8806/232098 objects misplaced (3.794%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T04:44:48.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4273245503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:44:48.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:48 smithi029 ceph-mon[223858]: pgmap v1141: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s; 23613/232098 objects degraded (10.174%); 8806/232098 objects misplaced (3.794%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T04:44:48.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4273245503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:44:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:48 smithi177 ceph-mon[195797]: pgmap v1141: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s; 23613/232098 objects degraded (10.174%); 8806/232098 objects misplaced (3.794%); 3.1 MiB/s, 48 objects/s recovering 2023-10-09T04:44:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4273245503' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:44:49.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 80160 80144 78.2265 32.25 0.00448379 0.0127543 2023-10-09T04:44:49.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4273245503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:49.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4273245503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: failed to encode map e297 with expected crc 2023-10-09T04:44:49.906 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:49 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:44:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4273245503' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:44:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: osdmap e297: 8 total, 8 up, 7 in 2023-10-09T04:44:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: failed to encode map e297 with expected crc 2023-10-09T04:44:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: failed to encode map e297 with expected crc 2023-10-09T04:44:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: failed to encode map e297 with expected crc 2023-10-09T04:44:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: failed to encode map e297 with expected crc 2023-10-09T04:44:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:49 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:50.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 80484 80468 77.3349 20.25 0.0101453 0.0129242 2023-10-09T04:44:50.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:50 smithi029 ceph-mon[216903]: pgmap v1143: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 23613/234942 objects degraded (10.051%); 8630/234942 objects misplaced (3.673%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:44:50.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/234942 objects degraded (10.051%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:50 smithi029 ceph-mon[223858]: pgmap v1143: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 23613/234942 objects degraded (10.051%); 8630/234942 objects misplaced (3.673%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:44:50.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/234942 objects degraded (10.051%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:50 smithi177 ceph-mon[195797]: pgmap v1143: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 475 op/s; 23613/234942 objects degraded (10.051%); 8630/234942 objects misplaced (3.673%); 4.2 MiB/s, 66 objects/s recovering 2023-10-09T04:44:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/234942 objects degraded (10.051%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:51.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 81016 81000 76.6658 33.25 0.0172693 0.0130341 2023-10-09T04:44:51.509 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:51.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:51.622 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:51.706 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:52.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 81656 81640 76.1187 40 0.00842562 0.0131313 2023-10-09T04:44:52.474 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:52.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:52.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:52.701 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:52.823 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:52 smithi029 ceph-mon[216903]: pgmap v1144: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 34 MiB/s wr, 543 op/s; 23613/239241 objects degraded (9.870%); 8318/239241 objects misplaced (3.477%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:44:52.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:52 smithi029 ceph-mon[223858]: pgmap v1144: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 34 MiB/s wr, 543 op/s; 23613/239241 objects degraded (9.870%); 8318/239241 objects misplaced (3.477%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:44:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:52 smithi177 ceph-mon[195797]: pgmap v1144: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 34 MiB/s wr, 543 op/s; 23613/239241 objects degraded (9.870%); 8318/239241 objects misplaced (3.477%); 5.5 MiB/s, 87 objects/s recovering 2023-10-09T04:44:53.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 82304 82288 75.595 40.5 0.0215287 0.0132231 2023-10-09T04:44:53.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:44:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:44:52] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-10-09T04:44:53.541 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:44:53.602 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:53.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:53.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:53.816 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:54.072 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:54.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:54.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 82902 82886 75.0413 37.375 0.0217028 0.0133199 2023-10-09T04:44:54.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:54.294 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:54.431 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:44:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:54.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:44:54.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:54.680 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:54.809 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:54.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:54 smithi029 ceph-mon[216903]: pgmap v1145: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 23613/243261 objects degraded (9.707%); 8109/243261 objects misplaced (3.333%); 6.0 MiB/s, 94 objects/s recovering 2023-10-09T04:44:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:54 smithi029 ceph-mon[223858]: pgmap v1145: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 23613/243261 objects degraded (9.707%); 8109/243261 objects misplaced (3.333%); 6.0 MiB/s, 94 objects/s recovering 2023-10-09T04:44:54.923 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:54.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:54 smithi177 ceph-mon[195797]: pgmap v1145: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 23613/243261 objects degraded (9.707%); 8109/243261 objects misplaced (3.333%); 6.0 MiB/s, 94 objects/s recovering 2023-10-09T04:44:54.973 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:55.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:55.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:55.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 83529 83513 74.5292 39.1875 0.00681891 0.0134076 2023-10-09T04:44:55.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:44:55.181 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:55.501 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:44:55.644 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:44:55.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/243261 objects degraded (9.707%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:55.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/243261 objects degraded (9.707%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:56.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/243261 objects degraded (9.707%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:44:56.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 84092 84076 73.9753 35.1875 0.0179758 0.0135122 2023-10-09T04:44:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:56 smithi029 ceph-mon[223858]: pgmap v1146: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 574 op/s; 23613/247575 objects degraded (9.538%); 7780/247575 objects misplaced (3.142%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:44:56.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:56 smithi029 ceph-mon[216903]: pgmap v1146: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 574 op/s; 23613/247575 objects degraded (9.538%); 7780/247575 objects misplaced (3.142%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:44:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:56 smithi177 ceph-mon[195797]: pgmap v1146: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 574 op/s; 23613/247575 objects degraded (9.538%); 7780/247575 objects misplaced (3.142%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:44:57.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 84660 84644 73.4411 35.5 0.0127171 0.0136112 2023-10-09T04:44:57.672 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:44:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:57.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:44:57.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:44:57.823 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:57.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:44:57.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:44:57.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:143840 10.12 10:4acbb7c5:::benchmark_data_smithi029_344401_object35469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:36:11.347554+0000", 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 526.546483352, 2023-10-09T04:44:57.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.28889141400000001, 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 143840 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347554+0000", 2023-10-09T04:44:57.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347554+0000", 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347553+0000", 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347559+0000", 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1319999999999999e-06 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347570+0000", 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1141000000000001e-05 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347577+0000", 2023-10-09T04:44:57.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347624+0000", 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7067000000000002e-05 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.347715+0000", 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0844e-05 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.348048+0000", 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033330300000000002 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.349685+0000", 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016366729999999999 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.417342+0000", 2023-10-09T04:44:57.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067657439 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.636345+0000", 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21900275899999999 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.636407+0000", 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1891999999999996e-05 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:36:11.636446+0000", 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8884000000000002e-05 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45542.0:196416 10.22 10:477c5541:::benchmark_data_smithi029_344401_object88045:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e225)", 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:37:04.320847+0000", 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 473.57319063699998, 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.31015875599999998, 2023-10-09T04:44:57.905 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45542", 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 196416 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320847+0000", 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320847+0000", 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320845+0000", 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320853+0000", 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2120000000000003e-06 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320873+0000", 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9426e-05 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320883+0000", 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0638e-05 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.320940+0000", 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6541e-05 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.321070+0000", 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012982799999999999 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.321464+0000", 2023-10-09T04:44:57.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039443300000000002 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.322919+0000", 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001454482 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.323713+0000", 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00079469399999999998 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.630915+0000", 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30720181099999999 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.630964+0000", 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9029000000000001e-05 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:37:04.631006+0000", 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1572000000000001e-05 2023-10-09T04:44:57.911 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:991 11.0 11:0ad1f868:::benchmark_data_smithi029_392508_object990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:02.468180+0000", 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 415.42585761999999, 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27276063699999997, 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.912 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 991 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.468180+0000", 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.468180+0000", 2023-10-09T04:44:57.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.468179+0000", 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.468202+0000", 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2549999999999999e-05 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.468210+0000", 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0849999999999997e-06 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.468216+0000", 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8950000000000003e-06 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.469697+0000", 2023-10-09T04:44:57.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014804830000000001 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.469884+0000", 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018725600000000001 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.470032+0000", 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014776700000000001 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.471081+0000", 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010498969999999999 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.471479+0000", 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00039749799999999998 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.740860+0000", 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26938137499999998 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.740902+0000", 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1754e-05 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:02.740941+0000", 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8646999999999998e-05 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:64301 11.c 11:3bb6d29d:::benchmark_data_smithi029_392508_object64300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e244)", 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:38:42.131547+0000", 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 375.76249055, 2023-10-09T04:44:57.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38340660900000001, 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 64301 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.131547+0000", 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.131547+0000", 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.131546+0000", 2023-10-09T04:44:57.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.131576+0000", 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0766000000000003e-05 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.321542+0000", 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18996550100000001 2023-10-09T04:44:57.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.321558+0000", 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6127e-05 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.321603+0000", 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4829000000000001e-05 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.321731+0000", 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012781399999999999 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.321836+0000", 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010574299999999999 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.323923+0000", 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002086432 2023-10-09T04:44:57.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.324264+0000", 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034098400000000001 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.514842+0000", 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.190578252 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.514906+0000", 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4238000000000004e-05 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:38:42.514954+0000", 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7388000000000001e-05 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89754 11.c 11:33b1c36a:::benchmark_data_smithi029_392508_object89753:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.381745+0000", 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.51229306800002, 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2012519429999999, 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89754 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.927 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381745+0000", 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381745+0000", 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381743+0000", 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381770+0000", 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6614999999999998e-05 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381783+0000", 2023-10-09T04:44:57.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3284e-05 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381792+0000", 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3379999999999994e-06 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381841+0000", 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9251e-05 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.382005+0000", 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164098 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.382174+0000", 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016878800000000001 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.403904+0000", 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021729721 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.403920+0000", 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6359e-05 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390650+0000", 2023-10-09T04:44:57.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98672965800000001 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390663+0000", 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2821e-05 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.582897+0000", 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.192233664 2023-10-09T04:44:57.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.582997+0000", 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100061 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89755 11.c 11:31713af8:::benchmark_data_smithi029_392508_object89754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.381808+0000", 2023-10-09T04:44:57.934 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.51222940600002, 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2052710119999999, 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89755 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381808+0000", 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381808+0000", 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381808+0000", 2023-10-09T04:44:57.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381828+0000", 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0469999999999999e-05 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381835+0000", 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9020000000000002e-06 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.381841+0000", 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4569999999999996e-06 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.393901+0000", 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012060611000000001 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.394058+0000", 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015636 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.394210+0000", 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015195000000000001 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.404239+0000", 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010029384000000001 2023-10-09T04:44:57.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.404248+0000", 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4020000000000008e-06 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390722+0000", 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98647359700000004 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390735+0000", 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2897999999999999e-05 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.586966+0000", 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.19623155 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.587079+0000", 2023-10-09T04:44:57.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011287 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89763 11.c 11:3492c3f6:::benchmark_data_smithi029_392508_object89762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.402548+0000", 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.491489887, 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2047692169999999, 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.942 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89763 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.402548+0000", 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.402548+0000", 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.402546+0000", 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.402570+0000", 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4159999999999999e-05 2023-10-09T04:44:57.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.402584+0000", 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3345e-05 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.402593+0000", 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0659999999999993e-06 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.405109+0000", 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025159620000000001 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.405117+0000", 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4009999999999997e-06 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390771+0000", 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98565397899999996 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390780+0000", 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9099999999999994e-06 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.587110+0000", 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.196330312 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.587242+0000", 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013180900000000001 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.587425+0000", 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018300900000000001 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.601940+0000", 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014514976000000001 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.602107+0000", 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166451 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.603529+0000", 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001421982 2023-10-09T04:44:57.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.603588+0000", 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9465000000000001e-05 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.607317+0000", 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003728968 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.950 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89767 11.c 11:3aa43d68:::benchmark_data_smithi029_392508_object89766:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.406338+0000", 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.48769923499998, 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2095970290000002, 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.951 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89767 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406338+0000", 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406338+0000", 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406337+0000", 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406364+0000", 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7614999999999999e-05 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406376+0000", 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1941e-05 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406386+0000", 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7899999999999994e-06 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406647+0000", 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026059199999999997 2023-10-09T04:44:57.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406657+0000", 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0319000000000001e-05 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390806+0000", 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98414847000000005 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390814+0000", 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9279999999999993e-06 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.590844+0000", 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.200030543 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.591044+0000", 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000199693 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.591269+0000", 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000224797 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.602148+0000", 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010879666 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.607335+0000", 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051864709999999998 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.612251+0000", 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049165859999999997 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.612323+0000", 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1612000000000005e-05 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.615935+0000", 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036125969999999999 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.959 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89768 11.c 11:3f4b2466:::benchmark_data_smithi029_392508_object89767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.406401+0000", 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.48763689100002, 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.213401996, 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89768 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406401+0000", 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406401+0000", 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406400+0000", 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406423+0000", 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2869000000000001e-05 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406431+0000", 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8779999999999998e-06 2023-10-09T04:44:57.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406437+0000", 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.976e-06 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406681+0000", 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024439600000000002 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.406690+0000", 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3059999999999998e-06 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390839+0000", 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.984149512 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390847+0000", 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8040000000000002e-06 2023-10-09T04:44:57.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.594557+0000", 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.203710238 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.594748+0000", 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019118299999999999 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:44:57.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.594966+0000", 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021729200000000001 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.607346+0000", 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012379927000000001 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.612158+0000", 2023-10-09T04:44:57.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048122909999999998 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.616045+0000", 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038875770000000001 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.616123+0000", 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8016999999999995e-05 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.619803+0000", 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036792919999999998 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89773 11.c 11:3ce93dda:::benchmark_data_smithi029_392508_object89772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.410794+0000", 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.48324329000002, 2023-10-09T04:44:57.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2131538320000002, 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89773 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.410794+0000", 2023-10-09T04:44:57.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.410794+0000", 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.410794+0000", 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.410818+0000", 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3674000000000001e-05 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.410826+0000", 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0989999999999999e-06 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.410832+0000", 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5980000000000002e-06 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.411004+0000", 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017173199999999999 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.411030+0000", 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5887000000000001e-05 2023-10-09T04:44:57.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390872+0000", 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97984203800000003 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390880+0000", 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9719999999999992e-06 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.598170+0000", 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2072901890000001 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.598374+0000", 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000204293 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.598582+0000", 2023-10-09T04:44:57.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020769199999999999 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.607355+0000", 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087733789999999996 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.612210+0000", 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048542669999999998 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.619910+0000", 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076999770000000002 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.619963+0000", 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3658e-05 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.623948+0000", 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039848970000000003 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89774 11.e 11:7416953e:::benchmark_data_smithi029_392508_object89773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.415064+0000", 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.47897349099998, 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4967752750000001, 2023-10-09T04:44:57.977 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89774 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.415064+0000", 2023-10-09T04:44:57.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.415064+0000", 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.415062+0000", 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.415092+0000", 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9377999999999999e-05 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.415108+0000", 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6538e-05 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.415118+0000", 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6930000000000002e-06 2023-10-09T04:44:57.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.417158+0000", 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020396749999999999 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.417172+0000", 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4884e-05 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390046+0000", 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97287378899999999 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390062+0000", 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.552e-05 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.833155+0000", 2023-10-09T04:44:57.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.443093669 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.835664+0000", 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025084930000000001 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.835829+0000", 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016531799999999999 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.850495+0000", 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014666287 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.860563+0000", 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010067664 2023-10-09T04:44:57.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.907516+0000", 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.046952920000000002 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.907580+0000", 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4369999999999995e-05 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.911839+0000", 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042590420000000002 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89793 11.e 11:70a06ee6:::benchmark_data_smithi029_392508_object89792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.439598+0000", 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.45443977399998, 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.476248623, 2023-10-09T04:44:57.986 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89793 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439598+0000", 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439598+0000", 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439597+0000", 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439644+0000", 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7620999999999997e-05 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439660+0000", 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5478999999999999e-05 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439668+0000", 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4780000000000008e-06 2023-10-09T04:44:57.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439734+0000", 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5792999999999998e-05 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.439742+0000", 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4800000000000004e-06 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390126+0000", 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95038394400000004 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390135+0000", 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2669999999999995e-06 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.843589+0000", 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4534538719999999 2023-10-09T04:44:57.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.843748+0000", 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015937699999999999 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.843910+0000", 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016196200000000001 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.850533+0000", 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066231220000000004 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.860608+0000", 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010074375 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.911919+0000", 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.051311631000000003 2023-10-09T04:44:57.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.911974+0000", 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4344e-05 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.915847+0000", 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003872931 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:57.994 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89800 11.e 11:7f13af3e:::benchmark_data_smithi029_392508_object89799:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.460090+0000", 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.433947934, 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4598401700000001, 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:44:57.995 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89800 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460090+0000", 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460090+0000", 2023-10-09T04:44:57.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460089+0000", 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460130+0000", 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1721999999999998e-05 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460143+0000", 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3018e-05 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460149+0000", 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1840000000000004e-06 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460209+0000", 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9772999999999998e-05 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.460223+0000", 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4253000000000001e-05 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390200+0000", 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.92997697000000001 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:57.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.390213+0000", 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2872e-05 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.847069+0000", 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.456855732 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.847234+0000", 2023-10-09T04:44:58.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164656 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.847351+0000", 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117465 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.860552+0000", 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013201034 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.860614+0000", 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1624000000000005e-05 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.915940+0000", 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.055325768999999997 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.915997+0000", 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7286e-05 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.919930+0000", 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039330229999999999 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:180853 11.b e273/237 11:d6b074f4:::benchmark_data_smithi029_392508_object80297:head v 273'11145, mlcod=273'11143)", 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:05.387548+0000", 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.50648997100001, 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.363381326, 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.387548+0000", 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.387548+0000", 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.387547+0000", 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.387553+0000", 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6670000000000001e-06 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.387563+0000", 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0327e-05 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.387572+0000", 2023-10-09T04:44:58.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7630000000000006e-06 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.389780+0000", 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022084919999999998 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.389800+0000", 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9292000000000001e-05 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.750884+0000", 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36108483099999999 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.750929+0000", 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4566e-05 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:180858 11.b e273/237 11:d58ed901:::benchmark_data_smithi029_392508_object80302:head v 273'11146, mlcod=273'11143)", 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:05.391659+0000", 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.502378862, 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.35929436599999998, 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.391659+0000", 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.391659+0000", 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.391658+0000", 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.391665+0000", 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7349999999999999e-06 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.391676+0000", 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1627000000000001e-05 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.391684+0000", 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4340000000000003e-06 2023-10-09T04:44:58.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.393951+0000", 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022672109999999999 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.393971+0000", 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9621999999999999e-05 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.750933+0000", 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.35696278999999997 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.750953+0000", 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.978e-05 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:180857 11.f e273/237 11:f28d926a:::benchmark_data_smithi029_392508_object80301:head v 273'11285, mlcod=273'11282)", 2023-10-09T04:44:58.013 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:05.396321+0000", 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.497716438, 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.355547944, 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396321+0000", 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396321+0000", 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396320+0000", 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396335+0000", 2023-10-09T04:44:58.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4859e-05 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396349+0000", 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3940999999999999e-05 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396357+0000", 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2050000000000002e-06 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396409+0000", 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2314000000000001e-05 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.396430+0000", 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0724e-05 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.751832+0000", 2023-10-09T04:44:58.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.35540215400000003 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.751869+0000", 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6996000000000003e-05 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:180863 11.c e273/262 11:3c0ed102:::benchmark_data_smithi029_392508_object80307:head v 273'11214, mlcod=246'5583)", 2023-10-09T04:44:58.018 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:05.397675+0000", 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.496362291, 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36414607599999999, 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397675+0000", 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397675+0000", 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397675+0000", 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397682+0000", 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7730000000000002e-06 2023-10-09T04:44:58.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397691+0000", 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.577e-06 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397697+0000", 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2090000000000002e-06 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397743+0000", 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5429000000000002e-05 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397766+0000", 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3659000000000002e-05 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.761782+0000", 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.364015174 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.761821+0000", 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9844e-05 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.023 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:180859 11.b e273/237 11:d78bf015:::benchmark_data_smithi029_392508_object80303:head v 273'11147, mlcod=273'11143)", 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:05.397718+0000", 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.49631959999999, 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36189703400000001, 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397718+0000", 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397718+0000", 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397718+0000", 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397722+0000", 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2030000000000002e-06 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397728+0000", 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3199999999999996e-06 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.397733+0000", 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.34e-06 2023-10-09T04:44:58.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.398253+0000", 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052009199999999999 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.398270+0000", 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7762999999999999e-05 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.759573+0000", 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36130255900000002 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.759615+0000", 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2110000000000002e-05 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.028 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.45869.0:180862 11.f e273/237 11:fd8841e5:::benchmark_data_smithi029_392508_object80306:head v 273'11286, mlcod=273'11282)", 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:05.400090+0000", 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 172.493947784, 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36005236800000001, 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.400090+0000", 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.400090+0000", 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.400089+0000", 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.400096+0000", 2023-10-09T04:44:58.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8669999999999996e-06 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.747595+0000", 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34749963099999998 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.747624+0000", 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8487999999999999e-05 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.747713+0000", 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9248999999999999e-05 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.747728+0000", 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4455e-05 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.760104+0000", 2023-10-09T04:44:58.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012376465 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:05.760142+0000", 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8123000000000001e-05 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:80240 12.9 e297/293 12:9f43efeb:::benchmark_data_smithi029_442458_object80239:head v 297'5038, mlcod=0'0)", 2023-10-09T04:44:58.033 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:44:49.483571+0000", 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 8.4104664679999992, 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27411632800000002, 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.483571+0000", 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.483571+0000", 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.483570+0000", 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:44:58.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.483598+0000", 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8119e-05 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.727032+0000", 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.24343427500000001 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.727042+0000", 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7990000000000005e-06 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.727085+0000", 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3251000000000003e-05 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.727119+0000", 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4267999999999998e-05 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:44:58.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.757644+0000", 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030524175000000001 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.757688+0000", 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3883e-05 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:44:58.038 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:44:58.039 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:44:58.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:44:58.143 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 85239 85223 72.928 36.1875 0.0245868 0.0137058 2023-10-09T04:44:58.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:44:58.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:58 smithi029 ceph-mon[216903]: pgmap v1147: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 572 op/s; 23613/249276 objects degraded (9.473%); 7780/249276 objects misplaced (3.121%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:44:58.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:58.666 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:44:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:58.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:58 smithi029 ceph-mon[223858]: pgmap v1147: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 572 op/s; 23613/249276 objects degraded (9.473%); 7780/249276 objects misplaced (3.121%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:44:58.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:58.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:44:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:58 smithi177 ceph-mon[195797]: pgmap v1147: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 36 MiB/s wr, 572 op/s; 23613/249276 objects degraded (9.473%); 7780/249276 objects misplaced (3.121%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:44:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:44:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:44:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:44:59.144 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 85918 85902 72.5161 42.4375 0.057095 0.0137842 2023-10-09T04:45:00.143 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 86455 86439 71.9968 33.5625 0.0348585 0.0138825 2023-10-09T04:45:00.159 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:00.392 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:00.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:00.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:00.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:00.617 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:00.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:45:00.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:00 smithi029 ceph-mon[216903]: pgmap v1148: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 33 MiB/s wr, 533 op/s; 23613/252000 objects degraded (9.370%); 7597/252000 objects misplaced (3.015%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T04:45:00.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/252000 objects degraded (9.370%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:00.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:00 smithi029 ceph-mon[223858]: pgmap v1148: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 33 MiB/s wr, 533 op/s; 23613/252000 objects degraded (9.370%); 7597/252000 objects misplaced (3.015%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T04:45:00.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/252000 objects degraded (9.370%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:01.008 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:00 smithi177 ceph-mon[195797]: pgmap v1148: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 33 MiB/s wr, 533 op/s; 23613/252000 objects degraded (9.370%); 7597/252000 objects misplaced (3.015%); 6.1 MiB/s, 96 objects/s recovering 2023-10-09T04:45:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/252000 objects degraded (9.370%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:01.084 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:01.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:01.144 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 87006 86990 71.5028 34.4375 0.0166156 0.0139813 2023-10-09T04:45:01.199 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:01.339 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:45:01.344 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:01.377 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:45:02.143 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 87609 87593 71.0635 37.6875 0.0334207 0.0140665 2023-10-09T04:45:02.556 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:02.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:02.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:02.784 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:02.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:45:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:45:02] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-10-09T04:45:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:02 smithi029 ceph-mon[223858]: pgmap v1149: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 37 MiB/s wr, 599 op/s; 23613/256527 objects degraded (9.205%); 7259/256527 objects misplaced (2.830%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T04:45:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:02 smithi029 ceph-mon[223858]: from='client.55960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:45:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:02 smithi029 ceph-mon[216903]: pgmap v1149: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 37 MiB/s wr, 599 op/s; 23613/256527 objects degraded (9.205%); 7259/256527 objects misplaced (2.830%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T04:45:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:02 smithi029 ceph-mon[216903]: from='client.55960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:45:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:02 smithi177 ceph-mon[195797]: pgmap v1149: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 37 MiB/s wr, 599 op/s; 23613/256527 objects degraded (9.205%); 7259/256527 objects misplaced (2.830%); 7.1 MiB/s, 114 objects/s recovering 2023-10-09T04:45:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:02 smithi177 ceph-mon[195797]: from='client.55960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:45:03.143 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 88252 88236 70.6677 40.1875 0.0279413 0.0141447 2023-10-09T04:45:04.150 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 88713 88697 70.1325 28.8125 0.01852 0.0142536 2023-10-09T04:45:05.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:04 smithi177 ceph-mon[195797]: pgmap v1150: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 23613/260928 objects degraded (9.050%); 7076/260928 objects misplaced (2.712%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T04:45:05.151 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:45:05.150278+0000 min lat: 0.00124565 max lat: 2.32654 avg lat: 0.0143387 2023-10-09T04:45:05.151 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:45:05.151 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 89307 89291 69.72 37.125 0.0182422 0.0143387 2023-10-09T04:45:05.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:04 smithi029 ceph-mon[216903]: pgmap v1150: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 23613/260928 objects degraded (9.050%); 7076/260928 objects misplaced (2.712%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T04:45:05.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:04 smithi029 ceph-mon[223858]: pgmap v1150: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 23613/260928 objects degraded (9.050%); 7076/260928 objects misplaced (2.712%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T04:45:06.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/260928 objects degraded (9.050%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:06.150 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 89925 89909 69.3363 38.625 0.0349832 0.0144176 2023-10-09T04:45:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/260928 objects degraded (9.050%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:06.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/260928 objects degraded (9.050%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:06 smithi029 ceph-mon[216903]: pgmap v1151: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 609 op/s; 23613/265197 objects degraded (8.904%); 6705/265197 objects misplaced (2.528%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:45:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:06 smithi029 ceph-mon[223858]: pgmap v1151: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 609 op/s; 23613/265197 objects degraded (8.904%); 6705/265197 objects misplaced (2.528%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:45:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:06 smithi177 ceph-mon[195797]: pgmap v1151: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 609 op/s; 23613/265197 objects degraded (8.904%); 6705/265197 objects misplaced (2.528%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:45:07.150 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 90616 90600 69.0175 43.1875 0.0110748 0.0144832 2023-10-09T04:45:08.010 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:08.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:08.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:08.123 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1149,"stamp":"2023-10-09T04:45:01.235437+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5602940832,"num_objects":85509,"num_object_clones":0,"num_object_copies":256527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23613,"num_objects_misplaced":7259,"num_objects_unfound":0,"num_objects_dirty":85509,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":85785,"num_write_kb":5473485,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1952,"num_bytes_recovered":128333180,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85701,"ondisk_log_size":85701,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19615820,"kb_used_data":14387092,"kb_used_omap":20,"kb_used_meta":5228651,"kb_avail":636600244,"statfs":{"total":671965249536,"available":651878649856,"internally_reserved":0,"allocated":14732382208,"data_stored":14726057675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":5354139278},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,9,2,11,5,4,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":471531520,"num_objects":7195,"num_object_clones":0,"num_object_copies":21585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1371,"num_objects_unfound":0,"num_objects_dirty":7195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7195,"num_write_kb":460480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1371,"num_bytes_recovered":89849856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003612"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":45,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599487+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:44:41.599487+0000","last_peered":"2023-10-09T04:44:41.599487+0000","last_clean":"2023-10-09T04:44:41.599487+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:44:41.599487+0000","last_undegraded":"2023-10-09T04:44:41.599487+0000","last_fullsized":"2023-10-09T04:44:41.599487+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":250,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.615466+0000","last_change":"2023-10-09T04:27:57.988720+0000","last_active":"2023-10-09T04:44:41.615466+0000","last_peered":"2023-10-09T04:44:41.615466+0000","last_clean":"2023-10-09T04:44:41.615466+0000","last_became_active":"2023-10-09T04:27:57.988430+0000","last_became_peered":"2023-10-09T04:27:57.988430+0000","last_unstale":"2023-10-09T04:44:41.615466+0000","last_undegraded":"2023-10-09T04:44:41.615466+0000","last_fullsized":"2023-10-09T04:44:41.615466+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:29:42.803785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":283,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.589770+0000","last_change":"2023-10-09T04:41:02.606146+0000","last_active":"2023-10-09T04:44:41.589770+0000","last_peered":"2023-10-09T04:44:41.589770+0000","last_clean":"2023-10-09T04:44:41.589770+0000","last_became_active":"2023-10-09T04:41:02.605877+0000","last_became_peered":"2023-10-09T04:41:02.605877+0000","last_unstale":"2023-10-09T04:44:41.589770+0000","last_undegraded":"2023-10-09T04:44:41.589770+0000","last_fullsized":"2023-10-09T04:44:41.589770+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:34:32.087395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":210,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597967+0000","last_change":"2023-10-09T04:34:22.267058+0000","last_active":"2023-10-09T04:44:41.597967+0000","last_peered":"2023-10-09T04:44:41.597967+0000","last_clean":"2023-10-09T04:44:41.597967+0000","last_became_active":"2023-10-09T04:34:22.266282+0000","last_became_peered":"2023-10-09T04:34:22.266282+0000","last_unstale":"2023-10-09T04:44:41.597967+0000","last_undegraded":"2023-10-09T04:44:41.597967+0000","last_fullsized":"2023-10-09T04:44:41.597967+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:45:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":287,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:48.583132+0000","last_change":"2023-10-09T04:29:51.056700+0000","last_active":"2023-10-09T04:44:48.583132+0000","last_peered":"2023-10-09T04:44:48.583132+0000","last_clean":"2023-10-09T04:44:48.583132+0000","last_became_active":"2023-10-09T04:29:51.056372+0000","last_became_peered":"2023-10-09T04:29:51.056372+0000","last_unstale":"2023-10-09T04:44:48.583132+0000","last_undegraded":"2023-10-09T04:44:48.583132+0000","last_fullsized":"2023-10-09T04:44:48.583132+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:09.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596407+0000","last_change":"2023-10-09T04:44:38.775587+0000","last_active":"2023-10-09T04:44:41.596407+0000","last_peered":"2023-10-09T04:44:41.596407+0000","last_clean":"2023-10-09T04:44:41.596407+0000","last_became_active":"2023-10-09T04:44:38.775298+0000","last_became_peered":"2023-10-09T04:44:38.775298+0000","last_unstale":"2023-10-09T04:44:41.596407+0000","last_undegraded":"2023-10-09T04:44:41.596407+0000","last_fullsized":"2023-10-09T04:44:41.596407+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:58:44.311958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":209,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596258+0000","last_change":"2023-10-09T04:34:22.267601+0000","last_active":"2023-10-09T04:44:41.596258+0000","last_peered":"2023-10-09T04:44:41.596258+0000","last_clean":"2023-10-09T04:44:41.596258+0000","last_became_active":"2023-10-09T04:34:22.267247+0000","last_became_peered":"2023-10-09T04:34:22.267247+0000","last_unstale":"2023-10-09T04:44:41.596258+0000","last_undegraded":"2023-10-09T04:44:41.596258+0000","last_fullsized":"2023-10-09T04:44:41.596258+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.361756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.610862+0000","last_change":"2023-10-09T04:44:38.772081+0000","last_active":"2023-10-09T04:44:41.610862+0000","last_peered":"2023-10-09T04:44:41.610862+0000","last_clean":"2023-10-09T04:44:41.610862+0000","last_became_active":"2023-10-09T04:44:38.771763+0000","last_became_peered":"2023-10-09T04:44:38.771763+0000","last_unstale":"2023-10-09T04:44:41.610862+0000","last_undegraded":"2023-10-09T04:44:41.610862+0000","last_fullsized":"2023-10-09T04:44:41.610862+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:16.923225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595458+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:44:41.595458+0000","last_peered":"2023-10-09T04:44:41.595458+0000","last_clean":"2023-10-09T04:44:41.595458+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:44:41.595458+0000","last_undegraded":"2023-10-09T04:44:41.595458+0000","last_fullsized":"2023-10-09T04:44:41.595458+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":233,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.612900+0000","last_change":"2023-10-09T04:44:38.772938+0000","last_active":"2023-10-09T04:44:41.612900+0000","last_peered":"2023-10-09T04:44:41.612900+0000","last_clean":"2023-10-09T04:44:41.612900+0000","last_became_active":"2023-10-09T04:44:38.772612+0000","last_became_peered":"2023-10-09T04:44:38.772612+0000","last_unstale":"2023-10-09T04:44:41.612900+0000","last_undegraded":"2023-10-09T04:44:41.612900+0000","last_fullsized":"2023-10-09T04:44:41.612900+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:30:06.253005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":291,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597172+0000","last_change":"2023-10-09T04:41:02.730765+0000","last_active":"2023-10-09T04:44:41.597172+0000","last_peered":"2023-10-09T04:44:41.597172+0000","last_clean":"2023-10-09T04:44:41.597172+0000","last_became_active":"2023-10-09T04:41:02.611177+0000","last_became_peered":"2023-10-09T04:41:02.611177+0000","last_unstale":"2023-10-09T04:44:41.597172+0000","last_undegraded":"2023-10-09T04:44:41.597172+0000","last_fullsized":"2023-10-09T04:44:41.597172+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:52:08.538705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":213,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:42.077733+0000","last_change":"2023-10-09T04:44:38.768067+0000","last_active":"2023-10-09T04:44:42.077733+0000","last_peered":"2023-10-09T04:44:42.077733+0000","last_clean":"2023-10-09T04:44:42.077733+0000","last_became_active":"2023-10-09T04:44:38.767706+0000","last_became_peered":"2023-10-09T04:44:38.767706+0000","last_unstale":"2023-10-09T04:44:42.077733+0000","last_undegraded":"2023-10-09T04:44:42.077733+0000","last_fullsized":"2023-10-09T04:44:42.077733+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:33.801969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":75,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599530+0000","last_change":"2023-10-09T04:44:38.813366+0000","last_active":"2023-10-09T04:44:41.599530+0000","last_peered":"2023-10-09T04:44:41.599530+0000","last_clean":"2023-10-09T04:44:41.599530+0000","last_became_active":"2023-10-09T04:44:38.777069+0000","last_became_peered":"2023-10-09T04:44:38.777069+0000","last_unstale":"2023-10-09T04:44:41.599530+0000","last_undegraded":"2023-10-09T04:44:41.599530+0000","last_fullsized":"2023-10-09T04:44:41.599530+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:39.101540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":267,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.601788+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:44:41.601788+0000","last_peered":"2023-10-09T04:44:41.601788+0000","last_clean":"2023-10-09T04:44:41.601788+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:44:41.601788+0000","last_undegraded":"2023-10-09T04:44:41.601788+0000","last_fullsized":"2023-10-09T04:44:41.601788+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":307,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597015+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:44:41.597015+0000","last_peered":"2023-10-09T04:44:41.597015+0000","last_clean":"2023-10-09T04:44:41.597015+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:44:41.597015+0000","last_undegraded":"2023-10-09T04:44:41.597015+0000","last_fullsized":"2023-10-09T04:44:41.597015+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.619245+0000","last_change":"2023-10-09T04:27:57.987345+0000","last_active":"2023-10-09T04:44:41.619245+0000","last_peered":"2023-10-09T04:44:41.619245+0000","last_clean":"2023-10-09T04:44:41.619245+0000","last_became_active":"2023-10-09T04:27:57.986960+0000","last_became_peered":"2023-10-09T04:27:57.986960+0000","last_unstale":"2023-10-09T04:44:41.619245+0000","last_undegraded":"2023-10-09T04:44:41.619245+0000","last_fullsized":"2023-10-09T04:44:41.619245+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:26:16.387785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":209,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596874+0000","last_change":"2023-10-09T04:34:22.266429+0000","last_active":"2023-10-09T04:44:41.596874+0000","last_peered":"2023-10-09T04:44:41.596874+0000","last_clean":"2023-10-09T04:44:41.596874+0000","last_became_active":"2023-10-09T04:34:22.266178+0000","last_became_peered":"2023-10-09T04:34:22.266178+0000","last_unstale":"2023-10-09T04:44:41.596874+0000","last_undegraded":"2023-10-09T04:44:41.596874+0000","last_fullsized":"2023-10-09T04:44:41.596874+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:19.790253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":256,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595188+0000","last_change":"2023-10-09T04:27:57.992866+0000","last_active":"2023-10-09T04:44:41.595188+0000","last_peered":"2023-10-09T04:44:41.595188+0000","last_clean":"2023-10-09T04:44:41.595188+0000","last_became_active":"2023-10-09T04:27:57.992553+0000","last_became_peered":"2023-10-09T04:27:57.992553+0000","last_unstale":"2023-10-09T04:44:41.595188+0000","last_undegraded":"2023-10-09T04:44:41.595188+0000","last_fullsized":"2023-10-09T04:44:41.595188+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:28.976105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":249,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597590+0000","last_change":"2023-10-09T04:44:38.778382+0000","last_active":"2023-10-09T04:44:41.597590+0000","last_peered":"2023-10-09T04:44:41.597590+0000","last_clean":"2023-10-09T04:44:41.597590+0000","last_became_active":"2023-10-09T04:44:38.778069+0000","last_became_peered":"2023-10-09T04:44:38.778069+0000","last_unstale":"2023-10-09T04:44:41.597590+0000","last_undegraded":"2023-10-09T04:44:41.597590+0000","last_fullsized":"2023-10-09T04:44:41.597590+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:52:09.818809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":212,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.614736+0000","last_change":"2023-10-09T04:34:22.265498+0000","last_active":"2023-10-09T04:44:41.614736+0000","last_peered":"2023-10-09T04:44:41.614736+0000","last_clean":"2023-10-09T04:44:41.614736+0000","last_became_active":"2023-10-09T04:34:22.265187+0000","last_became_peered":"2023-10-09T04:34:22.265187+0000","last_unstale":"2023-10-09T04:44:41.614736+0000","last_undegraded":"2023-10-09T04:44:41.614736+0000","last_fullsized":"2023-10-09T04:44:41.614736+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:55.369607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":38,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.594315+0000","last_change":"2023-10-09T04:44:38.787971+0000","last_active":"2023-10-09T04:44:41.594315+0000","last_peered":"2023-10-09T04:44:41.594315+0000","last_clean":"2023-10-09T04:44:41.594315+0000","last_became_active":"2023-10-09T04:44:38.775892+0000","last_became_peered":"2023-10-09T04:44:38.775892+0000","last_unstale":"2023-10-09T04:44:41.594315+0000","last_undegraded":"2023-10-09T04:44:41.594315+0000","last_fullsized":"2023-10-09T04:44:41.594315+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:29:12.443809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:48.583695+0000","last_change":"2023-10-09T04:27:57.986461+0000","last_active":"2023-10-09T04:44:48.583695+0000","last_peered":"2023-10-09T04:44:48.583695+0000","last_clean":"2023-10-09T04:44:48.583695+0000","last_became_active":"2023-10-09T04:27:57.986222+0000","last_became_peered":"2023-10-09T04:27:57.986222+0000","last_unstale":"2023-10-09T04:44:48.583695+0000","last_undegraded":"2023-10-09T04:44:48.583695+0000","last_fullsized":"2023-10-09T04:44:48.583695+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:28.659135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"297'5458","reported_seq":5492,"reported_epoch":297,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:44:59.290922+0000","last_change":"2023-10-09T04:44:40.036208+0000","last_active":"2023-10-09T04:44:59.290922+0000","last_peered":"2023-10-09T04:44:59.290922+0000","last_clean":"2023-10-09T04:44:37.624385+0000","last_became_active":"2023-10-09T04:44:40.023059+0000","last_became_peered":"2023-10-09T04:44:40.023059+0000","last_unstale":"2023-10-09T04:44:59.290922+0000","last_undegraded":"2023-10-09T04:44:39.944445+0000","last_fullsized":"2023-10-09T04:44:39.771566+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5458,"log_dups_size":0,"ondisk_log_size":5458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:06.349239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357695488,"num_objects":5458,"num_object_clones":0,"num_object_copies":16374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4751,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5458,"num_write_kb":349312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":707},{"shards":"2,4,6","objects":4751}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":313,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596938+0000","last_change":"2023-10-09T04:41:02.627041+0000","last_active":"2023-10-09T04:44:41.596938+0000","last_peered":"2023-10-09T04:44:41.596938+0000","last_clean":"2023-10-09T04:44:41.596938+0000","last_became_active":"2023-10-09T04:41:02.609580+0000","last_became_peered":"2023-10-09T04:41:02.609580+0000","last_unstale":"2023-10-09T04:44:41.596938+0000","last_undegraded":"2023-10-09T04:44:41.596938+0000","last_fullsized":"2023-10-09T04:44:41.596938+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:31.452824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.594206+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:44:41.594206+0000","last_peered":"2023-10-09T04:44:41.594206+0000","last_clean":"2023-10-09T04:44:41.594206+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:44:41.594206+0000","last_undegraded":"2023-10-09T04:44:41.594206+0000","last_fullsized":"2023-10-09T04:44:41.594206+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"297'5389","reported_seq":5416,"reported_epoch":297,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:44:57.989292+0000","last_change":"2023-10-09T04:44:40.054868+0000","last_active":"2023-10-09T04:44:57.989292+0000","last_peered":"2023-10-09T04:44:57.989292+0000","last_clean":"2023-10-09T04:44:37.740580+0000","last_became_active":"2023-10-09T04:44:40.036044+0000","last_became_peered":"2023-10-09T04:44:40.036044+0000","last_unstale":"2023-10-09T04:44:57.989292+0000","last_undegraded":"2023-10-09T04:44:39.825710+0000","last_fullsized":"2023-10-09T04:44:39.772664+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5389,"log_dups_size":0,"ondisk_log_size":5389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:27:22.818306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353173504,"num_objects":5389,"num_object_clones":0,"num_object_copies":16167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4694,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5389,"num_write_kb":344896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":4694},{"shards":"3,7","objects":695}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":286,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597892+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:44:41.597892+0000","last_peered":"2023-10-09T04:44:41.597892+0000","last_clean":"2023-10-09T04:44:41.597892+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:44:41.597892+0000","last_undegraded":"2023-10-09T04:44:41.597892+0000","last_fullsized":"2023-10-09T04:44:41.597892+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":274,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599363+0000","last_change":"2023-10-09T04:27:57.988386+0000","last_active":"2023-10-09T04:44:41.599363+0000","last_peered":"2023-10-09T04:44:41.599363+0000","last_clean":"2023-10-09T04:44:41.599363+0000","last_became_active":"2023-10-09T04:27:57.988025+0000","last_became_peered":"2023-10-09T04:27:57.988025+0000","last_unstale":"2023-10-09T04:44:41.599363+0000","last_undegraded":"2023-10-09T04:44:41.599363+0000","last_fullsized":"2023-10-09T04:44:41.599363+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:59.531685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"297'5338","reported_seq":5359,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:59.270921+0000","last_change":"2023-10-09T04:43:34.547460+0000","last_active":"2023-10-09T04:44:59.270921+0000","last_peered":"2023-10-09T04:44:59.270921+0000","last_clean":"2023-10-09T04:44:59.270921+0000","last_became_active":"2023-10-09T04:43:34.547150+0000","last_became_peered":"2023-10-09T04:43:34.547150+0000","last_unstale":"2023-10-09T04:44:59.270921+0000","last_undegraded":"2023-10-09T04:44:59.270921+0000","last_fullsized":"2023-10-09T04:44:59.270921+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5338,"log_dups_size":0,"ondisk_log_size":5338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:43.085276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349831168,"num_objects":5338,"num_object_clones":0,"num_object_copies":16014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5338,"num_write_kb":341632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":254,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597029+0000","last_change":"2023-10-09T04:44:38.793625+0000","last_active":"2023-10-09T04:44:41.597029+0000","last_peered":"2023-10-09T04:44:41.597029+0000","last_clean":"2023-10-09T04:44:41.597029+0000","last_became_active":"2023-10-09T04:44:38.778994+0000","last_became_peered":"2023-10-09T04:44:38.778994+0000","last_unstale":"2023-10-09T04:44:41.597029+0000","last_undegraded":"2023-10-09T04:44:41.597029+0000","last_fullsized":"2023-10-09T04:44:41.597029+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:11:24.482041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":213,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:42.077720+0000","last_change":"2023-10-09T04:44:38.775465+0000","last_active":"2023-10-09T04:44:42.077720+0000","last_peered":"2023-10-09T04:44:42.077720+0000","last_clean":"2023-10-09T04:44:42.077720+0000","last_became_active":"2023-10-09T04:44:38.775246+0000","last_became_peered":"2023-10-09T04:44:38.775246+0000","last_unstale":"2023-10-09T04:44:42.077720+0000","last_undegraded":"2023-10-09T04:44:42.077720+0000","last_fullsized":"2023-10-09T04:44:42.077720+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.286190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.3","version":"297'5398","reported_seq":5419,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:59.282226+0000","last_change":"2023-10-09T04:43:34.881088+0000","last_active":"2023-10-09T04:44:59.282226+0000","last_peered":"2023-10-09T04:44:59.282226+0000","last_clean":"2023-10-09T04:44:59.282226+0000","last_became_active":"2023-10-09T04:43:34.880601+0000","last_became_peered":"2023-10-09T04:43:34.880601+0000","last_unstale":"2023-10-09T04:44:59.282226+0000","last_undegraded":"2023-10-09T04:44:59.282226+0000","last_fullsized":"2023-10-09T04:44:59.282226+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5398,"log_dups_size":0,"ondisk_log_size":5398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:27.274950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353763328,"num_objects":5398,"num_object_clones":0,"num_object_copies":16194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5398,"num_write_kb":345472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":267,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595800+0000","last_change":"2023-10-09T04:44:38.775807+0000","last_active":"2023-10-09T04:44:41.595800+0000","last_peered":"2023-10-09T04:44:41.595800+0000","last_clean":"2023-10-09T04:44:41.595800+0000","last_became_active":"2023-10-09T04:44:38.775568+0000","last_became_peered":"2023-10-09T04:44:38.775568+0000","last_unstale":"2023-10-09T04:44:41.595800+0000","last_undegraded":"2023-10-09T04:44:41.595800+0000","last_fullsized":"2023-10-09T04:44:41.595800+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:26.592236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":154,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.594884+0000","last_change":"2023-10-09T04:34:22.263525+0000","last_active":"2023-10-09T04:44:41.594884+0000","last_peered":"2023-10-09T04:44:41.594884+0000","last_clean":"2023-10-09T04:44:41.594884+0000","last_became_active":"2023-10-09T04:34:22.263147+0000","last_became_peered":"2023-10-09T04:34:22.263147+0000","last_unstale":"2023-10-09T04:44:41.594884+0000","last_undegraded":"2023-10-09T04:44:41.594884+0000","last_fullsized":"2023-10-09T04:44:41.594884+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:45.775834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.4","version":"297'5305","reported_seq":6687,"reported_epoch":297,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:44:57.984101+0000","last_change":"2023-10-09T04:44:40.046213+0000","last_active":"2023-10-09T04:44:57.984101+0000","last_peered":"2023-10-09T04:44:57.984101+0000","last_clean":"2023-10-09T04:44:37.730195+0000","last_became_active":"2023-10-09T04:44:39.998920+0000","last_became_peered":"2023-10-09T04:44:39.998920+0000","last_unstale":"2023-10-09T04:44:57.984101+0000","last_undegraded":"2023-10-09T04:44:57.984101+0000","last_fullsized":"2023-10-09T04:44:39.772464+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:50.004218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347668480,"num_objects":5305,"num_object_clones":0,"num_object_copies":15915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3982,"num_objects_unfound":0,"num_objects_dirty":5305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5305,"num_write_kb":339520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":677,"num_bytes_recovered":44367872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"1,3","objects":1323},{"shards":"1,3,6","objects":3982}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":295,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.619247+0000","last_change":"2023-10-09T04:29:51.063648+0000","last_active":"2023-10-09T04:44:41.619247+0000","last_peered":"2023-10-09T04:44:41.619247+0000","last_clean":"2023-10-09T04:44:41.619247+0000","last_became_active":"2023-10-09T04:29:51.056287+0000","last_became_peered":"2023-10-09T04:29:51.056287+0000","last_unstale":"2023-10-09T04:44:41.619247+0000","last_undegraded":"2023-10-09T04:44:41.619247+0000","last_fullsized":"2023-10-09T04:44:41.619247+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:25.958096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":55,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599596+0000","last_change":"2023-10-09T04:44:38.767167+0000","last_active":"2023-10-09T04:44:41.599596+0000","last_peered":"2023-10-09T04:44:41.599596+0000","last_clean":"2023-10-09T04:44:41.599596+0000","last_became_active":"2023-10-09T04:44:38.766851+0000","last_became_peered":"2023-10-09T04:44:38.766851+0000","last_unstale":"2023-10-09T04:44:41.599596+0000","last_undegraded":"2023-10-09T04:44:41.599596+0000","last_fullsized":"2023-10-09T04:44:41.599596+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:55:38.953502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"297'5286","reported_seq":5313,"reported_epoch":297,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:44:57.010496+0000","last_change":"2023-10-09T04:44:40.013455+0000","last_active":"2023-10-09T04:44:57.010496+0000","last_peered":"2023-10-09T04:44:57.010496+0000","last_clean":"2023-10-09T04:44:37.740389+0000","last_became_active":"2023-10-09T04:44:40.002929+0000","last_became_peered":"2023-10-09T04:44:40.002929+0000","last_unstale":"2023-10-09T04:44:57.010496+0000","last_undegraded":"2023-10-09T04:44:39.808323+0000","last_fullsized":"2023-10-09T04:44:39.770758+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5286,"log_dups_size":0,"ondisk_log_size":5286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:01.593599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346423296,"num_objects":5286,"num_object_clones":0,"num_object_copies":15858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4685,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5286,"num_write_kb":338304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":601},{"shards":"1,2,6","objects":4685}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":248,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599536+0000","last_change":"2023-10-09T04:44:38.767613+0000","last_active":"2023-10-09T04:44:41.599536+0000","last_peered":"2023-10-09T04:44:41.599536+0000","last_clean":"2023-10-09T04:44:41.599536+0000","last_became_active":"2023-10-09T04:44:38.766848+0000","last_became_peered":"2023-10-09T04:44:38.766848+0000","last_unstale":"2023-10-09T04:44:41.599536+0000","last_undegraded":"2023-10-09T04:44:41.599536+0000","last_fullsized":"2023-10-09T04:44:41.599536+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:46.404623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599830+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:44:41.599830+0000","last_peered":"2023-10-09T04:44:41.599830+0000","last_clean":"2023-10-09T04:44:41.599830+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:44:41.599830+0000","last_undegraded":"2023-10-09T04:44:41.599830+0000","last_fullsized":"2023-10-09T04:44:41.599830+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"297'5208","reported_seq":5230,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:57.276488+0000","last_change":"2023-10-09T04:43:34.881151+0000","last_active":"2023-10-09T04:44:57.276488+0000","last_peered":"2023-10-09T04:44:57.276488+0000","last_clean":"2023-10-09T04:44:57.276488+0000","last_became_active":"2023-10-09T04:43:34.880832+0000","last_became_peered":"2023-10-09T04:43:34.880832+0000","last_unstale":"2023-10-09T04:44:57.276488+0000","last_undegraded":"2023-10-09T04:44:57.276488+0000","last_fullsized":"2023-10-09T04:44:57.276488+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5208,"log_dups_size":0,"ondisk_log_size":5208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:48.504273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341311488,"num_objects":5208,"num_object_clones":0,"num_object_copies":15624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5208,"num_write_kb":333312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599244+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:44:41.599244+0000","last_peered":"2023-10-09T04:44:41.599244+0000","last_clean":"2023-10-09T04:44:41.599244+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:44:41.599244+0000","last_undegraded":"2023-10-09T04:44:41.599244+0000","last_fullsized":"2023-10-09T04:44:41.599244+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":267,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.601942+0000","last_change":"2023-10-09T04:44:38.765747+0000","last_active":"2023-10-09T04:44:41.601942+0000","last_peered":"2023-10-09T04:44:41.601942+0000","last_clean":"2023-10-09T04:44:41.601942+0000","last_became_active":"2023-10-09T04:44:38.765459+0000","last_became_peered":"2023-10-09T04:44:38.765459+0000","last_unstale":"2023-10-09T04:44:41.601942+0000","last_undegraded":"2023-10-09T04:44:41.601942+0000","last_fullsized":"2023-10-09T04:44:41.601942+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:33.564640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"297'5398","reported_seq":5419,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:59.234110+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:44:59.234110+0000","last_peered":"2023-10-09T04:44:59.234110+0000","last_clean":"2023-10-09T04:44:59.234110+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:44:59.234110+0000","last_undegraded":"2023-10-09T04:44:59.234110+0000","last_fullsized":"2023-10-09T04:44:59.234110+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5398,"log_dups_size":0,"ondisk_log_size":5398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353763328,"num_objects":5398,"num_object_clones":0,"num_object_copies":16194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5398,"num_write_kb":345472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.610739+0000","last_change":"2023-10-09T04:27:57.986421+0000","last_active":"2023-10-09T04:44:41.610739+0000","last_peered":"2023-10-09T04:44:41.610739+0000","last_clean":"2023-10-09T04:44:41.610739+0000","last_became_active":"2023-10-09T04:27:57.986122+0000","last_became_peered":"2023-10-09T04:27:57.986122+0000","last_unstale":"2023-10-09T04:44:41.610739+0000","last_undegraded":"2023-10-09T04:44:41.610739+0000","last_fullsized":"2023-10-09T04:44:41.610739+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:00.865911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":282,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.614914+0000","last_change":"2023-10-09T04:41:02.607747+0000","last_active":"2023-10-09T04:44:41.614914+0000","last_peered":"2023-10-09T04:44:41.614914+0000","last_clean":"2023-10-09T04:44:41.614914+0000","last_became_active":"2023-10-09T04:41:02.607455+0000","last_became_peered":"2023-10-09T04:41:02.607455+0000","last_unstale":"2023-10-09T04:44:41.614914+0000","last_undegraded":"2023-10-09T04:44:41.614914+0000","last_fullsized":"2023-10-09T04:44:41.614914+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:54.024631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"297'5388","reported_seq":5409,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:57.079832+0000","last_change":"2023-10-09T04:43:34.883699+0000","last_active":"2023-10-09T04:44:57.079832+0000","last_peered":"2023-10-09T04:44:57.079832+0000","last_clean":"2023-10-09T04:44:57.079832+0000","last_became_active":"2023-10-09T04:43:34.882682+0000","last_became_peered":"2023-10-09T04:43:34.882682+0000","last_unstale":"2023-10-09T04:44:57.079832+0000","last_undegraded":"2023-10-09T04:44:57.079832+0000","last_fullsized":"2023-10-09T04:44:57.079832+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5388,"log_dups_size":0,"ondisk_log_size":5388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:24:59.341363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353107968,"num_objects":5388,"num_object_clones":0,"num_object_copies":16164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5388,"num_write_kb":344832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":174,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597261+0000","last_change":"2023-10-09T04:39:40.395636+0000","last_active":"2023-10-09T04:44:41.597261+0000","last_peered":"2023-10-09T04:44:41.597261+0000","last_clean":"2023-10-09T04:44:41.597261+0000","last_became_active":"2023-10-09T04:39:40.395370+0000","last_became_peered":"2023-10-09T04:39:40.395370+0000","last_unstale":"2023-10-09T04:44:41.597261+0000","last_undegraded":"2023-10-09T04:44:41.597261+0000","last_fullsized":"2023-10-09T04:44:41.597261+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.928656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":291,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595273+0000","last_change":"2023-10-09T04:36:47.550883+0000","last_active":"2023-10-09T04:44:41.595273+0000","last_peered":"2023-10-09T04:44:41.595273+0000","last_clean":"2023-10-09T04:44:41.595273+0000","last_became_active":"2023-10-09T04:36:47.550635+0000","last_became_peered":"2023-10-09T04:36:47.550635+0000","last_unstale":"2023-10-09T04:44:41.595273+0000","last_undegraded":"2023-10-09T04:44:41.595273+0000","last_fullsized":"2023-10-09T04:44:41.595273+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:22:58.666003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":268,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.594596+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:44:41.594596+0000","last_peered":"2023-10-09T04:44:41.594596+0000","last_clean":"2023-10-09T04:44:41.594596+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:44:41.594596+0000","last_undegraded":"2023-10-09T04:44:41.594596+0000","last_fullsized":"2023-10-09T04:44:41.594596+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"297'5384","reported_seq":5405,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:59.283315+0000","last_change":"2023-10-09T04:43:34.881122+0000","last_active":"2023-10-09T04:44:59.283315+0000","last_peered":"2023-10-09T04:44:59.283315+0000","last_clean":"2023-10-09T04:44:59.283315+0000","last_became_active":"2023-10-09T04:43:34.880680+0000","last_became_peered":"2023-10-09T04:43:34.880680+0000","last_unstale":"2023-10-09T04:44:59.283315+0000","last_undegraded":"2023-10-09T04:44:59.283315+0000","last_fullsized":"2023-10-09T04:44:59.283315+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:08.954210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352845824,"num_objects":5384,"num_object_clones":0,"num_object_copies":16152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5384,"num_write_kb":344576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":271,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.594740+0000","last_change":"2023-10-09T04:44:38.775908+0000","last_active":"2023-10-09T04:44:41.594740+0000","last_peered":"2023-10-09T04:44:41.594740+0000","last_clean":"2023-10-09T04:44:41.594740+0000","last_became_active":"2023-10-09T04:44:38.775498+0000","last_became_peered":"2023-10-09T04:44:38.775498+0000","last_unstale":"2023-10-09T04:44:41.594740+0000","last_undegraded":"2023-10-09T04:44:41.594740+0000","last_fullsized":"2023-10-09T04:44:41.594740+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:04:47.621095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":55,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599986+0000","last_change":"2023-10-09T04:44:38.766371+0000","last_active":"2023-10-09T04:44:41.599986+0000","last_peered":"2023-10-09T04:44:41.599986+0000","last_clean":"2023-10-09T04:44:41.599986+0000","last_became_active":"2023-10-09T04:44:38.766111+0000","last_became_peered":"2023-10-09T04:44:38.766111+0000","last_unstale":"2023-10-09T04:44:41.599986+0000","last_undegraded":"2023-10-09T04:44:41.599986+0000","last_fullsized":"2023-10-09T04:44:41.599986+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:34.327675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":268,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599606+0000","last_change":"2023-10-09T04:39:40.401114+0000","last_active":"2023-10-09T04:44:41.599606+0000","last_peered":"2023-10-09T04:44:41.599606+0000","last_clean":"2023-10-09T04:44:41.599606+0000","last_became_active":"2023-10-09T04:39:40.400673+0000","last_became_peered":"2023-10-09T04:39:40.400673+0000","last_unstale":"2023-10-09T04:44:41.599606+0000","last_undegraded":"2023-10-09T04:44:41.599606+0000","last_fullsized":"2023-10-09T04:44:41.599606+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:26:44.664970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"297'5229","reported_seq":7711,"reported_epoch":297,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:44:59.379028+0000","last_change":"2023-10-09T04:44:40.018230+0000","last_active":"2023-10-09T04:44:59.379028+0000","last_peered":"2023-10-09T04:44:59.379028+0000","last_clean":"2023-10-09T04:44:37.720807+0000","last_became_active":"2023-10-09T04:44:39.958141+0000","last_became_peered":"2023-10-09T04:44:39.958141+0000","last_unstale":"2023-10-09T04:44:59.379028+0000","last_undegraded":"2023-10-09T04:44:59.379028+0000","last_fullsized":"2023-10-09T04:44:39.776234+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5229,"log_dups_size":0,"ondisk_log_size":5229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:06.051523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342687744,"num_objects":5229,"num_object_clones":0,"num_object_copies":15687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3277,"num_objects_unfound":0,"num_objects_dirty":5229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5229,"num_write_kb":334656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1226,"num_bytes_recovered":80347136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5","objects":1952},{"shards":"2,5,6","objects":3277}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":254,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.601496+0000","last_change":"2023-10-09T04:36:47.574307+0000","last_active":"2023-10-09T04:44:41.601496+0000","last_peered":"2023-10-09T04:44:41.601496+0000","last_clean":"2023-10-09T04:44:41.601496+0000","last_became_active":"2023-10-09T04:36:47.572813+0000","last_became_peered":"2023-10-09T04:36:47.572813+0000","last_unstale":"2023-10-09T04:44:41.601496+0000","last_undegraded":"2023-10-09T04:44:41.601496+0000","last_fullsized":"2023-10-09T04:44:41.601496+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:35.520996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":262,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.602801+0000","last_change":"2023-10-09T04:44:38.765629+0000","last_active":"2023-10-09T04:44:41.602801+0000","last_peered":"2023-10-09T04:44:41.602801+0000","last_clean":"2023-10-09T04:44:41.602801+0000","last_became_active":"2023-10-09T04:44:38.765243+0000","last_became_peered":"2023-10-09T04:44:38.765243+0000","last_unstale":"2023-10-09T04:44:41.602801+0000","last_undegraded":"2023-10-09T04:44:41.602801+0000","last_fullsized":"2023-10-09T04:44:41.602801+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:58:47.027834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":169,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596335+0000","last_change":"2023-10-09T04:39:40.401220+0000","last_active":"2023-10-09T04:44:41.596335+0000","last_peered":"2023-10-09T04:44:41.596335+0000","last_clean":"2023-10-09T04:44:41.596335+0000","last_became_active":"2023-10-09T04:39:40.400910+0000","last_became_peered":"2023-10-09T04:39:40.400910+0000","last_unstale":"2023-10-09T04:44:41.596335+0000","last_undegraded":"2023-10-09T04:44:41.596335+0000","last_fullsized":"2023-10-09T04:44:41.596335+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:38.717094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"297'5324","reported_seq":5345,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:59.287690+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:44:59.287690+0000","last_peered":"2023-10-09T04:44:59.287690+0000","last_clean":"2023-10-09T04:44:59.287690+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:44:59.287690+0000","last_undegraded":"2023-10-09T04:44:59.287690+0000","last_fullsized":"2023-10-09T04:44:59.287690+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348913664,"num_objects":5324,"num_object_clones":0,"num_object_copies":15972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5324,"num_write_kb":340736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":278,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.594958+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:44:41.594958+0000","last_peered":"2023-10-09T04:44:41.594958+0000","last_clean":"2023-10-09T04:44:41.594958+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:44:41.594958+0000","last_undegraded":"2023-10-09T04:44:41.594958+0000","last_fullsized":"2023-10-09T04:44:41.594958+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":715,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.617265+0000","last_change":"2023-10-09T04:44:41.616991+0000","last_active":"2023-10-09T04:44:41.617265+0000","last_peered":"2023-10-09T04:44:41.617265+0000","last_clean":"2023-10-09T04:44:41.617265+0000","last_became_active":"2023-10-09T04:44:41.616662+0000","last_became_peered":"2023-10-09T04:44:41.616662+0000","last_unstale":"2023-10-09T04:44:41.617265+0000","last_undegraded":"2023-10-09T04:44:41.617265+0000","last_fullsized":"2023-10-09T04:44:41.617265+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:07.416849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":233,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.615592+0000","last_change":"2023-10-09T04:44:38.774169+0000","last_active":"2023-10-09T04:44:41.615592+0000","last_peered":"2023-10-09T04:44:41.615592+0000","last_clean":"2023-10-09T04:44:41.615592+0000","last_became_active":"2023-10-09T04:44:38.773823+0000","last_became_peered":"2023-10-09T04:44:38.773823+0000","last_unstale":"2023-10-09T04:44:41.615592+0000","last_undegraded":"2023-10-09T04:44:41.615592+0000","last_fullsized":"2023-10-09T04:44:41.615592+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:29.305304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":230,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599788+0000","last_change":"2023-10-09T04:44:38.777706+0000","last_active":"2023-10-09T04:44:41.599788+0000","last_peered":"2023-10-09T04:44:41.599788+0000","last_clean":"2023-10-09T04:44:41.599788+0000","last_became_active":"2023-10-09T04:44:38.777415+0000","last_became_peered":"2023-10-09T04:44:38.777415+0000","last_unstale":"2023-10-09T04:44:41.599788+0000","last_undegraded":"2023-10-09T04:44:41.599788+0000","last_fullsized":"2023-10-09T04:44:41.599788+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:30.938179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"297'5314","reported_seq":5335,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:59.305770+0000","last_change":"2023-10-09T04:43:34.880958+0000","last_active":"2023-10-09T04:44:59.305770+0000","last_peered":"2023-10-09T04:44:59.305770+0000","last_clean":"2023-10-09T04:44:59.305770+0000","last_became_active":"2023-10-09T04:43:34.880522+0000","last_became_peered":"2023-10-09T04:43:34.880522+0000","last_unstale":"2023-10-09T04:44:59.305770+0000","last_undegraded":"2023-10-09T04:44:59.305770+0000","last_fullsized":"2023-10-09T04:44:59.305770+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:04.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348258304,"num_objects":5314,"num_object_clones":0,"num_object_copies":15942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5314,"num_write_kb":340096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":297,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596289+0000","last_change":"2023-10-09T04:29:51.057078+0000","last_active":"2023-10-09T04:44:41.596289+0000","last_peered":"2023-10-09T04:44:41.596289+0000","last_clean":"2023-10-09T04:44:41.596289+0000","last_became_active":"2023-10-09T04:29:51.056796+0000","last_became_peered":"2023-10-09T04:29:51.056796+0000","last_unstale":"2023-10-09T04:44:41.596289+0000","last_undegraded":"2023-10-09T04:44:41.596289+0000","last_fullsized":"2023-10-09T04:44:41.596289+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:43.927980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599329+0000","last_change":"2023-10-09T04:41:02.605290+0000","last_active":"2023-10-09T04:44:41.599329+0000","last_peered":"2023-10-09T04:44:41.599329+0000","last_clean":"2023-10-09T04:44:41.599329+0000","last_became_active":"2023-10-09T04:41:02.603619+0000","last_became_peered":"2023-10-09T04:41:02.603619+0000","last_unstale":"2023-10-09T04:44:41.599329+0000","last_undegraded":"2023-10-09T04:44:41.599329+0000","last_fullsized":"2023-10-09T04:44:41.599329+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:54.013579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":174,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.602532+0000","last_change":"2023-10-09T04:36:47.540409+0000","last_active":"2023-10-09T04:44:41.602532+0000","last_peered":"2023-10-09T04:44:41.602532+0000","last_clean":"2023-10-09T04:44:41.602532+0000","last_became_active":"2023-10-09T04:36:47.539992+0000","last_became_peered":"2023-10-09T04:36:47.539992+0000","last_unstale":"2023-10-09T04:44:41.602532+0000","last_undegraded":"2023-10-09T04:44:41.602532+0000","last_fullsized":"2023-10-09T04:44:41.602532+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:06:56.526026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"297'5410","reported_seq":5440,"reported_epoch":297,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:44:58.012675+0000","last_change":"2023-10-09T04:44:40.029793+0000","last_active":"2023-10-09T04:44:58.012675+0000","last_peered":"2023-10-09T04:44:58.012675+0000","last_clean":"2023-10-09T04:44:37.770348+0000","last_became_active":"2023-10-09T04:44:40.015056+0000","last_became_peered":"2023-10-09T04:44:40.015056+0000","last_unstale":"2023-10-09T04:44:58.012675+0000","last_undegraded":"2023-10-09T04:44:39.826001+0000","last_fullsized":"2023-10-09T04:44:39.772399+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5410,"log_dups_size":0,"ondisk_log_size":5410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:12:15.109861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354549760,"num_objects":5410,"num_object_clones":0,"num_object_copies":16230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4803,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5410,"num_write_kb":346240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"2,3","objects":607},{"shards":"2,3,6","objects":4803}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":300,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597097+0000","last_change":"2023-10-09T04:41:02.611300+0000","last_active":"2023-10-09T04:44:41.597097+0000","last_peered":"2023-10-09T04:44:41.597097+0000","last_clean":"2023-10-09T04:44:41.597097+0000","last_became_active":"2023-10-09T04:41:02.611067+0000","last_became_peered":"2023-10-09T04:41:02.611067+0000","last_unstale":"2023-10-09T04:44:41.597097+0000","last_undegraded":"2023-10-09T04:44:41.597097+0000","last_fullsized":"2023-10-09T04:44:41.597097+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:34:43.558310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":253,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599584+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:44:41.599584+0000","last_peered":"2023-10-09T04:44:41.599584+0000","last_clean":"2023-10-09T04:44:41.599584+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:44:41.599584+0000","last_undegraded":"2023-10-09T04:44:41.599584+0000","last_fullsized":"2023-10-09T04:44:41.599584+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":141,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595029+0000","last_change":"2023-10-09T04:34:22.264227+0000","last_active":"2023-10-09T04:44:41.595029+0000","last_peered":"2023-10-09T04:44:41.595029+0000","last_clean":"2023-10-09T04:44:41.595029+0000","last_became_active":"2023-10-09T04:34:22.263761+0000","last_became_peered":"2023-10-09T04:34:22.263761+0000","last_unstale":"2023-10-09T04:44:41.595029+0000","last_undegraded":"2023-10-09T04:44:41.595029+0000","last_fullsized":"2023-10-09T04:44:41.595029+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:25:37.248877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"297'5329","reported_seq":5351,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:57.929796+0000","last_change":"2023-10-09T04:43:34.880794+0000","last_active":"2023-10-09T04:44:57.929796+0000","last_peered":"2023-10-09T04:44:57.929796+0000","last_clean":"2023-10-09T04:44:57.929796+0000","last_became_active":"2023-10-09T04:43:34.880481+0000","last_became_peered":"2023-10-09T04:43:34.880481+0000","last_unstale":"2023-10-09T04:44:57.929796+0000","last_undegraded":"2023-10-09T04:44:57.929796+0000","last_fullsized":"2023-10-09T04:44:57.929796+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5329,"log_dups_size":0,"ondisk_log_size":5329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:22:13.398120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349241344,"num_objects":5329,"num_object_clones":0,"num_object_copies":15987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5329,"num_write_kb":341056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":295,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.614989+0000","last_change":"2023-10-09T04:29:51.072682+0000","last_active":"2023-10-09T04:44:41.614989+0000","last_peered":"2023-10-09T04:44:41.614989+0000","last_clean":"2023-10-09T04:44:41.614989+0000","last_became_active":"2023-10-09T04:29:51.060069+0000","last_became_peered":"2023-10-09T04:29:51.060069+0000","last_unstale":"2023-10-09T04:44:41.614989+0000","last_undegraded":"2023-10-09T04:44:41.614989+0000","last_fullsized":"2023-10-09T04:44:41.614989+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:24:40.724692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":240,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.601913+0000","last_change":"2023-10-09T04:44:38.765825+0000","last_active":"2023-10-09T04:44:41.601913+0000","last_peered":"2023-10-09T04:44:41.601913+0000","last_clean":"2023-10-09T04:44:41.601913+0000","last_became_active":"2023-10-09T04:44:38.765517+0000","last_became_peered":"2023-10-09T04:44:38.765517+0000","last_unstale":"2023-10-09T04:44:41.601913+0000","last_undegraded":"2023-10-09T04:44:41.601913+0000","last_fullsized":"2023-10-09T04:44:41.601913+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:24:26.547437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":285,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.610787+0000","last_change":"2023-10-09T04:29:51.060110+0000","last_active":"2023-10-09T04:44:41.610787+0000","last_peered":"2023-10-09T04:44:41.610787+0000","last_clean":"2023-10-09T04:44:41.610787+0000","last_became_active":"2023-10-09T04:29:51.059769+0000","last_became_peered":"2023-10-09T04:29:51.059769+0000","last_unstale":"2023-10-09T04:44:41.610787+0000","last_undegraded":"2023-10-09T04:44:41.610787+0000","last_fullsized":"2023-10-09T04:44:41.610787+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:19.565929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"297'5327","reported_seq":5357,"reported_epoch":297,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:44:57.069250+0000","last_change":"2023-10-09T04:44:40.054779+0000","last_active":"2023-10-09T04:44:57.069250+0000","last_peered":"2023-10-09T04:44:57.069250+0000","last_clean":"2023-10-09T04:44:37.742164+0000","last_became_active":"2023-10-09T04:44:40.045697+0000","last_became_peered":"2023-10-09T04:44:40.045697+0000","last_unstale":"2023-10-09T04:44:57.069250+0000","last_undegraded":"2023-10-09T04:44:39.812603+0000","last_fullsized":"2023-10-09T04:44:39.771139+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5327,"log_dups_size":0,"ondisk_log_size":5327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:35.218914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349110272,"num_objects":5327,"num_object_clones":0,"num_object_copies":15981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5327,"num_write_kb":340928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3","objects":647},{"shards":"2,3,6","objects":4680}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":209,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.597010+0000","last_change":"2023-10-09T04:44:38.775676+0000","last_active":"2023-10-09T04:44:41.597010+0000","last_peered":"2023-10-09T04:44:41.597010+0000","last_clean":"2023-10-09T04:44:41.597010+0000","last_became_active":"2023-10-09T04:44:38.775281+0000","last_became_peered":"2023-10-09T04:44:38.775281+0000","last_unstale":"2023-10-09T04:44:41.597010+0000","last_undegraded":"2023-10-09T04:44:41.597010+0000","last_fullsized":"2023-10-09T04:44:41.597010+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:55.518293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":69,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.601965+0000","last_change":"2023-10-09T04:39:40.394664+0000","last_active":"2023-10-09T04:44:41.601965+0000","last_peered":"2023-10-09T04:44:41.601965+0000","last_clean":"2023-10-09T04:44:41.601965+0000","last_became_active":"2023-10-09T04:39:40.394206+0000","last_became_peered":"2023-10-09T04:39:40.394206+0000","last_unstale":"2023-10-09T04:44:41.601965+0000","last_undegraded":"2023-10-09T04:44:41.601965+0000","last_fullsized":"2023-10-09T04:44:41.601965+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:31.197658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599204+0000","last_change":"2023-10-09T04:44:38.772587+0000","last_active":"2023-10-09T04:44:41.599204+0000","last_peered":"2023-10-09T04:44:41.599204+0000","last_clean":"2023-10-09T04:44:41.599204+0000","last_became_active":"2023-10-09T04:44:38.772281+0000","last_became_peered":"2023-10-09T04:44:38.772281+0000","last_unstale":"2023-10-09T04:44:41.599204+0000","last_undegraded":"2023-10-09T04:44:41.599204+0000","last_fullsized":"2023-10-09T04:44:41.599204+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:48.547043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":137,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.601752+0000","last_change":"2023-10-09T04:34:22.264315+0000","last_active":"2023-10-09T04:44:41.601752+0000","last_peered":"2023-10-09T04:44:41.601752+0000","last_clean":"2023-10-09T04:44:41.601752+0000","last_became_active":"2023-10-09T04:34:22.263938+0000","last_became_peered":"2023-10-09T04:34:22.263938+0000","last_unstale":"2023-10-09T04:44:41.601752+0000","last_undegraded":"2023-10-09T04:44:41.601752+0000","last_fullsized":"2023-10-09T04:44:41.601752+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:41:45.432641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":247,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599667+0000","last_change":"2023-10-09T04:36:47.542346+0000","last_active":"2023-10-09T04:44:41.599667+0000","last_peered":"2023-10-09T04:44:41.599667+0000","last_clean":"2023-10-09T04:44:41.599667+0000","last_became_active":"2023-10-09T04:36:47.541569+0000","last_became_peered":"2023-10-09T04:36:47.541569+0000","last_unstale":"2023-10-09T04:44:41.599667+0000","last_undegraded":"2023-10-09T04:44:41.599667+0000","last_fullsized":"2023-10-09T04:44:41.599667+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:20.800926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":287,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595563+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:44:41.595563+0000","last_peered":"2023-10-09T04:44:41.595563+0000","last_clean":"2023-10-09T04:44:41.595563+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:44:41.595563+0000","last_undegraded":"2023-10-09T04:44:41.595563+0000","last_fullsized":"2023-10-09T04:44:41.595563+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":45,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.600025+0000","last_change":"2023-10-09T04:41:02.606490+0000","last_active":"2023-10-09T04:44:41.600025+0000","last_peered":"2023-10-09T04:44:41.600025+0000","last_clean":"2023-10-09T04:44:41.600025+0000","last_became_active":"2023-10-09T04:41:02.605280+0000","last_became_peered":"2023-10-09T04:41:02.605280+0000","last_unstale":"2023-10-09T04:44:41.600025+0000","last_undegraded":"2023-10-09T04:44:41.600025+0000","last_fullsized":"2023-10-09T04:44:41.600025+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:30:18.525998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":256,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.595358+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:44:41.595358+0000","last_peered":"2023-10-09T04:44:41.595358+0000","last_clean":"2023-10-09T04:44:41.595358+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:44:41.595358+0000","last_undegraded":"2023-10-09T04:44:41.595358+0000","last_fullsized":"2023-10-09T04:44:41.595358+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599592+0000","last_change":"2023-10-09T04:41:02.597300+0000","last_active":"2023-10-09T04:44:41.599592+0000","last_peered":"2023-10-09T04:44:41.599592+0000","last_clean":"2023-10-09T04:44:41.599592+0000","last_became_active":"2023-10-09T04:41:02.596911+0000","last_became_peered":"2023-10-09T04:41:02.596911+0000","last_unstale":"2023-10-09T04:44:41.599592+0000","last_undegraded":"2023-10-09T04:44:41.599592+0000","last_fullsized":"2023-10-09T04:44:41.599592+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:08.045263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":295,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.614575+0000","last_change":"2023-10-09T04:44:38.765992+0000","last_active":"2023-10-09T04:44:41.614575+0000","last_peered":"2023-10-09T04:44:41.614575+0000","last_clean":"2023-10-09T04:44:41.614575+0000","last_became_active":"2023-10-09T04:44:38.765630+0000","last_became_peered":"2023-10-09T04:44:38.765630+0000","last_unstale":"2023-10-09T04:44:41.614575+0000","last_undegraded":"2023-10-09T04:44:41.614575+0000","last_fullsized":"2023-10-09T04:44:41.614575+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:24:29.355316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":318,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-10-09T04:44:48.589613+0000","last_change":"2023-10-09T04:36:47.575657+0000","last_active":"2023-10-09T04:44:48.589613+0000","last_peered":"2023-10-09T04:44:48.589613+0000","last_clean":"2023-10-09T04:44:48.589613+0000","last_became_active":"2023-10-09T04:36:47.575233+0000","last_became_peered":"2023-10-09T04:36:47.575233+0000","last_unstale":"2023-10-09T04:44:48.589613+0000","last_undegraded":"2023-10-09T04:44:48.589613+0000","last_fullsized":"2023-10-09T04:44:48.589613+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:30.423128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":171,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.596487+0000","last_change":"2023-10-09T04:36:47.577435+0000","last_active":"2023-10-09T04:44:41.596487+0000","last_peered":"2023-10-09T04:44:41.596487+0000","last_clean":"2023-10-09T04:44:41.596487+0000","last_became_active":"2023-10-09T04:36:47.577046+0000","last_became_peered":"2023-10-09T04:36:47.577046+0000","last_unstale":"2023-10-09T04:44:41.596487+0000","last_undegraded":"2023-10-09T04:44:41.596487+0000","last_fullsized":"2023-10-09T04:44:41.596487+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:41:22.283435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":55,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-10-09T04:44:41.599228+0000","last_change":"2023-10-09T04:44:38.773645+0000","last_active":"2023-10-09T04:44:41.599228+0000","last_peered":"2023-10-09T04:44:41.599228+0000","last_clean":"2023-10-09T04:44:41.599228+0000","last_became_active":"2023-10-09T04:44:38.773301+0000","last_became_peered":"2023-10-09T04:44:38.773301+0000","last_unstale":"2023-10-09T04:44:41.599228+0000","last_undegraded":"2023-10-09T04:44:41.599228+0000","last_fullsized":"2023-10-09T04:44:41.599228+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:56:45.226978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":5602344960,"num_objects":85485,"num_object_clones":0,"num_object_copies":256455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23613,"num_objects_misplaced":7259,"num_objects_unfound":0,"num_objects_dirty":85485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":85485,"num_write_kb":5471040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1904,"num_bytes_recovered":124780544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16905469952,"data_stored":16905469952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85485,"ondisk_log_size":85485,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":10428,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234457,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3447056,"kb_used_data":2536072,"kb_used_omap":0,"kb_used_meta":910976,"kb_avail":90298096,"statfs":{"total":95995035648,"available":92465250304,"internally_reserved":0,"allocated":2596937728,"data_stored":2596322012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":932839424},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.5800000000000001,"15min":4.0430000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.4830000000000001,"5min":14.571999999999999,"15min":186.446},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.3460000000000001,"5min":2.4260000000000002,"15min":4.4329999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":6.1070000000000002,"5min":17.670999999999999,"15min":186.429},"last":5.2290000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:44:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.9830000000000001,"15min":5.3869999999999996},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":4.3639999999999999,"5min":19.298999999999999,"15min":264.62},"last":2.3490000000000002},{"interface":"front","average":{"1min":1.1359999999999999,"5min":2.2290000000000001,"15min":5.5570000000000004},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.23799999999999999},"max":{"1min":3.7010000000000001,"5min":19.251999999999999,"15min":264.65899999999999},"last":5.282}]},{"osd":2,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":7.4429999999999996,"15min":3.6349999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.294,"5min":184.72,"15min":184.72},"last":7.4489999999999998},{"interface":"front","average":{"1min":0.90600000000000003,"5min":7.8259999999999996,"15min":4.0720000000000001},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6070000000000002,"5min":184.697,"15min":184.697},"last":7.4359999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:26 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":2.3260000000000001,"15min":7.7670000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.0640000000000001,"5min":20.469999999999999,"15min":249.989},"last":4.1230000000000002},{"interface":"front","average":{"1min":1.2250000000000001,"5min":3.0760000000000001,"15min":8.1129999999999995},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.29799999999999999},"max":{"1min":5.5979999999999999,"5min":20.765000000000001,"15min":250.21700000000001},"last":5.5949999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":1.23,"15min":1.3999999999999999},"min":{"1min":0.27200000000000002,"5min":0.26900000000000002,"15min":0.23300000000000001},"max":{"1min":3.0449999999999999,"5min":14.257999999999999,"15min":14.257999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.77900000000000003,"15min":1.1319999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.21199999999999999},"max":{"1min":2.0129999999999999,"5min":7.2290000000000001,"15min":10.613},"last":0.32200000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.2230000000000001,"15min":1.109},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27900000000000003},"max":{"1min":2.4049999999999998,"5min":13.319000000000001,"15min":13.319000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.641,"15min":1.5549999999999999},"min":{"1min":0.309,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.9460000000000002,"5min":18.408000000000001,"15min":18.408000000000001},"last":0.34000000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.0700000000000001,"15min":1.2390000000000001},"min":{"1min":0.34699999999999998,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":1.069,"5min":11.388,"15min":11.388},"last":0.432},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.5900000000000001,"15min":1.635},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.30099999999999999},"max":{"1min":3.0190000000000001,"5min":18.311,"15min":18.311},"last":0.376}]}]},{"osd":6,"up_from":0,"seq":197568496091,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141180,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2441020,"kb_used_data":1766580,"kb_used_omap":4,"kb_used_meta":674427,"kb_avail":91304132,"statfs":{"total":95995035648,"available":93495431168,"internally_reserved":0,"allocated":1808977920,"data_stored":1807737084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":690614158},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":3.0019999999999998,"15min":4.0640000000000001},"min":{"1min":0.23599999999999999,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":21.021999999999998,"5min":75.620000000000005,"15min":97.308000000000007},"last":0.23200000000000001},{"interface":"front","average":{"1min":2.1419999999999999,"5min":2.8239999999999998,"15min":4.29},"min":{"1min":0.25800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":21.280999999999999,"5min":27.321000000000002,"15min":97.055999999999997},"last":3.387}]},{"osd":2,"last update":"Mon Oct 9 04:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":5.9089999999999998,"15min":3.6890000000000001},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.5529999999999999,"5min":263.60500000000002,"15min":263.60500000000002},"last":0.378},{"interface":"front","average":{"1min":0.66100000000000003,"5min":7.008,"15min":4.1790000000000003},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":3.3170000000000002,"5min":263.65199999999999,"15min":263.65199999999999},"last":3.4340000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:44:31 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":9.8119999999999994,"15min":6.5039999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.2409999999999997,"5min":379.80900000000003,"15min":379.80900000000003},"last":6.117},{"interface":"front","average":{"1min":0.69799999999999995,"5min":10.941000000000001,"15min":7.1189999999999998},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.19700000000000001},"max":{"1min":2.6019999999999999,"5min":379.95800000000003,"15min":379.95800000000003},"last":18.285}]},{"osd":4,"last update":"Mon Oct 9 04:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1819999999999999,"15min":2.464},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.245},"max":{"1min":7.266,"5min":14.449,"15min":170.072},"last":5.1360000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":2.6190000000000002,"15min":2.266},"min":{"1min":0.33200000000000002,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":4.9100000000000001,"5min":75.513999999999996,"15min":169.94499999999999},"last":1.081}]},{"osd":5,"last update":"Mon Oct 9 04:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.109,"15min":2.464},"min":{"1min":0.34999999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.3200000000000003,"5min":11.499000000000001,"15min":171.036},"last":3.4620000000000002},{"interface":"front","average":{"1min":1.7949999999999999,"5min":2.3140000000000001,"15min":2.6299999999999999},"min":{"1min":0.40899999999999997,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":7.3259999999999996,"5min":14.557,"15min":170.88},"last":5.194}]},{"osd":6,"last update":"Mon Oct 9 04:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":3.274,"15min":2.4609999999999999},"min":{"1min":0.38600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.3610000000000007,"5min":132.36099999999999,"15min":132.36099999999999},"last":3.4849999999999999},{"interface":"front","average":{"1min":1.9630000000000001,"5min":3.472,"15min":2.6389999999999998},"min":{"1min":0.434,"5min":0.35699999999999998,"15min":0.317},"max":{"1min":7.3959999999999999,"5min":132.17699999999999,"15min":132.17699999999999},"last":5.0880000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:44:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0979999999999999,"5min":3.7330000000000001,"15min":3.198},"min":{"1min":0.46300000000000002,"5min":0.40400000000000003,"15min":0.34100000000000003},"max":{"1min":7.3529999999999998,"5min":132.136,"15min":174.75999999999999},"last":5.165},{"interface":"front","average":{"1min":1.175,"5min":3.9020000000000001,"15min":2.883},"min":{"1min":0.48199999999999998,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":4.8609999999999998,"5min":132.12,"15min":174.779},"last":0.46200000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024237,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2888380,"kb_used_data":2051892,"kb_used_omap":5,"kb_used_meta":836474,"kb_avail":90856772,"statfs":{"total":95995035648,"available":93037334528,"internally_reserved":0,"allocated":2101137408,"data_stored":2100013086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":856550010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:44:35 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":4.508,"15min":5.1929999999999996},"min":{"1min":0.23599999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":2.8620000000000001,"5min":119.392,"15min":167.92699999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.2070000000000001,"5min":4.9790000000000001,"15min":5.6120000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":3.7639999999999998,"5min":119.34399999999999,"15min":169.78999999999999},"last":2.8159999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:44:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":3.2029999999999998,"15min":7.04},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.5449999999999999,"5min":98.938000000000002,"15min":131.30699999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.992,"5min":3.581,"15min":7.444},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.219},"max":{"1min":5.3959999999999999,"5min":98.975999999999999,"15min":129.922},"last":2.8639999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":5.4290000000000003,"15min":5.0700000000000003},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":4.1159999999999997,"5min":156.69800000000001,"15min":213.28399999999999},"last":0.626},{"interface":"front","average":{"1min":1.131,"5min":6.7160000000000002,"15min":5.7220000000000004},"min":{"1min":0.28199999999999997,"5min":0.26600000000000001,"15min":0.222},"max":{"1min":3.0750000000000002,"5min":161.40199999999999,"15min":212.27000000000001},"last":3.008}]},{"osd":4,"last update":"Mon Oct 9 04:44:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.4790000000000001,"15min":2.3530000000000002},"min":{"1min":0.40400000000000003,"5min":0.318,"15min":0.25700000000000001},"max":{"1min":5.5170000000000003,"5min":11.300000000000001,"15min":161.67099999999999},"last":3.1179999999999999},{"interface":"front","average":{"1min":1.613,"5min":1.742,"15min":2.6349999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":4.7759999999999998,"5min":14.109999999999999,"15min":160.74799999999999},"last":3.048}]},{"osd":5,"last update":"Mon Oct 9 04:44:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":3.1099999999999999,"15min":2.4630000000000001},"min":{"1min":0.35999999999999999,"5min":0.29199999999999998,"15min":0.27900000000000003},"max":{"1min":4.1479999999999997,"5min":136.322,"15min":136.322},"last":2.919},{"interface":"front","average":{"1min":1.542,"5min":2.9830000000000001,"15min":2.2429999999999999},"min":{"1min":0.33200000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.0339999999999998,"5min":136.15799999999999,"15min":136.15799999999999},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:44:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":2.923,"15min":2.7629999999999999},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":4.1909999999999998,"5min":78.438999999999993,"15min":108.63800000000001},"last":2.976},{"interface":"front","average":{"1min":1.9630000000000001,"5min":2.556,"15min":2.4020000000000001},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.9790000000000001,"5min":78.617999999999995,"15min":108.76300000000001},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:52 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.5870000000000002,"15min":2.0760000000000001},"min":{"1min":0.39100000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.1509999999999998,"5min":77.682000000000002,"15min":77.682000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.873,"5min":2.5800000000000001,"15min":2.093},"min":{"1min":0.46999999999999997,"5min":0.373,"15min":0.36399999999999999},"max":{"1min":5.4640000000000004,"5min":77.606999999999999,"15min":77.606999999999999},"last":0.53400000000000003}]}]},{"osd":2,"up_from":221,"seq":949187772515,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3796108,"kb_used_data":3379332,"kb_used_omap":7,"kb_used_meta":416760,"kb_avail":89949044,"statfs":{"total":95995035648,"available":92107821056,"internally_reserved":0,"allocated":3460435968,"data_stored":3459360422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":426762321},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,0,3,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":2.5710000000000002,"15min":7.2469999999999999},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":1.1459999999999999,"5min":13.813000000000001,"15min":72.608000000000004},"last":1.444},{"interface":"front","average":{"1min":0.51200000000000001,"5min":2.2610000000000001,"15min":6.7880000000000003},"min":{"1min":0.23599999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.5920000000000001,"5min":14.377000000000001,"15min":72.632000000000005},"last":5.0369999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":5.1379999999999999,"15min":4.9630000000000001},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.3639999999999999,"5min":121.848,"15min":121.848},"last":5.0880000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":5.3479999999999999,"15min":5.399},"min":{"1min":0.25,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.337,"5min":121.827,"15min":121.827},"last":2.3879999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":5.1299999999999999,"5min":10.228,"15min":6.7590000000000003},"min":{"1min":0.316,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":84.840999999999994,"5min":335.928,"15min":335.928},"last":1.863},{"interface":"front","average":{"1min":5.1109999999999998,"5min":10.731,"15min":6.915},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":85.147999999999996,"5min":335.68599999999998,"15min":335.68599999999998},"last":0.85599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":2.343,"15min":3.8839999999999999},"min":{"1min":0.34799999999999998,"5min":0.32800000000000001,"15min":0.315},"max":{"1min":4.6980000000000004,"5min":14.452999999999999,"15min":72.944999999999993},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":2.3149999999999999,"15min":4.0099999999999998},"min":{"1min":0.41799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0389999999999999,"5min":14.183,"15min":73.054000000000002},"last":0.90200000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":2.6139999999999999,"15min":3.738},"min":{"1min":0.41399999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.9199999999999999,"5min":65.388000000000005,"15min":72.986000000000004},"last":5.0629999999999997},{"interface":"front","average":{"1min":0.57799999999999996,"5min":2.8460000000000001,"15min":4.1710000000000003},"min":{"1min":0.39100000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.524,"5min":67.799000000000007,"15min":72.963999999999999},"last":1.2070000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":2.661,"15min":8.5060000000000002},"min":{"1min":0.40300000000000002,"5min":0.39000000000000001,"15min":0.33600000000000002},"max":{"1min":1.264,"5min":34.753,"15min":135.614},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.79100000000000004,"5min":2.5190000000000001,"15min":7.907},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.9020000000000001,"5min":34.796999999999997,"15min":130.196},"last":4.883}]},{"osd":7,"last update":"Mon Oct 9 04:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":2.6779999999999999,"15min":4.2590000000000003},"min":{"1min":0.44500000000000001,"5min":0.38500000000000001,"15min":0.378},"max":{"1min":2.7639999999999998,"5min":28.126999999999999,"15min":73.079999999999998},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.4670000000000001,"15min":3.702},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.8639999999999999,"5min":28.106000000000002,"15min":73.069000000000003},"last":5.0090000000000003}]}]},{"osd":3,"up_from":142,"seq":609885356238,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3372408,"kb_used_data":2777008,"kb_used_omap":2,"kb_used_meta":595389,"kb_avail":90372744,"statfs":{"total":95995035648,"available":92541689856,"internally_reserved":0,"allocated":2843656192,"data_stored":2842592227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":609678389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,4,3,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":2.7149999999999999,"15min":2.7490000000000001},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.20000000000000001},"max":{"1min":75.132999999999996,"5min":75.132999999999996,"15min":142.14500000000001},"last":1.1499999999999999},{"interface":"front","average":{"1min":4.5250000000000004,"5min":5.867,"15min":3.9239999999999999},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.21099999999999999},"max":{"1min":76.897999999999996,"5min":254.38499999999999,"15min":254.38499999999999},"last":1.3759999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":5.5279999999999996,"5min":3.4319999999999999,"15min":3.8100000000000001},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.192},"max":{"1min":92.644000000000005,"5min":92.644000000000005,"15min":180.02500000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":5.3090000000000002,"5min":2.9980000000000002,"15min":3.9430000000000001},"min":{"1min":0.22700000000000001,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":93.165999999999997,"5min":93.165999999999997,"15min":184.00899999999999},"last":0.89400000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":4.4690000000000003,"15min":3.125},"min":{"1min":0.29399999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.7370000000000001,"5min":112.613,"15min":177.28399999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.47399999999999998,"5min":6.7460000000000004,"15min":3.653},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.179,"5min":255.05199999999999,"15min":255.05199999999999},"last":2.7890000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":4.5759999999999996,"15min":2.8999999999999999},"min":{"1min":0.33000000000000002,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":6.3869999999999996,"5min":254.072,"15min":254.072},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.468,"5min":4.3929999999999998,"15min":2.7829999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.307},"max":{"1min":9.1769999999999996,"5min":253.49700000000001,"15min":253.49700000000001},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":5.2729999999999997,"15min":3.5179999999999998},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.27100000000000002},"max":{"1min":9.2129999999999992,"5min":253.57900000000001,"15min":253.57900000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.88300000000000001,"5min":5.1500000000000004,"15min":3.5859999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29799999999999999},"max":{"1min":5.6109999999999998,"5min":254.191,"15min":254.191},"last":1.462}]},{"osd":6,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":5.423,"15min":3.4289999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.31900000000000001},"max":{"1min":5.6639999999999997,"5min":254.14699999999999,"15min":254.14699999999999},"last":1.496},{"interface":"front","average":{"1min":1.3899999999999999,"5min":2.6899999999999999,"15min":2.823},"min":{"1min":0.44900000000000001,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":6.4619999999999997,"5min":50.308,"15min":123.505},"last":0.61499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:44:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":2.4129999999999998,"15min":2.427},"min":{"1min":0.46800000000000003,"5min":0.378,"15min":0.32300000000000001},"max":{"1min":6.4160000000000004,"5min":43.835000000000001,"15min":57.399000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.48,"5min":2.1709999999999998,"15min":2.286},"min":{"1min":0.41599999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":9.125,"5min":43.781999999999996,"15min":57.447000000000003},"last":0.501}]}]},{"osd":4,"up_from":30,"seq":128849019365,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2096264,"kb_used_data":1196032,"kb_used_omap":0,"kb_used_meta":900224,"kb_avail":91648888,"statfs":{"total":95995035648,"available":93848461312,"internally_reserved":0,"allocated":1224736768,"data_stored":1224128630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921829376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:48 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.851,"15min":5.3819999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":3.1619999999999999,"5min":23.114999999999998,"15min":219.577},"last":1.6120000000000001},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.4259999999999999,"15min":5.0609999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.6139999999999999,"5min":13.992000000000001,"15min":218.96000000000001},"last":2.2599999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:44:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":2.335,"15min":5.2199999999999998},"min":{"1min":0.33400000000000002,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":5.0259999999999998,"5min":37.356000000000002,"15min":142.74799999999999},"last":2.2040000000000002},{"interface":"front","average":{"1min":1.544,"5min":2.569,"15min":5.3890000000000002},"min":{"1min":0.30099999999999999,"5min":0.28100000000000003,"15min":0.27300000000000002},"max":{"1min":10.824,"5min":37.311999999999998,"15min":139.09700000000001},"last":6.2770000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:44:04 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":5.6699999999999999,"15min":4.1399999999999997},"min":{"1min":0.41399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.149,"5min":195.267,"15min":195.267},"last":3.9060000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":5.298,"15min":3.669},"min":{"1min":0.33800000000000002,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":2.173,"5min":195.15799999999999,"15min":195.15799999999999},"last":3.4590000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:44:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":7.4889999999999999,"15min":7.508},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.27900000000000003},"max":{"1min":3.1619999999999999,"5min":357.125,"15min":357.125},"last":2.363},{"interface":"front","average":{"1min":1.2829999999999999,"5min":7.3460000000000001,"15min":7.6550000000000002},"min":{"1min":0.36299999999999999,"5min":0.314,"15min":0.312},"max":{"1min":4.2069999999999999,"5min":359.02800000000002,"15min":359.02800000000002},"last":1.8480000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:44:22 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.92800000000000005,"15min":1.873},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":3.5779999999999998,"5min":9.5609999999999999,"15min":107.476},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.995,"15min":2.0830000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.23300000000000001},"max":{"1min":1.6060000000000001,"5min":15.394,"15min":107.414},"last":0.75}]},{"osd":6,"last update":"Mon Oct 9 04:44:48 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.175,"15min":1.9159999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29599999999999999},"max":{"1min":4.2640000000000002,"5min":9.6099999999999994,"15min":65.75},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.2010000000000001,"15min":1.893},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.2000000000000002,"5min":9.782,"15min":65.849999999999994},"last":0.44800000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.0169999999999999,"15min":1.6200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":2.024,"5min":8.6120000000000001,"15min":76.283000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":1.288,"15min":1.6739999999999999},"min":{"1min":0.40400000000000003,"5min":0.379,"15min":0.32100000000000001},"max":{"1min":4.2850000000000001,"5min":9.6310000000000002,"15min":76.259},"last":0.57599999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757729,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1574584,"kb_used_data":680176,"kb_used_omap":0,"kb_used_meta":894400,"kb_avail":92170568,"statfs":{"total":95995035648,"available":94382661632,"internally_reserved":0,"allocated":696500224,"data_stored":695904214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915865600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:44:43 2023","interfaces":[{"interface":"back","average":{"1min":11.423999999999999,"5min":4.2320000000000002,"15min":5.96},"min":{"1min":0.26900000000000002,"5min":0.23799999999999999,"15min":0.22},"max":{"1min":163.70699999999999,"5min":163.70699999999999,"15min":210.85499999999999},"last":2.1619999999999999},{"interface":"front","average":{"1min":11.489000000000001,"5min":4.7560000000000002,"15min":6.0869999999999997},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":163.756,"5min":163.756,"15min":210.23500000000001},"last":2.7490000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:44:43 2023","interfaces":[{"interface":"back","average":{"1min":4.8179999999999996,"5min":4.1559999999999997,"15min":4.8259999999999996},"min":{"1min":0.39800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":37.198,"5min":117.84099999999999,"15min":145.577},"last":3.2599999999999998},{"interface":"front","average":{"1min":4.5430000000000001,"5min":4.1710000000000003,"15min":5.1890000000000001},"min":{"1min":0.34699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":36.899000000000001,"5min":117.90600000000001,"15min":145.792},"last":2.363}]},{"osd":2,"last update":"Mon Oct 9 04:44:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":4.2759999999999998,"15min":5.5259999999999998},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.26800000000000002},"max":{"1min":40.686,"5min":66.984999999999999,"15min":90.763999999999996},"last":4.774},{"interface":"front","average":{"1min":3.613,"5min":4.4080000000000004,"15min":5.6820000000000004},"min":{"1min":0.31900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":40.539000000000001,"5min":67.686000000000007,"15min":91.340000000000003},"last":3.0129999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":5.2510000000000003,"15min":7.101},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":2.9750000000000001,"5min":226.43799999999999,"15min":268.63299999999998},"last":4.843},{"interface":"front","average":{"1min":0.89900000000000002,"5min":5.1189999999999998,"15min":6.8949999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.6850000000000001,"5min":226.63300000000001,"15min":268.92500000000001},"last":1.4279999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:44:19 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.622,"15min":2.3460000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.98,"5min":4.9390000000000001,"15min":361.09399999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.129,"15min":2.7829999999999999},"min":{"1min":0.26900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":2.036,"5min":7.4859999999999998,"15min":361.71300000000002},"last":0.91100000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:44:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":0.80600000000000005,"15min":1.2290000000000001},"min":{"1min":0.28499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":4.0730000000000004},{"interface":"front","average":{"1min":0.76800000000000002,"5min":1.073,"15min":1.3360000000000001},"min":{"1min":0.28899999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.5640000000000001,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:44:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.2589999999999999,"15min":1.3600000000000001},"min":{"1min":0.39200000000000002,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":2.403,"5min":13.268000000000001,"15min":13.268000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.94499999999999995,"15min":1.242},"min":{"1min":0.437,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.351,"5min":6.5129999999999999,"15min":15.257999999999999},"last":0.70999999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2097479680,"data_stored":2097479680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1814495232,"data_stored":1814495232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3474849792,"data_stored":3474849792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2849374208,"data_stored":2849374208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1226506240,"data_stored":1226506240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":695533568,"data_stored":695533568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2148007936,"data_stored":2148007936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2599223296,"data_stored":2599223296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:45:08.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-10-09T04:45:08.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:08.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 91247 91231 68.66 39.4375 0.0148199 0.014557 2023-10-09T04:45:08.304 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:08.473 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:08.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:08.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:08.724 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:08.833 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:08.899 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:08.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:08.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:08.997 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:45:09.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:09.051 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:09.077 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:45:09.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:09.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:09.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 91676 91660 68.162 26.8125 0.047787 0.0146631 2023-10-09T04:45:09.172 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:45:09.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[216903]: pgmap v1152: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 33 MiB/s wr, 535 op/s; 23613/266850 objects degraded (8.849%); 6705/266850 objects misplaced (2.513%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3790785880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[223858]: pgmap v1152: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 33 MiB/s wr, 535 op/s; 23613/266850 objects degraded (8.849%); 6705/266850 objects misplaced (2.513%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:09.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3790785880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:09.201 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:45:09.211 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:09.233 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:45:09.239 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:09.289 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:45:09.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:09.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:09 smithi177 ceph-mon[195797]: pgmap v1152: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 33 MiB/s wr, 535 op/s; 23613/266850 objects degraded (8.849%); 6705/266850 objects misplaced (2.513%); 5.6 MiB/s, 89 objects/s recovering 2023-10-09T04:45:09.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:09.298 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:09.299 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3790785880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:09.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:09.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:09.330 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:45:09.373 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:45:09.401 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:09.401 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.402 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:45:09.499 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:10.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 92255 92239 67.7859 36.1875 0.0272672 0.0147454 2023-10-09T04:45:10.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3020922603' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:10.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:10.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3020922603' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:10.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:45:10.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:45:10.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:45:10.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:10.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3020922603' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:10.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:11.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 92816 92800 67.4056 35.0625 0.0122764 0.0148289 2023-10-09T04:45:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[216903]: pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 567 op/s; 23613/269703 objects degraded (8.755%); 6481/269703 objects misplaced (2.403%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:45:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[216903]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:45:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[216903]: failed to encode map e298 with expected crc 2023-10-09T04:45:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/269703 objects degraded (8.755%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[223858]: pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 567 op/s; 23613/269703 objects degraded (8.755%); 6481/269703 objects misplaced (2.403%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:45:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[223858]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:45:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[223858]: failed to encode map e298 with expected crc 2023-10-09T04:45:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/269703 objects degraded (8.755%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:11 smithi177 ceph-mon[195797]: pgmap v1153: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 35 MiB/s wr, 567 op/s; 23613/269703 objects degraded (8.755%); 6481/269703 objects misplaced (2.403%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T04:45:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:11 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:11 smithi177 ceph-mon[195797]: osdmap e298: 8 total, 8 up, 7 in 2023-10-09T04:45:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:11 smithi177 ceph-mon[195797]: failed to encode map e298 with expected crc 2023-10-09T04:45:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/269703 objects degraded (8.755%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:12.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 93460 93444 67.0935 40.25 0.0195507 0.0148996 2023-10-09T04:45:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:12 smithi177 ceph-mon[195797]: failed to encode map e298 with expected crc 2023-10-09T04:45:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:12 smithi177 ceph-mon[195797]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 584 op/s; 23613/274086 objects degraded (8.615%); 6073/274086 objects misplaced (2.216%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T04:45:12.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:12 smithi029 ceph-mon[216903]: failed to encode map e298 with expected crc 2023-10-09T04:45:12.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:12 smithi029 ceph-mon[216903]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 584 op/s; 23613/274086 objects degraded (8.615%); 6073/274086 objects misplaced (2.216%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T04:45:12.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:12 smithi029 ceph-mon[223858]: failed to encode map e298 with expected crc 2023-10-09T04:45:12.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:12 smithi029 ceph-mon[223858]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 584 op/s; 23613/274086 objects degraded (8.615%); 6073/274086 objects misplaced (2.216%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T04:45:12.808 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:12.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:12.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:12.948 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:13.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:13.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:13.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:13.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 94120 94104 66.8 41.25 0.0188545 0.0149642 2023-10-09T04:45:13.175 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:45:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:45:12] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-10-09T04:45:13.176 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:13.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:45:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:13.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:13.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:13.453 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:14.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 94693 94677 66.4519 35.8125 0.0119436 0.0150426 2023-10-09T04:45:14.588 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:14 smithi029 ceph-mon[216903]: pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 23613/278412 objects degraded (8.481%); 5897/278412 objects misplaced (2.118%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T04:45:14.588 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:14 smithi029 ceph-mon[223858]: pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 23613/278412 objects degraded (8.481%); 5897/278412 objects misplaced (2.118%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:14.603 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:14 smithi177 ceph-mon[195797]: pgmap v1156: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 23613/278412 objects degraded (8.481%); 5897/278412 objects misplaced (2.118%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T04:45:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:15.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0068 2023-10-09T04:45:15.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 95247 2023-10-09T04:45:15.152 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 66.1387 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 42.9098 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 215.125 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1058 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 686.557 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3442 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0151157 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0361051 2023-10-09T04:45:15.153 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.32654 2023-10-09T04:45:15.154 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00124565 2023-10-09T04:45:15.158 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:45:15.573 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:15.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:15.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:15.756 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:16.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:16 smithi029 ceph-mon[223858]: pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 23613/283035 objects degraded (8.343%); 5466/283035 objects misplaced (1.931%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:45:16.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/283035 objects degraded (8.343%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:16.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:16 smithi029 ceph-mon[216903]: pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 23613/283035 objects degraded (8.343%); 5466/283035 objects misplaced (1.931%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:45:16.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/283035 objects degraded (8.343%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:16.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:45:16.690 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:16.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:16.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:16 smithi177 ceph-mon[195797]: pgmap v1157: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 23613/283035 objects degraded (8.343%); 5466/283035 objects misplaced (1.931%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:45:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/283035 objects degraded (8.343%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:16.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:16.824 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:16.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:16.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:16.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:17.124 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:17.733 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:45:17.733 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:45:18.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:45:18.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:18 smithi029 ceph-mon[216903]: pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 597 op/s; 23613/283317 objects degraded (8.334%); 5466/283317 objects misplaced (1.929%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:45:18.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:18 smithi029 ceph-mon[223858]: pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 597 op/s; 23613/283317 objects degraded (8.334%); 5466/283317 objects misplaced (1.929%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:45:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:18 smithi177 ceph-mon[195797]: pgmap v1158: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 36 MiB/s wr, 597 op/s; 23613/283317 objects degraded (8.334%); 5466/283317 objects misplaced (1.929%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:45:20.055 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:20.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:20.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:20.204 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:20.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:20.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:20.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:20.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:20.391 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:20 smithi029 ceph-mon[216903]: pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 32 MiB/s wr, 593 op/s; 23613/282936 objects degraded (8.346%); 5235/282936 objects misplaced (1.850%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:45:20.428 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:20.487 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:20.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:20.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:20.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:20 smithi177 ceph-mon[195797]: pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 32 MiB/s wr, 593 op/s; 23613/282936 objects degraded (8.346%); 5235/282936 objects misplaced (1.850%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:45:20.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:20.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:20 smithi029 ceph-mon[223858]: pgmap v1159: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 32 MiB/s wr, 593 op/s; 23613/282936 objects degraded (8.346%); 5235/282936 objects misplaced (1.850%); 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:45:20.678 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:21.422 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:21.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:21.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:21.564 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:21.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:21.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:21.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/282936 objects degraded (8.346%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:21.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/282936 objects degraded (8.346%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:21.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:21.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/282936 objects degraded (8.346%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:21.825 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:21.949 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:21.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:22.060 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:22.094 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:22.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:22.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:22.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:22.264 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:22.300 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:22.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:45:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:22.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:22.473 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:22.500 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:22.566 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:22.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:22.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:22.744 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:22.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:45:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:22 smithi177 ceph-mon[195797]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 94 B/s rd, 23 MiB/s wr, 546 op/s; 23613/279744 objects degraded (8.441%); 4705/279744 objects misplaced (1.682%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T04:45:22.805 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:22.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:22 smithi029 ceph-mon[223858]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 94 B/s rd, 23 MiB/s wr, 546 op/s; 23613/279744 objects degraded (8.441%); 4705/279744 objects misplaced (1.682%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T04:45:22.874 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:45:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:45:22] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2023-10-09T04:45:22.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:22 smithi029 ceph-mon[216903]: pgmap v1160: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 94 B/s rd, 23 MiB/s wr, 546 op/s; 23613/279744 objects degraded (8.441%); 4705/279744 objects misplaced (1.682%); 7.9 MiB/s, 125 objects/s recovering 2023-10-09T04:45:23.050 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:23.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:23.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:23.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:23.385 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:23.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:23.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:24.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:24.111 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:24.626 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pgp_num to 1 2023-10-09T04:45:24.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:24 smithi029 ceph-mon[216903]: pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 20 MiB/s wr, 620 op/s; 23613/275199 objects degraded (8.580%); 4487/275199 objects misplaced (1.630%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:45:24.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3248064481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:45:24.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:45:24.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:24 smithi029 ceph-mon[223858]: pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 20 MiB/s wr, 620 op/s; 23613/275199 objects degraded (8.580%); 4487/275199 objects misplaced (1.630%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:45:24.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3248064481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:45:24.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:45:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:24 smithi177 ceph-mon[195797]: pgmap v1161: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 20 MiB/s wr, 620 op/s; 23613/275199 objects degraded (8.580%); 4487/275199 objects misplaced (1.630%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:45:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3248064481' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:45:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:45:25.165 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:25.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:25.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:25.360 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:45:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[216903]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:45:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[216903]: failed to encode map e299 with expected crc 2023-10-09T04:45:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[216903]: failed to encode map e299 with expected crc 2023-10-09T04:45:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[216903]: failed to encode map e299 with expected crc 2023-10-09T04:45:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/275199 objects degraded (8.580%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:45:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[223858]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:45:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[223858]: failed to encode map e299 with expected crc 2023-10-09T04:45:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[223858]: failed to encode map e299 with expected crc 2023-10-09T04:45:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[223858]: failed to encode map e299 with expected crc 2023-10-09T04:45:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/275199 objects degraded (8.580%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:45:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:25 smithi177 ceph-mon[195797]: osdmap e299: 8 total, 8 up, 7 in 2023-10-09T04:45:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:25 smithi177 ceph-mon[195797]: failed to encode map e299 with expected crc 2023-10-09T04:45:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:25 smithi177 ceph-mon[195797]: failed to encode map e299 with expected crc 2023-10-09T04:45:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:25 smithi177 ceph-mon[195797]: failed to encode map e299 with expected crc 2023-10-09T04:45:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/275199 objects degraded (8.580%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:26.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:26 smithi029 ceph-mon[216903]: pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 581 op/s; 23613/271158 objects degraded (8.708%); 3822/271158 objects misplaced (1.410%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:26.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:26 smithi029 ceph-mon[223858]: pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 581 op/s; 23613/271158 objects degraded (8.708%); 3822/271158 objects misplaced (1.410%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:26 smithi177 ceph-mon[195797]: pgmap v1163: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 5.8 MiB/s wr, 581 op/s; 23613/271158 objects degraded (8.708%); 3822/271158 objects misplaced (1.410%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:27.873 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:45:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:27.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:27.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:28.099 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:28.624 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:28 smithi029 ceph-mon[216903]: pgmap v1164: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 3.7 MiB/s wr, 570 op/s; 23613/269730 objects degraded (8.754%); 3822/269730 objects misplaced (1.417%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:28.625 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:28 smithi029 ceph-mon[223858]: pgmap v1164: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 3.7 MiB/s wr, 570 op/s; 23613/269730 objects degraded (8.754%); 3822/269730 objects misplaced (1.417%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:28 smithi177 ceph-mon[195797]: pgmap v1164: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 3.7 MiB/s wr, 570 op/s; 23613/269730 objects degraded (8.754%); 3822/269730 objects misplaced (1.417%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:29.631 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:29.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:29.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:29.769 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:29.771 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:29.813 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:29.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:29.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:29.959 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:30.130 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:30.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:30.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:30.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:30.267 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:30.672 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:45:30.787 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:30.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:30.897 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 1.2 MiB/s wr, 562 op/s; 23613/267240 objects degraded (8.836%); 3593/267240 objects misplaced (1.344%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: failed to encode map e300 with expected crc 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: failed to encode map e300 with expected crc 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/267240 objects degraded (8.836%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/606052219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:30.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 1.2 MiB/s wr, 562 op/s; 23613/267240 objects degraded (8.836%); 3593/267240 objects misplaced (1.344%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: failed to encode map e300 with expected crc 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: failed to encode map e300 with expected crc 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/267240 objects degraded (8.836%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/606052219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:30.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:31.008 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: pgmap v1165: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 1.2 MiB/s wr, 562 op/s; 23613/267240 objects degraded (8.836%); 3593/267240 objects misplaced (1.344%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:45:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: osdmap e300: 8 total, 8 up, 7 in 2023-10-09T04:45:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: failed to encode map e300 with expected crc 2023-10-09T04:45:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: failed to encode map e300 with expected crc 2023-10-09T04:45:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/267240 objects degraded (8.836%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/606052219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:31.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:45:31.056 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:45:31.100 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:45:31.144 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:45:31.190 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:45:31.230 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:45:31.263 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:45:31.304 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:45:31.353 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.366 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:45:31.485 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:31.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:31.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:31.740 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:31.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:31.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:31.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:31.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:31.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: failed to encode map e301 with expected crc 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: failed to encode map e301 with expected crc 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: failed to encode map e301 with expected crc 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: failed to encode map e301 with expected crc 2023-10-09T04:45:31.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3222268928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:31.940 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:31.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:31.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:32.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:32.126 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:32.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:32.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:45:32.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3222268928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: osdmap e301: 8 total, 8 up, 7 in 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: failed to encode map e301 with expected crc 2023-10-09T04:45:32.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3222268928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:32.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:45:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:32.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:32.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:32.367 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:32 smithi177 ceph-mon[195797]: pgmap v1168: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 23613/262938 objects degraded (8.980%); 2721/262938 objects misplaced (1.035%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:45:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:32 smithi029 ceph-mon[223858]: pgmap v1168: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 23613/262938 objects degraded (8.980%); 2721/262938 objects misplaced (1.035%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:45:33.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:45:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:45:32] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-10-09T04:45:33.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:32 smithi029 ceph-mon[216903]: pgmap v1168: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 23613/262938 objects degraded (8.980%); 2721/262938 objects misplaced (1.035%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:45:34.798 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:45:34.798 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:45:34.798 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [1, 2, 3, 4, 5, 7, 0] 2023-10-09T04:45:34.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-09T04:45:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:34 smithi177 ceph-mon[195797]: pgmap v1169: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 23613/259056 objects degraded (9.115%); 2384/259056 objects misplaced (0.920%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:45:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:34 smithi029 ceph-mon[216903]: pgmap v1169: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 23613/259056 objects degraded (9.115%); 2384/259056 objects misplaced (0.920%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:45:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:34 smithi029 ceph-mon[223858]: pgmap v1169: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 503 op/s; 23613/259056 objects degraded (9.115%); 2384/259056 objects misplaced (0.920%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:45:35.728 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:35.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:35.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:35.938 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/259056 objects degraded (9.115%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:36.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/259056 objects degraded (9.115%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:36.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/259056 objects degraded (9.115%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:36 smithi177 ceph-mon[195797]: pgmap v1170: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 23613/255435 objects degraded (9.244%); 2384/255435 objects misplaced (0.933%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:45:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:36 smithi029 ceph-mon[216903]: pgmap v1170: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 23613/255435 objects degraded (9.244%); 2384/255435 objects misplaced (0.933%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:45:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:36 smithi029 ceph-mon[223858]: pgmap v1170: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 23613/255435 objects degraded (9.244%); 2384/255435 objects misplaced (0.933%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:45:37.406 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:37.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:37.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:37.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:37.686 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:37.742 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:37.840 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:37.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:37.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:37.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:38.083 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:38.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:38.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:38.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:38.577 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:45:38.578 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:45:38.578 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:45:38.578 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:45:38.578 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:45:38.652 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:38.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:38 smithi029 ceph-mon[216903]: pgmap v1171: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 23613/253968 objects degraded (9.298%); 2384/253968 objects misplaced (0.939%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T04:45:38.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:38 smithi029 ceph-mon[223858]: pgmap v1171: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 23613/253968 objects degraded (9.298%); 2384/253968 objects misplaced (0.939%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T04:45:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:38 smithi177 ceph-mon[195797]: pgmap v1171: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 23613/253968 objects degraded (9.298%); 2384/253968 objects misplaced (0.939%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T04:45:39.336 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:39.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:39.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:39.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:45:39.480 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:39.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:39.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:45:39.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:39.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:39.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:39.793 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:39.970 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:39.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:40.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:40.151 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:41.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:40 smithi177 ceph-mon[195797]: pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 23613/251364 objects degraded (9.394%); 2111/251364 objects misplaced (0.840%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:45:41.040 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/251364 objects degraded (9.394%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:40 smithi029 ceph-mon[216903]: pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 23613/251364 objects degraded (9.394%); 2111/251364 objects misplaced (0.840%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:45:41.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/251364 objects degraded (9.394%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:41.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:40 smithi029 ceph-mon[223858]: pgmap v1172: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 401 op/s; 23613/251364 objects degraded (9.394%); 2111/251364 objects misplaced (0.840%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:45:41.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/251364 objects degraded (9.394%), 5 pgs degraded (PG_DEGRADED) 2023-10-09T04:45:42.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:45:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:42.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:42.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:42.216 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:42.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:45:42.585 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:42.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:42.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:42.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:42 smithi029 ceph-mon[216903]: pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 23613/247515 objects degraded (9.540%); 2111/247515 objects misplaced (0.853%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:45:42.806 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:42.840 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:42.898 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:43.012 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:45:43.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:43.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:43.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:42 smithi177 ceph-mon[195797]: pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 23613/247515 objects degraded (9.540%); 2111/247515 objects misplaced (0.853%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:45:43.036 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:45:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:45:42] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-10-09T04:45:43.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:42 smithi029 ceph-mon[223858]: pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 484 op/s; 23613/247515 objects degraded (9.540%); 2111/247515 objects misplaced (0.853%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:45:43.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:43.259 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:43.702 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:45:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:43.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:43.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:43.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:43.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:43.913 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:44.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:44 smithi029 conmon[216878]: 2023-10-09T04:45:44.143+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e301 definitely_dead 0 2023-10-09T04:45:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:44.848 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.3. 2023-10-09T04:45:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:44 smithi029 ceph-mon[216903]: pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 23613/243225 objects degraded (9.708%); 1821/243225 objects misplaced (0.749%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:45:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1357298061' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:45:45.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:44 smithi029 ceph-mon[216903]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:45:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:44 smithi029 ceph-mon[223858]: pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 23613/243225 objects degraded (9.708%); 1821/243225 objects misplaced (0.749%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:45:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1357298061' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:45:45.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:44 smithi029 ceph-mon[223858]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:45:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:44 smithi177 ceph-mon[195797]: pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 23613/243225 objects degraded (9.708%); 1821/243225 objects misplaced (0.749%); 4.7 MiB/s, 74 objects/s recovering 2023-10-09T04:45:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1357298061' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-09T04:45:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:44 smithi177 ceph-mon[195797]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1357298061' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[223858]: osdmap e302: 8 total, 8 up, 6 in 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[223858]: failed to encode map e302 with expected crc 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[223858]: failed to encode map e302 with expected crc 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/243225 objects degraded (9.708%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[223858]: osdmap e303: 8 total, 8 up, 6 in 2023-10-09T04:45:46.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1357298061' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:45:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[216903]: osdmap e302: 8 total, 8 up, 6 in 2023-10-09T04:45:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[216903]: failed to encode map e302 with expected crc 2023-10-09T04:45:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[216903]: failed to encode map e302 with expected crc 2023-10-09T04:45:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/243225 objects degraded (9.708%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:45 smithi029 ceph-mon[216903]: osdmap e303: 8 total, 8 up, 6 in 2023-10-09T04:45:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1357298061' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-09T04:45:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:45 smithi177 ceph-mon[195797]: osdmap e302: 8 total, 8 up, 6 in 2023-10-09T04:45:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:45 smithi177 ceph-mon[195797]: failed to encode map e302 with expected crc 2023-10-09T04:45:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:45 smithi177 ceph-mon[195797]: failed to encode map e302 with expected crc 2023-10-09T04:45:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/243225 objects degraded (9.708%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:45 smithi177 ceph-mon[195797]: osdmap e303: 8 total, 8 up, 6 in 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 23613/239088 objects degraded (9.876%); 1821/239088 objects misplaced (0.762%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[216903]: osdmap e304: 8 total, 8 up, 6 in 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 23613/239088 objects degraded (9.876%); 1821/239088 objects misplaced (0.762%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:46 smithi029 ceph-mon[223858]: osdmap e304: 8 total, 8 up, 6 in 2023-10-09T04:45:47.207 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:47.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: pgmap v1176: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 23613/239088 objects degraded (9.876%); 1821/239088 objects misplaced (0.762%); 3.5 MiB/s, 56 objects/s recovering 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: failed to encode map e303 with expected crc 2023-10-09T04:45:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:46 smithi177 ceph-mon[195797]: osdmap e304: 8 total, 8 up, 6 in 2023-10-09T04:45:47.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:47.352 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:47.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:47.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:47.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:47.537 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:47.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:47.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:47.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:47.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:47.719 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:47.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:45:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:47.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:47.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:47.899 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:47.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:47.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: 12.8 starting backfill to osd.4 from (0'0,0'0] MAX to 301'6990 2023-10-09T04:45:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: 12.b starting backfill to osd.1 from (0'0,0'0] MAX to 301'7087 2023-10-09T04:45:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: 12.1 starting backfill to osd.5 from (0'0,0'0] MAX to 301'7028 2023-10-09T04:45:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: 12.4 starting backfill to osd.4 from (0'0,0'0] MAX to 301'6932 2023-10-09T04:45:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:47 smithi177 ceph-mon[195797]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.053 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:48.080 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:48.126 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:48.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:48.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: 12.8 starting backfill to osd.4 from (0'0,0'0] MAX to 301'6990 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: 12.b starting backfill to osd.1 from (0'0,0'0] MAX to 301'7087 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: 12.1 starting backfill to osd.5 from (0'0,0'0] MAX to 301'7028 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: 12.4 starting backfill to osd.4 from (0'0,0'0] MAX to 301'6932 2023-10-09T04:45:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[223858]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: 12.8 starting backfill to osd.4 from (0'0,0'0] MAX to 301'6990 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: 12.b starting backfill to osd.1 from (0'0,0'0] MAX to 301'7087 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: 12.1 starting backfill to osd.5 from (0'0,0'0] MAX to 301'7028 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: 12.4 starting backfill to osd.4 from (0'0,0'0] MAX to 301'6932 2023-10-09T04:45:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:47 smithi029 ceph-mon[216903]: failed to encode map e304 with expected crc 2023-10-09T04:45:48.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:48.266 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:49.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:48 smithi029 ceph-mon[223858]: pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 23613/239088 objects degraded (9.876%); 1821/239088 objects misplaced (0.762%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:45:49.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:48 smithi029 ceph-mon[216903]: pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 23613/239088 objects degraded (9.876%); 1821/239088 objects misplaced (0.762%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:45:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:48 smithi177 ceph-mon[195797]: pgmap v1179: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 23613/239088 objects degraded (9.876%); 1821/239088 objects misplaced (0.762%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T04:45:49.631 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:49.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:49.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:49.762 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:49.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:49.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:49.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:49.945 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:49.994 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:49.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:50.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:50.131 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:50.180 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:50.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:50.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:50.284 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:50.310 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:50.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:50.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:50.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:50.496 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:50.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:50 smithi029 ceph-mon[223858]: pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 23613/238812 objects degraded (9.888%); 1821/238812 objects misplaced (0.763%) 2023-10-09T04:45:50.852 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23613/238812 objects degraded (9.888%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:50 smithi029 ceph-mon[216903]: pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 23613/238812 objects degraded (9.888%); 1821/238812 objects misplaced (0.763%) 2023-10-09T04:45:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23613/238812 objects degraded (9.888%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:50 smithi177 ceph-mon[195797]: pgmap v1180: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 245 op/s; 23613/238812 objects degraded (9.888%); 1821/238812 objects misplaced (0.763%) 2023-10-09T04:45:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23613/238812 objects degraded (9.888%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:53.029 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:53.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:53.093 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:52 smithi029 ceph-mon[216903]: pgmap v1181: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 33531/237549 objects degraded (14.115%); 13320/237549 objects misplaced (5.607%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T04:45:53.093 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:52 smithi029 ceph-mon[223858]: pgmap v1181: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 33531/237549 objects degraded (14.115%); 13320/237549 objects misplaced (5.607%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T04:45:53.094 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:45:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:45:52] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-10-09T04:45:53.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:53.145 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:45:53.171 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:53.242 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:53.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:52 smithi177 ceph-mon[195797]: pgmap v1181: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 79 op/s; 33531/237549 objects degraded (14.115%); 13320/237549 objects misplaced (5.607%); 3.9 MiB/s, 63 objects/s recovering 2023-10-09T04:45:53.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:53.378 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:53.534 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:53.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:53.640 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:45:53.666 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:53.676 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:45:53.698 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:45:53.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:45:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:53.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:45:53.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:53.863 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1625980722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:54.117 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:54.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1625980722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:54.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1625980722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:45:54.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:54.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:54.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:54.248 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:54.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:54.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:54.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:54.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:54.440 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:54.493 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:45:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:54.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:54.597 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:45:54.627 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:54.678 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:45:54.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:54.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:45:54.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:54.809 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:54.859 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:54.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:54.879 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:45:54.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:45:54.999 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:55.049 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:45:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:55.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:45:55.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:45:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:54 smithi029 ceph-mon[216903]: pgmap v1182: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 33531/237510 objects degraded (14.118%); 13320/237510 objects misplaced (5.608%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:45:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3902327975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:54 smithi029 ceph-mon[223858]: pgmap v1182: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 33531/237510 objects degraded (14.118%); 13320/237510 objects misplaced (5.608%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:45:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3902327975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:55.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:54 smithi177 ceph-mon[195797]: pgmap v1182: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 65 op/s; 33531/237510 objects degraded (14.118%); 13320/237510 objects misplaced (5.608%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T04:45:55.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3902327975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:45:55.180 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:55.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:45:55.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:55.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:45:55.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:45:55.361 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:55.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:45:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:55.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:45:55.513 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:55.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:55.583 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:55.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:55.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:45:55.724 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:45:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[216903]: failed to encode map e305 with expected crc 2023-10-09T04:45:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3902327975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[216903]: osdmap e305: 8 total, 8 up, 6 in 2023-10-09T04:45:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 33531/237510 objects degraded (14.118%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:45:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[223858]: failed to encode map e305 with expected crc 2023-10-09T04:45:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3902327975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[223858]: osdmap e305: 8 total, 8 up, 6 in 2023-10-09T04:45:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 33531/237510 objects degraded (14.118%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:55 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:45:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:55 smithi177 ceph-mon[195797]: failed to encode map e305 with expected crc 2023-10-09T04:45:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3902327975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:45:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:55 smithi177 ceph-mon[195797]: osdmap e305: 8 total, 8 up, 6 in 2023-10-09T04:45:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 33531/237510 objects degraded (14.118%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T04:45:56.499 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:45:56.520 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:45:56.541 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:45:56.561 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:45:56.685 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:45:56.685 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:45:56.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:45:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:56 smithi029 ceph-mon[216903]: pgmap v1184: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 33528/237480 objects degraded (14.118%); 12904/237480 objects misplaced (5.434%); 5.8 MiB/s, 94 objects/s recovering 2023-10-09T04:45:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:56 smithi029 ceph-mon[223858]: pgmap v1184: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 33528/237480 objects degraded (14.118%); 12904/237480 objects misplaced (5.434%); 5.8 MiB/s, 94 objects/s recovering 2023-10-09T04:45:57.133 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:45:57.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:57.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:45:57.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:57.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:56 smithi177 ceph-mon[195797]: pgmap v1184: 89 pgs: 2 active+recovery_wait+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 33528/237480 objects degraded (14.118%); 12904/237480 objects misplaced (5.434%); 5.8 MiB/s, 94 objects/s recovering 2023-10-09T04:45:57.264 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:57.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:57.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:57.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:45:57.444 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:58.120 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 5, 7, 0] out_osds: [6, 3] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:45:58.121 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:45:58.121 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:45:58.121 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:45:58.121 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:45:58.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:45:58.487 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:45:58.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:58 smithi177 ceph-mon[195797]: pgmap v1185: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 40862/236154 objects degraded (17.303%); 27046/236154 objects misplaced (11.453%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:45:58.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:58.943 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:45:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:58 smithi029 ceph-mon[216903]: pgmap v1185: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 40862/236154 objects degraded (17.303%); 27046/236154 objects misplaced (11.453%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:45:59.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:59.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:45:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:59.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:58 smithi029 ceph-mon[223858]: pgmap v1185: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 40862/236154 objects degraded (17.303%); 27046/236154 objects misplaced (11.453%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:45:59.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:45:59.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:45:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:45:59.658 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:45:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:59.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:45:59.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:45:59.796 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:45:59.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:45:59.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:45:59.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:45:59.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:00.055 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:00.211 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:00.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:00.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:00.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:00.396 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:00.508 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:00.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:00.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:00.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:00.654 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:00.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:00.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:00.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:00.907 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:01.020 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:00 smithi177 ceph-mon[195797]: pgmap v1186: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 40862/236154 objects degraded (17.303%); 27046/236154 objects misplaced (11.453%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:46:01.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40862/236154 objects degraded (17.303%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:01.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:01.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:00 smithi029 ceph-mon[223858]: pgmap v1186: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 40862/236154 objects degraded (17.303%); 27046/236154 objects misplaced (11.453%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:46:01.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40862/236154 objects degraded (17.303%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:01.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:00 smithi029 ceph-mon[216903]: pgmap v1186: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 40862/236154 objects degraded (17.303%); 27046/236154 objects misplaced (11.453%); 9.1 MiB/s, 146 objects/s recovering 2023-10-09T04:46:01.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40862/236154 objects degraded (17.303%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:01.232 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:01.357 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:01.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:01.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:01.518 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:01.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:01.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:01.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:01.747 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:01.823 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:01.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:01.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:02.000 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:03.119 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:03.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:03.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:03.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:02 smithi029 ceph-mon[216903]: pgmap v1187: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 46 op/s; 40862/236154 objects degraded (17.303%); 26604/236154 objects misplaced (11.266%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T04:46:03.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:02 smithi029 ceph-mon[223858]: pgmap v1187: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 46 op/s; 40862/236154 objects degraded (17.303%); 26604/236154 objects misplaced (11.266%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T04:46:03.126 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:46:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:46:02] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-10-09T04:46:03.226 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:02 smithi177 ceph-mon[195797]: pgmap v1187: 89 pgs: 2 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 46 op/s; 40862/236154 objects degraded (17.303%); 26604/236154 objects misplaced (11.266%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T04:46:03.289 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:03.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:03.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:03.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:03.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:03.545 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:04.511 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:04.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:04.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:04.698 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:04.944 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:04.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:05.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:05.090 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:05.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:05.165 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:04 smithi177 ceph-mon[195797]: pgmap v1188: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 40691/235197 objects degraded (17.301%); 26219/235197 objects misplaced (11.148%); 12 MiB/s, 189 objects/s recovering 2023-10-09T04:46:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:05.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:04 smithi029 ceph-mon[216903]: pgmap v1188: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 40691/235197 objects degraded (17.301%); 26219/235197 objects misplaced (11.148%); 12 MiB/s, 189 objects/s recovering 2023-10-09T04:46:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:04 smithi029 ceph-mon[223858]: pgmap v1188: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 40691/235197 objects degraded (17.301%); 26219/235197 objects misplaced (11.148%); 12 MiB/s, 189 objects/s recovering 2023-10-09T04:46:05.267 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:05.292 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:05.292 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:05.349 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:05.631 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:05.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:05.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:05.817 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:05.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40691/235197 objects degraded (17.301%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:05.953 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:05.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:06.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:06.159 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40691/235197 objects degraded (17.301%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:06.162 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:06.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40691/235197 objects degraded (17.301%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:06.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:06.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:46:06.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:46:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:06.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:06.601 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:06.601 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2023-10-09T04:46:06.601 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:06.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:06.732 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:06.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:06.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:06.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:06 smithi177 ceph-mon[195797]: pgmap v1189: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 193 op/s; 40582/231438 objects degraded (17.535%); 25682/231438 objects misplaced (11.097%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:46:06.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:07.075 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:06 smithi029 ceph-mon[216903]: pgmap v1189: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 193 op/s; 40582/231438 objects degraded (17.535%); 25682/231438 objects misplaced (11.097%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:46:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:06 smithi029 ceph-mon[223858]: pgmap v1189: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 193 op/s; 40582/231438 objects degraded (17.535%); 25682/231438 objects misplaced (11.097%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:46:07.756 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:07.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:07.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:07.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:07.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:46:08.005 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:08.076 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:08.083 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:08.140 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:08.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:08.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:08.332 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:08.433 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:08.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:08.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:08.513 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:08.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:08.610 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:08.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:08.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:08.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:08.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:08.961 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:09.080 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:09.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:09.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:08 smithi029 ceph-mon[216903]: pgmap v1190: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 40172/229212 objects degraded (17.526%); 25246/229212 objects misplaced (11.014%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:46:09.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:08 smithi029 ceph-mon[223858]: pgmap v1190: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 40172/229212 objects degraded (17.526%); 25246/229212 objects misplaced (11.014%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:46:09.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:09.262 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:08 smithi177 ceph-mon[195797]: pgmap v1190: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 229 op/s; 40172/229212 objects degraded (17.526%); 25246/229212 objects misplaced (11.014%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:46:09.267 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:10.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:46:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:46:10.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:46:11.123 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:11.123 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[216903]: pgmap v1191: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 40172/228795 objects degraded (17.558%); 25246/228795 objects misplaced (11.034%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:46:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40172/228795 objects degraded (17.558%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[223858]: pgmap v1191: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 40172/228795 objects degraded (17.558%); 25246/228795 objects misplaced (11.034%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:46:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40172/228795 objects degraded (17.558%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:11 smithi177 ceph-mon[195797]: pgmap v1191: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 40172/228795 objects degraded (17.558%); 25246/228795 objects misplaced (11.034%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:46:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40172/228795 objects degraded (17.558%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:12.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:46:12.315 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:12.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:12.423 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:12.496 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:12.940 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:13.152 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:46:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:46:12] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-10-09T04:46:13.166 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 5, 7, 0] out_osds: [6, 3] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:46:13.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:46:13.167 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 2, 4, 5, 7, 0] 2023-10-09T04:46:13.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T04:46:13.338 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:13.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:13.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:13 smithi029 ceph-mon[216903]: pgmap v1192: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 40044/224589 objects degraded (17.830%); 24712/224589 objects misplaced (11.003%); 11 MiB/s, 168 objects/s recovering 2023-10-09T04:46:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:13 smithi029 ceph-mon[223858]: pgmap v1192: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 40044/224589 objects degraded (17.830%); 24712/224589 objects misplaced (11.003%); 11 MiB/s, 168 objects/s recovering 2023-10-09T04:46:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:13.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:13.507 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:13 smithi177 ceph-mon[195797]: pgmap v1192: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 40044/224589 objects degraded (17.830%); 24712/224589 objects misplaced (11.003%); 11 MiB/s, 168 objects/s recovering 2023-10-09T04:46:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:14.174 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:14.471 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:14.471 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:14.490 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:14.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:14.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:14.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:14.666 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:14.768 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:14.934 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:14.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:14.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:15.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:15.110 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:15.181 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:15 smithi177 ceph-mon[195797]: pgmap v1193: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 39627/222216 objects degraded (17.833%); 24302/222216 objects misplaced (10.936%); 10 MiB/s, 168 objects/s recovering 2023-10-09T04:46:15.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:15.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:15.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:15.397 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:15 smithi029 ceph-mon[216903]: pgmap v1193: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 39627/222216 objects degraded (17.833%); 24302/222216 objects misplaced (10.936%); 10 MiB/s, 168 objects/s recovering 2023-10-09T04:46:15.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:15 smithi029 ceph-mon[223858]: pgmap v1193: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 39627/222216 objects degraded (17.833%); 24302/222216 objects misplaced (10.936%); 10 MiB/s, 168 objects/s recovering 2023-10-09T04:46:15.401 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:15.580 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:15.580 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:15.587 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:15.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:15.653 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:46:15.699 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:15.702 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:46:15.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:15.750 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:46:15.814 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:46:15.882 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:46:15.938 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:46:15.982 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:46:16.027 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:46:16.037 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.037 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.037 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.037 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.038 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:46:16.351 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3487715340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:16.351 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 39627/222216 objects degraded (17.833%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:16.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3058469490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:16.352 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3487715340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:16.352 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 39627/222216 objects degraded (17.833%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:16.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3058469490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3487715340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:16.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 39627/222216 objects degraded (17.833%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:16.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3058469490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:46:16.593 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:46:17.004 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:17.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:17.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:17.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:46:17.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:17.157 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:17.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:17.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:17 smithi177 ceph-mon[195797]: pgmap v1194: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 472 op/s; 39490/218175 objects degraded (18.100%); 23765/218175 objects misplaced (10.893%); 11 MiB/s, 168 objects/s recovering 2023-10-09T04:46:17.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:17 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:17.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3058469490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:46:17.290 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:17 smithi177 ceph-mon[195797]: osdmap e306: 8 total, 8 up, 6 in 2023-10-09T04:46:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:17.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:17.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:17.487 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[216903]: pgmap v1194: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 472 op/s; 39490/218175 objects degraded (18.100%); 23765/218175 objects misplaced (10.893%); 11 MiB/s, 168 objects/s recovering 2023-10-09T04:46:17.487 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:17.488 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3058469490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:46:17.488 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[216903]: osdmap e306: 8 total, 8 up, 6 in 2023-10-09T04:46:17.488 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[223858]: pgmap v1194: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 472 op/s; 39490/218175 objects degraded (18.100%); 23765/218175 objects misplaced (10.893%); 11 MiB/s, 168 objects/s recovering 2023-10-09T04:46:17.488 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:17.488 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3058469490' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:46:17.488 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:17 smithi029 ceph-mon[223858]: osdmap e306: 8 total, 8 up, 6 in 2023-10-09T04:46:17.496 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:17.784 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:18.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:18 smithi177 ceph-mon[195797]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:18 smithi177 ceph-mon[195797]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:18 smithi177 ceph-mon[195797]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:18 smithi177 ceph-mon[195797]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.405 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:18 smithi177 ceph-mon[195797]: pgmap v1196: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 432 op/s; 39134/216240 objects degraded (18.097%); 23386/216240 objects misplaced (10.815%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[216903]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[216903]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[216903]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[216903]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[216903]: pgmap v1196: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 432 op/s; 39134/216240 objects degraded (18.097%); 23386/216240 objects misplaced (10.815%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[223858]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[223858]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[223858]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[223858]: failed to encode map e306 with expected crc 2023-10-09T04:46:18.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:18 smithi029 ceph-mon[223858]: pgmap v1196: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 432 op/s; 39134/216240 objects degraded (18.097%); 23386/216240 objects misplaced (10.815%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:20.608 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:20.608 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:46:20.608 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:46:20.608 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:46:20.608 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45542.0:175241 10.4 10:22ec89cb:::benchmark_data_smithi029_344401_object66870:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:36:38.440862+0000", 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 582.16190466900002, 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33252321200000001, 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45542", 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/935312906", 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175241 2023-10-09T04:46:20.609 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.440862+0000", 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.440862+0000", 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.440861+0000", 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.440864+0000", 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.48e-06 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.440869+0000", 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2190000000000003e-06 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.440873+0000", 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6679999999999999e-06 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.612 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.500124+0000", 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.059251166000000001 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.500218+0000", 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3840999999999998e-05 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.500668+0000", 2023-10-09T04:46:20.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000450266 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.502923+0000", 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00225538 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.752550+0000", 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.24962659300000001 2023-10-09T04:46:20.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.773311+0000", 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020761465999999999 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.773351+0000", 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9147999999999999e-05 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:36:38.773385+0000", 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4335000000000002e-05 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145894 11.e 11.813cce1e (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.593405+0000", 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 319.00936113199998, 2023-10-09T04:46:20.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0051563910000001, 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145894 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.593405+0000", 2023-10-09T04:46:20.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.593405+0000", 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.593403+0000", 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:46:20.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.593411+0000", 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4789999999999999e-06 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.593436+0000", 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5449999999999999e-05 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.593445+0000", 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9290000000000004e-06 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.595149+0000", 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001703654 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.595157+0000", 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9689999999999994e-06 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598546+0000", 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0033893190000001 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598562+0000", 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5386000000000001e-05 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145896 11.e 11.9af1872e (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.594847+0000", 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 319.00791916100002, 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0037517730000001, 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145896 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.622 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594847+0000", 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594847+0000", 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594846+0000", 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594852+0000", 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2059999999999998e-06 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594862+0000", 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0769e-05 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594869+0000", 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7410000000000003e-06 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.595177+0000", 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000308086 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.595182+0000", 2023-10-09T04:46:20.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9660000000000004e-06 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598585+0000", 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0034032319999999 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598599+0000", 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.359e-05 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.626 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145898 11.e 11.788e5c6e (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.594876+0000", 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 319.00789058999999, 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.003762804, 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145898 2023-10-09T04:46:20.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594876+0000", 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594876+0000", 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594875+0000", 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594878+0000", 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5720000000000001e-06 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594884+0000", 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0920000000000003e-06 2023-10-09T04:46:20.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.594888+0000", 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9099999999999998e-06 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.595201+0000", 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031276200000000001 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.595205+0000", 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5410000000000002e-06 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598618+0000", 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.003412663 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598639+0000", 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0696999999999999e-05 2023-10-09T04:46:20.631 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145922 11.e 11.90f9e93e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.625838+0000", 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 318.97692826000002, 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97283720100000004, 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.632 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145922 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.625838+0000", 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.625838+0000", 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.625837+0000", 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.625843+0000", 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.314e-06 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.625855+0000", 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2128e-05 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.625862+0000", 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8299999999999998e-06 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.629451+0000", 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035895129999999999 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.629462+0000", 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0178e-05 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598662+0000", 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96920093799999996 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598675+0000", 2023-10-09T04:46:20.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2865e-05 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145948 11.e 11.22becefe (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.877874+0000", 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 318.72489215299998, 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72084295600000003, 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.637 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145948 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877874+0000", 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877874+0000", 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877872+0000", 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877883+0000", 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0487e-05 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877907+0000", 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4247e-05 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877917+0000", 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9629999999999993e-06 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.877989+0000", 2023-10-09T04:46:20.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2229000000000003e-05 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.878000+0000", 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.04e-05 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598705+0000", 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72070548599999995 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598717+0000", 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.198e-05 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145958 11.e 11.af82a14e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.892404+0000", 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 318.71036282300003, 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70634927599999997, 2023-10-09T04:46:20.642 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145958 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892404+0000", 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892404+0000", 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892402+0000", 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892410+0000", 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3170000000000004e-06 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892429+0000", 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9014000000000001e-05 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892438+0000", 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2830000000000007e-06 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892515+0000", 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7169000000000006e-05 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.892524+0000", 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4509999999999998e-06 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598740+0000", 2023-10-09T04:46:20.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70621546099999999 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.598753+0000", 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2944e-05 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145974 11.e 11.df26314e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.937375+0000", 2023-10-09T04:46:20.647 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 318.66539179599999, 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66173008099999997, 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145974 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937375+0000", 2023-10-09T04:46:20.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937375+0000", 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937373+0000", 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.649 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937381+0000", 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6939999999999995e-06 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937399+0000", 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8410000000000002e-05 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937408+0000", 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5890000000000003e-06 2023-10-09T04:46:20.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937462+0000", 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4608000000000003e-05 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.937472+0000", 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1770000000000004e-06 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.599094+0000", 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.66162198900000002 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.599105+0000", 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1061e-05 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(11.c e279,262 11:3c0cd7fc:::benchmark_data_smithi029_392508_object11536:head,253'6894)", 2023-10-09T04:46:20.652 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:43:04.752076+0000", 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 195.85069009899999, 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36520602400000002, 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752076+0000", 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752076+0000", 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752075+0000", 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752082+0000", 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6329999999999999e-06 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752094+0000", 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2369e-05 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752101+0000", 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8210000000000004e-06 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752132+0000", 2023-10-09T04:46:20.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1335999999999999e-05 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.752136+0000", 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0640000000000004e-06 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:05.117282+0000", 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36514582499999998 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.656 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(11.c e279,262 11:3ff78155:::benchmark_data_smithi029_392508_object11548:head,253'6895)", 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:43:04.755401+0000", 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 195.847365274, 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36547470599999998, 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.755401+0000", 2023-10-09T04:46:20.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.755401+0000", 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.755401+0000", 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:04.755409+0000", 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3389999999999993e-06 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:05.117275+0000", 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.361865838 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:05.117290+0000", 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5630000000000001e-05 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:05.120823+0000", 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035325729999999998 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:05.120831+0000", 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3999999999999992e-06 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:43:05.120876+0000", 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4379999999999999e-05 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:80145 12.0 12:0c7d584b:::benchmark_data_smithi029_442458_object80144:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:44:49.004017+0000", 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 91.598748938, 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33737684000000001, 2023-10-09T04:46:20.661 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 80145 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004017+0000", 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004017+0000", 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004016+0000", 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004045+0000", 2023-10-09T04:46:20.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8532999999999999e-05 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004059+0000", 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.385e-05 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004064+0000", 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3419999999999998e-06 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004115+0000", 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1081000000000001e-05 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004224+0000", 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109047 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.004326+0000", 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010198499999999999 2023-10-09T04:46:20.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.008847+0000", 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045208660000000001 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.010407+0000", 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015602120000000001 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.337525+0000", 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32711817100000001 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.337594+0000", 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9170000000000004e-05 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:44:49.341394+0000", 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037998289999999998 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.667 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:105082 12.0 12:06168465:::benchmark_data_smithi029_442458_object9832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:32.861187+0000", 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 47.741579524999999, 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30157908999999999, 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.668 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 105082 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861187+0000", 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861187+0000", 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861186+0000", 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861191+0000", 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0350000000000001e-06 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861202+0000", 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.128e-05 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861209+0000", 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6610000000000003e-06 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861253+0000", 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4487999999999998e-05 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861389+0000", 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013580999999999999 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.861498+0000", 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109232 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.864467+0000", 2023-10-09T04:46:20.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029685100000000002 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:32.867041+0000", 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002574233 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:33.159318+0000", 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29227704500000001 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:33.159358+0000", 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9820000000000002e-05 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:33.162766+0000", 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034078509999999999 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.674 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:107176 12.2 12:4b039106:::benchmark_data_smithi029_442458_object11926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:36.657242+0000", 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 43.945524683999999, 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.306060889, 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107176 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.657242+0000", 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.657242+0000", 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.657241+0000", 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:20.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.657245+0000", 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1579999999999998e-06 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.657254+0000", 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2140000000000002e-06 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.657259+0000", 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.028e-06 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.659687+0000", 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024283500000000001 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.659811+0000", 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012345800000000001 2023-10-09T04:46:20.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.659925+0000", 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114067 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.663222+0000", 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003297038 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.663794+0000", 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057198299999999997 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.963206+0000", 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29941216599999998 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.963265+0000", 2023-10-09T04:46:20.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8798000000000002e-05 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.963303+0000", 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7790000000000002e-05 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:107188 12.2 12:4c3a2751:::benchmark_data_smithi029_442458_object11938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:36.676353+0000", 2023-10-09T04:46:20.681 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 43.926413576000002, 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29419230499999999, 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107188 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676353+0000", 2023-10-09T04:46:20.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676353+0000", 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676351+0000", 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676358+0000", 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7390000000000002e-06 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676373+0000", 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5087e-05 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676379+0000", 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2979999999999999e-06 2023-10-09T04:46:20.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676417+0000", 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8841000000000003e-05 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676520+0000", 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102827 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:46:20.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.676683+0000", 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016258900000000001 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.679812+0000", 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031292109999999998 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682102+0000", 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002289761 2023-10-09T04:46:20.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.970444+0000", 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28834220599999999 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.970490+0000", 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6356000000000001e-05 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.970545+0000", 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4809999999999999e-05 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:107196 12.2 12:48782e62:::benchmark_data_smithi029_442458_object11946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:36.682252+0000", 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 43.920513903, 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28887886499999998, 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.688 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 107196 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682252+0000", 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682252+0000", 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682251+0000", 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682257+0000", 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4520000000000002e-06 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682272+0000", 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4632e-05 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682278+0000", 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0410000000000003e-06 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682312+0000", 2023-10-09T04:46:20.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.383e-05 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682404+0000", 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2328000000000001e-05 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,7", 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.682491+0000", 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7305000000000005e-05 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.692 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.686737+0000", 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042456509999999996 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.687704+0000", 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00096716100000000002 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.971055+0000", 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.283351242 2023-10-09T04:46:20.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.971084+0000", 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8702999999999999e-05 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:36.971131+0000", 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7175999999999997e-05 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:108951 12.0 12:0b9a7341:::benchmark_data_smithi029_442458_object13701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:40.119958+0000", 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 40.482807954999998, 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.22069816, 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 108951 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.695 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.119958+0000", 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.119958+0000", 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.119957+0000", 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.119963+0000", 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8590000000000001e-06 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.119979+0000", 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6211000000000001e-05 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.119987+0000", 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3159999999999999e-06 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.120041+0000", 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4397000000000001e-05 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.120210+0000", 2023-10-09T04:46:20.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000168658 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.120368+0000", 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000158527 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.123815+0000", 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034462030000000001 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.126799+0000", 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002984076 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.336909+0000", 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21011044800000001 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.336951+0000", 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1974999999999997e-05 2023-10-09T04:46:20.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:40.340657+0000", 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037055180000000001 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:119045 12.0 12:066830aa:::benchmark_data_smithi029_442458_object23795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:16.453362+0000", 2023-10-09T04:46:20.701 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.1494042579999997, 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26017526499999999, 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119045 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453362+0000", 2023-10-09T04:46:20.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453362+0000", 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453361+0000", 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453368+0000", 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8970000000000002e-06 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453381+0000", 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3597999999999999e-05 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453388+0000", 2023-10-09T04:46:20.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7819999999999996e-06 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.459547+0000", 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061587250000000003 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.459720+0000", 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000173208 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.459894+0000", 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017445699999999999 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.466781+0000", 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068864180000000001 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.466825+0000", 2023-10-09T04:46:20.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4558000000000003e-05 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.711231+0000", 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.244405716 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.711292+0000", 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1247000000000005e-05 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.713537+0000", 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022451300000000001 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:119046 12.0 12:0239b7ce:::benchmark_data_smithi029_442458_object23796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:16.453833+0000", 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.1489328939999996, 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.26192895100000002, 2023-10-09T04:46:20.708 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119046 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453833+0000", 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453833+0000", 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453833+0000", 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453841+0000", 2023-10-09T04:46:20.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2770000000000005e-06 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453851+0000", 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0382e-05 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.453857+0000", 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2530000000000004e-06 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.463413+0000", 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095565930000000004 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.463564+0000", 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015124100000000001 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.463730+0000", 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000165198 2023-10-09T04:46:20.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.469096+0000", 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053667940000000003 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.472733+0000", 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036362690000000001 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.713587+0000", 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.240853971 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.713634+0000", 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7379000000000002e-05 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.715762+0000", 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021284329999999999 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.714 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:119058 12.0 12:0c0ee911:::benchmark_data_smithi029_442458_object23808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:16.478938+0000", 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.1238284370000002, 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.239006838, 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.715 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119058 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.478938+0000", 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.478938+0000", 2023-10-09T04:46:20.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.478936+0000", 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.478944+0000", 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.123e-06 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.478960+0000", 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5415000000000001e-05 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.478966+0000", 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.302e-06 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.479018+0000", 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.185e-05 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.479178+0000", 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016040499999999999 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.479337+0000", 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000158598 2023-10-09T04:46:20.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.482798+0000", 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034609829999999999 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.482963+0000", 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016499400000000001 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.715809+0000", 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.23284621999999999 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.715856+0000", 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7205e-05 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.717945+0000", 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020883429999999999 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.721 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:119064 12.0 12:083d3f76:::benchmark_data_smithi029_442458_object23814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e305)", 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:16.499206+0000", 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.1035606409999996, 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.22106113599999999, 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:20.722 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 119064 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499206+0000", 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499206+0000", 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499198+0000", 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999928 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499216+0000", 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7606999999999999e-05 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499248+0000", 2023-10-09T04:46:20.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2521999999999998e-05 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499261+0000", 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2614e-05 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499375+0000", 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114103 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499696+0000", 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032041799999999999 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.499867+0000", 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000171733 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.502660+0000", 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002792545 2023-10-09T04:46:20.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.502922+0000", 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026177500000000001 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.717998+0000", 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21507654000000001 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:46:20.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.718046+0000", 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7321000000000003e-05 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:16.720267+0000", 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002221345 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:20.728 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[223858]: pgmap v1197: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 428 op/s; 39134/215919 objects degraded (18.124%); 23386/215919 objects misplaced (10.831%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 39134/215919 objects degraded (18.124%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[216903]: pgmap v1197: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 428 op/s; 39134/215919 objects degraded (18.124%); 23386/215919 objects misplaced (10.831%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:21.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 39134/215919 objects degraded (18.124%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:21.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:21 smithi177 ceph-mon[195797]: pgmap v1197: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 428 op/s; 39134/215919 objects degraded (18.124%); 23386/215919 objects misplaced (10.831%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:21 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:21 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 39134/215919 objects degraded (18.124%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:21.592 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:21.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:21.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:21.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:46:21.779 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:22.328 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:22.609 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:22.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:22.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:22.752 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:22.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:22.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:22.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:22.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:22.996 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:46:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:46:22] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-10-09T04:46:23.005 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:23.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:46:23.112 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:23 smithi177 ceph-mon[195797]: pgmap v1198: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 39024/212274 objects degraded (18.384%); 22889/212274 objects misplaced (10.783%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:23.221 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:23.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:23.325 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:23.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:23.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:23 smithi029 ceph-mon[216903]: pgmap v1198: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 39024/212274 objects degraded (18.384%); 22889/212274 objects misplaced (10.783%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:23.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:23 smithi029 ceph-mon[223858]: pgmap v1198: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 39024/212274 objects degraded (18.384%); 22889/212274 objects misplaced (10.783%); 9.9 MiB/s, 158 objects/s recovering 2023-10-09T04:46:23.463 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:23.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:23.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:23.603 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:23.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:23.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:24.000 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:24.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:24.142 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:23 smithi029 conmon[216878]: 2023-10-09T04:46:23.771+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e306 definitely_dead 0 2023-10-09T04:46:24.142 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3897470939' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:46:24.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:46:24.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:24 smithi029 ceph-mon[216903]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:46:24.143 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3897470939' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:46:24.143 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:46:24.143 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:24 smithi029 ceph-mon[223858]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:46:24.160 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.0. 2023-10-09T04:46:24.163 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:24.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3897470939' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:46:24.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:46:24.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:24 smithi177 ceph-mon[195797]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:46:24.495 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122037 12.c 12:351c053a:::benchmark_data_smithi029_442458_object26787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.129316+0000", 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.3658942, 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36592333399999999, 2023-10-09T04:46:24.496 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for peered", 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122037 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129316+0000", 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129316+0000", 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:24.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129314+0000", 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129320+0000", 2023-10-09T04:46:24.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5210000000000002e-06 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129331+0000", 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1483e-05 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129337+0000", 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2429999999999996e-06 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129388+0000", 2023-10-09T04:46:24.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1019000000000001e-05 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129534+0000", 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000146379 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129661+0000", 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126556 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.132910+0000", 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032498010000000001 2023-10-09T04:46:24.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.146328+0000", 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013417584 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159582+0000", 2023-10-09T04:46:24.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013253811000000001 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159593+0000", 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0739e-05 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:24.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122044 12.c 12:3399bf74:::benchmark_data_smithi029_442458_object26794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.149660+0000", 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.345549567, 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.345646236, 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for peered", 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:24.506 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122044 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149660+0000", 2023-10-09T04:46:24.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149660+0000", 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149659+0000", 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149664+0000", 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3359999999999997e-06 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:24.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149677+0000", 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2931e-05 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149685+0000", 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1640000000000001e-06 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149735+0000", 2023-10-09T04:46:24.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9997000000000003e-05 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149890+0000", 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015476199999999999 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.150010+0000", 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120862 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159617+0000", 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096066729999999996 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159623+0000", 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1959999999999996e-06 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:46:24.512 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 2 2023-10-09T04:46:24.513 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:24.563 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:24.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:24.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:24.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:24.699 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:24.747 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: pgmap v1199: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 406 op/s; 38648/210006 objects degraded (18.403%); 22483/210006 objects misplaced (10.706%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: osdmap e307: 8 total, 8 up, 5 in 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[216903]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: pgmap v1199: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 406 op/s; 38648/210006 objects degraded (18.403%); 22483/210006 objects misplaced (10.706%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: osdmap e307: 8 total, 8 up, 5 in 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:25 smithi029 ceph-mon[223858]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.520 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:25.522 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: pgmap v1199: 89 pgs: 3 active+recovery_wait+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 406 op/s; 38648/210006 objects degraded (18.403%); 22483/210006 objects misplaced (10.706%); 9.8 MiB/s, 157 objects/s recovering 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: osdmap e307: 8 total, 8 up, 5 in 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.523 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:25 smithi177 ceph-mon[195797]: failed to encode map e307 with expected crc 2023-10-09T04:46:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:25.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:25.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:25.669 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:25.725 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:25.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:25.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:25.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:25.857 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: osdmap e308: 8 total, 8 up, 5 in 2023-10-09T04:46:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: 12.8 continuing backfill to osd.4 from (288'600,306'7620] MIN to 306'7620 2023-10-09T04:46:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: 12.8 starting backfill to osd.7 from (0'0,0'0] MAX to 306'7620 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38648/210006 objects degraded (18.403%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: osdmap e308: 8 total, 8 up, 5 in 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: 12.8 continuing backfill to osd.4 from (288'600,306'7620] MIN to 306'7620 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: 12.8 starting backfill to osd.7 from (0'0,0'0] MAX to 306'7620 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38648/210006 objects degraded (18.403%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:26.469 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: osdmap e308: 8 total, 8 up, 5 in 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: 12.8 continuing backfill to osd.4 from (288'600,306'7620] MIN to 306'7620 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: 12.8 starting backfill to osd.7 from (0'0,0'0] MAX to 306'7620 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.470 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: failed to encode map e308 with expected crc 2023-10-09T04:46:26.471 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38648/210006 objects degraded (18.403%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:26.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:46:27.126 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:27.157 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: pgmap v1202: 89 pgs: 1 remapped+peering, 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 44 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 409 op/s; 28489/206406 objects degraded (13.802%); 23571/206406 objects misplaced (11.420%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: osdmap e309: 8 total, 8 up, 5 in 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: 12.e starting backfill to osd.5 from (0'0,0'0] MAX to 306'7545 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 306'7652 2023-10-09T04:46:27.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[223858]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 306'7543 2023-10-09T04:46:27.306 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:27.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:27.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: pgmap v1202: 89 pgs: 1 remapped+peering, 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 44 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 409 op/s; 28489/206406 objects degraded (13.802%); 23571/206406 objects misplaced (11.420%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: osdmap e309: 8 total, 8 up, 5 in 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: 12.e starting backfill to osd.5 from (0'0,0'0] MAX to 306'7545 2023-10-09T04:46:27.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 306'7652 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:27 smithi177 ceph-mon[195797]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 306'7543 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: pgmap v1202: 89 pgs: 1 remapped+peering, 1 active+recovering+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 44 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 409 op/s; 28489/206406 objects degraded (13.802%); 23571/206406 objects misplaced (11.420%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: osdmap e309: 8 total, 8 up, 5 in 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: failed to encode map e309 with expected crc 2023-10-09T04:46:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: 12.e starting backfill to osd.5 from (0'0,0'0] MAX to 306'7545 2023-10-09T04:46:27.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 306'7652 2023-10-09T04:46:27.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:27 smithi029 ceph-mon[216903]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 306'7543 2023-10-09T04:46:27.493 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:27.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:27.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:27.777 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:27.852 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:28.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:28.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:28.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:28.205 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:28.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:28.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:28.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:28.505 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:28.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:28.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:28.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:28.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:28.784 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:28.876 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:28.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:28.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:29.005 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:29.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:29.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:29.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:29.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:29 smithi029 ceph-mon[216903]: pgmap v1204: 89 pgs: 1 remapped+peering, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 44 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 378 op/s; 40233/205449 objects degraded (19.583%); 24302/205449 objects misplaced (11.829%); 12 MiB/s, 192 objects/s recovering 2023-10-09T04:46:29.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:29.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:29.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:29 smithi029 ceph-mon[223858]: pgmap v1204: 89 pgs: 1 remapped+peering, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 44 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 378 op/s; 40233/205449 objects degraded (19.583%); 24302/205449 objects misplaced (11.829%); 12 MiB/s, 192 objects/s recovering 2023-10-09T04:46:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:29.199 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:29.254 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:29.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:29.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:29 smithi177 ceph-mon[195797]: pgmap v1204: 89 pgs: 1 remapped+peering, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 44 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 378 op/s; 40233/205449 objects degraded (19.583%); 24302/205449 objects misplaced (11.829%); 12 MiB/s, 192 objects/s recovering 2023-10-09T04:46:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:29.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:29.415 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:29.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:29.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:29.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:29.800 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:29.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:30.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:30.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:30.105 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:30.119 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:30.228 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:30.316 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:30.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:30.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:30.378 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:30.379 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:46:30.379 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:46:30.379 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:46:30.379 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:30.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:30.543 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:30.666 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:46:30.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:30.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:30.699 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:46:30.735 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:46:30.771 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:46:30.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:30.831 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:46:30.856 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:30.887 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:46:30.923 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:46:30.952 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:30.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:30.961 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:46:30.965 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:46:31.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:31.087 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:31.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:31 smithi029 ceph-mon[216903]: pgmap v1205: 89 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 265 op/s; 40233/205221 objects degraded (19.605%); 28359/205221 objects misplaced (13.819%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:46:31.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40233/205221 objects degraded (19.605%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:31.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2469111698' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:31.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:31 smithi029 ceph-mon[223858]: pgmap v1205: 89 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 265 op/s; 40233/205221 objects degraded (19.605%); 28359/205221 objects misplaced (13.819%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:46:31.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40233/205221 objects degraded (19.605%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:31.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2469111698' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:31 smithi177 ceph-mon[195797]: pgmap v1205: 89 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 43 active+clean, 2 active+recovery_wait+degraded+remapped, 35 peering; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 265 op/s; 40233/205221 objects degraded (19.605%); 28359/205221 objects misplaced (13.819%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:46:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40233/205221 objects degraded (19.605%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2469111698' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:31.596 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:31.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:31.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:46:31.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:31.830 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:31.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:32.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:32.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:32.225 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:32.251 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 5, 7] out_osds: [6, 3, 0] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:46:32.251 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:46:32.251 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:46:32.252 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:46:32.252 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:46:32.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:46:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:32 smithi177 ceph-mon[195797]: pgmap v1206: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 321 op/s; 52615/203085 objects degraded (25.908%); 35415/203085 objects misplaced (17.439%); 13 MiB/s, 204 objects/s recovering 2023-10-09T04:46:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:32 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:46:32.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:32.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:32.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:32.424 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:32.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:32.536 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:32.587 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:32.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:32.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:32 smithi029 ceph-mon[216903]: pgmap v1206: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 321 op/s; 52615/203085 objects degraded (25.908%); 35415/203085 objects misplaced (17.439%); 13 MiB/s, 204 objects/s recovering 2023-10-09T04:46:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:32 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:46:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:32 smithi029 ceph-mon[223858]: pgmap v1206: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 321 op/s; 52615/203085 objects degraded (25.908%); 35415/203085 objects misplaced (17.439%); 13 MiB/s, 204 objects/s recovering 2023-10-09T04:46:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:32 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:46:32.692 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:32.734 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:32.786 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:32.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:32.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:32.965 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:33.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:33.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:33.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:46:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:46:32] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:46:33.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:33.299 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:33.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:33.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:33.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:33.517 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:33.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:33.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:33.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:33.878 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:33.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:33.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:33.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:34.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:34.188 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:34.294 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:34.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:34.303 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:34.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:34 smithi029 ceph-mon[216903]: pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 225 op/s; 52029/200880 objects degraded (25.901%); 34284/200880 objects misplaced (17.067%); 15 MiB/s, 244 objects/s recovering 2023-10-09T04:46:34.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:34.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:34.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:34 smithi029 ceph-mon[223858]: pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 225 op/s; 52029/200880 objects degraded (25.901%); 34284/200880 objects misplaced (17.067%); 15 MiB/s, 244 objects/s recovering 2023-10-09T04:46:34.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:34.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:34.399 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:34.466 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:34.592 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:34 smithi177 ceph-mon[195797]: pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 225 op/s; 52029/200880 objects degraded (25.901%); 34284/200880 objects misplaced (17.067%); 15 MiB/s, 244 objects/s recovering 2023-10-09T04:46:34.592 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:34.592 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:34.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:34.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:34.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:34.803 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:34.896 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:34.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:35.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:35.115 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:35.188 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:35.188 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:35.227 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:35.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:35.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:35.349 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:35.449 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:35.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:46:35.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:46:35.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:46:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:46:35.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:35.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:46:35.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:46:35.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:36.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:46:36.399 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:46:36.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:36 smithi029 ceph-mon[216903]: pgmap v1208: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 288 op/s; 51847/197763 objects degraded (26.217%); 33586/197763 objects misplaced (16.983%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:46:36.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3262160149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:36.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 52029/200880 objects degraded (25.901%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:36.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:36 smithi029 ceph-mon[223858]: pgmap v1208: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 288 op/s; 51847/197763 objects degraded (26.217%); 33586/197763 objects misplaced (16.983%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:46:36.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3262160149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:36.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 52029/200880 objects degraded (25.901%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:36.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:36.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:36.677 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:36 smithi177 ceph-mon[195797]: pgmap v1208: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 288 op/s; 51847/197763 objects degraded (26.217%); 33586/197763 objects misplaced (16.983%); 16 MiB/s, 253 objects/s recovering 2023-10-09T04:46:36.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3262160149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:36.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 52029/200880 objects degraded (25.901%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:36.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:36.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:36.827 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:36.914 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:36.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:36.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:37.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:37.066 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:37.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:46:37.150 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:37.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:37.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:37.292 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:37.357 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:37.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:37.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:37.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.505 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:37.613 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:37.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:37.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:37.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3262160149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: osdmap e310: 8 total, 8 up, 5 in 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[223858]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3262160149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: osdmap e310: 8 total, 8 up, 5 in 2023-10-09T04:46:37.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.630 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:37 smithi029 ceph-mon[216903]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.728 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.744 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:37.744 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2023-10-09T04:46:37.744 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:37.762 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3262160149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: osdmap e310: 8 total, 8 up, 5 in 2023-10-09T04:46:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:37 smithi177 ceph-mon[195797]: failed to encode map e310 with expected crc 2023-10-09T04:46:37.842 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:37.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:37.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:37.999 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:38.070 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:38.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:38.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:38.232 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:38.291 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:38.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:38.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:38.424 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:38.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:38.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:38.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:38.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:38.633 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:38 smithi177 ceph-mon[195797]: pgmap v1210: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 312 op/s; 51267/196059 objects degraded (26.149%); 33281/196059 objects misplaced (16.975%); 16 MiB/s, 264 objects/s recovering 2023-10-09T04:46:38.639 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:38.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:38 smithi029 ceph-mon[216903]: pgmap v1210: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 312 op/s; 51267/196059 objects degraded (26.149%); 33281/196059 objects misplaced (16.975%); 16 MiB/s, 264 objects/s recovering 2023-10-09T04:46:38.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:38 smithi029 ceph-mon[223858]: pgmap v1210: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 312 op/s; 51267/196059 objects degraded (26.149%); 33281/196059 objects misplaced (16.975%); 16 MiB/s, 264 objects/s recovering 2023-10-09T04:46:38.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:38.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:38.891 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:38.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:39.036 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:39.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:39.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:39.283 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:39.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:39.385 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:39.476 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:39.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:39.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:39.671 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:39.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:39.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:39.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:39.940 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:39.986 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:39.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:40.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:40.118 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:40.196 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:40.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:40.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:40.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:40.357 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:40.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:40 smithi029 ceph-mon[216903]: pgmap v1211: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 51267/195714 objects degraded (26.195%); 32544/195714 objects misplaced (16.628%); 20 MiB/s, 314 objects/s recovering 2023-10-09T04:46:40.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:40.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:40.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:40.646 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:40 smithi029 ceph-mon[223858]: pgmap v1211: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 51267/195714 objects degraded (26.195%); 32544/195714 objects misplaced (16.628%); 20 MiB/s, 314 objects/s recovering 2023-10-09T04:46:40.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:40.753 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:40 smithi177 ceph-mon[195797]: pgmap v1211: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 51267/195714 objects degraded (26.195%); 32544/195714 objects misplaced (16.628%); 20 MiB/s, 314 objects/s recovering 2023-10-09T04:46:40.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:40.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:40.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:40.974 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:41.178 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:41.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:46:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:41.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:41.294 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:41.294 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:41.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:41.369 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 51267/195714 objects degraded (26.195%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:41.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:41.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:41.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:41.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 51267/195714 objects degraded (26.195%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:41.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 51267/195714 objects degraded (26.195%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:41.694 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:41.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:41.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:41.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:41.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:41.905 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:41.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:41.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:42.022 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:42.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:42.162 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:42.208 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:46:42.249 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:46:42.295 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:46:42.341 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:42.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:42.356 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:46:42.398 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:46:42.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:46:42.443 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:46:42.451 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:42.476 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:46:42.513 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:46:42.521 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.521 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.521 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.521 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.521 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.521 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.522 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:46:42.567 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:42 smithi029 ceph-mon[216903]: pgmap v1212: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 367 op/s; 51002/192048 objects degraded (26.557%); 31756/192048 objects misplaced (16.535%); 20 MiB/s, 316 objects/s recovering 2023-10-09T04:46:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1176865334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:42 smithi029 ceph-mon[223858]: pgmap v1212: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 367 op/s; 51002/192048 objects degraded (26.557%); 31756/192048 objects misplaced (16.535%); 20 MiB/s, 316 objects/s recovering 2023-10-09T04:46:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1176865334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:42.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:42.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:42 smithi177 ceph-mon[195797]: pgmap v1212: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 367 op/s; 51002/192048 objects degraded (26.557%); 31756/192048 objects misplaced (16.535%); 20 MiB/s, 316 objects/s recovering 2023-10-09T04:46:42.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1176865334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:42.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:42.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:42.933 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:43.158 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:43.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:46:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:46:42] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:46:43.923 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 5, 7] out_osds: [6, 3, 0] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:46:43.923 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:46:43.924 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_7 2023-10-09T04:46:43.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:46:44.030 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:44.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:44.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:44.192 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:44.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:44 smithi177 ceph-mon[195797]: pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 50444/190008 objects degraded (26.548%); 30612/190008 objects misplaced (16.111%); 20 MiB/s, 320 objects/s recovering 2023-10-09T04:46:44.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:44.809 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:44 smithi029 ceph-mon[216903]: pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 50444/190008 objects degraded (26.548%); 30612/190008 objects misplaced (16.111%); 20 MiB/s, 320 objects/s recovering 2023-10-09T04:46:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:44.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:44 smithi029 ceph-mon[223858]: pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 362 op/s; 50444/190008 objects degraded (26.548%); 30612/190008 objects misplaced (16.111%); 20 MiB/s, 320 objects/s recovering 2023-10-09T04:46:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:45.478 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:45.478 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:46:45.478 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:46:45.479 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:46:45.479 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:45.633 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:45.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:45.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:45.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:45.921 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:45.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:45.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:45.930 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:46.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:46.086 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:46.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:46.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:46.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:46.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:46.318 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:46.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:46.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:46:46.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:46 smithi029 ceph-mon[216903]: pgmap v1214: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 373 op/s; 50080/186540 objects degraded (26.847%); 30026/186540 objects misplaced (16.096%); 19 MiB/s, 322 objects/s recovering 2023-10-09T04:46:46.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 50080/186540 objects degraded (26.847%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:46.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:46 smithi029 ceph-mon[223858]: pgmap v1214: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 373 op/s; 50080/186540 objects degraded (26.847%); 30026/186540 objects misplaced (16.096%); 19 MiB/s, 322 objects/s recovering 2023-10-09T04:46:46.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 50080/186540 objects degraded (26.847%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:46.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:46.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:46.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:46.829 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:46.917 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:46.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:47.022 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:46 smithi177 ceph-mon[195797]: pgmap v1214: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 373 op/s; 50080/186540 objects degraded (26.847%); 30026/186540 objects misplaced (16.096%); 19 MiB/s, 322 objects/s recovering 2023-10-09T04:46:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 50080/186540 objects degraded (26.847%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:46:47.072 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:47.148 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:47.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:47.255 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:47.298 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:47.367 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:47.380 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:47.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:47.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:47.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:46:47.586 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:47.622 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:46:47.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4140381297' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:46:47.646 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:46:47.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4140381297' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:46:47.647 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:46:47.717 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:47.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:47.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:47.851 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:47.904 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:47.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:47.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4140381297' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:46:47.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:46:48.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:48.034 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: pgmap v1215: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 49551/184836 objects degraded (26.808%); 29684/184836 objects misplaced (16.060%); 20 MiB/s, 325 objects/s recovering 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: osdmap e311: 8 total, 8 up, 5 in 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: pgmap v1215: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 49551/184836 objects degraded (26.808%); 29684/184836 objects misplaced (16.060%); 20 MiB/s, 325 objects/s recovering 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: osdmap e311: 8 total, 8 up, 5 in 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:48 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: pgmap v1215: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 49551/184836 objects degraded (26.808%); 29684/184836 objects misplaced (16.060%); 20 MiB/s, 325 objects/s recovering 2023-10-09T04:46:48.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:46:48.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:46:48.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: osdmap e311: 8 total, 8 up, 5 in 2023-10-09T04:46:48.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:48.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:48 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.423 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:49.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:49.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:49.528 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:49.588 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:46:49.696 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:26.532093+0000", 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 23.161830231, 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00079991399999999999, 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532093+0000", 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532093+0000", 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532092+0000", 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532109+0000", 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7368e-05 2023-10-09T04:46:49.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532131+0000", 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2226e-05 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532140+0000", 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2250000000000006e-06 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532227+0000", 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6731999999999996e-05 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.532893+0000", 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00066569800000000005 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.700 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:27.265167+0000", 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 22.428756394000001, 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015889039999999999, 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265167+0000", 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265167+0000", 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265165+0000", 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265182+0000", 2023-10-09T04:46:49.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7201e-05 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265202+0000", 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9650999999999999e-05 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265210+0000", 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5180000000000004e-06 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.265267+0000", 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6419000000000003e-05 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.266756+0000", 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001488767 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:27.799837+0000", 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 21.894086103999999, 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043397239999999997, 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799837+0000", 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799837+0000", 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799836+0000", 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799854+0000", 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8139e-05 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799876+0000", 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1835e-05 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799884+0000", 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5089999999999993e-06 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.802185+0000", 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023006260000000001 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.804177+0000", 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00199204 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:29.300210+0000", 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 20.393712607000001, 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021015579999999999, 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300210+0000", 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300210+0000", 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300209+0000", 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300222+0000", 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2945999999999999e-05 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300238+0000", 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5403000000000001e-05 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300245+0000", 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2169999999999997e-06 2023-10-09T04:46:49.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.300288+0000", 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2660999999999997e-05 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:29.302312+0000", 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020243510000000002 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.712 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:30.277470+0000", 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 19.41645321, 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00074955900000000001, 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277470+0000", 2023-10-09T04:46:49.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277470+0000", 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277468+0000", 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277490+0000", 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1746000000000002e-05 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277512+0000", 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1616000000000001e-05 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277520+0000", 2023-10-09T04:46:49.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5499999999999995e-06 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.277572+0000", 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1397999999999998e-05 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.278219+0000", 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00064774199999999998 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.716 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:30.782203+0000", 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 18.911719644000001, 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00093996900000000003, 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782203+0000", 2023-10-09T04:46:49.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782203+0000", 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782202+0000", 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782219+0000", 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7394999999999999e-05 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782240+0000", 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0999e-05 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782248+0000", 2023-10-09T04:46:49.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7910000000000005e-06 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.782319+0000", 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1167000000000005e-05 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:30.783143+0000", 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082406099999999996 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.720 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:42.734211+0000", 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.9597124680000002, 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001863775, 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734211+0000", 2023-10-09T04:46:49.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734211+0000", 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734209+0000", 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734227+0000", 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7625000000000002e-05 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734249+0000", 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2217000000000001e-05 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734257+0000", 2023-10-09T04:46:49.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6769999999999995e-06 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.734329+0000", 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1964000000000004e-05 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.736074+0000", 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001745482 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.724 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:42.910518+0000", 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.7834048720000002, 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00090640000000000002, 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910518+0000", 2023-10-09T04:46:49.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910518+0000", 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910517+0000", 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910537+0000", 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0386000000000001e-05 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910559+0000", 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1787e-05 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910567+0000", 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9140000000000008e-06 2023-10-09T04:46:49.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.910644+0000", 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6676999999999998e-05 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:42.911425+0000", 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00078109999999999996 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.728 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:43.438763+0000", 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 6.2551605429999997, 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065351089999999999, 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438763+0000", 2023-10-09T04:46:49.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438763+0000", 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438761+0000", 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438778+0000", 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7088e-05 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438801+0000", 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2906999999999999e-05 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438809+0000", 2023-10-09T04:46:49.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5750000000000003e-06 2023-10-09T04:46:49.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438888+0000", 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9142999999999998e-05 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.445298+0000", 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006409756 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.733 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:45.172373+0000", 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.5215500139999998, 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00067986299999999995, 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172373+0000", 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172373+0000", 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172371+0000", 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172391+0000", 2023-10-09T04:46:49.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0400000000000001e-05 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172415+0000", 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3665000000000001e-05 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172424+0000", 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3789999999999995e-06 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.172480+0000", 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5770000000000003e-05 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.173053+0000", 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057281499999999996 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:45.665788+0000", 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.0281352909999999, 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00127512, 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665788+0000", 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665788+0000", 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665786+0000", 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665804+0000", 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7997999999999999e-05 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665828+0000", 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3104000000000001e-05 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665836+0000", 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2179999999999999e-06 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.665906+0000", 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9851999999999995e-05 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.667063+0000", 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001157269 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:45.835363+0000", 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.8585599400000001, 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001423475, 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835363+0000", 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835363+0000", 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835361+0000", 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835382+0000", 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0203999999999998e-05 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835404+0000", 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2583e-05 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835412+0000", 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8520000000000004e-06 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.835485+0000", 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3157000000000004e-05 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:45.836787+0000", 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013014669999999999 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:46.110564+0000", 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.5833587800000002, 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00070008099999999997, 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.746 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110564+0000", 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110564+0000", 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110563+0000", 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110580+0000", 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7041000000000002e-05 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110601+0000", 2023-10-09T04:46:49.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1083000000000001e-05 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110609+0000", 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2560000000000002e-06 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.110693+0000", 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3688000000000002e-05 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.111264+0000", 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057175599999999998 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:46.144180+0000", 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.5497434920000002, 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013121229999999999, 2023-10-09T04:46:49.750 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144180+0000", 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144180+0000", 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144179+0000", 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144193+0000", 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4011e-05 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144209+0000", 2023-10-09T04:46:49.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6439999999999998e-05 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144216+0000", 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3089999999999998e-06 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.144263+0000", 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6782e-05 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.145492+0000", 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012286059999999999 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:46.194607+0000", 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.4993158310000001, 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001362666, 2023-10-09T04:46:49.754 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194607+0000", 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194607+0000", 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194606+0000", 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194636+0000", 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9476e-05 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194651+0000", 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5554999999999999e-05 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194658+0000", 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3369999999999996e-06 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.194704+0000", 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5828999999999998e-05 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.195970+0000", 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001266337 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:46.212659+0000", 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.4812644160000001, 2023-10-09T04:46:49.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023849309999999999, 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.212659+0000", 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.212659+0000", 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.212658+0000", 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.212673+0000", 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5943e-05 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.212696+0000", 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2364000000000001e-05 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.212704+0000", 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2320000000000001e-06 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.214475+0000", 2023-10-09T04:46:49.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017711490000000001 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.215044+0000", 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00056840700000000005 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.762 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:46.364482+0000", 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.3294415050000001, 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015630920000000001, 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364482+0000", 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364482+0000", 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364480+0000", 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364499+0000", 2023-10-09T04:46:49.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9262e-05 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364522+0000", 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3027000000000002e-05 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364530+0000", 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9899999999999997e-06 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.364610+0000", 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0049000000000006e-05 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:46.366045+0000", 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014344830000000001 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:47.543196+0000", 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 2.15072749, 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012677229999999999, 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543196+0000", 2023-10-09T04:46:49.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543196+0000", 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543194+0000", 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543212+0000", 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7561999999999999e-05 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543235+0000", 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3519e-05 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543244+0000", 2023-10-09T04:46:49.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.543320+0000", 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5716999999999994e-05 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.544463+0000", 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011436420000000001 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.770 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e311/303 cost 1049576)", 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:47.711558+0000", 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 1.9823649109999999, 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00067730600000000002, 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.771 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711558+0000", 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711558+0000", 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711557+0000", 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:49.772 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711574+0000", 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6733000000000001e-05 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711596+0000", 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2308000000000001e-05 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.773 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711604+0000", 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0819999999999999e-06 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.711694+0000", 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9598000000000005e-05 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:47.712235+0000", 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00054174699999999998 2023-10-09T04:46:49.774 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e311/303 cost 1049576)", 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:49.038403+0000", 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.65551965599999995, 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032935899999999999, 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.775 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038403+0000", 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038403+0000", 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038402+0000", 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:49.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038419+0000", 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7041000000000002e-05 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038442+0000", 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3147e-05 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:49.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038451+0000", 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2609999999999993e-06 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.038544+0000", 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3448000000000004e-05 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:49.041697+0000", 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031530400000000002 2023-10-09T04:46:49.778 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.779 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.779 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.779 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:49.779 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:49.779 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:49.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.786 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[216903]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:49 smithi029 ceph-mon[223858]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.805 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:49.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:49.914 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:49.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:49 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:49 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:49 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:49 smithi177 ceph-mon[195797]: failed to encode map e311 with expected crc 2023-10-09T04:46:49.989 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:50.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:50.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:50.138 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:50.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:46:50.288 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:50.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:46:50.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:50.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:46:50.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:50.557 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:50.608 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:46:50.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:50 smithi177 ceph-mon[195797]: pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 373 op/s; 49551/184503 objects degraded (26.856%); 28933/184503 objects misplaced (15.682%); 20 MiB/s, 326 objects/s recovering 2023-10-09T04:46:50.608 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 49551/184503 objects degraded (26.856%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:50.645 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:46:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:50 smithi029 ceph-mon[223858]: pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 373 op/s; 49551/184503 objects degraded (26.856%); 28933/184503 objects misplaced (15.682%); 20 MiB/s, 326 objects/s recovering 2023-10-09T04:46:50.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 49551/184503 objects degraded (26.856%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:50.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:50 smithi029 ceph-mon[216903]: pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 373 op/s; 49551/184503 objects degraded (26.856%); 28933/184503 objects misplaced (15.682%); 20 MiB/s, 326 objects/s recovering 2023-10-09T04:46:50.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 49551/184503 objects degraded (26.856%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:50.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:46:51.602 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:51.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:46:51.864 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:51 smithi029 ceph-mon[216903]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:51.865 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:51 smithi029 ceph-mon[223858]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:51.922 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:51 smithi177 ceph-mon[195797]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:46:52.478 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:52.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:52.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:52.625 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:52 smithi177 ceph-mon[195797]: pgmap v1218: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 361 op/s; 48759/181206 objects degraded (26.908%); 28458/181206 objects misplaced (15.705%); 18 MiB/s, 351 objects/s recovering 2023-10-09T04:46:52.674 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:52.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:52.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:52.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:52 smithi029 ceph-mon[216903]: pgmap v1218: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 361 op/s; 48759/181206 objects degraded (26.908%); 28458/181206 objects misplaced (15.705%); 18 MiB/s, 351 objects/s recovering 2023-10-09T04:46:52.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:52 smithi029 ceph-mon[223858]: pgmap v1218: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 361 op/s; 48759/181206 objects degraded (26.908%); 28458/181206 objects misplaced (15.705%); 18 MiB/s, 351 objects/s recovering 2023-10-09T04:46:52.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:52.871 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:52.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:52.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:53.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:53.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:46:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:46:52] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:46:53.192 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:53.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:53.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:53.426 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:53.477 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:53.551 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1217,"stamp":"2023-10-09T04:46:49.302439+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4029487032,"num_objects":61501,"num_object_clones":0,"num_object_copies":184503,"num_objects_missing_on_primary":1541,"num_objects_missing":1541,"num_objects_degraded":49551,"num_objects_misplaced":28933,"num_objects_unfound":0,"num_objects_dirty":61501,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":129319,"num_write_kb":6098254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20188,"num_bytes_recovered":1312176310,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94562,"ondisk_log_size":94562,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":11514328,"kb_used_data":7671984,"kb_used_omap":12,"kb_used_meta":3841971,"kb_avail":457211432,"statfs":{"total":479975178240,"available":468184506368,"internally_reserved":0,"allocated":7856111616,"data_stored":7851676382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12321,"internal_metadata":3934179295},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,8,6,5,8,1,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-244908032,"num_objects":-3737,"num_object_clones":0,"num_object_copies":-11211,"num_objects_missing_on_primary":-92,"num_objects_missing":-92,"num_objects_degraded":-1716,"num_objects_misplaced":-3611,"num_objects_unfound":0,"num_objects_dirty":-3737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3737,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3264,"num_bytes_recovered":204668928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003432"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":58,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144620+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:46:26.144620+0000","last_peered":"2023-10-09T04:46:26.144620+0000","last_clean":"2023-10-09T04:46:26.144620+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:46:26.144620+0000","last_undegraded":"2023-10-09T04:46:26.144620+0000","last_fullsized":"2023-10-09T04:46:26.144620+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144304+0000","last_change":"2023-10-09T04:45:45.599027+0000","last_active":"2023-10-09T04:46:26.144304+0000","last_peered":"2023-10-09T04:46:26.144304+0000","last_clean":"2023-10-09T04:46:26.144304+0000","last_became_active":"2023-10-09T04:45:45.598650+0000","last_became_peered":"2023-10-09T04:45:45.598650+0000","last_unstale":"2023-10-09T04:46:26.144304+0000","last_undegraded":"2023-10-09T04:46:26.144304+0000","last_fullsized":"2023-10-09T04:46:26.144304+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:15.414379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":307,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T04:46:36.376285+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:46:36.376285+0000","last_peered":"2023-10-09T04:46:36.376285+0000","last_clean":"2023-10-09T04:46:36.376285+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:46:36.376285+0000","last_undegraded":"2023-10-09T04:46:36.376285+0000","last_fullsized":"2023-10-09T04:46:36.376285+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":243,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140219+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:46:26.140219+0000","last_peered":"2023-10-09T04:46:26.140219+0000","last_clean":"2023-10-09T04:46:26.140219+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:46:26.140219+0000","last_undegraded":"2023-10-09T04:46:26.140219+0000","last_fullsized":"2023-10-09T04:46:26.140219+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":27,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.132668+0000","last_change":"2023-10-09T04:46:25.173258+0000","last_active":"2023-10-09T04:46:26.132668+0000","last_peered":"2023-10-09T04:46:26.132668+0000","last_clean":"2023-10-09T04:46:26.132668+0000","last_became_active":"2023-10-09T04:46:25.172947+0000","last_became_peered":"2023-10-09T04:46:25.172947+0000","last_unstale":"2023-10-09T04:46:26.132668+0000","last_undegraded":"2023-10-09T04:46:26.132668+0000","last_fullsized":"2023-10-09T04:46:26.132668+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:38.532832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139609+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:46:26.139609+0000","last_peered":"2023-10-09T04:46:26.139609+0000","last_clean":"2023-10-09T04:46:26.139609+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:46:26.139609+0000","last_undegraded":"2023-10-09T04:46:26.139609+0000","last_fullsized":"2023-10-09T04:46:26.139609+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":242,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139448+0000","last_change":"2023-10-09T04:46:25.155473+0000","last_active":"2023-10-09T04:46:26.139448+0000","last_peered":"2023-10-09T04:46:26.139448+0000","last_clean":"2023-10-09T04:46:26.139448+0000","last_became_active":"2023-10-09T04:46:25.155148+0000","last_became_peered":"2023-10-09T04:46:25.155148+0000","last_unstale":"2023-10-09T04:46:26.139448+0000","last_undegraded":"2023-10-09T04:46:26.139448+0000","last_fullsized":"2023-10-09T04:46:26.139448+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:38.576510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":267,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.141521+0000","last_change":"2023-10-09T04:45:45.611067+0000","last_active":"2023-10-09T04:46:26.141521+0000","last_peered":"2023-10-09T04:46:26.141521+0000","last_clean":"2023-10-09T04:46:26.141521+0000","last_became_active":"2023-10-09T04:45:45.610571+0000","last_became_peered":"2023-10-09T04:45:45.610571+0000","last_unstale":"2023-10-09T04:46:26.141521+0000","last_undegraded":"2023-10-09T04:46:26.141521+0000","last_fullsized":"2023-10-09T04:46:26.141521+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:14.381921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":270,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139207+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:46:26.139207+0000","last_peered":"2023-10-09T04:46:26.139207+0000","last_clean":"2023-10-09T04:46:26.139207+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:46:26.139207+0000","last_undegraded":"2023-10-09T04:46:26.139207+0000","last_fullsized":"2023-10-09T04:46:26.139207+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":250,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.145885+0000","last_change":"2023-10-09T04:46:25.193703+0000","last_active":"2023-10-09T04:46:26.145885+0000","last_peered":"2023-10-09T04:46:26.145885+0000","last_clean":"2023-10-09T04:46:26.145885+0000","last_became_active":"2023-10-09T04:46:25.193392+0000","last_became_peered":"2023-10-09T04:46:25.193392+0000","last_unstale":"2023-10-09T04:46:26.145885+0000","last_undegraded":"2023-10-09T04:46:26.145885+0000","last_fullsized":"2023-10-09T04:46:26.145885+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:56.086176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":336,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140069+0000","last_change":"2023-10-09T04:46:25.479989+0000","last_active":"2023-10-09T04:46:26.140069+0000","last_peered":"2023-10-09T04:46:26.140069+0000","last_clean":"2023-10-09T04:46:26.140069+0000","last_became_active":"2023-10-09T04:46:25.381247+0000","last_became_peered":"2023-10-09T04:46:25.381247+0000","last_unstale":"2023-10-09T04:46:26.140069+0000","last_undegraded":"2023-10-09T04:46:26.140069+0000","last_fullsized":"2023-10-09T04:46:26.140069+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:12.653312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":235,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.132436+0000","last_change":"2023-10-09T04:46:25.150997+0000","last_active":"2023-10-09T04:46:26.132436+0000","last_peered":"2023-10-09T04:46:26.132436+0000","last_clean":"2023-10-09T04:46:26.132436+0000","last_became_active":"2023-10-09T04:46:25.150370+0000","last_became_peered":"2023-10-09T04:46:25.150370+0000","last_unstale":"2023-10-09T04:46:26.132436+0000","last_undegraded":"2023-10-09T04:46:26.132436+0000","last_fullsized":"2023-10-09T04:46:26.132436+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:07.231212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":88,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144317+0000","last_change":"2023-10-09T04:44:38.813366+0000","last_active":"2023-10-09T04:46:26.144317+0000","last_peered":"2023-10-09T04:46:26.144317+0000","last_clean":"2023-10-09T04:46:26.144317+0000","last_became_active":"2023-10-09T04:44:38.777069+0000","last_became_peered":"2023-10-09T04:44:38.777069+0000","last_unstale":"2023-10-09T04:46:26.144317+0000","last_undegraded":"2023-10-09T04:46:26.144317+0000","last_fullsized":"2023-10-09T04:46:26.144317+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:39.101540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":280,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.143066+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:46:26.143066+0000","last_peered":"2023-10-09T04:46:26.143066+0000","last_clean":"2023-10-09T04:46:26.143066+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:46:26.143066+0000","last_undegraded":"2023-10-09T04:46:26.143066+0000","last_fullsized":"2023-10-09T04:46:26.143066+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":320,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139934+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:46:26.139934+0000","last_peered":"2023-10-09T04:46:26.139934+0000","last_clean":"2023-10-09T04:46:26.139934+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:46:26.139934+0000","last_undegraded":"2023-10-09T04:46:26.139934+0000","last_fullsized":"2023-10-09T04:46:26.139934+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139967+0000","last_change":"2023-10-09T04:45:45.607791+0000","last_active":"2023-10-09T04:46:26.139967+0000","last_peered":"2023-10-09T04:46:26.139967+0000","last_clean":"2023-10-09T04:46:26.139967+0000","last_became_active":"2023-10-09T04:45:45.607501+0000","last_became_peered":"2023-10-09T04:45:45.607501+0000","last_unstale":"2023-10-09T04:46:26.139967+0000","last_undegraded":"2023-10-09T04:46:26.139967+0000","last_fullsized":"2023-10-09T04:46:26.139967+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:05.067640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":242,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.138566+0000","last_change":"2023-10-09T04:46:25.168444+0000","last_active":"2023-10-09T04:46:26.138566+0000","last_peered":"2023-10-09T04:46:26.138566+0000","last_clean":"2023-10-09T04:46:26.138566+0000","last_became_active":"2023-10-09T04:46:25.168143+0000","last_became_peered":"2023-10-09T04:46:25.168143+0000","last_unstale":"2023-10-09T04:46:26.138566+0000","last_undegraded":"2023-10-09T04:46:26.138566+0000","last_fullsized":"2023-10-09T04:46:26.138566+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:56.166900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":289,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139475+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:46:26.139475+0000","last_peered":"2023-10-09T04:46:26.139475+0000","last_clean":"2023-10-09T04:46:26.139475+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:46:26.139475+0000","last_undegraded":"2023-10-09T04:46:26.139475+0000","last_fullsized":"2023-10-09T04:46:26.139475+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":272,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.145796+0000","last_change":"2023-10-09T04:46:25.175418+0000","last_active":"2023-10-09T04:46:26.145796+0000","last_peered":"2023-10-09T04:46:26.145796+0000","last_clean":"2023-10-09T04:46:26.145796+0000","last_became_active":"2023-10-09T04:46:25.174958+0000","last_became_peered":"2023-10-09T04:46:25.174958+0000","last_unstale":"2023-10-09T04:46:26.145796+0000","last_undegraded":"2023-10-09T04:46:26.145796+0000","last_fullsized":"2023-10-09T04:46:26.145796+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:35.362263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140555+0000","last_change":"2023-10-09T04:46:25.170147+0000","last_active":"2023-10-09T04:46:26.140555+0000","last_peered":"2023-10-09T04:46:26.140555+0000","last_clean":"2023-10-09T04:46:26.140555+0000","last_became_active":"2023-10-09T04:46:25.169835+0000","last_became_peered":"2023-10-09T04:46:25.169835+0000","last_unstale":"2023-10-09T04:46:26.140555+0000","last_undegraded":"2023-10-09T04:46:26.140555+0000","last_fullsized":"2023-10-09T04:46:26.140555+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:01.393349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":91,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139754+0000","last_change":"2023-10-09T04:46:25.271462+0000","last_active":"2023-10-09T04:46:26.139754+0000","last_peered":"2023-10-09T04:46:26.139754+0000","last_clean":"2023-10-09T04:46:26.139754+0000","last_became_active":"2023-10-09T04:46:25.166617+0000","last_became_peered":"2023-10-09T04:46:25.166617+0000","last_unstale":"2023-10-09T04:46:26.139754+0000","last_undegraded":"2023-10-09T04:46:26.139754+0000","last_fullsized":"2023-10-09T04:46:26.139754+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:30.657523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140615+0000","last_change":"2023-10-09T04:45:45.603131+0000","last_active":"2023-10-09T04:46:26.140615+0000","last_peered":"2023-10-09T04:46:26.140615+0000","last_clean":"2023-10-09T04:46:26.140615+0000","last_became_active":"2023-10-09T04:45:45.602862+0000","last_became_peered":"2023-10-09T04:45:45.602862+0000","last_unstale":"2023-10-09T04:46:26.140615+0000","last_undegraded":"2023-10-09T04:46:26.140615+0000","last_fullsized":"2023-10-09T04:46:26.140615+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:22.358682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"310'8139","reported_seq":15865,"reported_epoch":310,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:44.477003+0000","last_change":"2023-10-09T04:46:25.281764+0000","last_active":"2023-10-09T04:46:44.477003+0000","last_peered":"2023-10-09T04:46:44.477003+0000","last_clean":"2023-10-09T04:44:37.624385+0000","last_became_active":"2023-10-09T04:44:40.023059+0000","last_became_peered":"2023-10-09T04:44:40.023059+0000","last_unstale":"2023-10-09T04:46:44.477003+0000","last_undegraded":"2023-10-09T04:44:39.944445+0000","last_fullsized":"2023-10-09T04:44:39.771566+0000","mapping_epoch":293,"log_start":"288'2272","ondisk_log_start":"288'2272","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:27:06.349239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253165568,"num_objects":3863,"num_object_clones":0,"num_object_copies":11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8139,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3837,"num_bytes_recovered":242221056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":2948}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":340,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140573+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:46:26.140573+0000","last_peered":"2023-10-09T04:46:26.140573+0000","last_clean":"2023-10-09T04:46:26.140573+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:46:26.140573+0000","last_undegraded":"2023-10-09T04:46:26.140573+0000","last_fullsized":"2023-10-09T04:46:26.140573+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":262,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139740+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:46:26.139740+0000","last_peered":"2023-10-09T04:46:26.139740+0000","last_clean":"2023-10-09T04:46:26.139740+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:46:26.139740+0000","last_undegraded":"2023-10-09T04:46:26.139740+0000","last_fullsized":"2023-10-09T04:46:26.139740+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"310'8074","reported_seq":10217,"reported_epoch":310,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:46:44.627723+0000","last_change":"2023-10-09T04:46:25.178557+0000","last_active":"2023-10-09T04:46:44.627723+0000","last_peered":"2023-10-09T04:46:44.627723+0000","last_clean":"2023-10-09T04:44:37.740580+0000","last_became_active":"2023-10-09T04:45:46.774175+0000","last_became_peered":"2023-10-09T04:45:46.774175+0000","last_unstale":"2023-10-09T04:46:44.627723+0000","last_undegraded":"2023-10-09T04:46:25.175568+0000","last_fullsized":"2023-10-09T04:46:44.627723+0000","mapping_epoch":303,"log_start":"289'2806","ondisk_log_start":"289'2806","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:40.805571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258211840,"num_objects":3940,"num_object_clones":0,"num_object_copies":11820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3648,"num_objects_misplaced":3940,"num_objects_unfound":0,"num_objects_dirty":3940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8074,"num_write_kb":384448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1046,"num_bytes_recovered":68550656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"1,3,7","objects":1313},{"shards":"3,6,7","objects":2627}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":299,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140141+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:46:26.140141+0000","last_peered":"2023-10-09T04:46:26.140141+0000","last_clean":"2023-10-09T04:46:26.140141+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:46:26.140141+0000","last_undegraded":"2023-10-09T04:46:26.140141+0000","last_fullsized":"2023-10-09T04:46:26.140141+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":307,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.142053+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:46:26.142053+0000","last_peered":"2023-10-09T04:46:26.142053+0000","last_clean":"2023-10-09T04:46:26.142053+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:46:26.142053+0000","last_undegraded":"2023-10-09T04:46:26.142053+0000","last_fullsized":"2023-10-09T04:46:26.142053+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"310'8033","reported_seq":8082,"reported_epoch":310,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:44.155496+0000","last_change":"2023-10-09T04:46:25.245840+0000","last_active":"2023-10-09T04:46:44.155496+0000","last_peered":"2023-10-09T04:46:44.155496+0000","last_clean":"2023-10-09T04:45:44.791779+0000","last_became_active":"2023-10-09T04:45:46.998769+0000","last_became_peered":"2023-10-09T04:45:46.998769+0000","last_unstale":"2023-10-09T04:46:44.155496+0000","last_undegraded":"2023-10-09T04:45:46.613330+0000","last_fullsized":"2023-10-09T04:45:46.574934+0000","mapping_epoch":303,"log_start":"288'2229","ondisk_log_start":"288'2229","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5804,"log_dups_size":0,"ondisk_log_size":5804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:00.584863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251068416,"num_objects":3831,"num_object_clones":0,"num_object_copies":11493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5931,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8033,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":3831},{"shards":"4,7","objects":-1096}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":281,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139874+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:46:26.139874+0000","last_peered":"2023-10-09T04:46:26.139874+0000","last_clean":"2023-10-09T04:46:26.139874+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:46:26.139874+0000","last_undegraded":"2023-10-09T04:46:26.139874+0000","last_fullsized":"2023-10-09T04:46:26.139874+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":226,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.132348+0000","last_change":"2023-10-09T04:44:38.775465+0000","last_active":"2023-10-09T04:46:26.132348+0000","last_peered":"2023-10-09T04:46:26.132348+0000","last_clean":"2023-10-09T04:46:26.132348+0000","last_became_active":"2023-10-09T04:44:38.775246+0000","last_became_peered":"2023-10-09T04:44:38.775246+0000","last_unstale":"2023-10-09T04:46:26.132348+0000","last_undegraded":"2023-10-09T04:46:26.132348+0000","last_fullsized":"2023-10-09T04:46:26.132348+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.286190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.3","version":"310'8075","reported_seq":8980,"reported_epoch":310,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:46:44.593866+0000","last_change":"2023-10-09T04:46:26.476463+0000","last_active":"2023-10-09T04:46:44.593866+0000","last_peered":"2023-10-09T04:46:44.593866+0000","last_clean":"2023-10-09T04:45:44.834828+0000","last_became_active":"2023-10-09T04:46:26.386549+0000","last_became_peered":"2023-10-09T04:46:26.386549+0000","last_unstale":"2023-10-09T04:46:44.593866+0000","last_undegraded":"2023-10-09T04:46:26.372334+0000","last_fullsized":"2023-10-09T04:46:44.593866+0000","mapping_epoch":308,"log_start":"289'2880","ondisk_log_start":"289'2880","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:01.311399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252903424,"num_objects":3859,"num_object_clones":0,"num_object_copies":11577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3859,"num_objects_misplaced":5541,"num_objects_unfound":0,"num_objects_dirty":3859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8075,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":426,"num_bytes_recovered":27721728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,0],"avail_no_missing":["7","0","3"],"object_location_counts":[{"shards":"0,3,7","objects":3859}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":300,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.369467+0000","last_change":"2023-10-09T04:46:25.150945+0000","last_active":"2023-10-09T04:46:26.369467+0000","last_peered":"2023-10-09T04:46:26.369467+0000","last_clean":"2023-10-09T04:46:26.369467+0000","last_became_active":"2023-10-09T04:46:25.150622+0000","last_became_peered":"2023-10-09T04:46:25.150622+0000","last_unstale":"2023-10-09T04:46:26.369467+0000","last_undegraded":"2023-10-09T04:46:26.369467+0000","last_fullsized":"2023-10-09T04:46:26.369467+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:02.718023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":61,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.142861+0000","last_change":"2023-10-09T04:46:25.173123+0000","last_active":"2023-10-09T04:46:26.142861+0000","last_peered":"2023-10-09T04:46:26.142861+0000","last_clean":"2023-10-09T04:46:26.142861+0000","last_became_active":"2023-10-09T04:46:25.172534+0000","last_became_peered":"2023-10-09T04:46:25.172534+0000","last_unstale":"2023-10-09T04:46:26.142861+0000","last_undegraded":"2023-10-09T04:46:26.142861+0000","last_fullsized":"2023-10-09T04:46:26.142861+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:00.581088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"310'8046","reported_seq":14328,"reported_epoch":310,"state":"active+recovering+remapped","last_fresh":"2023-10-09T04:46:44.620548+0000","last_change":"2023-10-09T04:46:25.493554+0000","last_active":"2023-10-09T04:46:44.620548+0000","last_peered":"2023-10-09T04:46:44.620548+0000","last_clean":"2023-10-09T04:44:37.730195+0000","last_became_active":"2023-10-09T04:45:47.970624+0000","last_became_peered":"2023-10-09T04:45:47.970624+0000","last_unstale":"2023-10-09T04:46:44.620548+0000","last_undegraded":"2023-10-09T04:46:44.620548+0000","last_fullsized":"2023-10-09T04:46:44.620548+0000","mapping_epoch":303,"log_start":"289'2836","ondisk_log_start":"289'2836","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5210,"log_dups_size":0,"ondisk_log_size":5210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:58.727121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252051456,"num_objects":3846,"num_object_clones":0,"num_object_copies":11538,"num_objects_missing_on_primary":1541,"num_objects_missing":1541,"num_objects_degraded":0,"num_objects_misplaced":3083,"num_objects_unfound":0,"num_objects_dirty":3846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8046,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3116,"num_bytes_recovered":204210176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,3],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1541},{"shards":"1,3,6,7","objects":1},{"shards":"1,3,7","objects":2304}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":49,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.143176+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:46:26.143176+0000","last_peered":"2023-10-09T04:46:26.143176+0000","last_clean":"2023-10-09T04:46:26.143176+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:46:26.143176+0000","last_undegraded":"2023-10-09T04:46:26.143176+0000","last_fullsized":"2023-10-09T04:46:26.143176+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":78,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144456+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:46:26.144456+0000","last_peered":"2023-10-09T04:46:26.144456+0000","last_clean":"2023-10-09T04:46:26.144456+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:46:26.144456+0000","last_undegraded":"2023-10-09T04:46:26.144456+0000","last_fullsized":"2023-10-09T04:46:26.144456+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"310'8094","reported_seq":11335,"reported_epoch":310,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:46.531076+0000","last_change":"2023-10-09T04:46:25.469213+0000","last_active":"2023-10-09T04:46:46.531076+0000","last_peered":"2023-10-09T04:46:46.531076+0000","last_clean":"2023-10-09T04:44:37.740389+0000","last_became_active":"2023-10-09T04:46:25.449027+0000","last_became_peered":"2023-10-09T04:46:25.449027+0000","last_unstale":"2023-10-09T04:46:46.531076+0000","last_undegraded":"2023-10-09T04:46:25.190986+0000","last_fullsized":"2023-10-09T04:46:25.133749+0000","mapping_epoch":307,"log_start":"288'1473","ondisk_log_start":"288'1473","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:53.663910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247988224,"num_objects":3784,"num_object_clones":0,"num_object_copies":11352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8094,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1593,"num_bytes_recovered":104202240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"0,1,2,6","objects":420},{"shards":"1,2","objects":-502},{"shards":"1,2,3","objects":1254},{"shards":"1,2,6","objects":2110}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":271,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144702+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:46:26.144702+0000","last_peered":"2023-10-09T04:46:26.144702+0000","last_clean":"2023-10-09T04:46:26.144702+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:46:26.144702+0000","last_undegraded":"2023-10-09T04:46:26.144702+0000","last_fullsized":"2023-10-09T04:46:26.144702+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":58,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144028+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:46:26.144028+0000","last_peered":"2023-10-09T04:46:26.144028+0000","last_clean":"2023-10-09T04:46:26.144028+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:46:26.144028+0000","last_undegraded":"2023-10-09T04:46:26.144028+0000","last_fullsized":"2023-10-09T04:46:26.144028+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"311'8037","reported_seq":13918,"reported_epoch":311,"state":"active+remapped+backfilling","last_fresh":"2023-10-09T04:46:47.688661+0000","last_change":"2023-10-09T04:46:26.572325+0000","last_active":"2023-10-09T04:46:47.688661+0000","last_peered":"2023-10-09T04:46:47.688661+0000","last_clean":"2023-10-09T04:46:24.119479+0000","last_became_active":"2023-10-09T04:46:26.518287+0000","last_became_peered":"2023-10-09T04:46:26.518287+0000","last_unstale":"2023-10-09T04:46:47.688661+0000","last_undegraded":"2023-10-09T04:46:47.688661+0000","last_fullsized":"2023-10-09T04:46:47.688661+0000","mapping_epoch":308,"log_start":"289'3328","ondisk_log_start":"289'3328","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:38.281893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245432320,"num_objects":3745,"num_object_clones":0,"num_object_copies":11235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1006,"num_objects_unfound":0,"num_objects_dirty":3745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8037,"num_write_kb":377024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2917,"num_bytes_recovered":191168512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,0],"avail_no_missing":["1","0","7"],"object_location_counts":[{"shards":"0,1,7","objects":3745}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":58,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.141832+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:46:26.141832+0000","last_peered":"2023-10-09T04:46:26.141832+0000","last_clean":"2023-10-09T04:46:26.141832+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:46:26.141832+0000","last_undegraded":"2023-10-09T04:46:26.141832+0000","last_fullsized":"2023-10-09T04:46:26.141832+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":290,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.166687+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:46:26.166687+0000","last_peered":"2023-10-09T04:46:26.166687+0000","last_clean":"2023-10-09T04:46:26.166687+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:46:26.166687+0000","last_undegraded":"2023-10-09T04:46:26.166687+0000","last_fullsized":"2023-10-09T04:46:26.166687+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"310'8065","reported_seq":8099,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T04:46:44.573044+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:46:44.573044+0000","last_peered":"2023-10-09T04:46:44.573044+0000","last_clean":"2023-10-09T04:46:44.573044+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:46:44.573044+0000","last_undegraded":"2023-10-09T04:46:44.573044+0000","last_fullsized":"2023-10-09T04:46:44.573044+0000","mapping_epoch":284,"log_start":"289'2796","ondisk_log_start":"289'2796","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5269,"log_dups_size":0,"ondisk_log_size":5269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256573440,"num_objects":3915,"num_object_clones":0,"num_object_copies":11745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8065,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":104,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139207+0000","last_change":"2023-10-09T04:45:45.611560+0000","last_active":"2023-10-09T04:46:26.139207+0000","last_peered":"2023-10-09T04:46:26.139207+0000","last_clean":"2023-10-09T04:46:26.139207+0000","last_became_active":"2023-10-09T04:45:45.611247+0000","last_became_peered":"2023-10-09T04:45:45.611247+0000","last_unstale":"2023-10-09T04:46:26.139207+0000","last_undegraded":"2023-10-09T04:46:26.139207+0000","last_fullsized":"2023-10-09T04:46:26.139207+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:52.196556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":256,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144920+0000","last_change":"2023-10-09T04:45:45.604134+0000","last_active":"2023-10-09T04:46:26.144920+0000","last_peered":"2023-10-09T04:46:26.144920+0000","last_clean":"2023-10-09T04:46:26.144920+0000","last_became_active":"2023-10-09T04:45:45.603767+0000","last_became_peered":"2023-10-09T04:45:45.603767+0000","last_unstale":"2023-10-09T04:46:26.144920+0000","last_undegraded":"2023-10-09T04:46:26.144920+0000","last_fullsized":"2023-10-09T04:46:26.144920+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:14.801140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.a","version":"310'8267","reported_seq":9675,"reported_epoch":310,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:46.617885+0000","last_change":"2023-10-09T04:46:26.440123+0000","last_active":"2023-10-09T04:46:46.617885+0000","last_peered":"2023-10-09T04:46:46.617885+0000","last_clean":"2023-10-09T04:45:44.754196+0000","last_became_active":"2023-10-09T04:46:26.422562+0000","last_became_peered":"2023-10-09T04:46:26.422562+0000","last_unstale":"2023-10-09T04:46:46.617885+0000","last_undegraded":"2023-10-09T04:46:26.243542+0000","last_fullsized":"2023-10-09T04:46:26.131676+0000","mapping_epoch":308,"log_start":"288'1697","ondisk_log_start":"288'1697","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:06.608939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251461632,"num_objects":3837,"num_object_clones":0,"num_object_copies":11511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11201,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8267,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":452,"num_bytes_recovered":29491200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":3836},{"shards":"1,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":197,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140139+0000","last_change":"2023-10-09T04:46:25.169137+0000","last_active":"2023-10-09T04:46:26.140139+0000","last_peered":"2023-10-09T04:46:26.140139+0000","last_clean":"2023-10-09T04:46:26.140139+0000","last_became_active":"2023-10-09T04:46:25.168834+0000","last_became_peered":"2023-10-09T04:46:25.168834+0000","last_unstale":"2023-10-09T04:46:26.140139+0000","last_undegraded":"2023-10-09T04:46:26.140139+0000","last_fullsized":"2023-10-09T04:46:26.140139+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:15.860221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":314,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139557+0000","last_change":"2023-10-09T04:45:45.824913+0000","last_active":"2023-10-09T04:46:26.139557+0000","last_peered":"2023-10-09T04:46:26.139557+0000","last_clean":"2023-10-09T04:46:26.139557+0000","last_became_active":"2023-10-09T04:45:45.824572+0000","last_became_peered":"2023-10-09T04:45:45.824572+0000","last_unstale":"2023-10-09T04:46:26.139557+0000","last_undegraded":"2023-10-09T04:46:26.139557+0000","last_fullsized":"2023-10-09T04:46:26.139557+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:36.276403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":281,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.138710+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:46:26.138710+0000","last_peered":"2023-10-09T04:46:26.138710+0000","last_clean":"2023-10-09T04:46:26.138710+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:46:26.138710+0000","last_undegraded":"2023-10-09T04:46:26.138710+0000","last_fullsized":"2023-10-09T04:46:26.138710+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"310'8049","reported_seq":8097,"reported_epoch":310,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:44.240170+0000","last_change":"2023-10-09T04:46:26.525026+0000","last_active":"2023-10-09T04:46:44.240170+0000","last_peered":"2023-10-09T04:46:44.240170+0000","last_clean":"2023-10-09T04:46:24.128327+0000","last_became_active":"2023-10-09T04:46:26.494164+0000","last_became_peered":"2023-10-09T04:46:26.494164+0000","last_unstale":"2023-10-09T04:46:44.240170+0000","last_undegraded":"2023-10-09T04:46:26.365928+0000","last_fullsized":"2023-10-09T04:46:26.143463+0000","mapping_epoch":308,"log_start":"288'1496","ondisk_log_start":"288'1496","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6553,"log_dups_size":0,"ondisk_log_size":6553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:51.675602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250413056,"num_objects":3821,"num_object_clones":0,"num_object_copies":11463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5933,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8049,"num_write_kb":379840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":3821},{"shards":"2,5","objects":-432}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":294,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.138577+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:46:26.138577+0000","last_peered":"2023-10-09T04:46:26.138577+0000","last_clean":"2023-10-09T04:46:26.138577+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:46:26.138577+0000","last_undegraded":"2023-10-09T04:46:26.138577+0000","last_fullsized":"2023-10-09T04:46:26.138577+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":78,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.142523+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:46:26.142523+0000","last_peered":"2023-10-09T04:46:26.142523+0000","last_clean":"2023-10-09T04:46:26.142523+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:46:26.142523+0000","last_undegraded":"2023-10-09T04:46:26.142523+0000","last_fullsized":"2023-10-09T04:46:26.142523+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.131721+0000","last_change":"2023-10-09T04:45:45.611766+0000","last_active":"2023-10-09T04:46:26.131721+0000","last_peered":"2023-10-09T04:46:26.131721+0000","last_clean":"2023-10-09T04:46:26.131721+0000","last_became_active":"2023-10-09T04:45:45.611242+0000","last_became_peered":"2023-10-09T04:45:45.611242+0000","last_unstale":"2023-10-09T04:46:26.131721+0000","last_undegraded":"2023-10-09T04:46:26.131721+0000","last_fullsized":"2023-10-09T04:46:26.131721+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:23.336238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"310'7801","reported_seq":16861,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T04:46:44.233733+0000","last_change":"2023-10-09T04:45:30.672535+0000","last_active":"2023-10-09T04:46:44.233733+0000","last_peered":"2023-10-09T04:46:44.233733+0000","last_clean":"2023-10-09T04:46:44.233733+0000","last_became_active":"2023-10-09T04:45:30.663157+0000","last_became_peered":"2023-10-09T04:45:30.663157+0000","last_unstale":"2023-10-09T04:46:44.233733+0000","last_undegraded":"2023-10-09T04:46:44.233733+0000","last_fullsized":"2023-10-09T04:46:44.233733+0000","mapping_epoch":300,"log_start":"288'1206","ondisk_log_start":"288'1206","created":284,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:31.926989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250281984,"num_objects":3819,"num_object_clones":0,"num_object_copies":11457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7801,"num_write_kb":371840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4503,"num_bytes_recovered":295108608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":286,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.133195+0000","last_change":"2023-10-09T04:46:25.161784+0000","last_active":"2023-10-09T04:46:26.133195+0000","last_peered":"2023-10-09T04:46:26.133195+0000","last_clean":"2023-10-09T04:46:26.133195+0000","last_became_active":"2023-10-09T04:46:25.161498+0000","last_became_peered":"2023-10-09T04:46:25.161498+0000","last_unstale":"2023-10-09T04:46:26.133195+0000","last_undegraded":"2023-10-09T04:46:26.133195+0000","last_fullsized":"2023-10-09T04:46:26.133195+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:01.500231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":285,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.132070+0000","last_change":"2023-10-09T04:46:25.165536+0000","last_active":"2023-10-09T04:46:26.132070+0000","last_peered":"2023-10-09T04:46:26.132070+0000","last_clean":"2023-10-09T04:46:26.132070+0000","last_became_active":"2023-10-09T04:46:25.165269+0000","last_became_peered":"2023-10-09T04:46:25.165269+0000","last_unstale":"2023-10-09T04:46:26.132070+0000","last_undegraded":"2023-10-09T04:46:26.132070+0000","last_fullsized":"2023-10-09T04:46:26.132070+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:30.362732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139528+0000","last_change":"2023-10-09T04:46:25.177443+0000","last_active":"2023-10-09T04:46:26.139528+0000","last_peered":"2023-10-09T04:46:26.139528+0000","last_clean":"2023-10-09T04:46:26.139528+0000","last_became_active":"2023-10-09T04:46:25.176635+0000","last_became_peered":"2023-10-09T04:46:25.176635+0000","last_unstale":"2023-10-09T04:46:26.139528+0000","last_undegraded":"2023-10-09T04:46:26.139528+0000","last_fullsized":"2023-10-09T04:46:26.139528+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:43.367744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"310'7945","reported_seq":7980,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T04:46:44.569134+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:46:44.569134+0000","last_peered":"2023-10-09T04:46:44.569134+0000","last_clean":"2023-10-09T04:46:44.569134+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:46:44.569134+0000","last_undegraded":"2023-10-09T04:46:44.569134+0000","last_fullsized":"2023-10-09T04:46:44.569134+0000","mapping_epoch":284,"log_start":"289'2769","ondisk_log_start":"289'2769","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5176,"log_dups_size":0,"ondisk_log_size":5176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255066136,"num_objects":3893,"num_object_clones":0,"num_object_copies":11679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3893,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7945,"num_write_kb":378753,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":291,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139033+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:46:26.139033+0000","last_peered":"2023-10-09T04:46:26.139033+0000","last_clean":"2023-10-09T04:46:26.139033+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:46:26.139033+0000","last_undegraded":"2023-10-09T04:46:26.139033+0000","last_fullsized":"2023-10-09T04:46:26.139033+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":738,"reported_epoch":309,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:26.415415+0000","last_change":"2023-10-09T04:46:26.415415+0000","last_active":"2023-10-09T04:46:26.415415+0000","last_peered":"2023-10-09T04:46:26.415415+0000","last_clean":"2023-10-09T04:46:16.698879+0000","last_became_active":"2023-10-09T04:46:26.415059+0000","last_became_peered":"2023-10-09T04:46:26.415059+0000","last_unstale":"2023-10-09T04:46:26.415415+0000","last_undegraded":"2023-10-09T04:46:26.138726+0000","last_fullsized":"2023-10-09T04:46:26.138125+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:23.417129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":240,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.138793+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:46:26.138793+0000","last_peered":"2023-10-09T04:46:26.138793+0000","last_clean":"2023-10-09T04:46:26.138793+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:46:26.138793+0000","last_undegraded":"2023-10-09T04:46:26.138793+0000","last_fullsized":"2023-10-09T04:46:26.138793+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":253,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.143955+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:46:26.143955+0000","last_peered":"2023-10-09T04:46:26.143955+0000","last_clean":"2023-10-09T04:46:26.143955+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:46:26.143955+0000","last_undegraded":"2023-10-09T04:46:26.143955+0000","last_fullsized":"2023-10-09T04:46:26.143955+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"310'7955","reported_seq":7999,"reported_epoch":310,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:46:44.231465+0000","last_change":"2023-10-09T04:46:26.351968+0000","last_active":"2023-10-09T04:46:44.231465+0000","last_peered":"2023-10-09T04:46:44.231465+0000","last_clean":"2023-10-09T04:46:24.123151+0000","last_became_active":"2023-10-09T04:46:26.320651+0000","last_became_peered":"2023-10-09T04:46:26.320651+0000","last_unstale":"2023-10-09T04:46:44.231465+0000","last_undegraded":"2023-10-09T04:46:44.231465+0000","last_fullsized":"2023-10-09T04:46:44.231465+0000","mapping_epoch":308,"log_start":"288'2176","ondisk_log_start":"288'2176","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:25.867280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253296640,"num_objects":3865,"num_object_clones":0,"num_object_copies":11595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3865,"num_objects_unfound":0,"num_objects_dirty":3865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7955,"num_write_kb":378240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":3865}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":320,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139112+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:46:26.139112+0000","last_peered":"2023-10-09T04:46:26.139112+0000","last_clean":"2023-10-09T04:46:26.139112+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:46:26.139112+0000","last_undegraded":"2023-10-09T04:46:26.139112+0000","last_fullsized":"2023-10-09T04:46:26.139112+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":78,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.141975+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:46:26.141975+0000","last_peered":"2023-10-09T04:46:26.141975+0000","last_clean":"2023-10-09T04:46:26.141975+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:46:26.141975+0000","last_undegraded":"2023-10-09T04:46:26.141975+0000","last_fullsized":"2023-10-09T04:46:26.141975+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.576826+0000","last_change":"2023-10-09T04:46:25.174089+0000","last_active":"2023-10-09T04:46:26.576826+0000","last_peered":"2023-10-09T04:46:26.576826+0000","last_clean":"2023-10-09T04:46:26.576826+0000","last_became_active":"2023-10-09T04:46:25.173414+0000","last_became_peered":"2023-10-09T04:46:25.173414+0000","last_unstale":"2023-10-09T04:46:26.576826+0000","last_undegraded":"2023-10-09T04:46:26.576826+0000","last_fullsized":"2023-10-09T04:46:26.576826+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.877749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"310'8190","reported_seq":10454,"reported_epoch":310,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:46:46.638298+0000","last_change":"2023-10-09T04:45:59.650149+0000","last_active":"2023-10-09T04:46:46.638298+0000","last_peered":"2023-10-09T04:46:46.638298+0000","last_clean":"2023-10-09T04:44:37.770348+0000","last_became_active":"2023-10-09T04:45:59.017846+0000","last_became_peered":"2023-10-09T04:45:59.017846+0000","last_unstale":"2023-10-09T04:46:46.638298+0000","last_undegraded":"2023-10-09T04:45:56.425429+0000","last_fullsized":"2023-10-09T04:46:46.638298+0000","mapping_epoch":303,"log_start":"288'1603","ondisk_log_start":"288'1603","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:00.633461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249036800,"num_objects":3800,"num_object_clones":0,"num_object_copies":11400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3696,"num_objects_misplaced":3800,"num_objects_unfound":0,"num_objects_dirty":3800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8190,"num_write_kb":383680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72351744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,4","objects":1193},{"shards":"2,3,6","objects":2607}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":323,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139971+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:46:26.139971+0000","last_peered":"2023-10-09T04:46:26.139971+0000","last_clean":"2023-10-09T04:46:26.139971+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:46:26.139971+0000","last_undegraded":"2023-10-09T04:46:26.139971+0000","last_fullsized":"2023-10-09T04:46:26.139971+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":267,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.131653+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:46:26.131653+0000","last_peered":"2023-10-09T04:46:26.131653+0000","last_clean":"2023-10-09T04:46:26.131653+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:46:26.131653+0000","last_undegraded":"2023-10-09T04:46:26.131653+0000","last_fullsized":"2023-10-09T04:46:26.131653+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":164,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139179+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:46:26.139179+0000","last_peered":"2023-10-09T04:46:26.139179+0000","last_clean":"2023-10-09T04:46:26.139179+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:46:26.139179+0000","last_undegraded":"2023-10-09T04:46:26.139179+0000","last_fullsized":"2023-10-09T04:46:26.139179+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"310'8071","reported_seq":8122,"reported_epoch":310,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:46:46.584148+0000","last_change":"2023-10-09T04:46:25.302033+0000","last_active":"2023-10-09T04:46:46.584148+0000","last_peered":"2023-10-09T04:46:46.584148+0000","last_clean":"2023-10-09T04:45:44.820208+0000","last_became_active":"2023-10-09T04:46:25.282101+0000","last_became_peered":"2023-10-09T04:46:25.282101+0000","last_unstale":"2023-10-09T04:46:46.584148+0000","last_undegraded":"2023-10-09T04:46:46.584148+0000","last_fullsized":"2023-10-09T04:46:46.584148+0000","mapping_epoch":307,"log_start":"288'1503","ondisk_log_start":"288'1503","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:16.931727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252248064,"num_objects":3849,"num_object_clones":0,"num_object_copies":11547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7698,"num_objects_unfound":0,"num_objects_dirty":3849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8071,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,0,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":3849}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":58,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140691+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:46:26.140691+0000","last_peered":"2023-10-09T04:46:26.140691+0000","last_clean":"2023-10-09T04:46:26.140691+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:46:26.140691+0000","last_undegraded":"2023-10-09T04:46:26.140691+0000","last_fullsized":"2023-10-09T04:46:26.140691+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":273,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.166602+0000","last_change":"2023-10-09T04:46:25.166210+0000","last_active":"2023-10-09T04:46:26.166602+0000","last_peered":"2023-10-09T04:46:26.166602+0000","last_clean":"2023-10-09T04:46:26.166602+0000","last_became_active":"2023-10-09T04:46:25.165948+0000","last_became_peered":"2023-10-09T04:46:25.165948+0000","last_unstale":"2023-10-09T04:46:26.166602+0000","last_undegraded":"2023-10-09T04:46:26.166602+0000","last_fullsized":"2023-10-09T04:46:26.166602+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:51.272111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":156,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139288+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:46:26.139288+0000","last_peered":"2023-10-09T04:46:26.139288+0000","last_clean":"2023-10-09T04:46:26.139288+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:46:26.139288+0000","last_undegraded":"2023-10-09T04:46:26.139288+0000","last_fullsized":"2023-10-09T04:46:26.139288+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"310'8178","reported_seq":11602,"reported_epoch":310,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:46:46.485113+0000","last_change":"2023-10-09T04:46:25.450073+0000","last_active":"2023-10-09T04:46:46.485113+0000","last_peered":"2023-10-09T04:46:46.485113+0000","last_clean":"2023-10-09T04:44:37.742164+0000","last_became_active":"2023-10-09T04:45:46.956446+0000","last_became_peered":"2023-10-09T04:45:46.956446+0000","last_unstale":"2023-10-09T04:46:46.485113+0000","last_undegraded":"2023-10-09T04:46:25.447165+0000","last_fullsized":"2023-10-09T04:45:46.571293+0000","mapping_epoch":303,"log_start":"288'1603","ondisk_log_start":"288'1603","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6575,"log_dups_size":0,"ondisk_log_size":6575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:11.914538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249692160,"num_objects":3810,"num_object_clones":0,"num_object_copies":11430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8428,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8178,"num_write_kb":383616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1123,"num_bytes_recovered":73531392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"1,2,3","objects":1314},{"shards":"2,3,6","objects":2496}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":232,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.140523+0000","last_change":"2023-10-09T04:45:45.602634+0000","last_active":"2023-10-09T04:46:26.140523+0000","last_peered":"2023-10-09T04:46:26.140523+0000","last_clean":"2023-10-09T04:46:26.140523+0000","last_became_active":"2023-10-09T04:45:45.602317+0000","last_became_peered":"2023-10-09T04:45:45.602317+0000","last_unstale":"2023-10-09T04:46:26.140523+0000","last_undegraded":"2023-10-09T04:46:26.140523+0000","last_fullsized":"2023-10-09T04:46:26.140523+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:13.963587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.166774+0000","last_change":"2023-10-09T04:46:25.210175+0000","last_active":"2023-10-09T04:46:26.166774+0000","last_peered":"2023-10-09T04:46:26.166774+0000","last_clean":"2023-10-09T04:46:26.166774+0000","last_became_active":"2023-10-09T04:46:25.174609+0000","last_became_peered":"2023-10-09T04:46:25.174609+0000","last_unstale":"2023-10-09T04:46:26.166774+0000","last_undegraded":"2023-10-09T04:46:26.166774+0000","last_fullsized":"2023-10-09T04:46:26.166774+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:29.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.141743+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:46:26.141743+0000","last_peered":"2023-10-09T04:46:26.141743+0000","last_clean":"2023-10-09T04:46:26.141743+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:46:26.141743+0000","last_undegraded":"2023-10-09T04:46:26.141743+0000","last_fullsized":"2023-10-09T04:46:26.141743+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":160,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.143000+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:46:26.143000+0000","last_peered":"2023-10-09T04:46:26.143000+0000","last_clean":"2023-10-09T04:46:26.143000+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:46:26.143000+0000","last_undegraded":"2023-10-09T04:46:26.143000+0000","last_fullsized":"2023-10-09T04:46:26.143000+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":279,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-10-09T04:46:36.393992+0000","last_change":"2023-10-09T04:46:25.160157+0000","last_active":"2023-10-09T04:46:36.393992+0000","last_peered":"2023-10-09T04:46:36.393992+0000","last_clean":"2023-10-09T04:46:36.393992+0000","last_became_active":"2023-10-09T04:46:25.159850+0000","last_became_peered":"2023-10-09T04:46:25.159850+0000","last_unstale":"2023-10-09T04:46:36.393992+0000","last_undegraded":"2023-10-09T04:46:36.393992+0000","last_fullsized":"2023-10-09T04:46:36.393992+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:07.475665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":300,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.369294+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:46:26.369294+0000","last_peered":"2023-10-09T04:46:26.369294+0000","last_clean":"2023-10-09T04:46:26.369294+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:46:26.369294+0000","last_undegraded":"2023-10-09T04:46:26.369294+0000","last_fullsized":"2023-10-09T04:46:26.369294+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":68,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.142593+0000","last_change":"2023-10-09T04:45:45.609849+0000","last_active":"2023-10-09T04:46:26.142593+0000","last_peered":"2023-10-09T04:46:26.142593+0000","last_clean":"2023-10-09T04:46:26.142593+0000","last_became_active":"2023-10-09T04:45:45.609639+0000","last_became_peered":"2023-10-09T04:45:45.609639+0000","last_unstale":"2023-10-09T04:46:26.142593+0000","last_undegraded":"2023-10-09T04:46:26.142593+0000","last_fullsized":"2023-10-09T04:46:26.142593+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:26.113652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":269,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139648+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:46:26.139648+0000","last_peered":"2023-10-09T04:46:26.139648+0000","last_clean":"2023-10-09T04:46:26.139648+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:46:26.139648+0000","last_undegraded":"2023-10-09T04:46:26.139648+0000","last_fullsized":"2023-10-09T04:46:26.139648+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.144775+0000","last_change":"2023-10-09T04:46:25.172010+0000","last_active":"2023-10-09T04:46:26.144775+0000","last_peered":"2023-10-09T04:46:26.144775+0000","last_clean":"2023-10-09T04:46:26.144775+0000","last_became_active":"2023-10-09T04:46:25.171776+0000","last_became_peered":"2023-10-09T04:46:25.171776+0000","last_unstale":"2023-10-09T04:46:26.144775+0000","last_undegraded":"2023-10-09T04:46:26.144775+0000","last_fullsized":"2023-10-09T04:46:26.144775+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:40.329129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":19,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.132563+0000","last_change":"2023-10-09T04:45:45.611105+0000","last_active":"2023-10-09T04:46:26.132563+0000","last_peered":"2023-10-09T04:46:26.132563+0000","last_clean":"2023-10-09T04:46:26.132563+0000","last_became_active":"2023-10-09T04:45:45.610758+0000","last_became_peered":"2023-10-09T04:45:45.610758+0000","last_unstale":"2023-10-09T04:46:26.132563+0000","last_undegraded":"2023-10-09T04:46:26.132563+0000","last_fullsized":"2023-10-09T04:46:26.132563+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:59.031709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":258,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.133145+0000","last_change":"2023-10-09T04:46:25.153824+0000","last_active":"2023-10-09T04:46:26.133145+0000","last_peered":"2023-10-09T04:46:26.133145+0000","last_clean":"2023-10-09T04:46:26.133145+0000","last_became_active":"2023-10-09T04:46:25.153497+0000","last_became_peered":"2023-10-09T04:46:25.153497+0000","last_unstale":"2023-10-09T04:46:26.133145+0000","last_undegraded":"2023-10-09T04:46:26.133145+0000","last_fullsized":"2023-10-09T04:46:26.133145+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:00.144937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":200,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.139710+0000","last_change":"2023-10-09T04:46:25.217389+0000","last_active":"2023-10-09T04:46:26.139710+0000","last_peered":"2023-10-09T04:46:26.139710+0000","last_clean":"2023-10-09T04:46:26.139710+0000","last_became_active":"2023-10-09T04:46:25.162993+0000","last_became_peered":"2023-10-09T04:46:25.162993+0000","last_unstale":"2023-10-09T04:46:26.139710+0000","last_undegraded":"2023-10-09T04:46:26.139710+0000","last_fullsized":"2023-10-09T04:46:26.139710+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:13.549717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-10-09T04:46:26.141739+0000","last_change":"2023-10-09T04:45:45.610893+0000","last_active":"2023-10-09T04:46:26.141739+0000","last_peered":"2023-10-09T04:46:26.141739+0000","last_clean":"2023-10-09T04:46:26.141739+0000","last_became_active":"2023-10-09T04:45:45.610260+0000","last_became_peered":"2023-10-09T04:45:45.610260+0000","last_unstale":"2023-10-09T04:46:26.141739+0000","last_undegraded":"2023-10-09T04:46:26.141739+0000","last_fullsized":"2023-10-09T04:46:26.141739+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:22.552117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4028891160,"num_objects":61477,"num_object_clones":0,"num_object_copies":184431,"num_objects_missing_on_primary":1541,"num_objects_missing":1541,"num_objects_degraded":49549,"num_objects_misplaced":28933,"num_objects_unfound":0,"num_objects_dirty":61477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129019,"num_write_kb":6095809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20123,"num_bytes_recovered":1308622848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13811986432,"data_stored":13811974216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94346,"ondisk_log_size":94346,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":11254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":17338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234478,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3032036,"kb_used_data":2133340,"kb_used_omap":0,"kb_used_meta":898688,"kb_avail":90713116,"statfs":{"total":95995035648,"available":92890230784,"internally_reserved":0,"allocated":2184540160,"data_stored":2183880884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920256512},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,2,0,2,1,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:40 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":1.262,"15min":3.903},"min":{"1min":0.25600000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.9139999999999997,"5min":11.144,"15min":186.446},"last":0.248},{"interface":"front","average":{"1min":2.496,"5min":1.919,"15min":4.3559999999999999},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":10.394,"5min":10.42,"15min":186.429},"last":0.871}]},{"osd":1,"last update":"Mon Oct 9 04:46:40 2023","interfaces":[{"interface":"back","average":{"1min":3.4380000000000002,"5min":3.1259999999999999,"15min":4.718},"min":{"1min":0.253,"5min":0.253,"15min":0.22800000000000001},"max":{"1min":47.82,"5min":92.046000000000006,"15min":264.62},"last":0.36799999999999999},{"interface":"front","average":{"1min":3.085,"5min":2.9670000000000001,"15min":4.8319999999999999},"min":{"1min":0.29699999999999999,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":47.701000000000001,"5min":91.593000000000004,"15min":264.65899999999999},"last":0.42799999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:46:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6760000000000002,"5min":3.8170000000000002,"15min":3.7599999999999998},"min":{"1min":0.34000000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":32.362000000000002,"5min":156.053,"15min":184.72},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.6779999999999999,"5min":4.1870000000000003,"15min":4.1310000000000002},"min":{"1min":0.37,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":32.377000000000002,"5min":156.036,"15min":184.697},"last":0.96999999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.6160000000000001,"15min":6.4850000000000003},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.032,"5min":11.339,"15min":249.989},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.4620000000000002,"5min":2.5910000000000002,"15min":6.9870000000000001},"min":{"1min":0.36299999999999999,"5min":0.33300000000000002,"15min":0.29799999999999999},"max":{"1min":11.117000000000001,"5min":15.837999999999999,"15min":250.21700000000001},"last":0.90200000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.306,"15min":1.403},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.0629999999999997,"5min":14.257999999999999,"15min":14.257999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.79100000000000004,"15min":1.145},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":2.9209999999999998,"5min":7.2290000000000001,"15min":10.613},"last":1.0580000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.288,"15min":1.133},"min":{"1min":0.315,"5min":0.28199999999999997,"15min":0.27900000000000003},"max":{"1min":2.9500000000000002,"5min":13.319000000000001,"15min":13.319000000000001},"last":0.497},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.7829999999999999,"15min":1.5700000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":5.1420000000000003,"5min":18.408000000000001,"15min":18.408000000000001},"last":0.83599999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":1.1240000000000001,"15min":1.25},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":2.1520000000000001,"5min":11.388,"15min":11.388},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.7709999999999999,"15min":1.6659999999999999},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":5.1200000000000001,"5min":18.311,"15min":18.311},"last":0.93799999999999994}]}]},{"osd":6,"up_from":0,"seq":197568496112,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141202,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2105224,"kb_used_data":1388928,"kb_used_omap":4,"kb_used_meta":716283,"kb_avail":91639928,"statfs":{"total":95995035648,"available":93839286272,"internally_reserved":0,"allocated":1422262272,"data_stored":1420927751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":733474702},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":2.109,"15min":3.867},"min":{"1min":0.22600000000000001,"5min":0.14799999999999999,"15min":0.14799999999999999},"max":{"1min":11.653,"5min":21.021999999999998,"15min":97.308000000000007},"last":0.318},{"interface":"front","average":{"1min":2.3940000000000001,"5min":2.9740000000000002,"15min":4.0650000000000004},"min":{"1min":0.33400000000000002,"5min":0.193,"15min":0.193},"max":{"1min":10.765000000000001,"5min":27.321000000000002,"15min":97.055999999999997},"last":0.58299999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:46:03 2023","interfaces":[{"interface":"back","average":{"1min":2.6920000000000002,"5min":6.7140000000000004,"15min":4.0220000000000002},"min":{"1min":0.28100000000000003,"5min":0.21199999999999999,"15min":0.20899999999999999},"max":{"1min":11.545999999999999,"5min":263.60500000000002,"15min":263.60500000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":3.157,"5min":6.7370000000000001,"15min":4.4649999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":21.646000000000001,"5min":263.65199999999999,"15min":263.65199999999999},"last":0.26100000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:46:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":5.4729999999999999,"15min":5.7169999999999996},"min":{"1min":0.29799999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5780000000000003,"5min":379.80900000000003,"15min":379.80900000000003},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.694,"5min":5.6020000000000003,"15min":6.4029999999999996},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":10.442,"5min":379.95800000000003,"15min":379.95800000000003},"last":2.8370000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":3.177,"5min":2.1419999999999999,"15min":2.5579999999999998},"min":{"1min":0.34000000000000002,"5min":0.307,"15min":0.245},"max":{"1min":19.959,"5min":19.959,"15min":170.072},"last":2.3319999999999999},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.726,"15min":2.3130000000000002},"min":{"1min":0.35899999999999999,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":11.598000000000001,"5min":12.297000000000001,"15min":169.94499999999999},"last":7.7450000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":2.0529999999999999,"15min":2.4929999999999999},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":6.3840000000000003,"5min":11.499000000000001,"15min":171.036},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.391,"5min":2.254,"15min":2.734},"min":{"1min":0.44700000000000001,"5min":0.36399999999999999,"15min":0.27500000000000002},"max":{"1min":20.021000000000001,"5min":20.021000000000001,"15min":170.88},"last":0.56399999999999995}]},{"osd":6,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":3.3010000000000002,"15min":2.5049999999999999},"min":{"1min":0.35799999999999998,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":6.4290000000000003,"5min":132.36099999999999,"15min":132.36099999999999},"last":6.5389999999999997},{"interface":"front","average":{"1min":3.363,"5min":3.3719999999999999,"15min":2.7429999999999999},"min":{"1min":0.46999999999999997,"5min":0.39400000000000002,"15min":0.317},"max":{"1min":20.164999999999999,"5min":132.17699999999999,"15min":132.17699999999999},"last":6.5899999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:45:52 2023","interfaces":[{"interface":"back","average":{"1min":3.141,"5min":3.556,"15min":3.278},"min":{"1min":0.45700000000000002,"5min":0.42599999999999999,"15min":0.34100000000000003},"max":{"1min":20.055,"5min":132.136,"15min":174.75999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.629,"5min":2.8969999999999998,"15min":2.9159999999999999},"min":{"1min":0.38800000000000001,"5min":0.377,"15min":0.36199999999999999},"max":{"1min":11.627000000000001,"5min":132.12,"15min":174.779},"last":0.72099999999999997}]}]},{"osd":0,"up_from":0,"seq":390842024259,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772535,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2892548,"kb_used_data":2447612,"kb_used_omap":7,"kb_used_meta":444664,"kb_avail":90852604,"statfs":{"total":95995035648,"available":93033066496,"internally_reserved":0,"allocated":2506354688,"data_stored":2505203943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":455336017},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":2.2549999999999999,"15min":6.1390000000000002},"min":{"1min":0.188,"5min":0.183,"15min":0.183},"max":{"1min":14.513999999999999,"5min":14.513999999999999,"15min":72.608000000000004},"last":0.20699999999999999},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.7609999999999999,"15min":5.7409999999999997},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":11.510999999999999,"5min":11.526,"15min":72.632000000000005},"last":0.24399999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6400000000000001,"5min":5.077,"15min":4.5899999999999999},"min":{"1min":0.29799999999999999,"5min":0.217,"15min":0.217},"max":{"1min":12.159000000000001,"5min":121.848,"15min":121.848},"last":1.9350000000000001},{"interface":"front","average":{"1min":2.9409999999999998,"5min":5.0570000000000004,"15min":4.9980000000000002},"min":{"1min":0.35099999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.5009999999999994,"5min":121.827,"15min":121.827},"last":1.5609999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":9.1289999999999996,"15min":7.3879999999999999},"min":{"1min":0.315,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":7.6289999999999996,"5min":225.85499999999999,"15min":335.928},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.7229999999999999,"5min":9.2650000000000006,"15min":7.4889999999999999},"min":{"1min":0.26100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":8.4440000000000008,"5min":233.53299999999999,"15min":335.68599999999998},"last":0.36799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":2.2290000000000001,"15min":3.5049999999999999},"min":{"1min":0.39700000000000002,"5min":0.34399999999999997,"15min":0.315},"max":{"1min":7.5250000000000004,"5min":13.641,"15min":72.944999999999993},"last":4.6890000000000001},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.2599999999999998,"15min":3.5739999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.3460000000000001,"5min":14.183,"15min":73.054000000000002},"last":5.8630000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.073,"5min":2.5209999999999999,"15min":3.3959999999999999},"min":{"1min":0.33000000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.254,"5min":65.388000000000005,"15min":72.986000000000004},"last":2.6160000000000001},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.8300000000000001,"15min":3.77},"min":{"1min":0.42799999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":10.279,"5min":67.799000000000007,"15min":72.963999999999999},"last":2.5390000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":2.5600000000000001,"15min":7.1059999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33600000000000002},"max":{"1min":8.391,"5min":34.753,"15min":135.614},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.0979999999999999,"5min":2.3519999999999999,"15min":6.6310000000000002},"min":{"1min":0.45200000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.279999999999999,"5min":34.796999999999997,"15min":130.196},"last":0.51900000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:46:02 2023","interfaces":[{"interface":"back","average":{"1min":2.4790000000000001,"5min":2.7029999999999998,"15min":3.823},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.4160000000000004,"5min":28.126999999999999,"15min":73.079999999999998},"last":0.89600000000000002},{"interface":"front","average":{"1min":2.302,"5min":2.552,"15min":3.4239999999999999},"min":{"1min":0.441,"5min":0.379,"15min":0.379},"max":{"1min":10.211,"5min":28.106000000000002,"15min":73.069000000000003},"last":0.88700000000000001}]}]},{"osd":3,"up_from":0,"seq":609885356260,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019386,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1955744,"kb_used_data":1063704,"kb_used_omap":0,"kb_used_meta":892032,"kb_avail":91789408,"statfs":{"total":95995035648,"available":93992353792,"internally_reserved":0,"allocated":1089232896,"data_stored":1088577702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":913440768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,3,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1190000000000002,"5min":1.9690000000000001,"15min":5.1790000000000003},"min":{"1min":0.247,"5min":0.247,"15min":0.23000000000000001},"max":{"1min":9.1170000000000009,"5min":23.114999999999998,"15min":219.577},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.3359999999999999,"5min":1.55,"15min":5.0839999999999996},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6980000000000004,"5min":13.992000000000001,"15min":218.96000000000001},"last":10.061}]},{"osd":1,"last update":"Mon Oct 9 04:45:44 2023","interfaces":[{"interface":"back","average":{"1min":5.3159999999999998,"5min":3.0179999999999998,"15min":4.6840000000000002},"min":{"1min":0.28299999999999997,"5min":0.26400000000000001,"15min":0.24099999999999999},"max":{"1min":50.837000000000003,"5min":50.837000000000003,"15min":142.74799999999999},"last":1.946},{"interface":"front","average":{"1min":4.6150000000000002,"5min":3.1339999999999999,"15min":4.7850000000000001},"min":{"1min":0.29799999999999999,"5min":0.28100000000000003,"15min":0.27300000000000002},"max":{"1min":47.844999999999999,"5min":47.844999999999999,"15min":139.09700000000001},"last":0.54900000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:46:05 2023","interfaces":[{"interface":"back","average":{"1min":7.8200000000000003,"5min":5.4409999999999998,"15min":4.3330000000000002},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":98.221999999999994,"5min":195.267,"15min":195.267},"last":0.499},{"interface":"front","average":{"1min":7.3049999999999997,"5min":4.7750000000000004,"15min":3.8559999999999999},"min":{"1min":0.38700000000000001,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":94.219999999999999,"5min":195.15799999999999,"15min":195.15799999999999},"last":10.035}]},{"osd":3,"last update":"Mon Oct 9 04:46:23 2023","interfaces":[{"interface":"back","average":{"1min":10.170999999999999,"5min":5.0720000000000001,"15min":6.3109999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":141.334,"5min":146.02199999999999,"15min":357.125},"last":10.087},{"interface":"front","average":{"1min":11.423999999999999,"5min":5.1660000000000004,"15min":6.3979999999999997},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.312},"max":{"1min":141.18299999999999,"5min":147.75700000000001,"15min":359.02800000000002},"last":0.83099999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:46:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":1.26,"15min":1.974},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":24.234999999999999,"5min":24.234999999999999,"15min":107.476},"last":6.3540000000000001},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.1140000000000001,"15min":1.9410000000000001},"min":{"1min":0.33800000000000002,"5min":0.28599999999999998,"15min":0.23300000000000001},"max":{"1min":7.2999999999999998,"5min":15.394,"15min":107.414},"last":9.9909999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0859999999999999,"5min":1.387,"15min":1.794},"min":{"1min":0.36799999999999999,"5min":0.29799999999999999,"15min":0.29599999999999999},"max":{"1min":22.920999999999999,"5min":22.920999999999999,"15min":65.75},"last":3.0579999999999998},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.244,"15min":1.7190000000000001},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.0380000000000003,"5min":9.782,"15min":65.849999999999994},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:46:12 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.1970000000000001,"15min":1.669},"min":{"1min":0.35199999999999998,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":7.3200000000000003,"5min":8.6120000000000001,"15min":76.283000000000001},"last":11.493},{"interface":"front","average":{"1min":2.4449999999999998,"5min":1.7110000000000001,"15min":1.798},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":22.858000000000001,"5min":22.858000000000001,"15min":76.259},"last":0.80500000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757750,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528776,"kb_used_data":638400,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":92216376,"statfs":{"total":95995035648,"available":94429569024,"internally_reserved":0,"allocated":653721600,"data_stored":653086102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":4.0979999999999999,"15min":5.7439999999999998},"min":{"1min":0.27900000000000003,"5min":0.23799999999999999,"15min":0.22},"max":{"1min":8.0860000000000003,"5min":163.70699999999999,"15min":210.85499999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.7,"5min":4.569,"15min":5.8940000000000001},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.24099999999999999},"max":{"1min":5.8840000000000003,"5min":163.756,"15min":210.23500000000001},"last":0.27800000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:45:48 2023","interfaces":[{"interface":"back","average":{"1min":6.8899999999999997,"5min":3.9060000000000001,"15min":4.859},"min":{"1min":0.35499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":69.244,"5min":73.296000000000006,"15min":145.577},"last":10.140000000000001},{"interface":"front","average":{"1min":6.7400000000000002,"5min":4.0110000000000001,"15min":5.1589999999999998},"min":{"1min":0.35499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":69.459999999999994,"5min":73.045000000000002,"15min":145.792},"last":0.86199999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:46:00 2023","interfaces":[{"interface":"back","average":{"1min":7.0339999999999998,"5min":4.6559999999999997,"15min":5.508},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":42.121000000000002,"5min":66.984999999999999,"15min":90.763999999999996},"last":10.19},{"interface":"front","average":{"1min":7.5030000000000001,"5min":4.8739999999999997,"15min":5.6859999999999999},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":40.133000000000003,"5min":67.686000000000007,"15min":91.340000000000003},"last":0.89400000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:46:20 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":3.5329999999999999,"15min":6.5599999999999996},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":13.711,"5min":67.129999999999995,"15min":268.63299999999998},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.88500000000000001,"5min":2.677,"15min":6.125},"min":{"1min":0.43099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.1389999999999998,"5min":67.189999999999998,"15min":268.92500000000001},"last":10.227}]},{"osd":4,"last update":"Mon Oct 9 04:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.65300000000000002,"15min":2.3540000000000001},"min":{"1min":0.307,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.351,"5min":4.6929999999999996,"15min":361.09399999999999},"last":10.268000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.94399999999999995,"15min":2.794},"min":{"1min":0.34799999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.944,"5min":7.4859999999999998,"15min":361.71300000000002},"last":10.384}]},{"osd":6,"last update":"Mon Oct 9 04:45:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.82199999999999995,"15min":1.228},"min":{"1min":0.32900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.0730000000000004,"5min":13.162000000000001,"15min":13.162000000000001},"last":10.295999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.1259999999999999,"15min":1.3520000000000001},"min":{"1min":0.35799999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.169,"5min":13.231999999999999,"15min":13.231999999999999},"last":0.35099999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:46:19 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.121,"15min":1.3500000000000001},"min":{"1min":0.40000000000000002,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":3.391,"5min":13.268000000000001,"15min":13.268000000000001},"last":2.5960000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.875,"15min":1.2290000000000001},"min":{"1min":0.41999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.4880000000000004,"5min":5.3040000000000003,"15min":15.257999999999999},"last":10.321999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1607270400,"data_stored":1607270400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1412632576,"data_stored":1412628504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2499219456,"data_stored":2499215384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2373255168,"data_stored":2373255168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1085865984,"data_stored":1085865984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":651165696,"data_stored":651165696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2002583552,"data_stored":2002583552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2179993600,"data_stored":2179989528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:46:53.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_7 pgp_num 16 2023-10-09T04:46:53.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:53.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:53.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:53.685 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:53.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:53.761 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:53.852 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:46:53.852 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:46:53.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36225.0:131922 12.2 e313/303 12:4901c654:::benchmark_data_smithi029_442458_object36672:head v 313'8249, mlcod=288'1665)", 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.845142+0000", 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0048242340000000002, 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048570990000000001, 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845142+0000", 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845142+0000", 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845141+0000", 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845151+0000", 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4120000000000007e-06 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845163+0000", 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2713e-05 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:53.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845170+0000", 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5400000000000001e-06 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845277+0000", 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107431 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845304+0000", 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6491000000000001e-05 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.856 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36225.0:131927 12.e e313/308 12:7015dadb:::benchmark_data_smithi029_442458_object36677:head v 313'8170, mlcod=313'8168)", 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.845188+0000", 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0047783349999999999, 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048772210000000002, 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.857 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845188+0000", 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845188+0000", 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845188+0000", 2023-10-09T04:46:53.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845194+0000", 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.801e-06 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845201+0000", 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3350000000000001e-06 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845206+0000", 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8899999999999998e-06 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.848690+0000", 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003484502 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:53.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.848717+0000", 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7050999999999999e-05 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131910 12.3 12:cf05f2fd:::benchmark_data_smithi029_442458_object36660:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.812317+0000", 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.037649576999999997, 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.037815493999999998, 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:46:53.861 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131910 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812317+0000", 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812317+0000", 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812316+0000", 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812322+0000", 2023-10-09T04:46:53.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0329999999999997e-06 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812334+0000", 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2492e-05 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812340+0000", 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4210000000000003e-06 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.812401+0000", 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1016e-05 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.813187+0000", 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00078625199999999998 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.825040+0000", 2023-10-09T04:46:53.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011852791999999999 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.825153+0000", 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011291299999999999 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.825321+0000", 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000168111 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.839906+0000", 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014585646000000001 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.840759+0000", 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000852475 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36225.0:131916 12.c e313/308 12:3e3ce2cd:::benchmark_data_smithi029_442458_object36666:head v 313'8261, mlcod=289'2333)", 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.834548+0000", 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.015418507, 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015680803, 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.834548+0000", 2023-10-09T04:46:53.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.834548+0000", 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.834548+0000", 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.834553+0000", 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0640000000000001e-06 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.834558+0000", 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4820000000000002e-06 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:53.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.834562+0000", 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.67e-06 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.837703+0000", 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031408289999999999 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.837728+0000", 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4871e-05 2023-10-09T04:46:53.871 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131925 12.3 12:c69aa10d:::benchmark_data_smithi029_442458_object36675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.844476+0000", 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0054902730000000004, 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058147349999999997, 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for degraded object", 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:46:53.872 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131925 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844476+0000", 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:46:53.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844476+0000", 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844474+0000", 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844483+0000", 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9809999999999992e-06 2023-10-09T04:46:53.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844514+0000", 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1043000000000001e-05 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844523+0000", 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8040000000000007e-06 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:46:53.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.844579+0000", 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6140999999999997e-05 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.845494+0000", 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00091481799999999997 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:46:53.876 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 5 2023-10-09T04:46:53.877 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:46:53.884 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:53 smithi177 ceph-mon[195797]: osdmap e312: 8 total, 8 up, 5 in 2023-10-09T04:46:53.884 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:53 smithi177 ceph-mon[195797]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.884 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:53 smithi177 ceph-mon[195797]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.884 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:53 smithi177 ceph-mon[195797]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.885 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:53 smithi177 ceph-mon[195797]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.885 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:53 smithi177 ceph-mon[195797]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[216903]: osdmap e312: 8 total, 8 up, 5 in 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[216903]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[216903]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[216903]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[216903]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[216903]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[223858]: osdmap e312: 8 total, 8 up, 5 in 2023-10-09T04:46:53.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[223858]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[223858]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[223858]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[223858]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:53 smithi029 ceph-mon[223858]: failed to encode map e312 with expected crc 2023-10-09T04:46:53.979 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:53.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:54.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:54.149 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:54.227 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:54.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:54.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:54.366 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:54.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:46:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:54.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:46:54.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:54.581 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:54.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:54 smithi177 ceph-mon[195797]: pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 306 op/s; 48197/179178 objects degraded (26.899%); 27257/179178 objects misplaced (15.212%); 19 MiB/s, 379 objects/s recovering 2023-10-09T04:46:54.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:54 smithi177 ceph-mon[195797]: failed to encode map e313 with expected crc 2023-10-09T04:46:54.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:54 smithi177 ceph-mon[195797]: osdmap e313: 8 total, 8 up, 5 in 2023-10-09T04:46:54.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:54 smithi177 ceph-mon[195797]: failed to encode map e313 with expected crc 2023-10-09T04:46:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[216903]: pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 306 op/s; 48197/179178 objects degraded (26.899%); 27257/179178 objects misplaced (15.212%); 19 MiB/s, 379 objects/s recovering 2023-10-09T04:46:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[216903]: failed to encode map e313 with expected crc 2023-10-09T04:46:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[216903]: osdmap e313: 8 total, 8 up, 5 in 2023-10-09T04:46:54.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[216903]: failed to encode map e313 with expected crc 2023-10-09T04:46:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[223858]: pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean, 3 active+recovery_wait+degraded+remapped; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 306 op/s; 48197/179178 objects degraded (26.899%); 27257/179178 objects misplaced (15.212%); 19 MiB/s, 379 objects/s recovering 2023-10-09T04:46:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[223858]: failed to encode map e313 with expected crc 2023-10-09T04:46:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[223858]: osdmap e313: 8 total, 8 up, 5 in 2023-10-09T04:46:54.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:54 smithi029 ceph-mon[223858]: failed to encode map e313 with expected crc 2023-10-09T04:46:55.005 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:46:55.117 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:46:55.117 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:46:55.123 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:55.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:55.180 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:46:55.220 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:46:55.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:46:55.267 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:46:55.290 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:55.295 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:46:55.339 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:46:55.374 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:46:55.391 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.391 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.391 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.391 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.391 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.392 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.392 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.392 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:46:55.644 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:46:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:55.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:46:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[223858]: osdmap e314: 8 total, 8 up, 5 in 2023-10-09T04:46:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[223858]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[223858]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[223858]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2590334650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 48197/179178 objects degraded (26.899%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:55.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[216903]: osdmap e314: 8 total, 8 up, 5 in 2023-10-09T04:46:55.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[216903]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[216903]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[216903]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2590334650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:55.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 48197/179178 objects degraded (26.899%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:55.676 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:46:55.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:55.861 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:55.963 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:55.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:55.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:55 smithi177 ceph-mon[195797]: osdmap e314: 8 total, 8 up, 5 in 2023-10-09T04:46:55.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:55 smithi177 ceph-mon[195797]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:55 smithi177 ceph-mon[195797]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:55 smithi177 ceph-mon[195797]: failed to encode map e314 with expected crc 2023-10-09T04:46:55.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2590334650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:46:55.970 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 48197/179178 objects degraded (26.899%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:46:56.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:56.097 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:56.635 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:56.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:56.741 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:46:56.753 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:46:56.780 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:56.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:46:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:56.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:46:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:56 smithi029 ceph-mon[216903]: pgmap v1223: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean, 3 active+recovery_wait+degraded+remapped; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 476 op/s; 42303/175920 objects degraded (24.047%); 30356/175920 objects misplaced (17.256%); 19 MiB/s, 451 objects/s recovering 2023-10-09T04:46:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:56 smithi029 ceph-mon[223858]: pgmap v1223: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean, 3 active+recovery_wait+degraded+remapped; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 476 op/s; 42303/175920 objects degraded (24.047%); 30356/175920 objects misplaced (17.256%); 19 MiB/s, 451 objects/s recovering 2023-10-09T04:46:56.972 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:56.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:56 smithi177 ceph-mon[195797]: pgmap v1223: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean, 3 active+recovery_wait+degraded+remapped; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 476 op/s; 42303/175920 objects degraded (24.047%); 30356/175920 objects misplaced (17.256%); 19 MiB/s, 451 objects/s recovering 2023-10-09T04:46:56.999 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:57.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:57.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:57.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:57.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:46:57.246 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:57.471 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:46:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:57.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:46:57.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:46:57.628 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:57.724 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:46:57.745 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:46:57.748 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3759025933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:57.748 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:57 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:57.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3759025933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:57.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:57 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:57.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:46:57.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3759025933' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:57.783 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:57 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:46:57.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:57.881 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:57.983 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:57.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:58.088 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:46:58.174 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:58.242 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:46:58.243 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:46:58.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:46:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:58.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:46:58.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:46:58.389 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:46:58.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: pgmap v1224: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 375 op/s; 41820/174447 objects degraded (23.973%); 30061/174447 objects misplaced (17.232%); 19 MiB/s, 357 objects/s recovering 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: osdmap e315: 8 total, 8 up, 5 in 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:58.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:46:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:58.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:46:58.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: pgmap v1224: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 375 op/s; 41820/174447 objects degraded (23.973%); 30061/174447 objects misplaced (17.232%); 19 MiB/s, 357 objects/s recovering 2023-10-09T04:46:58.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:46:58.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:58.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: osdmap e315: 8 total, 8 up, 5 in 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: pgmap v1224: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 375 op/s; 41820/174447 objects degraded (23.973%); 30061/174447 objects misplaced (17.232%); 19 MiB/s, 357 objects/s recovering 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:46:58.979 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: osdmap e315: 8 total, 8 up, 5 in 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: failed to encode map e315 with expected crc 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:46:58.980 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:46:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:46:59.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:46:59.822 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:46:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:46:59.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:46:59.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:46:59.989 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:00.103 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:00.106 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:47:00.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:00.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:00.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:00.280 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:00.356 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:00.450 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:00.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:00.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:00.661 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:00.741 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:00 smithi029 ceph-mon[216903]: pgmap v1226: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 279 op/s; 41820/174138 objects degraded (24.015%); 29871/174138 objects misplaced (17.154%); 8.6 MiB/s, 198 objects/s recovering 2023-10-09T04:47:00.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 41820/174138 objects degraded (24.015%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:00.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:00 smithi029 ceph-mon[223858]: pgmap v1226: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 279 op/s; 41820/174138 objects degraded (24.015%); 29871/174138 objects misplaced (17.154%); 8.6 MiB/s, 198 objects/s recovering 2023-10-09T04:47:00.742 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 41820/174138 objects degraded (24.015%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:00.792 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:00.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:00.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:00.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:00.953 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:00 smithi177 ceph-mon[195797]: pgmap v1226: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 279 op/s; 41820/174138 objects degraded (24.015%); 29871/174138 objects misplaced (17.154%); 8.6 MiB/s, 198 objects/s recovering 2023-10-09T04:47:00.953 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 41820/174138 objects degraded (24.015%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:00.965 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:01.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:01.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:01.703 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 12 pgp_num to 16 2023-10-09T04:47:01.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:01.774 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:01.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:47:02.022 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:02.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4246871004' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:47:02.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:01 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:47:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:02.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:02.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4246871004' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:47:02.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:01 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:47:02.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4246871004' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:47:02.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:01 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-09T04:47:02.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:02.195 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:02.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:02.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:02.400 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:02.459 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:02.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:02 smithi177 ceph-mon[195797]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 378 op/s; 41581/170478 objects degraded (24.391%); 29211/170478 objects misplaced (17.135%); 10 MiB/s, 211 objects/s recovering 2023-10-09T04:47:02.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:02 smithi177 ceph-mon[195797]: failed to encode map e316 with expected crc 2023-10-09T04:47:02.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:02 smithi177 ceph-mon[195797]: failed to encode map e316 with expected crc 2023-10-09T04:47:02.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:47:02.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:02 smithi177 ceph-mon[195797]: osdmap e316: 8 total, 8 up, 5 in 2023-10-09T04:47:02.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:02 smithi177 ceph-mon[195797]: failed to encode map e316 with expected crc 2023-10-09T04:47:02.800 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:47:02.800 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:47:03.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[216903]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 378 op/s; 41581/170478 objects degraded (24.391%); 29211/170478 objects misplaced (17.135%); 10 MiB/s, 211 objects/s recovering 2023-10-09T04:47:03.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[216903]: failed to encode map e316 with expected crc 2023-10-09T04:47:03.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[216903]: failed to encode map e316 with expected crc 2023-10-09T04:47:03.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[216903]: osdmap e316: 8 total, 8 up, 5 in 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[216903]: failed to encode map e316 with expected crc 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[223858]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 378 op/s; 41581/170478 objects degraded (24.391%); 29211/170478 objects misplaced (17.135%); 10 MiB/s, 211 objects/s recovering 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[223858]: failed to encode map e316 with expected crc 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[223858]: failed to encode map e316 with expected crc 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[223858]: osdmap e316: 8 total, 8 up, 5 in 2023-10-09T04:47:03.077 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:02 smithi029 ceph-mon[223858]: failed to encode map e316 with expected crc 2023-10-09T04:47:03.078 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:47:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:47:02] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:47:03.335 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:03.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:03.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:03.496 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:03.589 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:03.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:03.695 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:03.792 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:03.822 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:03.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:03.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:03.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:03.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:47:03.980 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:47:04.026 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:47:04.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:04.065 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:47:04.113 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:47:04.135 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:04.158 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:47:04.203 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:47:04.242 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:47:04.271 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:47:04.277 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:04.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:04.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.313 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.314 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.314 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:47:04.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:04.439 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:04.544 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:04.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:04.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:04.707 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:04 smithi177 ceph-mon[195797]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 323 op/s; 40927/168153 objects degraded (24.339%); 28804/168153 objects misplaced (17.130%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:47:04.707 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3090147859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:04.761 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:04.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:04.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:04.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:04 smithi029 ceph-mon[223858]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 323 op/s; 40927/168153 objects degraded (24.339%); 28804/168153 objects misplaced (17.130%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:47:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3090147859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:04 smithi029 ceph-mon[216903]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 323 op/s; 40927/168153 objects degraded (24.339%); 28804/168153 objects misplaced (17.130%); 10 MiB/s, 162 objects/s recovering 2023-10-09T04:47:04.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3090147859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:04.988 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:05.101 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:05.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:05.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:05.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:05.302 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:05.337 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:05.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:05.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:05.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:05.531 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:05.731 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40927/168153 objects degraded (24.339%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:05.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40927/168153 objects degraded (24.339%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:05.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40927/168153 objects degraded (24.339%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:05.883 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:05.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:05.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:06.020 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:06.749 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:06.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:06.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:06.851 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:06.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:06.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:06 smithi029 ceph-mon[216903]: pgmap v1230: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 422 op/s; 40690/164298 objects degraded (24.766%); 28093/164298 objects misplaced (17.099%); 11 MiB/s, 182 objects/s recovering 2023-10-09T04:47:06.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:06 smithi029 ceph-mon[223858]: pgmap v1230: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 422 op/s; 40690/164298 objects degraded (24.766%); 28093/164298 objects misplaced (17.099%); 11 MiB/s, 182 objects/s recovering 2023-10-09T04:47:06.954 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:07.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:06 smithi177 ceph-mon[195797]: pgmap v1230: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 422 op/s; 40690/164298 objects degraded (24.766%); 28093/164298 objects misplaced (17.099%); 11 MiB/s, 182 objects/s recovering 2023-10-09T04:47:07.297 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:07.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:07.401 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:07.470 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:07.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:07.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:07.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:07.764 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:07.870 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:07.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:07.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:08.034 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:08.124 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:08.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:08.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:08.296 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:08.346 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:08.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:08.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:08.480 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 5, 7] out_osds: [6, 3, 0] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:47:08.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:47:08.482 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:08.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:08.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:08.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:08.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:47:08.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:08.734 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:08.771 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:47:08.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:08 smithi029 ceph-mon[223858]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 40073/162402 objects degraded (24.675%); 27679/162402 objects misplaced (17.044%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:47:08.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2523422510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:47:08.813 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:08 smithi029 ceph-mon[216903]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 40073/162402 objects degraded (24.675%); 27679/162402 objects misplaced (17.044%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:47:08.813 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2523422510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:47:08.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:08.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:08.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:08 smithi177 ceph-mon[195797]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 40073/162402 objects degraded (24.675%); 27679/162402 objects misplaced (17.044%); 12 MiB/s, 190 objects/s recovering 2023-10-09T04:47:08.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2523422510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:47:08.953 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:09.013 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:09.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:09.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:09.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:09.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:09.306 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:10.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:09 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:10.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2523422510' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:47:10.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:09 smithi177 ceph-mon[195797]: osdmap e317: 8 total, 8 up, 5 in 2023-10-09T04:47:10.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:09 smithi177 ceph-mon[195797]: failed to encode map e317 with expected crc 2023-10-09T04:47:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2523422510' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:47:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[223858]: osdmap e317: 8 total, 8 up, 5 in 2023-10-09T04:47:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[223858]: failed to encode map e317 with expected crc 2023-10-09T04:47:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2523422510' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:47:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[216903]: osdmap e317: 8 total, 8 up, 5 in 2023-10-09T04:47:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:09 smithi029 ceph-mon[216903]: failed to encode map e317 with expected crc 2023-10-09T04:47:10.246 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:10.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:10.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:10.446 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:10.552 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:10.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:10.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:10.605 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:10.653 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:10.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:10.712 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:47:10.712 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:47:10.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.36225.0:138886 12.9 e317/303 ondisk, result = 0)", 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.700316+0000", 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0099905099999999993, 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010018815, 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.700316+0000", 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.700316+0000", 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.700315+0000", 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.700318+0000", 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.57e-06 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.700326+0000", 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5229999999999995e-06 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.700332+0000", 2023-10-09T04:47:10.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.169e-06 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708843+0000", 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085116970000000004 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708851+0000", 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9820000000000008e-06 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.716 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:138823 12.c e317/308 12:3693590a:::benchmark_data_smithi029_442458_object43573:head v 317'8724, mlcod=289'2784)", 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.706751+0000", 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0035546789999999998, 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003654058, 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706751+0000", 2023-10-09T04:47:10.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706751+0000", 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706750+0000", 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706764+0000", 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4266e-05 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706778+0000", 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3929e-05 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706788+0000", 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9869999999999994e-06 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706831+0000", 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3423000000000001e-05 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.706860+0000", 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8813e-05 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138912 12.9 12.c71d0549 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.710393+0000", 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 4294967295.9999127, 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5113999999999999e-05, 2023-10-09T04:47:10.721 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138912 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.710393+0000", 2023-10-09T04:47:10.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.710393+0000", 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.710392+0000", 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.710400+0000", 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1620000000000008e-06 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.710413+0000", 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3488e-05 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.710422+0000", 2023-10-09T04:47:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4370000000000007e-06 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138896 12.9 12.cf1b6e9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.667174+0000", 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.043132115999999998, 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043353338999999998, 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:47:10.726 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138896 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.667174+0000", 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.667174+0000", 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.667174+0000", 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.667177+0000", 2023-10-09T04:47:10.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1779999999999999e-06 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.667183+0000", 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9710000000000001e-06 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.667187+0000", 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3209999999999998e-06 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.729 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138886 12.9 12:94aa1ea5:::benchmark_data_smithi029_442458_object43636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.640362+0000", 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.069944574999999995, 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.070228360000000004, 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.730 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138886 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.640362+0000", 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.640362+0000", 2023-10-09T04:47:10.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.640360+0000", 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.640367+0000", 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5089999999999996e-06 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.640382+0000", 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4839e-05 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.640392+0000", 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0259e-05 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650360+0000", 2023-10-09T04:47:10.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099680340000000006 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650951+0000", 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059069099999999996 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.689667+0000", 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038716069999999998 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.689782+0000", 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114691 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.689952+0000", 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000170268 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.705346+0000", 2023-10-09T04:47:10.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015393946 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708816+0000", 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003469597 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708859+0000", 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3129e-05 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708933+0000", 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4492000000000004e-05 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138900 12.5 12:ac480bbb:::benchmark_data_smithi029_442458_object43650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.675898+0000", 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.034408444000000003, 2023-10-09T04:47:10.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.034804845000000001, 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138900 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675898+0000", 2023-10-09T04:47:10.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675898+0000", 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675896+0000", 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675903+0000", 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1110000000000002e-06 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675916+0000", 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2554e-05 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675923+0000", 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6000000000000001e-06 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.678704+0000", 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027805339999999999 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.678867+0000", 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016294899999999999 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679023+0000", 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000156532 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.698591+0000", 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019567523 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:10.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708929+0000", 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010338388 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138901 12.5 12:a42a54e0:::benchmark_data_smithi029_442458_object43651:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.675934+0000", 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.034371997000000001, 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.034873500000000002, 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:47:10.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138901 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675934+0000", 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675934+0000", 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675934+0000", 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675936+0000", 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5000000000000002e-06 2023-10-09T04:47:10.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675942+0000", 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7780000000000004e-06 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.675947+0000", 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5510000000000001e-06 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.683731+0000", 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077848659999999997 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.683887+0000", 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015569800000000001 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.684046+0000", 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015847800000000001 2023-10-09T04:47:10.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.698595+0000", 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014549849 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708974+0000", 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010378373 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.748 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138903 12.5 12:ab238cb3:::benchmark_data_smithi029_442458_object43653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.679798+0000", 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.030508038000000001, 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031143974000000001, 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138903 2023-10-09T04:47:10.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679798+0000", 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679798+0000", 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679796+0000", 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679804+0000", 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9300000000000003e-06 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679822+0000", 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7621e-05 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.679830+0000", 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2339999999999994e-06 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.695126+0000", 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015295908 2023-10-09T04:47:10.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.695302+0000", 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017618500000000001 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.695480+0000", 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017760399999999999 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.708870+0000", 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013390658 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.709379+0000", 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00050825999999999996 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:138890 12.9 12:9fcef406:::benchmark_data_smithi029_442458_object43640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.650203+0000", 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.060103043000000002, 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.060866768000000002, 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 138890 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:10.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650203+0000", 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650203+0000", 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:10.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650202+0000", 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650209+0000", 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7230000000000004e-06 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650221+0000", 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1205e-05 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650229+0000", 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2700000000000004e-06 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.650984+0000", 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00075533999999999996 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:47:10.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.651781+0000", 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000796878 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.701611+0000", 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.049830147999999998 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.701738+0000", 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000126897 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:10.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.701923+0000", 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018438999999999999 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 9 2023-10-09T04:47:10.760 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:47:10.761 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:10 smithi029 ceph-mon[216903]: pgmap v1233: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 40073/161946 objects degraded (24.745%); 27679/161946 objects misplaced (17.091%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:47:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40073/161946 objects degraded (24.745%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:10.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:10 smithi029 ceph-mon[223858]: pgmap v1233: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 40073/161946 objects degraded (24.745%); 27679/161946 objects misplaced (17.091%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:47:10.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40073/161946 objects degraded (24.745%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:10.961 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:10.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:11.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:10 smithi177 ceph-mon[195797]: pgmap v1233: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 40073/161946 objects degraded (24.745%); 27679/161946 objects misplaced (17.091%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T04:47:11.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40073/161946 objects degraded (24.745%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:11.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:11.130 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:12.245 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:12.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:12.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:47:12.357 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:12.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:47:12.412 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:12.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:12.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:12.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:12.685 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:12.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:12.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:12.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:12.938 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:12 smithi177 ceph-mon[195797]: pgmap v1234: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 369 op/s; 39862/157863 objects degraded (25.251%); 26962/157863 objects misplaced (17.079%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:47:13.114 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:47:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:47:12] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-10-09T04:47:13.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:12 smithi029 ceph-mon[216903]: pgmap v1234: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 369 op/s; 39862/157863 objects degraded (25.251%); 26962/157863 objects misplaced (17.079%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:47:13.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:12 smithi029 ceph-mon[223858]: pgmap v1234: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 369 op/s; 39862/157863 objects degraded (25.251%); 26962/157863 objects misplaced (17.079%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T04:47:13.123 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:13.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:13.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:13.296 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:13.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:13.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:13.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:13.510 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:14.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:14.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:14.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:14.321 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:14 smithi029 ceph-mon[216903]: pgmap v1235: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 426 op/s; 39177/155340 objects degraded (25.220%); 26537/155340 objects misplaced (17.083%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:47:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:14 smithi029 ceph-mon[223858]: pgmap v1235: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 426 op/s; 39177/155340 objects degraded (25.220%); 26537/155340 objects misplaced (17.083%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:47:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:15.020 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:14 smithi177 ceph-mon[195797]: pgmap v1235: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.2 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 426 op/s; 39177/155340 objects degraded (25.220%); 26537/155340 objects misplaced (17.083%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:47:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:15.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:15.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:15.269 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:15.503 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:15.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:15.576 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:15.610 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:15.612 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:15.656 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:47:15.672 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:15.706 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:47:15.743 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:47:15.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:47:15.791 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:15.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:15.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:15.812 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:47:15.854 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:47:15.900 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:47:15.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:15.942 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:47:15.956 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.957 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:47:15.964 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:16.040 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:16.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:16.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:16.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:16.261 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:16.414 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:16.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:16.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:16.570 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:16.640 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:47:16.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:16.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:16.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:16.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:16 smithi177 ceph-mon[195797]: pgmap v1236: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.1 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 438 op/s; 38934/151131 objects degraded (25.762%); 25778/151131 objects misplaced (17.057%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:47:16.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38934/151131 objects degraded (25.762%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:16.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2989871911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:16.853 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2502721339' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:47:16.863 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[223858]: pgmap v1236: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.1 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 438 op/s; 38934/151131 objects degraded (25.762%); 25778/151131 objects misplaced (17.057%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:47:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38934/151131 objects degraded (25.762%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2989871911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2502721339' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:47:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[216903]: pgmap v1236: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.1 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 438 op/s; 38934/151131 objects degraded (25.762%); 25778/151131 objects misplaced (17.057%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:47:16.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38934/151131 objects degraded (25.762%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:16.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2989871911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:16.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2502721339' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:47:16.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:16.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:17.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:17.157 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:17.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:17.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:17.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:17.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:17.452 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:17.504 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:17.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:17.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:17.681 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:17.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2502721339' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:47:17.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:17 smithi177 ceph-mon[195797]: osdmap e318: 8 total, 8 up, 5 in 2023-10-09T04:47:17.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:17 smithi177 ceph-mon[195797]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:17 smithi177 ceph-mon[195797]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:17 smithi177 ceph-mon[195797]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:17 smithi177 ceph-mon[195797]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2502721339' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[216903]: osdmap e318: 8 total, 8 up, 5 in 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[216903]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[216903]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[216903]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[216903]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2502721339' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:47:17.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[223858]: osdmap e318: 8 total, 8 up, 5 in 2023-10-09T04:47:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[223858]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[223858]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[223858]: failed to encode map e318 with expected crc 2023-10-09T04:47:17.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:17 smithi029 ceph-mon[223858]: failed to encode map e318 with expected crc 2023-10-09T04:47:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:18 smithi029 ceph-mon[223858]: pgmap v1238: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.0 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 38186/148953 objects degraded (25.636%); 25339/148953 objects misplaced (17.011%); 13 MiB/s, 208 objects/s recovering 2023-10-09T04:47:18.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:18 smithi029 ceph-mon[216903]: pgmap v1238: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.0 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 38186/148953 objects degraded (25.636%); 25339/148953 objects misplaced (17.011%); 13 MiB/s, 208 objects/s recovering 2023-10-09T04:47:19.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:18 smithi177 ceph-mon[195797]: pgmap v1238: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.0 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 38186/148953 objects degraded (25.636%); 25339/148953 objects misplaced (17.011%); 13 MiB/s, 208 objects/s recovering 2023-10-09T04:47:20.170 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:20.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:20.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:20.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:20.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:47:20.329 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:20.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T04:47:20.646 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:20.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:20.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:20.746 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:20 smithi029 ceph-mon[216903]: pgmap v1239: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.0 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 445 op/s; 38186/148572 objects degraded (25.702%); 25339/148572 objects misplaced (17.055%); 11 MiB/s, 178 objects/s recovering 2023-10-09T04:47:20.747 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38186/148572 objects degraded (25.702%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:20.747 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:20 smithi029 ceph-mon[223858]: pgmap v1239: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.0 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 445 op/s; 38186/148572 objects degraded (25.702%); 25339/148572 objects misplaced (17.055%); 11 MiB/s, 178 objects/s recovering 2023-10-09T04:47:20.747 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38186/148572 objects degraded (25.702%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:20.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:20.822 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:20.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:20.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:20.925 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:20 smithi177 ceph-mon[195797]: pgmap v1239: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 3.0 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 445 op/s; 38186/148572 objects degraded (25.702%); 25339/148572 objects misplaced (17.055%); 11 MiB/s, 178 objects/s recovering 2023-10-09T04:47:20.925 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38186/148572 objects degraded (25.702%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:21.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:21.095 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:21.205 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:21.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:21.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:21.358 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:21.374 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:47:21.374 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:47:21.429 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:21.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:21.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:21.558 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:21.590 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:21.676 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:22.665 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:47:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:22 smithi029 ceph-mon[216903]: pgmap v1240: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.9 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 37952/144606 objects degraded (26.245%); 24519/144606 objects misplaced (16.956%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:47:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2878033422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:22 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:22 smithi029 ceph-mon[223858]: pgmap v1240: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.9 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 37952/144606 objects degraded (26.245%); 24519/144606 objects misplaced (16.956%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:47:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2878033422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:22 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:22.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:47:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:47:22] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:47:23.030 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:23.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:22 smithi177 ceph-mon[195797]: pgmap v1240: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.9 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 37952/144606 objects degraded (26.245%); 24519/144606 objects misplaced (16.956%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:47:23.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2878033422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:23.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:22 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:23.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:23.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:23.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:23.197 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:23.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:47:23.324 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:23.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:23.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:23.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:23.513 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: osdmap e319: 8 total, 8 up, 5 in 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[223858]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:23.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: osdmap e319: 8 total, 8 up, 5 in 2023-10-09T04:47:23.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: failed to encode map e319 with expected crc 2023-10-09T04:47:23.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:23 smithi029 ceph-mon[216903]: failed to encode map e319 with expected crc 2023-10-09T04:47:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:24.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:24.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: osdmap e319: 8 total, 8 up, 5 in 2023-10-09T04:47:24.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: failed to encode map e319 with expected crc 2023-10-09T04:47:24.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: failed to encode map e319 with expected crc 2023-10-09T04:47:24.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: failed to encode map e319 with expected crc 2023-10-09T04:47:24.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:23 smithi177 ceph-mon[195797]: failed to encode map e319 with expected crc 2023-10-09T04:47:24.281 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:24.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:24.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:24.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:24.456 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:24.576 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:24.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:24.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:24.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:24.748 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:24 smithi029 ceph-mon[216903]: pgmap v1242: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.9 GiB data, 9.5 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 368 op/s; 37261/142287 objects degraded (26.187%); 24127/142287 objects misplaced (16.957%); 11 MiB/s, 174 objects/s recovering 2023-10-09T04:47:24.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:24 smithi029 ceph-mon[223858]: pgmap v1242: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.9 GiB data, 9.5 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 368 op/s; 37261/142287 objects degraded (26.187%); 24127/142287 objects misplaced (16.957%); 11 MiB/s, 174 objects/s recovering 2023-10-09T04:47:24.758 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:24 smithi177 ceph-mon[195797]: pgmap v1242: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.9 GiB data, 9.5 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 368 op/s; 37261/142287 objects degraded (26.187%); 24127/142287 objects misplaced (16.957%); 11 MiB/s, 174 objects/s recovering 2023-10-09T04:47:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37261/142287 objects degraded (26.187%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37261/142287 objects degraded (26.187%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:26.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37261/142287 objects degraded (26.187%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:26.494 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:47:26.803 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:26.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:26 smithi029 ceph-mon[216903]: pgmap v1243: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 36972/137904 objects degraded (26.810%); 23293/137904 objects misplaced (16.891%); 14 MiB/s, 222 objects/s recovering 2023-10-09T04:47:26.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:26 smithi029 ceph-mon[223858]: pgmap v1243: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 36972/137904 objects degraded (26.810%); 23293/137904 objects misplaced (16.891%); 14 MiB/s, 222 objects/s recovering 2023-10-09T04:47:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:26.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:26.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:27.024 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:27.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:26 smithi177 ceph-mon[195797]: pgmap v1243: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 36972/137904 objects degraded (26.810%); 23293/137904 objects misplaced (16.891%); 14 MiB/s, 222 objects/s recovering 2023-10-09T04:47:27.173 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:27.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:27.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:27.299 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:27.348 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:27.448 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:47:27.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:27.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:27.492 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:47:27.533 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:47:27.576 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:47:27.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:27.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:47:27.668 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:47:27.675 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:27.704 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3869719087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:27.716 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:47:27.775 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:47:27.779 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.780 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:47:27.785 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:27.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3869719087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:27.893 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3869719087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:28.580 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:28.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:28.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:28.728 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T04:47:28.761 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:28 smithi029 ceph-mon[223858]: pgmap v1244: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 437 op/s; 36338/135807 objects degraded (26.757%); 22845/135807 objects misplaced (16.822%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:47:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3091970324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:47:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:28 smithi029 ceph-mon[216903]: pgmap v1244: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 437 op/s; 36338/135807 objects degraded (26.757%); 22845/135807 objects misplaced (16.822%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:47:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3091970324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:47:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:28 smithi177 ceph-mon[195797]: pgmap v1244: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 437 op/s; 36338/135807 objects degraded (26.757%); 22845/135807 objects misplaced (16.822%); 12 MiB/s, 187 objects/s recovering 2023-10-09T04:47:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3091970324' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T04:47:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3091970324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:47:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: osdmap e320: 8 total, 8 up, 5 in 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:29 smithi177 ceph-mon[195797]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3091970324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:47:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: osdmap e320: 8 total, 8 up, 5 in 2023-10-09T04:47:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[223858]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3091970324' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: osdmap e320: 8 total, 8 up, 5 in 2023-10-09T04:47:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:29 smithi029 ceph-mon[216903]: failed to encode map e320 with expected crc 2023-10-09T04:47:30.207 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:30.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:30.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:30.341 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:30.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:30.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:30.566 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:30.644 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:30.803 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:30.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:30 smithi029 ceph-mon[216903]: pgmap v1246: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 384 op/s; 36338/135378 objects degraded (26.842%); 22845/135378 objects misplaced (16.875%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:47:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 36338/135378 objects degraded (26.842%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:30 smithi029 ceph-mon[223858]: pgmap v1246: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 384 op/s; 36338/135378 objects degraded (26.842%); 22845/135378 objects misplaced (16.875%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:47:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 36338/135378 objects degraded (26.842%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:30.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:30.962 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:30 smithi177 ceph-mon[195797]: pgmap v1246: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.8 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 384 op/s; 36338/135378 objects degraded (26.842%); 22845/135378 objects misplaced (16.875%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:47:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 36338/135378 objects degraded (26.842%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:31.582 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:31.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:31.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:31.788 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:31.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:47:32.264 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:32.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:32.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:32.448 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:32.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:32 smithi029 ceph-mon[216903]: pgmap v1247: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.7 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 447 op/s; 36061/130995 objects degraded (27.529%); 21899/130995 objects misplaced (16.717%); 11 MiB/s, 178 objects/s recovering 2023-10-09T04:47:33.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:32 smithi177 ceph-mon[195797]: pgmap v1247: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.7 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 447 op/s; 36061/130995 objects degraded (27.529%); 21899/130995 objects misplaced (16.717%); 11 MiB/s, 178 objects/s recovering 2023-10-09T04:47:33.133 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89779 11.d 11:b6440cbe:::benchmark_data_smithi029_392508_object89778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.422302+0000", 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 501.480658721, 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2239466000000001, 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.134 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89779 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422302+0000", 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422302+0000", 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422300+0000", 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422328+0000", 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7678000000000002e-05 2023-10-09T04:47:33.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422345+0000", 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.751e-05 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422354+0000", 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7600000000000008e-06 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422420+0000", 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6419000000000002e-05 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.422433+0000", 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2203e-05 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394893+0000", 2023-10-09T04:47:33.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.972460455 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394908+0000", 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4713e-05 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.633240+0000", 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.238332558 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.633379+0000", 2023-10-09T04:47:33.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013892099999999999 2023-10-09T04:47:33.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.633492+0000", 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011317899999999999 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641640+0000", 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081479450000000002 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641688+0000", 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8102999999999997e-05 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641701+0000", 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3006999999999999e-05 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.641763+0000", 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1185000000000005e-05 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646248+0000", 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044856119999999999 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89791 11.d 11:b817d833:::benchmark_data_smithi029_392508_object89790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.438144+0000", 2023-10-09T04:47:33.144 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 501.46481685100002, 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.239980504, 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89791 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438144+0000", 2023-10-09T04:47:33.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438144+0000", 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438142+0000", 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438168+0000", 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6075999999999999e-05 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438178+0000", 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6059999999999992e-06 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438197+0000", 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8683000000000001e-05 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438265+0000", 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8207999999999995e-05 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.438279+0000", 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3917000000000001e-05 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394933+0000", 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95665392599999999 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394942+0000", 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4579999999999999e-06 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646427+0000", 2023-10-09T04:47:33.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2514851170000001 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646596+0000", 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000169013 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.646783+0000", 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000186331 2023-10-09T04:47:33.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.660928+0000", 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014145061 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.660997+0000", 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9444999999999997e-05 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.673775+0000", 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012778273 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.673841+0000", 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5617999999999994e-05 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.678124+0000", 2023-10-09T04:47:33.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042831040000000003 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:89794 11.d 11:b962c3ba:::benchmark_data_smithi029_392508_object89793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e247)", 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:39:11.440680+0000", 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 501.462280174, 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2416845639999998, 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.153 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89794 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440680+0000", 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440680+0000", 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440679+0000", 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440701+0000", 2023-10-09T04:47:33.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1917999999999999e-05 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440716+0000", 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5790999999999998e-05 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440723+0000", 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7170000000000002e-06 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440772+0000", 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8900999999999998e-05 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:11.440779+0000", 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1999999999999997e-06 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394965+0000", 2023-10-09T04:47:33.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95418614000000002 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:12.394974+0000", 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2199999999999992e-06 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.649824+0000", 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.254850244 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.649952+0000", 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012858499999999999 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.650115+0000", 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016252899999999999 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.660967+0000", 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010852185 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.661004+0000", 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7236000000000001e-05 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.678184+0000", 2023-10-09T04:47:33.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017179507 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.678245+0000", 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1424e-05 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:39:13.682365+0000", 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041193979999999998 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.161 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145927 11.d 11:be473cf7:::benchmark_data_smithi029_392508_object45371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.631539+0000", 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 391.27142119199999, 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2090651059999999, 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:47:33.162 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145927 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631539+0000", 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631539+0000", 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631538+0000", 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631545+0000", 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.306e-06 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631562+0000", 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7039999999999999e-05 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631571+0000", 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6710000000000005e-06 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631637+0000", 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5897999999999998e-05 2023-10-09T04:47:33.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631651+0000", 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4246e-05 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.757167+0000", 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1255164500000001 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.757305+0000", 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137119 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.757422+0000", 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117538 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.768164+0000", 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010741516 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.780860+0000", 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012696146 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.839627+0000", 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.058767698 2023-10-09T04:47:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.839695+0000", 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7335999999999994e-05 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.840604+0000", 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00090948899999999998 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145928 11.d 11:b60f9d16:::benchmark_data_smithi029_392508_object45372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.631947+0000", 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 391.271013697, 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.209480036, 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:47:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145928 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631947+0000", 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631947+0000", 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631946+0000", 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631951+0000", 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8459999999999999e-06 2023-10-09T04:47:33.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631963+0000", 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2666e-05 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.631970+0000", 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6510000000000004e-06 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.632035+0000", 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4949000000000004e-05 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.632045+0000", 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6190000000000006e-06 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.762300+0000", 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.130255303 2023-10-09T04:47:33.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.762418+0000", 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117657 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.762510+0000", 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2128999999999998e-05 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771289+0000", 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087791169999999995 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.793784+0000", 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022494858 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.840683+0000", 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.046899376999999999 2023-10-09T04:47:33.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.840747+0000", 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3975999999999999e-05 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.841427+0000", 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00067968599999999998 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:145934 11.d 11:b5fb235c:::benchmark_data_smithi029_392508_object45378:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:41:01.638048+0000", 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 391.26491190600001, 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.204085243, 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:47:33.178 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 145934 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638048+0000", 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638048+0000", 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638047+0000", 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638053+0000", 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638068+0000", 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4608e-05 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638076+0000", 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1300000000000001e-06 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638154+0000", 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8053999999999995e-05 2023-10-09T04:47:33.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:01.638166+0000", 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2418999999999999e-05 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.765706+0000", 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.127539327 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.765786+0000", 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0094999999999998e-05 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.765863+0000", 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7152999999999998e-05 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.771298+0000", 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054349400000000001 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.793840+0000", 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022542736000000001 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.841500+0000", 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.047659941999999997 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.841556+0000", 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5578999999999998e-05 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:41:02.842134+0000", 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057772800000000001 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.185 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111563 12.9 12:9be70b01:::benchmark_data_smithi029_442458_object16313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.831091+0000", 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 108.07186958, 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1843307190000001, 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111563 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831091+0000", 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831091+0000", 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831089+0000", 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831096+0000", 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0729999999999999e-06 2023-10-09T04:47:33.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831110+0000", 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4085e-05 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831119+0000", 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6300000000000004e-06 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831170+0000", 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0921999999999998e-05 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831328+0000", 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015812000000000001 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:33.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831474+0000", 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014625900000000001 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.841353+0000", 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098784049999999998 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.874819+0000", 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033466796 2023-10-09T04:47:33.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.874830+0000", 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0947e-05 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.571773+0000", 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.69694232 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.571790+0000", 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7193e-05 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.977734+0000", 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.405944565 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.977900+0000", 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000165594 2023-10-09T04:47:33.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:47.015346+0000", 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037445604 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:47.015421+0000", 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5888000000000003e-05 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.194 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111564 12.5 12:a384f7d0:::benchmark_data_smithi029_442458_object16314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.831137+0000", 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 108.07182337099999, 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1753254260000001, 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.195 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111564 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831137+0000", 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831137+0000", 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831136+0000", 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831140+0000", 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3830000000000001e-06 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831147+0000", 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5190000000000003e-06 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.831154+0000", 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.528e-06 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.845145+0000", 2023-10-09T04:47:33.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013990997999999999 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.845312+0000", 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016751299999999999 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.845459+0000", 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000146329 2023-10-09T04:47:33.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.867088+0000", 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021629092999999999 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.874781+0000", 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076927419999999998 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.874793+0000", 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2269999999999999e-05 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.985887+0000", 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1110939630000001 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:47:33.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.986086+0000", 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019895500000000001 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.006330+0000", 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020244661000000001 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.006462+0000", 2023-10-09T04:47:33.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013199199999999999 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111571 12.5 12:a318946b:::benchmark_data_smithi029_442458_object16321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.842534+0000", 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 108.06042628500001, 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1720195470000001, 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.203 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111571 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.842534+0000", 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.842534+0000", 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.842533+0000", 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.842537+0000", 2023-10-09T04:47:33.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3719999999999998e-06 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.842549+0000", 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.221e-05 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.842560+0000", 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0908e-05 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.862843+0000", 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020282528000000001 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.863043+0000", 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020027599999999999 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.863220+0000", 2023-10-09T04:47:33.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017665500000000001 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.876484+0000", 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013264608000000001 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.876497+0000", 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3023000000000001e-05 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.989911+0000", 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1134132750000001 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.990100+0000", 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000189412 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.014466+0000", 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024365924000000001 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.014554+0000", 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7585999999999998e-05 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111573 12.a 12:50c4b64d:::benchmark_data_smithi029_442458_object16323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.844411+0000", 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 108.05854948699999, 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.235898454, 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111573 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844411+0000", 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844411+0000", 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844410+0000", 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844415+0000", 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7869999999999998e-06 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844426+0000", 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0772e-05 2023-10-09T04:47:33.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844434+0000", 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8110000000000003e-06 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844486+0000", 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1795999999999997e-05 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844633+0000", 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014704599999999999 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.844758+0000", 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000125093 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.861228+0000", 2023-10-09T04:47:33.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016469979999999999 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.861242+0000", 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4166e-05 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.574584+0000", 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.71334222000000003 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.574598+0000", 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3909e-05 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:47.029891+0000", 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4552930559999999 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:47.030060+0000", 2023-10-09T04:47:33.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000169183 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:47.080193+0000", 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.050132212000000002 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:47.080309+0000", 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000116618 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.218 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111557 12.8 12:1309151c:::benchmark_data_smithi029_442458_object16307:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.853349+0000", 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 108.04961110399999, 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9053686320000001, 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.219 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111557 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.853349+0000", 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.853349+0000", 2023-10-09T04:47:33.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.853348+0000", 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.853353+0000", 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6679999999999998e-06 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.853365+0000", 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2140999999999999e-05 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.853376+0000", 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0421000000000001e-05 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.865682+0000", 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012306246999999999 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.865694+0000", 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1984000000000001e-05 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.574146+0000", 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.70845175299999996 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.574158+0000", 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2544999999999999e-05 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.756644+0000", 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.18248522 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:46.758718+0000", 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020742489999999998 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122036 12.5 12:a763c3cf:::benchmark_data_smithi029_442458_object26786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.127682+0000", 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.775278809, 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.36064501, 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.225 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122036 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127682+0000", 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127682+0000", 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127681+0000", 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127684+0000", 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.01e-06 2023-10-09T04:47:33.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127689+0000", 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3349999999999997e-06 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127694+0000", 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6e-06 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.128425+0000", 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00073117500000000003 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.128619+0000", 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019422100000000001 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.128811+0000", 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000191063 2023-10-09T04:47:33.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.145846+0000", 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017035444 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.186137+0000", 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040290576000000002 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.186148+0000", 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.148e-05 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.468986+0000", 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.28283817 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.469100+0000", 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114224 2023-10-09T04:47:33.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488260+0000", 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019159944000000002 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488327+0000", 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6188999999999999e-05 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.232 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122038 12.8 12:1f7ced3b:::benchmark_data_smithi029_442458_object26788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.127709+0000", 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.775251140999998, 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1741744889999999, 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.233 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122038 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127709+0000", 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127709+0000", 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127709+0000", 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127712+0000", 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7049999999999999e-06 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127717+0000", 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5480000000000001e-06 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127721+0000", 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2019999999999997e-06 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.130077+0000", 2023-10-09T04:47:33.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023553340000000002 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.130210+0000", 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013299000000000001 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3,4", 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.130331+0000", 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012139200000000001 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.144238+0000", 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013906903 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.148640+0000", 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044022389999999996 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.164279+0000", 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015638840000000001 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.164291+0000", 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2386e-05 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.301796+0000", 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1375048670000001 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.301884+0000", 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7451e-05 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122039 12.a 12:58d58751:::benchmark_data_smithi029_442458_object26789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.127736+0000", 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.775224561000002, 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3285342710000001, 2023-10-09T04:47:33.240 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122039 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127736+0000", 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127736+0000", 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127735+0000", 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127738+0000", 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3350000000000001e-06 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127743+0000", 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9949999999999996e-06 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127747+0000", 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1829999999999996e-06 2023-10-09T04:47:33.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.127875+0000", 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000127737 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.128095+0000", 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022026399999999999 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:47:33.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.128273+0000", 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017813200000000001 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.148666+0000", 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020393261999999999 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.171525+0000", 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022858719 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.171548+0000", 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2447e-05 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.135157+0000", 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96360899200000005 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.135169+0000", 2023-10-09T04:47:33.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2901999999999999e-05 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.439852+0000", 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.30468282 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.439981+0000", 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012885500000000001 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.456192+0000", 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01621117 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.456270+0000", 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7781999999999995e-05 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122048 12.8 12:12f0bcb9:::benchmark_data_smithi029_442458_object26798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.185132+0000", 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.717827987000007, 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4418036910000001, 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122048 2023-10-09T04:47:33.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.185132+0000", 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.185132+0000", 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.185131+0000", 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.185136+0000", 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.091e-06 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.185145+0000", 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1090000000000004e-06 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.185153+0000", 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0099999999999995e-06 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.186095+0000", 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00094254699999999996 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.186105+0000", 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7210000000000006e-06 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.302153+0000", 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.116047799 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.302433+0000", 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000280084 2023-10-09T04:47:33.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3,4,7", 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.303020+0000", 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00058734 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318138+0000", 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015117644 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318204+0000", 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6106999999999996e-05 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318246+0000", 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1773000000000001e-05 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.322885+0000", 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046390420000000003 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626814+0000", 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30392884799999997 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626854+0000", 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0685000000000001e-05 2023-10-09T04:47:33.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626936+0000", 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1559999999999998e-05 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122058 12.8 12:106bae43:::benchmark_data_smithi029_442458_object26808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.226895+0000", 2023-10-09T04:47:33.257 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.67606542, 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4001806860000001, 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122058 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226895+0000", 2023-10-09T04:47:33.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226895+0000", 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226893+0000", 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226900+0000", 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1299999999999998e-06 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226914+0000", 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4209000000000001e-05 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226921+0000", 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7719999999999997e-06 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226979+0000", 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8300000000000001e-05 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.226992+0000", 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2737e-05 2023-10-09T04:47:33.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.310055+0000", 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.083063769 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.310195+0000", 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013925499999999999 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3,4,7", 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.310670+0000", 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00047512299999999998 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318280+0000", 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076101720000000001 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.322348+0000", 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004067813 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.322918+0000", 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057041899999999996 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.332756+0000", 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098381690000000008 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626971+0000", 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29421475699999999 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627002+0000", 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0522999999999999e-05 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627076+0000", 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4048999999999995e-05 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122066 12.a 12:56028c12:::benchmark_data_smithi029_442458_object26816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.251856+0000", 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.651104821999994, 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2493437410000001, 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.266 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122066 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251856+0000", 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251856+0000", 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251854+0000", 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251860+0000", 2023-10-09T04:47:33.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.139e-06 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251873+0000", 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2645e-05 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251879+0000", 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4600000000000002e-06 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251940+0000", 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1016e-05 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.251954+0000", 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4406999999999999e-05 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.135197+0000", 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.88324251799999998 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.135205+0000", 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1329999999999999e-06 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.442382+0000", 2023-10-09T04:47:33.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.307177705 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.443037+0000", 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00065511599999999997 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.458907+0000", 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015869260999999999 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.459047+0000", 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014016399999999999 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.459702+0000", 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00065530000000000004 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.475495+0000", 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015792520000000001 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.476165+0000", 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00067038600000000003 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.498381+0000", 2023-10-09T04:47:33.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022215609000000001 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.498436+0000", 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5574e-05 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.501199+0000", 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027629780000000001 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.275 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122070 12.8 12:1cf21352:::benchmark_data_smithi029_442458_object26820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.267097+0000", 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.635863350999998, 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3601213539999999, 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.276 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122070 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267097+0000", 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267097+0000", 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267095+0000", 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267104+0000", 2023-10-09T04:47:33.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6079999999999996e-06 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267121+0000", 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7186000000000001e-05 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267129+0000", 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267185+0000", 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5263999999999998e-05 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.267198+0000", 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3098000000000001e-05 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314214+0000", 2023-10-09T04:47:33.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0470168129999999 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314459+0000", 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024409400000000001 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3,4,7", 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314599+0000", 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014023400000000001 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.322840+0000", 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082408859999999994 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.322849+0000", 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1370000000000008e-06 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.329165+0000", 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0063162069999999999 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.332800+0000", 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003634548 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627098+0000", 2023-10-09T04:47:33.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29429877500000001 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627127+0000", 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9064000000000002e-05 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627218+0000", 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0939000000000005e-05 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.284 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122072 12.5 12:a5afa03a:::benchmark_data_smithi029_442458_object26822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.273149+0000", 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.629811828000001, 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2150552000000001, 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.285 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122072 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273149+0000", 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273149+0000", 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273147+0000", 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273153+0000", 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5770000000000001e-06 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273165+0000", 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2203e-05 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273173+0000", 2023-10-09T04:47:33.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0490000000000004e-06 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273241+0000", 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8150000000000003e-05 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.273254+0000", 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2819e-05 2023-10-09T04:47:33.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.471516+0000", 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.198262425 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.471635+0000", 2023-10-09T04:47:33.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011814 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.472248+0000", 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00061310100000000001 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.485552+0000", 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013304122 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.485594+0000", 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2114999999999999e-05 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.485617+0000", 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3340999999999999e-05 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.485661+0000", 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3705999999999998e-05 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488204+0000", 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002542823 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.293 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122077 12.8 12:17464ea6:::benchmark_data_smithi029_442458_object26827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.298482+0000", 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 68.604478295999996, 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3288604230000001, 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122077 2023-10-09T04:47:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298482+0000", 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298482+0000", 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298481+0000", 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298487+0000", 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6359999999999997e-06 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:47:33.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298502+0000", 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4482e-05 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298511+0000", 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6980000000000004e-06 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298557+0000", 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6560000000000001e-05 2023-10-09T04:47:33.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.298564+0000", 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3730000000000004e-06 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318391+0000", 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0198267599999999 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:47:33.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318527+0000", 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013576800000000001 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,3,4,7", 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318667+0000", 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013986399999999999 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.322981+0000", 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043137920000000003 2023-10-09T04:47:33.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.325465+0000", 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024845850000000001 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.329212+0000", 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037469869999999998 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.332837+0000", 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003625236 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627250+0000", 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29441278799999998 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627280+0000", 2023-10-09T04:47:33.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9804000000000001e-05 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.627342+0000", 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2409999999999994e-05 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:47:33.302 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:47:33.303 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:32 smithi029 ceph-mon[223858]: pgmap v1247: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.7 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 447 op/s; 36061/130995 objects degraded (27.529%); 21899/130995 objects misplaced (16.717%); 11 MiB/s, 178 objects/s recovering 2023-10-09T04:47:33.304 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:47:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:47:32] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:47:33.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:47:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:34.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:34.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:34.421 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:34.492 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:34.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:47:34.591 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:34.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:34.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:34.697 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:34.739 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:34.767 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:34.794 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:34.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:34 smithi029 ceph-mon[216903]: pgmap v1248: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 462 op/s; 35325/128397 objects degraded (27.512%); 21473/128397 objects misplaced (16.724%); 12 MiB/s, 197 objects/s recovering 2023-10-09T04:47:34.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:34 smithi029 ceph-mon[223858]: pgmap v1248: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 462 op/s; 35325/128397 objects degraded (27.512%); 21473/128397 objects misplaced (16.724%); 12 MiB/s, 197 objects/s recovering 2023-10-09T04:47:34.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:34.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:34.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:34.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:34 smithi177 ceph-mon[195797]: pgmap v1248: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 462 op/s; 35325/128397 objects degraded (27.512%); 21473/128397 objects misplaced (16.724%); 12 MiB/s, 197 objects/s recovering 2023-10-09T04:47:35.044 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:35.373 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:35.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:35.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:35.570 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:35.831 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:47:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4288053005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:47:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35325/128397 objects degraded (27.512%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:36.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:36.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4288053005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:47:36.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35325/128397 objects degraded (27.512%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:36.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:47:36.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4288053005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:47:36.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35325/128397 objects degraded (27.512%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:36.467 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:36.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:36.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:36.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:36.637 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:36.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:36.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:36.801 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: pgmap v1249: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 459 op/s; 35010/124113 objects degraded (28.208%); 20439/124113 objects misplaced (16.468%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:47:36.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:36.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: failed to encode map e321 with expected crc 2023-10-09T04:47:36.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4288053005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:47:36.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: osdmap e321: 8 total, 8 up, 5 in 2023-10-09T04:47:36.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: failed to encode map e321 with expected crc 2023-10-09T04:47:36.802 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[223858]: failed to encode map e321 with expected crc 2023-10-09T04:47:36.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:36.844 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: pgmap v1249: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 459 op/s; 35010/124113 objects degraded (28.208%); 20439/124113 objects misplaced (16.468%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: failed to encode map e321 with expected crc 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4288053005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: osdmap e321: 8 total, 8 up, 5 in 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: failed to encode map e321 with expected crc 2023-10-09T04:47:37.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:36 smithi177 ceph-mon[195797]: failed to encode map e321 with expected crc 2023-10-09T04:47:37.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: pgmap v1249: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 459 op/s; 35010/124113 objects degraded (28.208%); 20439/124113 objects misplaced (16.468%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:47:37.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:47:37.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: failed to encode map e321 with expected crc 2023-10-09T04:47:37.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4288053005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:47:37.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: osdmap e321: 8 total, 8 up, 5 in 2023-10-09T04:47:37.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: failed to encode map e321 with expected crc 2023-10-09T04:47:37.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:36 smithi029 ceph-mon[216903]: failed to encode map e321 with expected crc 2023-10-09T04:47:38.754 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:38.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:38.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:38.816 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:38 smithi029 ceph-mon[216903]: pgmap v1251: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 34318/121881 objects degraded (28.157%); 19979/121881 objects misplaced (16.392%); 16 MiB/s, 251 objects/s recovering 2023-10-09T04:47:38.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:38.932 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:39.037 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:39.062 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:39.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:39.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:39.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:38 smithi029 ceph-mon[223858]: pgmap v1251: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 34318/121881 objects degraded (28.157%); 19979/121881 objects misplaced (16.392%); 16 MiB/s, 251 objects/s recovering 2023-10-09T04:47:39.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:39.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:38 smithi177 ceph-mon[195797]: pgmap v1251: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 34318/121881 objects degraded (28.157%); 19979/121881 objects misplaced (16.392%); 16 MiB/s, 251 objects/s recovering 2023-10-09T04:47:39.290 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:39.410 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:39.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:39.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:39.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:39.625 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:39.651 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:47:39.700 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:47:39.738 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:39.756 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:47:39.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:39.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:39.798 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:47:39.849 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:47:39.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:39.886 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:47:39.914 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:47:39.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/621081939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:39.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/621081939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:39.955 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:47:39.956 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:47:39.983 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:47:40.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:40.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:40.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:40.177 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/621081939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:40.277 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:40.351 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:40.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:40.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:40.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:40.659 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:40.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:47:40.913 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:40 smithi029 ceph-mon[216903]: pgmap v1252: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 34318/121437 objects degraded (28.260%); 19979/121437 objects misplaced (16.452%); 14 MiB/s, 216 objects/s recovering 2023-10-09T04:47:40.913 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1608585570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:47:40.913 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 34318/121437 objects degraded (28.260%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:40.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:40 smithi029 ceph-mon[223858]: pgmap v1252: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 34318/121437 objects degraded (28.260%); 19979/121437 objects misplaced (16.452%); 14 MiB/s, 216 objects/s recovering 2023-10-09T04:47:40.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1608585570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:47:40.914 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 34318/121437 objects degraded (28.260%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:41.018 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:47:41.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:40 smithi177 ceph-mon[195797]: pgmap v1252: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.5 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 34318/121437 objects degraded (28.260%); 19979/121437 objects misplaced (16.452%); 14 MiB/s, 216 objects/s recovering 2023-10-09T04:47:41.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1608585570' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:47:41.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 34318/121437 objects degraded (28.260%), 9 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1608585570' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:47:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[216903]: osdmap e322: 8 total, 8 up, 5 in 2023-10-09T04:47:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[216903]: failed to encode map e322 with expected crc 2023-10-09T04:47:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1608585570' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:47:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[223858]: osdmap e322: 8 total, 8 up, 5 in 2023-10-09T04:47:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[223858]: failed to encode map e322 with expected crc 2023-10-09T04:47:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1608585570' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:47:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:41 smithi177 ceph-mon[195797]: osdmap e322: 8 total, 8 up, 5 in 2023-10-09T04:47:42.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:41 smithi177 ceph-mon[195797]: failed to encode map e322 with expected crc 2023-10-09T04:47:42.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:42.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:42.706 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:42.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:42.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:42.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:42.844 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:43.087 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:43 smithi029 ceph-mon[216903]: pgmap v1254: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.4 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 34014/117246 objects degraded (29.011%); 18867/117246 objects misplaced (16.092%); 15 MiB/s, 232 objects/s recovering 2023-10-09T04:47:43.087 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:47:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:47:42] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:47:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:43 smithi029 ceph-mon[223858]: pgmap v1254: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.4 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 34014/117246 objects degraded (29.011%); 18867/117246 objects misplaced (16.092%); 15 MiB/s, 232 objects/s recovering 2023-10-09T04:47:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:43 smithi177 ceph-mon[195797]: pgmap v1254: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 3 active+recovery_wait+degraded+remapped; 2.4 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 464 op/s; 34014/117246 objects degraded (29.011%); 18867/117246 objects misplaced (16.092%); 15 MiB/s, 232 objects/s recovering 2023-10-09T04:47:43.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:47:43.723 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:43.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:43.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:43.950 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:43.955 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:44.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:44.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:44.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:44.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:44.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:44.351 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:44.510 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:44.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:44.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:44.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:47:44.585 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:47:44.585 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:47:44.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:44.690 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:44.800 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:44.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:44.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:44.907 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:44.992 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:45.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:45 smithi029 ceph-mon[216903]: pgmap v1255: 89 pgs: 2 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 31094/114780 objects degraded (27.090%); 18396/114780 objects misplaced (16.027%); 12 MiB/s, 199 objects/s recovering 2023-10-09T04:47:45.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:45 smithi029 ceph-mon[223858]: pgmap v1255: 89 pgs: 2 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 31094/114780 objects degraded (27.090%); 18396/114780 objects misplaced (16.027%); 12 MiB/s, 199 objects/s recovering 2023-10-09T04:47:45.490 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:45.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:45 smithi177 ceph-mon[195797]: pgmap v1255: 89 pgs: 2 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.3 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 31094/114780 objects degraded (27.090%); 18396/114780 objects misplaced (16.027%); 12 MiB/s, 199 objects/s recovering 2023-10-09T04:47:45.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:45.634 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:45.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:45.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:45.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:45.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:45.852 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:46.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:46.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:46.121 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:46.184 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 31094/114780 objects degraded (27.090%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:46.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 31094/114780 objects degraded (27.090%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:46.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 31094/114780 objects degraded (27.090%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:47 smithi029 ceph-mon[216903]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 471 op/s; 30824/110631 objects degraded (27.862%); 19746/110631 objects misplaced (17.849%); 15 MiB/s, 234 objects/s recovering 2023-10-09T04:47:47.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:47 smithi029 ceph-mon[223858]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 471 op/s; 30824/110631 objects degraded (27.862%); 19746/110631 objects misplaced (17.849%); 15 MiB/s, 234 objects/s recovering 2023-10-09T04:47:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:47 smithi177 ceph-mon[195797]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 471 op/s; 30824/110631 objects degraded (27.862%); 19746/110631 objects misplaced (17.849%); 15 MiB/s, 234 objects/s recovering 2023-10-09T04:47:48.168 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:48.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:48.277 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:48.391 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:48.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:48.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:48.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:48.671 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:48.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:47:48.779 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:48.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:48.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:48.942 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:48.952 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:49.021 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:49.027 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:49.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:49.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:49.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:49.140 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:49 smithi029 ceph-mon[216903]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 446 op/s; 30258/108480 objects degraded (27.893%); 18998/108480 objects misplaced (17.513%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:47:49.202 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:49.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:49 smithi177 ceph-mon[195797]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 446 op/s; 30258/108480 objects degraded (27.893%); 18998/108480 objects misplaced (17.513%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:47:49.400 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:49 smithi029 ceph-mon[223858]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 446 op/s; 30258/108480 objects degraded (27.893%); 18998/108480 objects misplaced (17.513%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:47:49.426 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:49.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:49.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:47:49.576 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:49.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:47:49.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:49.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:47:49.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:47:49.929 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:50.172 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:47:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4195520693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4195520693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4195520693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 447 op/s; 30258/108006 objects degraded (28.015%); 18998/108006 objects misplaced (17.590%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: osdmap e323: 8 total, 8 up, 5 in 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 30258/108006 objects degraded (28.015%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 447 op/s; 30258/108006 objects degraded (28.015%); 18998/108006 objects misplaced (17.590%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: osdmap e323: 8 total, 8 up, 5 in 2023-10-09T04:47:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 30258/108006 objects degraded (28.015%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.2 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 447 op/s; 30258/108006 objects degraded (28.015%); 18998/108006 objects misplaced (17.590%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: osdmap e323: 8 total, 8 up, 5 in 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: failed to encode map e323 with expected crc 2023-10-09T04:47:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 30258/108006 objects degraded (28.015%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:52.483 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:47:52.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:52.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:47:52.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:47:52.650 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:53.164 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:47:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:47:52] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-10-09T04:47:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:53 smithi029 ceph-mon[216903]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 462 op/s; 29968/103353 objects degraded (28.996%); 18163/103353 objects misplaced (17.574%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:47:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:53 smithi029 ceph-mon[223858]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 462 op/s; 29968/103353 objects degraded (28.996%); 18163/103353 objects misplaced (17.574%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:47:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:53 smithi177 ceph-mon[195797]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.1 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 462 op/s; 29968/103353 objects degraded (28.996%); 18163/103353 objects misplaced (17.574%); 13 MiB/s, 212 objects/s recovering 2023-10-09T04:47:54.871 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:47:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:54.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:47:54.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:55.047 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:55.148 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:55.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:55.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:47:55.309 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:55 smithi177 ceph-mon[195797]: pgmap v1261: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 477 op/s; 29318/100452 objects degraded (29.186%); 17289/100452 objects misplaced (17.211%); 14 MiB/s, 230 objects/s recovering 2023-10-09T04:47:55.313 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:55.331 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:47:55.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:47:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:55.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:47:55.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:55.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:55 smithi029 ceph-mon[223858]: pgmap v1261: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 477 op/s; 29318/100452 objects degraded (29.186%); 17289/100452 objects misplaced (17.211%); 14 MiB/s, 230 objects/s recovering 2023-10-09T04:47:55.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:55 smithi029 ceph-mon[216903]: pgmap v1261: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 477 op/s; 29318/100452 objects degraded (29.186%); 17289/100452 objects misplaced (17.211%); 14 MiB/s, 230 objects/s recovering 2023-10-09T04:47:55.632 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:55.840 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:47:55.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:55.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:47:55.951 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:56.006 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:56.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:56.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:56.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:47:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 29318/100452 objects degraded (29.186%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:56.288 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:56.500 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:47:56.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:56.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:47:56.511 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 29318/100452 objects degraded (29.186%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:56.513 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 29318/100452 objects degraded (29.186%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:47:56.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:47:56.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:56.661 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:47:56.668 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:56.678 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:47:56.732 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:47:56.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:56.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:56.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:47:56.810 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:47:56.851 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:47:56.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:47:56.898 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:47:56.919 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.919 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.919 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.920 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.920 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.920 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.920 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.920 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:47:56.929 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:57.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:47:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:57.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:47:57.165 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:47:57.227 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:57 smithi177 ceph-mon[195797]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 501 op/s; 28991/95589 objects degraded (30.329%); 16420/95589 objects misplaced (17.178%); 14 MiB/s, 221 objects/s recovering 2023-10-09T04:47:57.227 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/30759697' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:57.227 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:57 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:57.227 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3378356567' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:57.230 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:57.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:47:57.251 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:47:57.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:47:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:47:57.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:47:57.443 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:47:57.535 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[223858]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 501 op/s; 28991/95589 objects degraded (30.329%); 16420/95589 objects misplaced (17.178%); 14 MiB/s, 221 objects/s recovering 2023-10-09T04:47:57.535 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/30759697' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:57.535 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:57.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3378356567' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:57.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[216903]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 501 op/s; 28991/95589 objects degraded (30.329%); 16420/95589 objects misplaced (17.178%); 14 MiB/s, 221 objects/s recovering 2023-10-09T04:47:57.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/30759697' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:57.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:47:57.536 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3378356567' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:47:57.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:47:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:47:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: osdmap e324: 8 total, 8 up, 5 in 2023-10-09T04:47:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:58 smithi177 ceph-mon[195797]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: osdmap e324: 8 total, 8 up, 5 in 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[216903]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: osdmap e324: 8 total, 8 up, 5 in 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: failed to encode map e324 with expected crc 2023-10-09T04:47:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:58 smithi029 ceph-mon[223858]: failed to encode map e324 with expected crc 2023-10-09T04:47:59.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:59 smithi177 ceph-mon[195797]: pgmap v1264: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 623 op/s; 28295/93030 objects degraded (30.415%); 15486/93030 objects misplaced (16.646%); 18 MiB/s, 291 objects/s recovering 2023-10-09T04:47:59.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:59.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:47:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:59 smithi029 ceph-mon[216903]: pgmap v1264: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 623 op/s; 28295/93030 objects degraded (30.415%); 15486/93030 objects misplaced (16.646%); 18 MiB/s, 291 objects/s recovering 2023-10-09T04:47:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:59.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:47:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:47:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:59 smithi029 ceph-mon[223858]: pgmap v1264: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 623 op/s; 28295/93030 objects degraded (30.415%); 15486/93030 objects misplaced (16.646%); 18 MiB/s, 291 objects/s recovering 2023-10-09T04:47:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:47:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:47:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:00.315 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:00 smithi029 ceph-mon[216903]: pgmap v1265: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 390 op/s; 28295/92598 objects degraded (30.557%); 15486/92598 objects misplaced (16.724%); 12 MiB/s, 196 objects/s recovering 2023-10-09T04:48:00.318 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:00 smithi029 ceph-mon[223858]: pgmap v1265: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 390 op/s; 28295/92598 objects degraded (30.557%); 15486/92598 objects misplaced (16.724%); 12 MiB/s, 196 objects/s recovering 2023-10-09T04:48:00.671 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:00.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:00.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:00.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:00 smithi177 ceph-mon[195797]: pgmap v1265: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.9 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 390 op/s; 28295/92598 objects degraded (30.557%); 15486/92598 objects misplaced (16.724%); 12 MiB/s, 196 objects/s recovering 2023-10-09T04:48:00.779 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:00.850 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 28295/92598 objects degraded (30.557%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:01.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 28295/92598 objects degraded (30.557%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 28295/92598 objects degraded (30.557%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:02.325 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:02.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:02.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:02.505 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:02.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:02 smithi029 ceph-mon[216903]: pgmap v1266: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.8 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 505 op/s; 28027/88170 objects degraded (31.787%); 14617/88170 objects misplaced (16.578%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:48:02.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:02 smithi029 ceph-mon[223858]: pgmap v1266: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.8 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 505 op/s; 28027/88170 objects degraded (31.787%); 14617/88170 objects misplaced (16.578%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:48:02.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:48:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:02 smithi177 ceph-mon[195797]: pgmap v1266: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.8 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 505 op/s; 28027/88170 objects degraded (31.787%); 14617/88170 objects misplaced (16.578%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:48:02.799 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:02.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:02.910 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:02.964 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:02.968 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:48:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:48:02] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-10-09T04:48:03.038 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:03.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:03.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:03.241 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:03.351 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:03.407 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:03.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:03.488 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:48:03.489 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:48:03.489 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:48:03.489 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:48:03.489 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:48:03.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:03.598 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:04.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:04.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:04.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:04.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:04.551 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:04.671 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:04.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:04.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:04.778 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:04.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:04 smithi177 ceph-mon[195797]: pgmap v1267: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 27321/85251 objects degraded (32.048%); 13653/85251 objects misplaced (16.015%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:48:04.820 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:04.820 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:04 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:04.821 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:04.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:48:04.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:04.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:05.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:05.106 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:04 smithi029 ceph-mon[216903]: pgmap v1267: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 27321/85251 objects degraded (32.048%); 13653/85251 objects misplaced (16.015%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:48:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:04 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:04 smithi029 ceph-mon[223858]: pgmap v1267: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.7 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 506 op/s; 27321/85251 objects degraded (32.048%); 13653/85251 objects misplaced (16.015%); 15 MiB/s, 245 objects/s recovering 2023-10-09T04:48:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:04 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:05.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:05.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:05.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:05.369 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:06.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 27321/85251 objects degraded (32.048%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 27321/85251 objects degraded (32.048%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:06.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 27321/85251 objects degraded (32.048%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:06.334 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:06.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:06.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:48:06.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:06.522 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:06.610 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:06.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:06.697 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:06.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:06 smithi177 ceph-mon[195797]: pgmap v1268: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 519 op/s; 26949/79980 objects degraded (33.695%); 12682/79980 objects misplaced (15.856%); 15 MiB/s, 252 objects/s recovering 2023-10-09T04:48:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:48:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:48:06.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:06.815 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[223858]: pgmap v1268: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 519 op/s; 26949/79980 objects degraded (33.695%); 12682/79980 objects misplaced (15.856%); 15 MiB/s, 252 objects/s recovering 2023-10-09T04:48:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:48:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:48:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:07.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[216903]: pgmap v1268: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 519 op/s; 26949/79980 objects degraded (33.695%); 12682/79980 objects misplaced (15.856%); 15 MiB/s, 252 objects/s recovering 2023-10-09T04:48:07.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:48:07.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:48:07.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:08.032 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:08.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:08.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:08.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:08.185 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:08.214 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:08.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:08.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:08.397 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:08.481 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:08.659 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:08.668 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:08.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:08.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:08.873 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:08.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:08 smithi029 ceph-mon[216903]: pgmap v1269: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 508 op/s; 26341/77586 objects degraded (33.951%); 12374/77586 objects misplaced (15.949%); 13 MiB/s, 251 objects/s recovering 2023-10-09T04:48:08.880 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:08 smithi029 ceph-mon[223858]: pgmap v1269: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 508 op/s; 26341/77586 objects degraded (33.951%); 12374/77586 objects misplaced (15.949%); 13 MiB/s, 251 objects/s recovering 2023-10-09T04:48:09.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:08 smithi177 ceph-mon[195797]: pgmap v1269: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 508 op/s; 26341/77586 objects degraded (33.951%); 12374/77586 objects misplaced (15.949%); 13 MiB/s, 251 objects/s recovering 2023-10-09T04:48:10.920 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:10.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:10 smithi177 ceph-mon[195797]: pgmap v1270: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 26341/77121 objects degraded (34.155%); 12374/77121 objects misplaced (16.045%); 11 MiB/s, 212 objects/s recovering 2023-10-09T04:48:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 26341/77121 objects degraded (34.155%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:11.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:11.085 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:10 smithi029 ceph-mon[216903]: pgmap v1270: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 26341/77121 objects degraded (34.155%); 12374/77121 objects misplaced (16.045%); 11 MiB/s, 212 objects/s recovering 2023-10-09T04:48:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 26341/77121 objects degraded (34.155%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:10 smithi029 ceph-mon[223858]: pgmap v1270: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.6 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 26341/77121 objects degraded (34.155%); 12374/77121 objects misplaced (16.045%); 11 MiB/s, 212 objects/s recovering 2023-10-09T04:48:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 26341/77121 objects degraded (34.155%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:11.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:11.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:11.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:11.347 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:11.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:11.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:11.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:11.577 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:12.091 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:12.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:12.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:12.273 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:12.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:12.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:12.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:12.542 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:12.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:12.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:12.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:12.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:12.874 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:12.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:12 smithi029 ceph-mon[216903]: pgmap v1271: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 26039/72735 objects degraded (35.800%); 11451/72735 objects misplaced (15.743%); 14 MiB/s, 263 objects/s recovering 2023-10-09T04:48:12.909 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:48:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:48:12] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-10-09T04:48:12.910 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:12 smithi029 ceph-mon[223858]: pgmap v1271: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 26039/72735 objects degraded (35.800%); 11451/72735 objects misplaced (15.743%); 14 MiB/s, 263 objects/s recovering 2023-10-09T04:48:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:12 smithi177 ceph-mon[195797]: pgmap v1271: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.5 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 26039/72735 objects degraded (35.800%); 11451/72735 objects misplaced (15.743%); 14 MiB/s, 263 objects/s recovering 2023-10-09T04:48:13.788 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:48:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1801112056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:13 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1801112056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:13 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:14.286 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1801112056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:14.286 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:13 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:48:14.286 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:14.680 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:14.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:14.792 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:14.805 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:48:14.886 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[216903]: pgmap v1272: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 515 op/s; 25373/69606 objects degraded (36.452%); 11121/69606 objects misplaced (15.977%); 13 MiB/s, 286 objects/s recovering 2023-10-09T04:48:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:48:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[216903]: osdmap e325: 8 total, 8 up, 5 in 2023-10-09T04:48:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/813205684' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[223858]: pgmap v1272: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 515 op/s; 25373/69606 objects degraded (36.452%); 11121/69606 objects misplaced (15.977%); 13 MiB/s, 286 objects/s recovering 2023-10-09T04:48:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:48:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[223858]: osdmap e325: 8 total, 8 up, 5 in 2023-10-09T04:48:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/813205684' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:15.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:15.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:15.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:15.200 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:14 smithi177 ceph-mon[195797]: pgmap v1272: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 77 active+clean, 2 active+recovery_wait+degraded+remapped; 1.4 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 515 op/s; 25373/69606 objects degraded (36.452%); 11121/69606 objects misplaced (15.977%); 13 MiB/s, 286 objects/s recovering 2023-10-09T04:48:15.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:14 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:15.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:48:15.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:14 smithi177 ceph-mon[195797]: osdmap e325: 8 total, 8 up, 5 in 2023-10-09T04:48:15.201 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/813205684' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:48:15.203 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:15.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:15.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:15.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:15.442 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:15.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: failed to encode map e325 with expected crc 2023-10-09T04:48:15.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: failed to encode map e325 with expected crc 2023-10-09T04:48:15.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/813205684' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:48:15.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: osdmap e326: 8 total, 8 up, 5 in 2023-10-09T04:48:15.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: failed to encode map e326 with expected crc 2023-10-09T04:48:15.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: failed to encode map e326 with expected crc 2023-10-09T04:48:15.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: failed to encode map e326 with expected crc 2023-10-09T04:48:15.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3649018692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:15.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 25373/69606 objects degraded (36.452%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:16.042 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:16.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:16.132 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:48:16.132 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.133 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:48:16.141 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: failed to encode map e325 with expected crc 2023-10-09T04:48:16.141 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: failed to encode map e325 with expected crc 2023-10-09T04:48:16.141 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/813205684' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:48:16.141 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: osdmap e326: 8 total, 8 up, 5 in 2023-10-09T04:48:16.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: failed to encode map e326 with expected crc 2023-10-09T04:48:16.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: failed to encode map e326 with expected crc 2023-10-09T04:48:16.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: failed to encode map e326 with expected crc 2023-10-09T04:48:16.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3649018692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:16.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 25373/69606 objects degraded (36.452%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:16.152 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:16.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: failed to encode map e325 with expected crc 2023-10-09T04:48:16.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: failed to encode map e325 with expected crc 2023-10-09T04:48:16.191 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/813205684' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:48:16.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: osdmap e326: 8 total, 8 up, 5 in 2023-10-09T04:48:16.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: failed to encode map e326 with expected crc 2023-10-09T04:48:16.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: failed to encode map e326 with expected crc 2023-10-09T04:48:16.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: failed to encode map e326 with expected crc 2023-10-09T04:48:16.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3649018692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:16.192 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 25373/69606 objects degraded (36.452%), 8 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:16.194 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:16.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:16.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:16.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:16.475 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:16.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:16.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:16.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:16.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:16.751 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:16.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:16.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:16.996 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:16 smithi177 ceph-mon[195797]: failed to encode map e326 with expected crc 2023-10-09T04:48:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:16 smithi177 ceph-mon[195797]: failed to encode map e326 with expected crc 2023-10-09T04:48:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:16 smithi177 ceph-mon[195797]: pgmap v1275: 89 pgs: 2 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 538 op/s; 23988/64647 objects degraded (37.106%); 8841/64647 objects misplaced (13.676%); 13 MiB/s, 262 objects/s recovering 2023-10-09T04:48:17.062 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:16 smithi029 ceph-mon[223858]: failed to encode map e326 with expected crc 2023-10-09T04:48:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:16 smithi029 ceph-mon[223858]: failed to encode map e326 with expected crc 2023-10-09T04:48:17.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:16 smithi029 ceph-mon[223858]: pgmap v1275: 89 pgs: 2 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 538 op/s; 23988/64647 objects degraded (37.106%); 8841/64647 objects misplaced (13.676%); 13 MiB/s, 262 objects/s recovering 2023-10-09T04:48:17.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:16 smithi029 ceph-mon[216903]: failed to encode map e326 with expected crc 2023-10-09T04:48:17.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:16 smithi029 ceph-mon[216903]: failed to encode map e326 with expected crc 2023-10-09T04:48:17.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:16 smithi029 ceph-mon[216903]: pgmap v1275: 89 pgs: 2 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 538 op/s; 23988/64647 objects degraded (37.106%); 8841/64647 objects misplaced (13.676%); 13 MiB/s, 262 objects/s recovering 2023-10-09T04:48:17.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:17.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:17.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:17.382 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:17.449 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:48:17.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:48:17.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:17.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:17.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:17.615 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:18.655 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:18.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:18.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:18.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:48:18.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:18 smithi029 ceph-mon[216903]: pgmap v1276: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 23470/62178 objects degraded (37.746%); 9644/62178 objects misplaced (15.510%); 15 MiB/s, 320 objects/s recovering 2023-10-09T04:48:18.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:18 smithi029 ceph-mon[223858]: pgmap v1276: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 23470/62178 objects degraded (37.746%); 9644/62178 objects misplaced (15.510%); 15 MiB/s, 320 objects/s recovering 2023-10-09T04:48:18.875 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:18.936 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:48:18.937 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:48:19.069 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:19.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:19.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:19.261 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:19.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:18 smithi177 ceph-mon[195797]: pgmap v1276: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 23470/62178 objects degraded (37.746%); 9644/62178 objects misplaced (15.510%); 15 MiB/s, 320 objects/s recovering 2023-10-09T04:48:19.362 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:19.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:19.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:19.472 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:19.550 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:19.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:19.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:19.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:19.826 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:19.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:19.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:20.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:20.092 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:20.454 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:20.484 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:20.498 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:48:20.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:20.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:20.661 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:20.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:48:20.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:20.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:20.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:20.968 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:20 smithi029 ceph-mon[216903]: pgmap v1277: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 459 op/s; 23470/61698 objects degraded (38.040%); 9644/61698 objects misplaced (15.631%); 9.9 MiB/s, 243 objects/s recovering 2023-10-09T04:48:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23470/61698 objects degraded (38.040%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:21.020 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:20 smithi029 ceph-mon[223858]: pgmap v1277: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 459 op/s; 23470/61698 objects degraded (38.040%); 9644/61698 objects misplaced (15.631%); 9.9 MiB/s, 243 objects/s recovering 2023-10-09T04:48:21.021 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23470/61698 objects degraded (38.040%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:20 smithi177 ceph-mon[195797]: pgmap v1277: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 459 op/s; 23470/61698 objects degraded (38.040%); 9644/61698 objects misplaced (15.631%); 9.9 MiB/s, 243 objects/s recovering 2023-10-09T04:48:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23470/61698 objects degraded (38.040%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:21.207 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:21.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:21.218 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:21.316 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:21.408 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:22.733 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:22.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:22.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:22.910 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:22.911 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:22.997 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:22 smithi029 ceph-mon[216903]: pgmap v1278: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 560 op/s; 23275/56142 objects degraded (41.457%); 8714/56142 objects misplaced (15.521%); 12 MiB/s, 254 objects/s recovering 2023-10-09T04:48:22.997 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:48:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:48:22] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2023-10-09T04:48:22.997 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:22 smithi029 ceph-mon[223858]: pgmap v1278: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 560 op/s; 23275/56142 objects degraded (41.457%); 8714/56142 objects misplaced (15.521%); 12 MiB/s, 254 objects/s recovering 2023-10-09T04:48:23.089 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:23.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:23.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:23.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:23.273 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:22 smithi177 ceph-mon[195797]: pgmap v1278: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 560 op/s; 23275/56142 objects degraded (41.457%); 8714/56142 objects misplaced (15.521%); 12 MiB/s, 254 objects/s recovering 2023-10-09T04:48:23.483 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:24.447 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:24.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:24.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:24.619 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:24.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:24.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:24.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:24.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:24.903 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:24 smithi177 ceph-mon[195797]: pgmap v1279: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 587 op/s; 22869/52752 objects degraded (43.352%); 8205/52752 objects misplaced (15.554%); 11 MiB/s, 234 objects/s recovering 2023-10-09T04:48:25.064 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:25.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:25.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:24 smithi029 ceph-mon[216903]: pgmap v1279: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 587 op/s; 22869/52752 objects degraded (43.352%); 8205/52752 objects misplaced (15.554%); 11 MiB/s, 234 objects/s recovering 2023-10-09T04:48:25.112 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:24 smithi029 ceph-mon[223858]: pgmap v1279: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 587 op/s; 22869/52752 objects degraded (43.352%); 8205/52752 objects misplaced (15.554%); 11 MiB/s, 234 objects/s recovering 2023-10-09T04:48:25.170 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:25.216 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:25.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:25.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:25.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:25.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:25.449 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:25.865 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:25.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 22869/52752 objects degraded (43.352%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:25.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 22869/52752 objects degraded (43.352%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:25.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:25.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:26.003 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 22869/52752 objects degraded (43.352%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:26.591 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:26.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:26.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:26.702 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:26.749 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:26.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:26.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:26.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:26 smithi029 ceph-mon[216903]: pgmap v1280: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 984 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 22698/47286 objects degraded (48.002%); 7612/47286 objects misplaced (16.098%); 9.3 MiB/s, 278 objects/s recovering 2023-10-09T04:48:26.969 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:27.014 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:48:27.016 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:48:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:26 smithi177 ceph-mon[195797]: pgmap v1280: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 984 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 22698/47286 objects degraded (48.002%); 7612/47286 objects misplaced (16.098%); 9.3 MiB/s, 278 objects/s recovering 2023-10-09T04:48:27.034 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:27.117 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:27.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:26 smithi029 ceph-mon[223858]: pgmap v1280: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 984 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 625 op/s; 22698/47286 objects degraded (48.002%); 7612/47286 objects misplaced (16.098%); 9.3 MiB/s, 278 objects/s recovering 2023-10-09T04:48:27.232 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:27.259 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:27.305 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:27.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:27.412 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:27.478 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:27.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:27.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:27.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:27.759 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:27.890 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:27 smithi177 ceph-mon[195797]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:27.890 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1288203043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:27.917 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:48:28.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:27 smithi029 ceph-mon[223858]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:28.026 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1288203043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:28.027 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:27 smithi029 ceph-mon[216903]: from='client.56116 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:28.027 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1288203043' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: pgmap v1281: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 932 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 22370/44796 objects degraded (49.937%); 7099/44796 objects misplaced (15.847%); 7.3 MiB/s, 229 objects/s recovering 2023-10-09T04:48:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1288203043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: osdmap e327: 8 total, 8 up, 5 in 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: pgmap v1281: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 932 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 22370/44796 objects degraded (49.937%); 7099/44796 objects misplaced (15.847%); 7.3 MiB/s, 229 objects/s recovering 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1288203043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: osdmap e327: 8 total, 8 up, 5 in 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:29.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:29.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: pgmap v1281: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 932 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 22370/44796 objects degraded (49.937%); 7099/44796 objects misplaced (15.847%); 7.3 MiB/s, 229 objects/s recovering 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1288203043' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: osdmap e327: 8 total, 8 up, 5 in 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: failed to encode map e327 with expected crc 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:29.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:30.778 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1280,"stamp":"2023-10-09T04:48:25.347658+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1031935928,"num_objects":15762,"num_object_clones":0,"num_object_copies":47286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22698,"num_objects_misplaced":7612,"num_objects_unfound":0,"num_objects_dirty":15762,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":175058,"num_write_kb":6098254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42759,"num_bytes_recovered":2584230070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94301,"ondisk_log_size":94301,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":6525956,"kb_used_data":2266204,"kb_used_omap":12,"kb_used_meta":4259635,"kb_avail":462199804,"statfs":{"total":479975178240,"available":473292599296,"internally_reserved":0,"allocated":2320592896,"data_stored":2315745308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12321,"internal_metadata":4361867231},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-432406528,"num_objects":-6598,"num_object_clones":0,"num_object_copies":-19794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2675,"num_objects_misplaced":-3022,"num_objects_unfound":0,"num_objects_dirty":-6598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6598,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2937,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.556547"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":74,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.155733+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:48:14.155733+0000","last_peered":"2023-10-09T04:48:14.155733+0000","last_clean":"2023-10-09T04:48:14.155733+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:48:14.155733+0000","last_undegraded":"2023-10-09T04:48:14.155733+0000","last_fullsized":"2023-10-09T04:48:14.155733+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":119,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.146948+0000","last_change":"2023-10-09T04:45:45.599027+0000","last_active":"2023-10-09T04:48:14.146948+0000","last_peered":"2023-10-09T04:48:14.146948+0000","last_clean":"2023-10-09T04:48:14.146948+0000","last_became_active":"2023-10-09T04:45:45.598650+0000","last_became_peered":"2023-10-09T04:45:45.598650+0000","last_unstale":"2023-10-09T04:48:14.146948+0000","last_undegraded":"2023-10-09T04:48:14.146948+0000","last_fullsized":"2023-10-09T04:48:14.146948+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:15.414379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":323,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.792502+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:48:14.792502+0000","last_peered":"2023-10-09T04:48:14.792502+0000","last_clean":"2023-10-09T04:48:14.792502+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:48:14.792502+0000","last_undegraded":"2023-10-09T04:48:14.792502+0000","last_fullsized":"2023-10-09T04:48:14.792502+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":259,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134863+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:48:14.134863+0000","last_peered":"2023-10-09T04:48:14.134863+0000","last_clean":"2023-10-09T04:48:14.134863+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:48:14.134863+0000","last_undegraded":"2023-10-09T04:48:14.134863+0000","last_fullsized":"2023-10-09T04:48:14.134863+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":44,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.799851+0000","last_change":"2023-10-09T04:46:25.173258+0000","last_active":"2023-10-09T04:48:14.799851+0000","last_peered":"2023-10-09T04:48:14.799851+0000","last_clean":"2023-10-09T04:48:14.799851+0000","last_became_active":"2023-10-09T04:46:25.172947+0000","last_became_peered":"2023-10-09T04:46:25.172947+0000","last_unstale":"2023-10-09T04:48:14.799851+0000","last_undegraded":"2023-10-09T04:48:14.799851+0000","last_fullsized":"2023-10-09T04:48:14.799851+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:38.532832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":126,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140076+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:48:14.140076+0000","last_peered":"2023-10-09T04:48:14.140076+0000","last_clean":"2023-10-09T04:48:14.140076+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:48:14.140076+0000","last_undegraded":"2023-10-09T04:48:14.140076+0000","last_fullsized":"2023-10-09T04:48:14.140076+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":258,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.139937+0000","last_change":"2023-10-09T04:46:25.155473+0000","last_active":"2023-10-09T04:48:14.139937+0000","last_peered":"2023-10-09T04:48:14.139937+0000","last_clean":"2023-10-09T04:48:14.139937+0000","last_became_active":"2023-10-09T04:46:25.155148+0000","last_became_peered":"2023-10-09T04:46:25.155148+0000","last_unstale":"2023-10-09T04:48:14.139937+0000","last_undegraded":"2023-10-09T04:48:14.139937+0000","last_fullsized":"2023-10-09T04:48:14.139937+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:38.576510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":283,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.146425+0000","last_change":"2023-10-09T04:45:45.611067+0000","last_active":"2023-10-09T04:48:14.146425+0000","last_peered":"2023-10-09T04:48:14.146425+0000","last_clean":"2023-10-09T04:48:14.146425+0000","last_became_active":"2023-10-09T04:45:45.610571+0000","last_became_peered":"2023-10-09T04:45:45.610571+0000","last_unstale":"2023-10-09T04:48:14.146425+0000","last_undegraded":"2023-10-09T04:48:14.146425+0000","last_fullsized":"2023-10-09T04:48:14.146425+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:14.381921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":286,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.139685+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:48:14.139685+0000","last_peered":"2023-10-09T04:48:14.139685+0000","last_clean":"2023-10-09T04:48:14.139685+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:48:14.139685+0000","last_undegraded":"2023-10-09T04:48:14.139685+0000","last_fullsized":"2023-10-09T04:48:14.139685+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":266,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135441+0000","last_change":"2023-10-09T04:46:25.193703+0000","last_active":"2023-10-09T04:48:14.135441+0000","last_peered":"2023-10-09T04:48:14.135441+0000","last_clean":"2023-10-09T04:48:14.135441+0000","last_became_active":"2023-10-09T04:46:25.193392+0000","last_became_peered":"2023-10-09T04:46:25.193392+0000","last_unstale":"2023-10-09T04:48:14.135441+0000","last_undegraded":"2023-10-09T04:48:14.135441+0000","last_fullsized":"2023-10-09T04:48:14.135441+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:56.086176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":352,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135067+0000","last_change":"2023-10-09T04:46:25.479989+0000","last_active":"2023-10-09T04:48:14.135067+0000","last_peered":"2023-10-09T04:48:14.135067+0000","last_clean":"2023-10-09T04:48:14.135067+0000","last_became_active":"2023-10-09T04:46:25.381247+0000","last_became_peered":"2023-10-09T04:46:25.381247+0000","last_unstale":"2023-10-09T04:48:14.135067+0000","last_undegraded":"2023-10-09T04:48:14.135067+0000","last_fullsized":"2023-10-09T04:48:14.135067+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:12.653312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":252,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.137363+0000","last_change":"2023-10-09T04:46:25.150997+0000","last_active":"2023-10-09T04:48:14.137363+0000","last_peered":"2023-10-09T04:48:14.137363+0000","last_clean":"2023-10-09T04:48:14.137363+0000","last_became_active":"2023-10-09T04:46:25.150370+0000","last_became_peered":"2023-10-09T04:46:25.150370+0000","last_unstale":"2023-10-09T04:48:14.137363+0000","last_undegraded":"2023-10-09T04:48:14.137363+0000","last_fullsized":"2023-10-09T04:48:14.137363+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:07.231212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":104,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.147123+0000","last_change":"2023-10-09T04:44:38.813366+0000","last_active":"2023-10-09T04:48:14.147123+0000","last_peered":"2023-10-09T04:48:14.147123+0000","last_clean":"2023-10-09T04:48:14.147123+0000","last_became_active":"2023-10-09T04:44:38.777069+0000","last_became_peered":"2023-10-09T04:44:38.777069+0000","last_unstale":"2023-10-09T04:48:14.147123+0000","last_undegraded":"2023-10-09T04:48:14.147123+0000","last_fullsized":"2023-10-09T04:48:14.147123+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:39.101540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":296,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.149194+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:48:14.149194+0000","last_peered":"2023-10-09T04:48:14.149194+0000","last_clean":"2023-10-09T04:48:14.149194+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:48:14.149194+0000","last_undegraded":"2023-10-09T04:48:14.149194+0000","last_fullsized":"2023-10-09T04:48:14.149194+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":336,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135888+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:48:14.135888+0000","last_peered":"2023-10-09T04:48:14.135888+0000","last_clean":"2023-10-09T04:48:14.135888+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:48:14.135888+0000","last_undegraded":"2023-10-09T04:48:14.135888+0000","last_fullsized":"2023-10-09T04:48:14.135888+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":119,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134383+0000","last_change":"2023-10-09T04:45:45.607791+0000","last_active":"2023-10-09T04:48:14.134383+0000","last_peered":"2023-10-09T04:48:14.134383+0000","last_clean":"2023-10-09T04:48:14.134383+0000","last_became_active":"2023-10-09T04:45:45.607501+0000","last_became_peered":"2023-10-09T04:45:45.607501+0000","last_unstale":"2023-10-09T04:48:14.134383+0000","last_undegraded":"2023-10-09T04:48:14.134383+0000","last_fullsized":"2023-10-09T04:48:14.134383+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:05.067640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":258,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134565+0000","last_change":"2023-10-09T04:46:25.168444+0000","last_active":"2023-10-09T04:48:14.134565+0000","last_peered":"2023-10-09T04:48:14.134565+0000","last_clean":"2023-10-09T04:48:14.134565+0000","last_became_active":"2023-10-09T04:46:25.168143+0000","last_became_peered":"2023-10-09T04:46:25.168143+0000","last_unstale":"2023-10-09T04:48:14.134565+0000","last_undegraded":"2023-10-09T04:48:14.134565+0000","last_fullsized":"2023-10-09T04:48:14.134565+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:56.166900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.141072+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:48:14.141072+0000","last_peered":"2023-10-09T04:48:14.141072+0000","last_clean":"2023-10-09T04:48:14.141072+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:48:14.141072+0000","last_undegraded":"2023-10-09T04:48:14.141072+0000","last_fullsized":"2023-10-09T04:48:14.141072+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":288,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135397+0000","last_change":"2023-10-09T04:46:25.175418+0000","last_active":"2023-10-09T04:48:14.135397+0000","last_peered":"2023-10-09T04:48:14.135397+0000","last_clean":"2023-10-09T04:48:14.135397+0000","last_became_active":"2023-10-09T04:46:25.174958+0000","last_became_peered":"2023-10-09T04:46:25.174958+0000","last_unstale":"2023-10-09T04:48:14.135397+0000","last_undegraded":"2023-10-09T04:48:14.135397+0000","last_fullsized":"2023-10-09T04:48:14.135397+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:35.362263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":146,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134830+0000","last_change":"2023-10-09T04:46:25.170147+0000","last_active":"2023-10-09T04:48:14.134830+0000","last_peered":"2023-10-09T04:48:14.134830+0000","last_clean":"2023-10-09T04:48:14.134830+0000","last_became_active":"2023-10-09T04:46:25.169835+0000","last_became_peered":"2023-10-09T04:46:25.169835+0000","last_unstale":"2023-10-09T04:48:14.134830+0000","last_undegraded":"2023-10-09T04:48:14.134830+0000","last_fullsized":"2023-10-09T04:48:14.134830+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:01.393349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":107,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.139090+0000","last_change":"2023-10-09T04:46:25.271462+0000","last_active":"2023-10-09T04:48:14.139090+0000","last_peered":"2023-10-09T04:48:14.139090+0000","last_clean":"2023-10-09T04:48:14.139090+0000","last_became_active":"2023-10-09T04:46:25.166617+0000","last_became_peered":"2023-10-09T04:46:25.166617+0000","last_unstale":"2023-10-09T04:48:14.139090+0000","last_undegraded":"2023-10-09T04:48:14.139090+0000","last_fullsized":"2023-10-09T04:48:14.139090+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:30.657523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":119,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134943+0000","last_change":"2023-10-09T04:45:45.603131+0000","last_active":"2023-10-09T04:48:14.134943+0000","last_peered":"2023-10-09T04:48:14.134943+0000","last_clean":"2023-10-09T04:48:14.134943+0000","last_became_active":"2023-10-09T04:45:45.602862+0000","last_became_peered":"2023-10-09T04:45:45.602862+0000","last_unstale":"2023-10-09T04:48:14.134943+0000","last_undegraded":"2023-10-09T04:48:14.134943+0000","last_fullsized":"2023-10-09T04:48:14.134943+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:22.358682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"326'11080","reported_seq":20663,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.572982+0000","last_change":"2023-10-09T04:46:53.684121+0000","last_active":"2023-10-09T04:48:24.572982+0000","last_peered":"2023-10-09T04:48:24.572982+0000","last_clean":"2023-10-09T04:48:24.572982+0000","last_became_active":"2023-10-09T04:46:53.658692+0000","last_became_peered":"2023-10-09T04:46:53.658692+0000","last_unstale":"2023-10-09T04:48:24.572982+0000","last_undegraded":"2023-10-09T04:48:24.572982+0000","last_fullsized":"2023-10-09T04:48:24.572982+0000","mapping_epoch":312,"log_start":"297'5272","ondisk_log_start":"297'5272","created":284,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:44.589594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11080,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4752,"num_bytes_recovered":242221056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":356,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134867+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:48:14.134867+0000","last_peered":"2023-10-09T04:48:14.134867+0000","last_clean":"2023-10-09T04:48:14.134867+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:48:14.134867+0000","last_undegraded":"2023-10-09T04:48:14.134867+0000","last_fullsized":"2023-10-09T04:48:14.134867+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":278,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.138991+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:48:14.138991+0000","last_peered":"2023-10-09T04:48:14.138991+0000","last_clean":"2023-10-09T04:48:14.138991+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:48:14.138991+0000","last_undegraded":"2023-10-09T04:48:14.138991+0000","last_fullsized":"2023-10-09T04:48:14.138991+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"326'11077","reported_seq":20079,"reported_epoch":326,"state":"active+recovering+remapped","last_fresh":"2023-10-09T04:48:24.972753+0000","last_change":"2023-10-09T04:48:14.905207+0000","last_active":"2023-10-09T04:48:24.972753+0000","last_peered":"2023-10-09T04:48:24.972753+0000","last_clean":"2023-10-09T04:44:37.740580+0000","last_became_active":"2023-10-09T04:45:46.774175+0000","last_became_peered":"2023-10-09T04:45:46.774175+0000","last_unstale":"2023-10-09T04:48:24.972753+0000","last_undegraded":"2023-10-09T04:48:24.972753+0000","last_fullsized":"2023-10-09T04:48:24.972753+0000","mapping_epoch":303,"log_start":"297'5806","ondisk_log_start":"297'5806","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5271,"log_dups_size":0,"ondisk_log_size":5271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:40.805571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61407232,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11077,"num_write_kb":384448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4467,"num_bytes_recovered":240713728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"1,3,7","objects":937}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134754+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:48:14.134754+0000","last_peered":"2023-10-09T04:48:14.134754+0000","last_clean":"2023-10-09T04:48:14.134754+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:48:14.134754+0000","last_undegraded":"2023-10-09T04:48:14.134754+0000","last_fullsized":"2023-10-09T04:48:14.134754+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":323,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.148117+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:48:14.148117+0000","last_peered":"2023-10-09T04:48:14.148117+0000","last_clean":"2023-10-09T04:48:14.148117+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:48:14.148117+0000","last_undegraded":"2023-10-09T04:48:14.148117+0000","last_fullsized":"2023-10-09T04:48:14.148117+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"326'10903","reported_seq":10968,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:24.587068+0000","last_change":"2023-10-09T04:46:25.245840+0000","last_active":"2023-10-09T04:48:24.587068+0000","last_peered":"2023-10-09T04:48:24.587068+0000","last_clean":"2023-10-09T04:45:44.791779+0000","last_became_active":"2023-10-09T04:45:46.998769+0000","last_became_peered":"2023-10-09T04:45:46.998769+0000","last_unstale":"2023-10-09T04:48:24.587068+0000","last_undegraded":"2023-10-09T04:45:46.613330+0000","last_fullsized":"2023-10-09T04:45:46.574934+0000","mapping_epoch":303,"log_start":"297'5129","ondisk_log_start":"297'5129","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:00.584863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62980096,"num_objects":961,"num_object_clones":0,"num_object_copies":2883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5931,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10903,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":961},{"shards":"4,7","objects":-3966}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134887+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:48:14.134887+0000","last_peered":"2023-10-09T04:48:14.134887+0000","last_clean":"2023-10-09T04:48:14.134887+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:48:14.134887+0000","last_undegraded":"2023-10-09T04:48:14.134887+0000","last_fullsized":"2023-10-09T04:48:14.134887+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":243,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.137297+0000","last_change":"2023-10-09T04:44:38.775465+0000","last_active":"2023-10-09T04:48:14.137297+0000","last_peered":"2023-10-09T04:48:14.137297+0000","last_clean":"2023-10-09T04:48:14.137297+0000","last_became_active":"2023-10-09T04:44:38.775246+0000","last_became_peered":"2023-10-09T04:44:38.775246+0000","last_unstale":"2023-10-09T04:48:14.137297+0000","last_undegraded":"2023-10-09T04:48:14.137297+0000","last_fullsized":"2023-10-09T04:48:14.137297+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.286190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.3","version":"326'10990","reported_seq":17743,"reported_epoch":326,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:48:24.968903+0000","last_change":"2023-10-09T04:46:26.476463+0000","last_active":"2023-10-09T04:48:24.968903+0000","last_peered":"2023-10-09T04:48:24.968903+0000","last_clean":"2023-10-09T04:45:44.834828+0000","last_became_active":"2023-10-09T04:46:26.386549+0000","last_became_peered":"2023-10-09T04:46:26.386549+0000","last_unstale":"2023-10-09T04:48:24.968903+0000","last_undegraded":"2023-10-09T04:46:26.372334+0000","last_fullsized":"2023-10-09T04:48:24.968903+0000","mapping_epoch":308,"log_start":"297'5780","ondisk_log_start":"297'5780","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5210,"log_dups_size":0,"ondisk_log_size":5210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:01.311399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61865984,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":944,"num_objects_misplaced":2625,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10990,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3342,"num_bytes_recovered":218824704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,0],"avail_no_missing":["7","0","3"],"object_location_counts":[{"shards":"0,3,7","objects":943},{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":316,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140896+0000","last_change":"2023-10-09T04:46:25.150945+0000","last_active":"2023-10-09T04:48:14.140896+0000","last_peered":"2023-10-09T04:48:14.140896+0000","last_clean":"2023-10-09T04:48:14.140896+0000","last_became_active":"2023-10-09T04:46:25.150622+0000","last_became_peered":"2023-10-09T04:46:25.150622+0000","last_unstale":"2023-10-09T04:48:14.140896+0000","last_undegraded":"2023-10-09T04:48:14.140896+0000","last_fullsized":"2023-10-09T04:48:14.140896+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:02.718023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":77,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.148008+0000","last_change":"2023-10-09T04:46:25.173123+0000","last_active":"2023-10-09T04:48:14.148008+0000","last_peered":"2023-10-09T04:48:14.148008+0000","last_clean":"2023-10-09T04:48:14.148008+0000","last_became_active":"2023-10-09T04:46:25.172534+0000","last_became_peered":"2023-10-09T04:46:25.172534+0000","last_unstale":"2023-10-09T04:48:14.148008+0000","last_undegraded":"2023-10-09T04:48:14.148008+0000","last_fullsized":"2023-10-09T04:48:14.148008+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:00.581088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"326'10940","reported_seq":20323,"reported_epoch":326,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:24.960400+0000","last_change":"2023-10-09T04:47:41.808546+0000","last_active":"2023-10-09T04:48:24.960400+0000","last_peered":"2023-10-09T04:48:24.960400+0000","last_clean":"2023-10-09T04:44:37.730195+0000","last_became_active":"2023-10-09T04:45:47.970624+0000","last_became_peered":"2023-10-09T04:45:47.970624+0000","last_unstale":"2023-10-09T04:48:24.960400+0000","last_undegraded":"2023-10-09T04:48:24.960400+0000","last_fullsized":"2023-10-09T04:48:24.960400+0000","mapping_epoch":303,"log_start":"297'5736","ondisk_log_start":"297'5736","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5204,"log_dups_size":0,"ondisk_log_size":5204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:58.727121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62390272,"num_objects":952,"num_object_clones":0,"num_object_copies":2856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":952,"num_objects_unfound":0,"num_objects_dirty":952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10940,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4658,"num_bytes_recovered":305266688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,3],"avail_no_missing":["7","1","3","6"],"object_location_counts":[{"shards":"1,3,7","objects":952}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":65,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.149280+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:48:14.149280+0000","last_peered":"2023-10-09T04:48:14.149280+0000","last_clean":"2023-10-09T04:48:14.149280+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:48:14.149280+0000","last_undegraded":"2023-10-09T04:48:14.149280+0000","last_fullsized":"2023-10-09T04:48:14.149280+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.147270+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:48:14.147270+0000","last_peered":"2023-10-09T04:48:14.147270+0000","last_clean":"2023-10-09T04:48:14.147270+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:48:14.147270+0000","last_undegraded":"2023-10-09T04:48:14.147270+0000","last_fullsized":"2023-10-09T04:48:14.147270+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"326'10864","reported_seq":14122,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:21.911904+0000","last_change":"2023-10-09T04:46:25.469213+0000","last_active":"2023-10-09T04:48:21.911904+0000","last_peered":"2023-10-09T04:48:21.911904+0000","last_clean":"2023-10-09T04:44:37.740389+0000","last_became_active":"2023-10-09T04:46:25.449027+0000","last_became_peered":"2023-10-09T04:46:25.449027+0000","last_unstale":"2023-10-09T04:48:21.911904+0000","last_undegraded":"2023-10-09T04:46:25.190986+0000","last_fullsized":"2023-10-09T04:46:25.133749+0000","mapping_epoch":307,"log_start":"290'4273","ondisk_log_start":"290'4273","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:53.663910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10864,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1593,"num_bytes_recovered":104202240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":-3272},{"shards":"1,2,3","objects":1014}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":287,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.152962+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:48:14.152962+0000","last_peered":"2023-10-09T04:48:14.152962+0000","last_clean":"2023-10-09T04:48:14.152962+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:48:14.152962+0000","last_undegraded":"2023-10-09T04:48:14.152962+0000","last_fullsized":"2023-10-09T04:48:14.152962+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":74,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.146830+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:48:14.146830+0000","last_peered":"2023-10-09T04:48:14.146830+0000","last_clean":"2023-10-09T04:48:14.146830+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:48:14.146830+0000","last_undegraded":"2023-10-09T04:48:14.146830+0000","last_fullsized":"2023-10-09T04:48:14.146830+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"326'10760","reported_seq":18649,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:22.992851+0000","last_change":"2023-10-09T04:46:54.673986+0000","last_active":"2023-10-09T04:48:22.992851+0000","last_peered":"2023-10-09T04:48:22.992851+0000","last_clean":"2023-10-09T04:48:22.992851+0000","last_became_active":"2023-10-09T04:46:54.653505+0000","last_became_peered":"2023-10-09T04:46:54.653505+0000","last_unstale":"2023-10-09T04:48:22.992851+0000","last_undegraded":"2023-10-09T04:48:22.992851+0000","last_fullsized":"2023-10-09T04:48:22.992851+0000","mapping_epoch":313,"log_start":"298'6128","ondisk_log_start":"298'6128","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":4632,"log_dups_size":0,"ondisk_log_size":4632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:19.484546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66977792,"num_objects":1022,"num_object_clones":0,"num_object_copies":3066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10760,"num_write_kb":377024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3907,"num_bytes_recovered":256049152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":74,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.147965+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:48:14.147965+0000","last_peered":"2023-10-09T04:48:14.147965+0000","last_clean":"2023-10-09T04:48:14.147965+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:48:14.147965+0000","last_undegraded":"2023-10-09T04:48:14.147965+0000","last_fullsized":"2023-10-09T04:48:14.147965+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":306,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.149027+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:48:14.149027+0000","last_peered":"2023-10-09T04:48:14.149027+0000","last_clean":"2023-10-09T04:48:14.149027+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:48:14.149027+0000","last_undegraded":"2023-10-09T04:48:14.149027+0000","last_fullsized":"2023-10-09T04:48:14.149027+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"326'10993","reported_seq":11043,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.837731+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:48:24.837731+0000","last_peered":"2023-10-09T04:48:24.837731+0000","last_clean":"2023-10-09T04:48:24.837731+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:48:24.837731+0000","last_undegraded":"2023-10-09T04:48:24.837731+0000","last_fullsized":"2023-10-09T04:48:24.837731+0000","mapping_epoch":284,"log_start":"297'5796","ondisk_log_start":"297'5796","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5197,"log_dups_size":0,"ondisk_log_size":5197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64684032,"num_objects":987,"num_object_clones":0,"num_object_copies":2961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10993,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":120,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135209+0000","last_change":"2023-10-09T04:45:45.611560+0000","last_active":"2023-10-09T04:48:14.135209+0000","last_peered":"2023-10-09T04:48:14.135209+0000","last_clean":"2023-10-09T04:48:14.135209+0000","last_became_active":"2023-10-09T04:45:45.611247+0000","last_became_peered":"2023-10-09T04:45:45.611247+0000","last_unstale":"2023-10-09T04:48:14.135209+0000","last_undegraded":"2023-10-09T04:48:14.135209+0000","last_fullsized":"2023-10-09T04:48:14.135209+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:52.196556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":272,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.150402+0000","last_change":"2023-10-09T04:45:45.604134+0000","last_active":"2023-10-09T04:48:14.150402+0000","last_peered":"2023-10-09T04:48:14.150402+0000","last_clean":"2023-10-09T04:48:14.150402+0000","last_became_active":"2023-10-09T04:45:45.603767+0000","last_became_peered":"2023-10-09T04:45:45.603767+0000","last_unstale":"2023-10-09T04:48:14.150402+0000","last_undegraded":"2023-10-09T04:48:14.150402+0000","last_fullsized":"2023-10-09T04:48:14.150402+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:14.801140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.a","version":"326'11042","reported_seq":20793,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:21.919079+0000","last_change":"2023-10-09T04:46:26.440123+0000","last_active":"2023-10-09T04:48:21.919079+0000","last_peered":"2023-10-09T04:48:21.919079+0000","last_clean":"2023-10-09T04:45:44.754196+0000","last_became_active":"2023-10-09T04:46:26.422562+0000","last_became_peered":"2023-10-09T04:46:26.422562+0000","last_unstale":"2023-10-09T04:48:21.919079+0000","last_undegraded":"2023-10-09T04:46:26.243542+0000","last_fullsized":"2023-10-09T04:46:26.131676+0000","mapping_epoch":308,"log_start":"291'4497","ondisk_log_start":"291'4497","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:06.608939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69599232,"num_objects":1062,"num_object_clones":0,"num_object_copies":3186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5651,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11042,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3227,"num_bytes_recovered":211353600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1061},{"shards":"1,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":213,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134478+0000","last_change":"2023-10-09T04:46:25.169137+0000","last_active":"2023-10-09T04:48:14.134478+0000","last_peered":"2023-10-09T04:48:14.134478+0000","last_clean":"2023-10-09T04:48:14.134478+0000","last_became_active":"2023-10-09T04:46:25.168834+0000","last_became_peered":"2023-10-09T04:46:25.168834+0000","last_unstale":"2023-10-09T04:48:14.134478+0000","last_undegraded":"2023-10-09T04:48:14.134478+0000","last_fullsized":"2023-10-09T04:48:14.134478+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:15.860221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":330,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.141114+0000","last_change":"2023-10-09T04:45:45.824913+0000","last_active":"2023-10-09T04:48:14.141114+0000","last_peered":"2023-10-09T04:48:14.141114+0000","last_clean":"2023-10-09T04:48:14.141114+0000","last_became_active":"2023-10-09T04:45:45.824572+0000","last_became_peered":"2023-10-09T04:45:45.824572+0000","last_unstale":"2023-10-09T04:48:14.141114+0000","last_undegraded":"2023-10-09T04:48:14.141114+0000","last_fullsized":"2023-10-09T04:48:14.141114+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:36.276403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":297,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140296+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:48:14.140296+0000","last_peered":"2023-10-09T04:48:14.140296+0000","last_clean":"2023-10-09T04:48:14.140296+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:48:14.140296+0000","last_undegraded":"2023-10-09T04:48:14.140296+0000","last_fullsized":"2023-10-09T04:48:14.140296+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"326'10939","reported_seq":18643,"reported_epoch":326,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:24.511697+0000","last_change":"2023-10-09T04:46:52.203041+0000","last_active":"2023-10-09T04:48:24.511697+0000","last_peered":"2023-10-09T04:48:24.511697+0000","last_clean":"2023-10-09T04:46:24.128327+0000","last_became_active":"2023-10-09T04:46:26.494164+0000","last_became_peered":"2023-10-09T04:46:26.494164+0000","last_unstale":"2023-10-09T04:48:24.511697+0000","last_undegraded":"2023-10-09T04:46:26.365928+0000","last_fullsized":"2023-10-09T04:46:26.143463+0000","mapping_epoch":308,"log_start":"290'4396","ondisk_log_start":"290'4396","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6543,"log_dups_size":0,"ondisk_log_size":6543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:51.675602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61014016,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":796,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10939,"num_write_kb":379840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3822,"num_bytes_recovered":226557952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"2,5","objects":135}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":310,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.139104+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:48:14.139104+0000","last_peered":"2023-10-09T04:48:14.139104+0000","last_clean":"2023-10-09T04:48:14.139104+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:48:14.139104+0000","last_undegraded":"2023-10-09T04:48:14.139104+0000","last_fullsized":"2023-10-09T04:48:14.139104+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":94,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.147796+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:48:14.147796+0000","last_peered":"2023-10-09T04:48:14.147796+0000","last_clean":"2023-10-09T04:48:14.147796+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:48:14.147796+0000","last_undegraded":"2023-10-09T04:48:14.147796+0000","last_fullsized":"2023-10-09T04:48:14.147796+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":308,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.136179+0000","last_change":"2023-10-09T04:45:45.611766+0000","last_active":"2023-10-09T04:48:14.136179+0000","last_peered":"2023-10-09T04:48:14.136179+0000","last_clean":"2023-10-09T04:48:14.136179+0000","last_became_active":"2023-10-09T04:45:45.611242+0000","last_became_peered":"2023-10-09T04:45:45.611242+0000","last_unstale":"2023-10-09T04:48:14.136179+0000","last_undegraded":"2023-10-09T04:48:14.136179+0000","last_fullsized":"2023-10-09T04:48:14.136179+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:23.336238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"326'10634","reported_seq":19710,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.499961+0000","last_change":"2023-10-09T04:45:30.672535+0000","last_active":"2023-10-09T04:48:24.499961+0000","last_peered":"2023-10-09T04:48:24.499961+0000","last_clean":"2023-10-09T04:48:24.499961+0000","last_became_active":"2023-10-09T04:45:30.663157+0000","last_became_peered":"2023-10-09T04:45:30.663157+0000","last_unstale":"2023-10-09T04:48:24.499961+0000","last_undegraded":"2023-10-09T04:48:24.499961+0000","last_fullsized":"2023-10-09T04:48:24.499961+0000","mapping_epoch":300,"log_start":"290'4106","ondisk_log_start":"290'4106","created":284,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:31.926989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64618496,"num_objects":986,"num_object_clones":0,"num_object_copies":2958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10634,"num_write_kb":371840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4503,"num_bytes_recovered":295108608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":303,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.136802+0000","last_change":"2023-10-09T04:46:25.161784+0000","last_active":"2023-10-09T04:48:14.136802+0000","last_peered":"2023-10-09T04:48:14.136802+0000","last_clean":"2023-10-09T04:48:14.136802+0000","last_became_active":"2023-10-09T04:46:25.161498+0000","last_became_peered":"2023-10-09T04:46:25.161498+0000","last_unstale":"2023-10-09T04:48:14.136802+0000","last_undegraded":"2023-10-09T04:48:14.136802+0000","last_fullsized":"2023-10-09T04:48:14.136802+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:01.500231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":302,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.136854+0000","last_change":"2023-10-09T04:46:25.165536+0000","last_active":"2023-10-09T04:48:14.136854+0000","last_peered":"2023-10-09T04:48:14.136854+0000","last_clean":"2023-10-09T04:48:14.136854+0000","last_became_active":"2023-10-09T04:46:25.165269+0000","last_became_peered":"2023-10-09T04:46:25.165269+0000","last_unstale":"2023-10-09T04:48:14.136854+0000","last_undegraded":"2023-10-09T04:48:14.136854+0000","last_fullsized":"2023-10-09T04:48:14.136854+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:30.362732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":208,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140006+0000","last_change":"2023-10-09T04:46:25.177443+0000","last_active":"2023-10-09T04:48:14.140006+0000","last_peered":"2023-10-09T04:48:14.140006+0000","last_clean":"2023-10-09T04:48:14.140006+0000","last_became_active":"2023-10-09T04:46:25.176635+0000","last_became_peered":"2023-10-09T04:46:25.176635+0000","last_unstale":"2023-10-09T04:48:14.140006+0000","last_undegraded":"2023-10-09T04:48:14.140006+0000","last_fullsized":"2023-10-09T04:48:14.140006+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:43.367744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"326'10869","reported_seq":10920,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:24.919380+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:48:24.919380+0000","last_peered":"2023-10-09T04:48:24.919380+0000","last_clean":"2023-10-09T04:48:24.919380+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:48:24.919380+0000","last_undegraded":"2023-10-09T04:48:24.919380+0000","last_fullsized":"2023-10-09T04:48:24.919380+0000","mapping_epoch":284,"log_start":"297'5669","ondisk_log_start":"297'5669","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5200,"log_dups_size":0,"ondisk_log_size":5200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63438872,"num_objects":969,"num_object_clones":0,"num_object_copies":2907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":969,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10869,"num_write_kb":378753,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":307,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140699+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:48:14.140699+0000","last_peered":"2023-10-09T04:48:14.140699+0000","last_clean":"2023-10-09T04:48:14.140699+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:48:14.140699+0000","last_undegraded":"2023-10-09T04:48:14.140699+0000","last_fullsized":"2023-10-09T04:48:14.140699+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":754,"reported_epoch":325,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:14.134257+0000","last_change":"2023-10-09T04:46:26.415415+0000","last_active":"2023-10-09T04:48:14.134257+0000","last_peered":"2023-10-09T04:48:14.134257+0000","last_clean":"2023-10-09T04:46:16.698879+0000","last_became_active":"2023-10-09T04:46:26.415059+0000","last_became_peered":"2023-10-09T04:46:26.415059+0000","last_unstale":"2023-10-09T04:48:14.134257+0000","last_undegraded":"2023-10-09T04:46:26.138726+0000","last_fullsized":"2023-10-09T04:46:26.138125+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:23.417129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":256,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140374+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:48:14.140374+0000","last_peered":"2023-10-09T04:48:14.140374+0000","last_clean":"2023-10-09T04:48:14.140374+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:48:14.140374+0000","last_undegraded":"2023-10-09T04:48:14.140374+0000","last_fullsized":"2023-10-09T04:48:14.140374+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":269,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.146764+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:48:14.146764+0000","last_peered":"2023-10-09T04:48:14.146764+0000","last_clean":"2023-10-09T04:48:14.146764+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:48:14.146764+0000","last_undegraded":"2023-10-09T04:48:14.146764+0000","last_fullsized":"2023-10-09T04:48:14.146764+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"326'10857","reported_seq":10917,"reported_epoch":326,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:24.581161+0000","last_change":"2023-10-09T04:46:26.351968+0000","last_active":"2023-10-09T04:48:24.581161+0000","last_peered":"2023-10-09T04:48:24.581161+0000","last_clean":"2023-10-09T04:46:24.123151+0000","last_became_active":"2023-10-09T04:46:26.320651+0000","last_became_peered":"2023-10-09T04:46:26.320651+0000","last_unstale":"2023-10-09T04:48:24.581161+0000","last_undegraded":"2023-10-09T04:48:24.581161+0000","last_fullsized":"2023-10-09T04:48:24.581161+0000","mapping_epoch":308,"log_start":"297'5076","ondisk_log_start":"297'5076","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:25.867280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63111168,"num_objects":963,"num_object_clones":0,"num_object_copies":2889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":963,"num_objects_unfound":0,"num_objects_dirty":963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10857,"num_write_kb":378240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":963}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":336,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134135+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:48:14.134135+0000","last_peered":"2023-10-09T04:48:14.134135+0000","last_clean":"2023-10-09T04:48:14.134135+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:48:14.134135+0000","last_undegraded":"2023-10-09T04:48:14.134135+0000","last_fullsized":"2023-10-09T04:48:14.134135+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":94,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.148076+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:48:14.148076+0000","last_peered":"2023-10-09T04:48:14.148076+0000","last_clean":"2023-10-09T04:48:14.148076+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:48:14.148076+0000","last_undegraded":"2023-10-09T04:48:14.148076+0000","last_fullsized":"2023-10-09T04:48:14.148076+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135782+0000","last_change":"2023-10-09T04:46:25.174089+0000","last_active":"2023-10-09T04:48:14.135782+0000","last_peered":"2023-10-09T04:48:14.135782+0000","last_clean":"2023-10-09T04:48:14.135782+0000","last_became_active":"2023-10-09T04:46:25.173414+0000","last_became_peered":"2023-10-09T04:46:25.173414+0000","last_unstale":"2023-10-09T04:48:14.135782+0000","last_undegraded":"2023-10-09T04:48:14.135782+0000","last_fullsized":"2023-10-09T04:48:14.135782+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.877749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"326'11022","reported_seq":20695,"reported_epoch":326,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:21.928600+0000","last_change":"2023-10-09T04:48:14.871531+0000","last_active":"2023-10-09T04:48:21.928600+0000","last_peered":"2023-10-09T04:48:21.928600+0000","last_clean":"2023-10-09T04:44:37.770348+0000","last_became_active":"2023-10-09T04:45:59.017846+0000","last_became_peered":"2023-10-09T04:45:59.017846+0000","last_unstale":"2023-10-09T04:48:21.928600+0000","last_undegraded":"2023-10-09T04:48:21.928600+0000","last_fullsized":"2023-10-09T04:48:21.928600+0000","mapping_epoch":303,"log_start":"290'4403","ondisk_log_start":"290'4403","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:00.633461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63438848,"num_objects":968,"num_object_clones":0,"num_object_copies":2904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":968,"num_objects_unfound":0,"num_objects_dirty":968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11022,"num_write_kb":383680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4803,"num_bytes_recovered":243204096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":968}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":339,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134970+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:48:14.134970+0000","last_peered":"2023-10-09T04:48:14.134970+0000","last_clean":"2023-10-09T04:48:14.134970+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:48:14.134970+0000","last_undegraded":"2023-10-09T04:48:14.134970+0000","last_fullsized":"2023-10-09T04:48:14.134970+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":284,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.136110+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:48:14.136110+0000","last_peered":"2023-10-09T04:48:14.136110+0000","last_clean":"2023-10-09T04:48:14.136110+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:48:14.136110+0000","last_undegraded":"2023-10-09T04:48:14.136110+0000","last_fullsized":"2023-10-09T04:48:14.136110+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":180,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140825+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:48:14.140825+0000","last_peered":"2023-10-09T04:48:14.140825+0000","last_clean":"2023-10-09T04:48:14.140825+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:48:14.140825+0000","last_undegraded":"2023-10-09T04:48:14.140825+0000","last_fullsized":"2023-10-09T04:48:14.140825+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"326'10868","reported_seq":10936,"reported_epoch":326,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:21.848468+0000","last_change":"2023-10-09T04:46:25.302033+0000","last_active":"2023-10-09T04:48:21.848468+0000","last_peered":"2023-10-09T04:48:21.848468+0000","last_clean":"2023-10-09T04:45:44.820208+0000","last_became_active":"2023-10-09T04:46:25.282101+0000","last_became_peered":"2023-10-09T04:46:25.282101+0000","last_unstale":"2023-10-09T04:48:21.848468+0000","last_undegraded":"2023-10-09T04:48:21.848468+0000","last_fullsized":"2023-10-09T04:48:21.848468+0000","mapping_epoch":307,"log_start":"290'4303","ondisk_log_start":"290'4303","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:16.931727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68943872,"num_objects":1052,"num_object_clones":0,"num_object_copies":3156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2104,"num_objects_unfound":0,"num_objects_dirty":1052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10868,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,0,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1052}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":74,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135026+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:48:14.135026+0000","last_peered":"2023-10-09T04:48:14.135026+0000","last_clean":"2023-10-09T04:48:14.135026+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:48:14.135026+0000","last_undegraded":"2023-10-09T04:48:14.135026+0000","last_fullsized":"2023-10-09T04:48:14.135026+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":289,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.148954+0000","last_change":"2023-10-09T04:46:25.166210+0000","last_active":"2023-10-09T04:48:14.148954+0000","last_peered":"2023-10-09T04:48:14.148954+0000","last_clean":"2023-10-09T04:48:14.148954+0000","last_became_active":"2023-10-09T04:46:25.165948+0000","last_became_peered":"2023-10-09T04:46:25.165948+0000","last_unstale":"2023-10-09T04:48:14.148954+0000","last_undegraded":"2023-10-09T04:48:14.148954+0000","last_fullsized":"2023-10-09T04:48:14.148954+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:51.272111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":172,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135172+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:48:14.135172+0000","last_peered":"2023-10-09T04:48:14.135172+0000","last_clean":"2023-10-09T04:48:14.135172+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:48:14.135172+0000","last_undegraded":"2023-10-09T04:48:14.135172+0000","last_fullsized":"2023-10-09T04:48:14.135172+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"326'10920","reported_seq":21849,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:21.884564+0000","last_change":"2023-10-09T04:46:25.450073+0000","last_active":"2023-10-09T04:48:21.884564+0000","last_peered":"2023-10-09T04:48:21.884564+0000","last_clean":"2023-10-09T04:44:37.742164+0000","last_became_active":"2023-10-09T04:45:46.956446+0000","last_became_peered":"2023-10-09T04:45:46.956446+0000","last_unstale":"2023-10-09T04:48:21.884564+0000","last_undegraded":"2023-10-09T04:46:25.447165+0000","last_fullsized":"2023-10-09T04:45:46.571293+0000","mapping_epoch":303,"log_start":"290'4303","ondisk_log_start":"290'4303","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:11.914538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69992448,"num_objects":1068,"num_object_clones":0,"num_object_copies":3204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10920,"num_write_kb":383616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3619,"num_bytes_recovered":237109248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"1,2","objects":-246},{"shards":"1,2,3","objects":1068}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":248,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.134780+0000","last_change":"2023-10-09T04:45:45.602634+0000","last_active":"2023-10-09T04:48:14.134780+0000","last_peered":"2023-10-09T04:48:14.134780+0000","last_clean":"2023-10-09T04:48:14.134780+0000","last_became_active":"2023-10-09T04:45:45.602317+0000","last_became_peered":"2023-10-09T04:45:45.602317+0000","last_unstale":"2023-10-09T04:48:14.134780+0000","last_undegraded":"2023-10-09T04:48:14.134780+0000","last_fullsized":"2023-10-09T04:48:14.134780+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:13.963587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":112,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.149103+0000","last_change":"2023-10-09T04:46:25.210175+0000","last_active":"2023-10-09T04:48:14.149103+0000","last_peered":"2023-10-09T04:48:14.149103+0000","last_clean":"2023-10-09T04:48:14.149103+0000","last_became_active":"2023-10-09T04:46:25.174609+0000","last_became_peered":"2023-10-09T04:46:25.174609+0000","last_unstale":"2023-10-09T04:48:14.149103+0000","last_undegraded":"2023-10-09T04:48:14.149103+0000","last_fullsized":"2023-10-09T04:48:14.149103+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:29.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":94,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.147891+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:48:14.147891+0000","last_peered":"2023-10-09T04:48:14.147891+0000","last_clean":"2023-10-09T04:48:14.147891+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:48:14.147891+0000","last_undegraded":"2023-10-09T04:48:14.147891+0000","last_fullsized":"2023-10-09T04:48:14.147891+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":176,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.149045+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:48:14.149045+0000","last_peered":"2023-10-09T04:48:14.149045+0000","last_clean":"2023-10-09T04:48:14.149045+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:48:14.149045+0000","last_undegraded":"2023-10-09T04:48:14.149045+0000","last_fullsized":"2023-10-09T04:48:14.149045+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":295,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.136036+0000","last_change":"2023-10-09T04:46:25.160157+0000","last_active":"2023-10-09T04:48:14.136036+0000","last_peered":"2023-10-09T04:48:14.136036+0000","last_clean":"2023-10-09T04:48:14.136036+0000","last_became_active":"2023-10-09T04:46:25.159850+0000","last_became_peered":"2023-10-09T04:46:25.159850+0000","last_unstale":"2023-10-09T04:48:14.136036+0000","last_undegraded":"2023-10-09T04:48:14.136036+0000","last_fullsized":"2023-10-09T04:48:14.136036+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:07.475665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":316,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.139299+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:48:14.139299+0000","last_peered":"2023-10-09T04:48:14.139299+0000","last_clean":"2023-10-09T04:48:14.139299+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:48:14.139299+0000","last_undegraded":"2023-10-09T04:48:14.139299+0000","last_fullsized":"2023-10-09T04:48:14.139299+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.147869+0000","last_change":"2023-10-09T04:45:45.609849+0000","last_active":"2023-10-09T04:48:14.147869+0000","last_peered":"2023-10-09T04:48:14.147869+0000","last_clean":"2023-10-09T04:48:14.147869+0000","last_became_active":"2023-10-09T04:45:45.609639+0000","last_became_peered":"2023-10-09T04:45:45.609639+0000","last_unstale":"2023-10-09T04:48:14.147869+0000","last_undegraded":"2023-10-09T04:48:14.147869+0000","last_fullsized":"2023-10-09T04:48:14.147869+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:26.113652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":285,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.141159+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:48:14.141159+0000","last_peered":"2023-10-09T04:48:14.141159+0000","last_clean":"2023-10-09T04:48:14.141159+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:48:14.141159+0000","last_undegraded":"2023-10-09T04:48:14.141159+0000","last_fullsized":"2023-10-09T04:48:14.141159+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":94,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.153041+0000","last_change":"2023-10-09T04:46:25.172010+0000","last_active":"2023-10-09T04:48:14.153041+0000","last_peered":"2023-10-09T04:48:14.153041+0000","last_clean":"2023-10-09T04:48:14.153041+0000","last_became_active":"2023-10-09T04:46:25.171776+0000","last_became_peered":"2023-10-09T04:46:25.171776+0000","last_unstale":"2023-10-09T04:48:14.153041+0000","last_undegraded":"2023-10-09T04:48:14.153041+0000","last_fullsized":"2023-10-09T04:48:14.153041+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:40.329129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":36,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.135877+0000","last_change":"2023-10-09T04:45:45.611105+0000","last_active":"2023-10-09T04:48:14.135877+0000","last_peered":"2023-10-09T04:48:14.135877+0000","last_clean":"2023-10-09T04:48:14.135877+0000","last_became_active":"2023-10-09T04:45:45.610758+0000","last_became_peered":"2023-10-09T04:45:45.610758+0000","last_unstale":"2023-10-09T04:48:14.135877+0000","last_undegraded":"2023-10-09T04:48:14.135877+0000","last_fullsized":"2023-10-09T04:48:14.135877+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:59.031709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":275,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.801172+0000","last_change":"2023-10-09T04:46:25.153824+0000","last_active":"2023-10-09T04:48:14.801172+0000","last_peered":"2023-10-09T04:48:14.801172+0000","last_clean":"2023-10-09T04:48:14.801172+0000","last_became_active":"2023-10-09T04:46:25.153497+0000","last_became_peered":"2023-10-09T04:46:25.153497+0000","last_unstale":"2023-10-09T04:48:14.801172+0000","last_undegraded":"2023-10-09T04:48:14.801172+0000","last_fullsized":"2023-10-09T04:48:14.801172+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:00.144937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":216,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.140158+0000","last_change":"2023-10-09T04:46:25.217389+0000","last_active":"2023-10-09T04:48:14.140158+0000","last_peered":"2023-10-09T04:48:14.140158+0000","last_clean":"2023-10-09T04:48:14.140158+0000","last_became_active":"2023-10-09T04:46:25.162993+0000","last_became_peered":"2023-10-09T04:46:25.162993+0000","last_unstale":"2023-10-09T04:48:14.140158+0000","last_undegraded":"2023-10-09T04:48:14.140158+0000","last_fullsized":"2023-10-09T04:48:14.140158+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:13.549717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":94,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.146604+0000","last_change":"2023-10-09T04:45:45.610893+0000","last_active":"2023-10-09T04:48:14.146604+0000","last_peered":"2023-10-09T04:48:14.146604+0000","last_clean":"2023-10-09T04:48:14.146604+0000","last_became_active":"2023-10-09T04:45:45.610260+0000","last_became_peered":"2023-10-09T04:45:45.610260+0000","last_unstale":"2023-10-09T04:48:14.146604+0000","last_undegraded":"2023-10-09T04:48:14.146604+0000","last_fullsized":"2023-10-09T04:48:14.146604+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:22.552117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1031340056,"num_objects":15738,"num_object_clones":0,"num_object_copies":47214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22696,"num_objects_misplaced":7612,"num_objects_unfound":0,"num_objects_dirty":15738,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174758,"num_write_kb":6095809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42694,"num_bytes_recovered":2580676608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6455046144,"data_stored":6455033928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94085,"ondisk_log_size":94085,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":11254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":17338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234498,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810276,"kb_used_data":615004,"kb_used_omap":0,"kb_used_meta":1195264,"kb_avail":91934876,"statfs":{"total":95995035648,"available":94141313024,"internally_reserved":0,"allocated":629764096,"data_stored":629042515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1223950336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:40 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.1930000000000001,"15min":2.552},"min":{"1min":0.224,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.202,"5min":7.202,"15min":111.72199999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":3.5499999999999998,"5min":1.919,"15min":3.1059999999999999},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":19.152000000000001,"5min":19.152000000000001,"15min":111.76600000000001},"last":0.497}]},{"osd":1,"last update":"Mon Oct 9 04:47:41 2023","interfaces":[{"interface":"back","average":{"1min":8.5540000000000003,"5min":4.2640000000000002,"15min":4.1440000000000001},"min":{"1min":0.27500000000000002,"5min":0.253,"15min":0.253},"max":{"1min":128.601,"5min":128.601,"15min":264.62},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.5069999999999997,"5min":4.0129999999999999,"15min":4.2560000000000002},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":128.381,"5min":128.381,"15min":264.65899999999999},"last":0.27900000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":13.303000000000001,"5min":5.2450000000000001,"15min":4.8490000000000002},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":129.10400000000001,"5min":129.10400000000001,"15min":184.72},"last":33.924999999999997},{"interface":"front","average":{"1min":13.906000000000001,"5min":5.5090000000000003,"15min":5.234},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":129.15100000000001,"5min":129.15100000000001,"15min":184.697},"last":33.944000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:47:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":1.4259999999999999,"15min":4.6870000000000003},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.8630000000000004,"5min":7.8630000000000004,"15min":182.839},"last":0.441},{"interface":"front","average":{"1min":3.016,"5min":2.137,"15min":5.1639999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29799999999999999},"max":{"1min":11.705,"5min":13.968,"15min":182.678},"last":0.54500000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":1.5569999999999999,"15min":1.53},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.573,"5min":11.573,"15min":14.257999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.23,"5min":1.353,"15min":1.327},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":7.9749999999999996,"5min":10.497,"15min":10.497},"last":0.628}]},{"osd":5,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.143,"15min":1.2150000000000001},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4169999999999998,"5min":8.5709999999999997,"15min":13.319000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.492,"5min":1.6080000000000001,"15min":1.6890000000000001},"min":{"1min":0.33600000000000002,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":12.081,"5min":12.081,"15min":18.408000000000001},"last":0.59199999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":0.98699999999999999,"15min":1.2509999999999999},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":7.4459999999999997,"5min":7.4459999999999997,"15min":11.388},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.8090000000000002,"5min":1.5189999999999999,"15min":1.778},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":18.311},"last":0.60899999999999999}]}]},{"osd":6,"up_from":0,"seq":197568496132,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141221,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1245316,"kb_used_data":462524,"kb_used_omap":4,"kb_used_meta":782715,"kb_avail":92499836,"statfs":{"total":95995035648,"available":94719832064,"internally_reserved":0,"allocated":473624576,"data_stored":472171692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":801501070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.448,"15min":3.4710000000000001},"min":{"1min":0.22700000000000001,"5min":0.22600000000000001,"15min":0.14799999999999999},"max":{"1min":7.4829999999999997,"5min":21.021999999999998,"15min":97.308000000000007},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.6899999999999999,"15min":3.6659999999999999},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.193},"max":{"1min":8.3580000000000005,"5min":21.280999999999999,"15min":97.055999999999997},"last":2.1030000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:48:07 2023","interfaces":[{"interface":"back","average":{"1min":16.600999999999999,"5min":5.8769999999999998,"15min":4.9039999999999999},"min":{"1min":0.50800000000000001,"5min":0.21199999999999999,"15min":0.20899999999999999},"max":{"1min":151.41900000000001,"5min":151.41900000000001,"15min":263.60500000000002},"last":3.8069999999999999},{"interface":"front","average":{"1min":16.071000000000002,"5min":5.875,"15min":5.3070000000000004},"min":{"1min":0.42199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":146.749,"5min":146.749,"15min":263.65199999999999},"last":6.6859999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":4.6420000000000003,"15min":5.6660000000000004},"min":{"1min":0.28499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.6760000000000002,"5min":379.80900000000003,"15min":379.80900000000003},"last":3.774},{"interface":"front","average":{"1min":2.2050000000000001,"5min":4.9930000000000003,"15min":6.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":8.4190000000000005,"5min":379.95800000000003,"15min":379.95800000000003},"last":2.1709999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":2.2069999999999999,"15min":2.5779999999999998},"min":{"1min":0.379,"5min":0.307,"15min":0.245},"max":{"1min":7.4610000000000003,"5min":19.959,"15min":170.072},"last":9.9179999999999993},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.591,"15min":2.3639999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":9.2129999999999992,"5min":11.935,"15min":169.94499999999999},"last":3.7400000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.742,"15min":2.496},"min":{"1min":0.34200000000000003,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":9.782,"5min":9.782,"15min":171.036},"last":6.6550000000000002},{"interface":"front","average":{"1min":2.2789999999999999,"5min":2.298,"15min":2.79},"min":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.27500000000000002},"max":{"1min":7.5209999999999999,"5min":20.021000000000001,"15min":170.88},"last":10.035}]},{"osd":6,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.841,"15min":2.5470000000000002},"min":{"1min":0.40799999999999997,"5min":0.35799999999999998,"15min":0.32000000000000001},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":132.36099999999999},"last":2.0720000000000001},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.544,"15min":2.8610000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":9.7370000000000001,"5min":20.164999999999999,"15min":132.17699999999999},"last":10.006}]},{"osd":7,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6230000000000002,"5min":2.4129999999999998,"15min":3.339},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":8.4339999999999993,"5min":20.055,"15min":174.75999999999999},"last":9.9730000000000008},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.573,"15min":2.9540000000000002},"min":{"1min":0.39800000000000002,"5min":0.38800000000000001,"15min":0.36199999999999999},"max":{"1min":8.0459999999999994,"5min":16.238,"15min":174.779},"last":3.8370000000000002}]}]},{"osd":0,"up_from":0,"seq":390842024278,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772554,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1176436,"kb_used_data":689388,"kb_used_omap":7,"kb_used_meta":487032,"kb_avail":92568716,"statfs":{"total":95995035648,"available":94790365184,"internally_reserved":0,"allocated":705933312,"data_stored":704675491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":498720849},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.2130000000000001,"15min":5.1769999999999996},"min":{"1min":0.253,"5min":0.183,"15min":0.183},"max":{"1min":7.7130000000000001,"5min":25.899999999999999,"15min":72.608000000000004},"last":5.4290000000000003},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.901,"15min":4.7389999999999999},"min":{"1min":0.23699999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.6379999999999999,"5min":16.550000000000001,"15min":72.632000000000005},"last":10.007}]},{"osd":1,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4550000000000001,"5min":3.8500000000000001,"15min":4.7839999999999998},"min":{"1min":0.32200000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":17.843,"5min":162.124,"15min":162.124},"last":10.048},{"interface":"front","average":{"1min":4.0010000000000003,"5min":4.1710000000000003,"15min":5.2009999999999996},"min":{"1min":0.33300000000000002,"5min":0.223,"15min":0.223},"max":{"1min":21.329000000000001,"5min":164.68299999999999,"15min":164.68299999999999},"last":9.8450000000000006}]},{"osd":3,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":6,"15min":6.9749999999999996},"min":{"1min":0.31900000000000001,"5min":0.25600000000000001,"15min":0.21199999999999999},"max":{"1min":5.8170000000000002,"5min":161.12799999999999,"15min":335.928},"last":5.319},{"interface":"front","average":{"1min":2.9769999999999999,"5min":6.0019999999999998,"15min":7.1710000000000003},"min":{"1min":0.377,"5min":0.246,"15min":0.246},"max":{"1min":7.4580000000000002,"5min":161.386,"15min":335.68599999999998},"last":8.141}]},{"osd":4,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":2.2410000000000001,"15min":3.2429999999999999},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.315},"max":{"1min":7.6390000000000002,"5min":10.256,"15min":72.944999999999993},"last":4.351},{"interface":"front","average":{"1min":2.6789999999999998,"5min":2,"15min":3.2349999999999999},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9369999999999994,"5min":16.341000000000001,"15min":73.054000000000002},"last":5.2939999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":2.077,"15min":3.1379999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":12.388,"5min":12.98,"15min":72.986000000000004},"last":9.9000000000000004},{"interface":"front","average":{"1min":2.0979999999999999,"5min":2.0910000000000002,"15min":3.4470000000000001},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.32200000000000001},"max":{"1min":5.923,"5min":10.289,"15min":72.963999999999999},"last":4.3140000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":1.857,"15min":5.7329999999999997},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.33600000000000002},"max":{"1min":5.5670000000000002,"5min":16.425999999999998,"15min":135.614},"last":5.3339999999999996},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.7270000000000001,"15min":5.3230000000000004},"min":{"1min":0.39500000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.5709999999999997,"5min":10.977,"15min":130.196},"last":9.9429999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.702,"5min":2.3690000000000002,"15min":3.5390000000000001},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.7530000000000001,"5min":16.462,"15min":73.079999999999998},"last":5.2640000000000002},{"interface":"front","average":{"1min":3.2650000000000001,"5min":2.347,"15min":3.1840000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.6379999999999999,"5min":10.935,"15min":73.069000000000003},"last":9.9740000000000002}]}]},{"osd":3,"up_from":0,"seq":609885356279,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019406,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205664,"kb_used_data":309528,"kb_used_omap":0,"kb_used_meta":896128,"kb_avail":92539488,"statfs":{"total":95995035648,"available":94760435712,"internally_reserved":0,"allocated":316956672,"data_stored":316239173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917635072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":1.4330000000000001,"15min":4.1280000000000001},"min":{"1min":0.29299999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":12.795,"5min":12.795,"15min":171.13499999999999},"last":5.1970000000000001},{"interface":"front","average":{"1min":2.4199999999999999,"5min":1.716,"15min":4.1829999999999998},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":16.004000000000001,"5min":16.004000000000001,"15min":171.11799999999999},"last":0.60799999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":9.8460000000000001,"5min":5.3040000000000003,"15min":4.0389999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24099999999999999},"max":{"1min":144.65299999999999,"5min":144.65299999999999,"15min":144.65299999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":9.5960000000000001,"5min":5.0220000000000002,"15min":3.9780000000000002},"min":{"1min":0.33500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":146.50700000000001,"5min":146.50700000000001,"15min":146.50700000000001},"last":5.226}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":16.193000000000001,"5min":6.4290000000000003,"15min":5.2519999999999998},"min":{"1min":0.47199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":74.069999999999993,"5min":98.221999999999994,"15min":195.267},"last":5.165},{"interface":"front","average":{"1min":16.573,"5min":6.5599999999999996,"15min":5.0060000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":75.219999999999999,"5min":94.219999999999999,"15min":195.15799999999999},"last":4.1289999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:47:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0960000000000001,"5min":5.0860000000000003,"15min":6.3730000000000002},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":11.161,"5min":146.02199999999999,"15min":357.125},"last":3.4620000000000002},{"interface":"front","average":{"1min":1.8660000000000001,"5min":5.0570000000000004,"15min":6.3330000000000002},"min":{"1min":0.34499999999999997,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":7.0380000000000003,"5min":147.75700000000001,"15min":359.02800000000002},"last":4.0359999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:47:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9279999999999999,"5min":1.3660000000000001,"15min":2.012},"min":{"1min":0.32700000000000001,"5min":0.253,"15min":0.253},"max":{"1min":11.317,"5min":24.234999999999999,"15min":107.476},"last":3.9980000000000002},{"interface":"front","average":{"1min":2.2570000000000001,"5min":1.143,"15min":2.0230000000000001},"min":{"1min":0.33300000000000002,"5min":0.28599999999999998,"15min":0.24099999999999999},"max":{"1min":10.936999999999999,"5min":10.936999999999999,"15min":107.414},"last":1.659}]},{"osd":6,"last update":"Mon Oct 9 04:47:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.5409999999999999,"15min":1.8320000000000001},"min":{"1min":0.40799999999999997,"5min":0.29799999999999999,"15min":0.29599999999999999},"max":{"1min":12.122999999999999,"5min":22.920999999999999,"15min":65.75},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.2270000000000001,"15min":1.7430000000000001},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.095000000000001,"5min":15.095000000000001,"15min":65.849999999999994},"last":5.1079999999999997}]},{"osd":7,"last update":"Mon Oct 9 04:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.403,"5min":1.8580000000000001,"15min":1.5880000000000001},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":16.109000000000002,"5min":16.109000000000002,"15min":16.109000000000002},"last":1.95},{"interface":"front","average":{"1min":3.7069999999999999,"5min":2.2799999999999998,"15min":1.7390000000000001},"min":{"1min":0.45000000000000001,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":12.013999999999999,"5min":22.858000000000001,"15min":22.858000000000001},"last":5.2110000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757770,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1088264,"kb_used_data":189760,"kb_used_omap":0,"kb_used_meta":898496,"kb_avail":92656888,"statfs":{"total":95995035648,"available":94880653312,"internally_reserved":0,"allocated":194314240,"data_stored":193616437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920059904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5139999999999998,"5min":3.7269999999999999,"15min":5.0339999999999998},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":9.0749999999999993,"5min":163.70699999999999,"15min":210.85499999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.8260000000000001,"5min":3.629,"15min":5.1070000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.249},"max":{"1min":7.0599999999999996,"5min":163.756,"15min":210.23500000000001},"last":0.746}]},{"osd":1,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":7.2080000000000002,"5min":7.7009999999999996,"15min":5.4790000000000001},"min":{"1min":0.42899999999999999,"5min":0.318,"15min":0.252},"max":{"1min":98.331000000000003,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.372},{"interface":"front","average":{"1min":7.9880000000000004,"5min":7.6689999999999996,"15min":5.7080000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":98.358999999999995,"5min":211.435,"15min":211.435},"last":1.637}]},{"osd":2,"last update":"Mon Oct 9 04:48:03 2023","interfaces":[{"interface":"back","average":{"1min":9.3610000000000007,"5min":5.8159999999999998,"15min":5.4619999999999997},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":82.561999999999998,"5min":82.561999999999998,"15min":90.763999999999996},"last":1.5169999999999999},{"interface":"front","average":{"1min":10.454000000000001,"5min":5.8280000000000003,"15min":5.5670000000000002},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":83.082999999999998,"5min":83.082999999999998,"15min":91.340000000000003},"last":1.319}]},{"osd":3,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.766,"15min":5.1589999999999998},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":7.585,"5min":46.497,"15min":268.63299999999998},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.7070000000000001,"5min":2.0539999999999998,"15min":4.7560000000000002},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.9450000000000003,"5min":46.125999999999998,"15min":268.92500000000001},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.3320000000000001,"15min":2.5299999999999998},"min":{"1min":0.34100000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5650000000000004,"5min":10.268000000000001,"15min":361.09399999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":3.1970000000000001,"5min":1.4650000000000001,"15min":2.8860000000000001},"min":{"1min":0.33200000000000002,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":12.609999999999999,"5min":12.609999999999999,"15min":361.71300000000002},"last":0.32500000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.208,"15min":1.2849999999999999},"min":{"1min":0.378,"5min":0.28499999999999998,"15min":0.23100000000000001},"max":{"1min":3.8490000000000002,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.438},{"interface":"front","average":{"1min":1.925,"5min":1.1220000000000001,"15min":1.3879999999999999},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":7.3040000000000003,"5min":7.3040000000000003,"15min":13.231999999999999},"last":0.68700000000000006}]},{"osd":7,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":2.5249999999999999,"15min":1.764},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":7.1029999999999998,"5min":68.411000000000001,"15min":68.411000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.754,"15min":1.786},"min":{"1min":0.434,"5min":0.38300000000000001,"15min":0.30599999999999999},"max":{"1min":8.4499999999999993,"5min":68.435000000000002,"15min":68.435000000000002},"last":0.55400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":877592576,"data_stored":877592576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":466358272,"data_stored":466354200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":682430464,"data_stored":682426392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1634336768,"data_stored":1634336768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":309592064,"data_stored":309592064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":190251008,"data_stored":190251008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1674051584,"data_stored":1674051584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":620433408,"data_stored":620429336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:48:30.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:48:30.815 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:30.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:30.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:30.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:30.975 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:31.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:30 smithi029 ceph-mon[223858]: pgmap v1283: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 922 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 22370/44307 objects degraded (50.489%); 7099/44307 objects misplaced (16.022%); 7.1 MiB/s, 229 objects/s recovering 2023-10-09T04:48:31.111 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 22370/44307 objects degraded (50.489%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:31.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:30 smithi029 ceph-mon[216903]: pgmap v1283: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 922 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 22370/44307 objects degraded (50.489%); 7099/44307 objects misplaced (16.022%); 7.1 MiB/s, 229 objects/s recovering 2023-10-09T04:48:31.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 22370/44307 objects degraded (50.489%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:30 smithi177 ceph-mon[195797]: pgmap v1283: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 922 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 22370/44307 objects degraded (50.489%); 7099/44307 objects misplaced (16.022%); 7.1 MiB/s, 229 objects/s recovering 2023-10-09T04:48:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 22370/44307 objects degraded (50.489%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:31.310 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:31.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:31.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:31.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:31.485 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:31.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:31.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:31.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:31.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:31.745 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:31.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:48:31.809 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:48:31.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:31.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:31.847 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:48:31.884 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:48:31.909 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:31.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:31.948 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:48:31.994 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1279247305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:32.002 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:32.003 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:48:32.038 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:48:32.092 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:48:32.092 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:48:32.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.092 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:179587 12.2 e327/303 12:48a717ae:::benchmark_data_smithi029_442458_object84337:head v 327'11172, mlcod=288'1665)", 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:32.065981+0000", 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.024994091999999999, 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025025371000000001, 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.065981+0000", 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.065981+0000", 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.065981+0000", 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:48:32.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.065988+0000", 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7020000000000001e-06 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.066004+0000", 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5359e-05 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.066031+0000", 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6914e-05 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.068381+0000", 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002350151 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.068406+0000", 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5541000000000002e-05 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:179590 12.2 e327/303 12:4dfb5b07:::benchmark_data_smithi029_442458_object84340:head v 327'11173, mlcod=288'1665)", 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:32.076872+0000", 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.014103205000000001, 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014221777999999999, 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076872+0000", 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076872+0000", 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076872+0000", 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:48:32.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076884+0000", 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2588999999999999e-05 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076893+0000", 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3399999999999998e-06 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076899+0000", 2023-10-09T04:48:32.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5180000000000001e-06 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.078318+0000", 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001418854 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.078343+0000", 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4969000000000001e-05 2023-10-09T04:48:32.100 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:179596 12.2 e327/303 12:44d391ee:::benchmark_data_smithi029_442458_object84346:head v 327'11174, mlcod=288'1665)", 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:32.076911+0000", 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.014064576000000001, 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014244714, 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:48:32.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076911+0000", 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076911+0000", 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076911+0000", 2023-10-09T04:48:32.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076915+0000", 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6380000000000003e-06 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076921+0000", 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6099999999999997e-06 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.076925+0000", 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4769999999999997e-06 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.087102+0000", 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010176651 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.087129+0000", 2023-10-09T04:48:32.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7104e-05 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:179592 12.9 e327/303 12:9cf6bd4e:::benchmark_data_smithi029_442458_object84342:head v 327'11298, mlcod=291'4680)", 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:32.084579+0000", 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0063964360000000001, 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066415479999999997, 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:48:32.105 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084579+0000", 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084579+0000", 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084578+0000", 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084596+0000", 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8111e-05 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084609+0000", 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2395000000000001e-05 2023-10-09T04:48:32.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084617+0000", 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1429999999999998e-06 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.085555+0000", 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00093873299999999997 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.085582+0000", 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6956000000000001e-05 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:179588 12.5 e327/307 12:aeaa9262:::benchmark_data_smithi029_442458_object84338:head v 327'11214, mlcod=288'1653)", 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:32.084631+0000", 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0063448530000000001, 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066521779999999999, 2023-10-09T04:48:32.109 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084631+0000", 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084631+0000", 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084630+0000", 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084634+0000", 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.799e-06 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084640+0000", 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4090000000000002e-06 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084644+0000", 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1819999999999999e-06 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084902+0000", 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025862599999999997 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084930+0000", 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7250000000000002e-05 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.114 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:179599 12.9 e327/303 12:9a9fbf9d:::benchmark_data_smithi029_442458_object84349:head v 327'11299, mlcod=291'4680)", 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:32.084655+0000", 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0063206, 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066911549999999998, 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084655+0000", 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084655+0000", 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084655+0000", 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:48:32.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084662+0000", 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9920000000000001e-06 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084667+0000", 2023-10-09T04:48:32.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2569999999999998e-06 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.084671+0000", 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.98e-06 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:48:32.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.090342+0000", 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056714979999999996 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:32.090368+0000", 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5208999999999999e-05 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.119 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:48:32.120 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:48:32.120 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 6 2023-10-09T04:48:32.120 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:48:32.121 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:48:32.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:32.121 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:32.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:32.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1279247305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:32.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1279247305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:32.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.134 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.135 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.135 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.135 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.135 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.135 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:48:32.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:32.269 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:32.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:32.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:32.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:32.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:32.583 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:32.721 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:32.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:32.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:32.830 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:32.886 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:32.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:33.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:33.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:33.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:33.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:48:33.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:32 smithi177 ceph-mon[195797]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 807 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 22184/38781 objects degraded (57.203%); 7304/38781 objects misplaced (18.834%); 4.8 MiB/s, 225 objects/s recovering 2023-10-09T04:48:33.157 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:33.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:32 smithi029 ceph-mon[216903]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 807 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 22184/38781 objects degraded (57.203%); 7304/38781 objects misplaced (18.834%); 4.8 MiB/s, 225 objects/s recovering 2023-10-09T04:48:33.158 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:48:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:48:32] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-10-09T04:48:33.158 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:32 smithi029 ceph-mon[223858]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 807 MiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 22184/38781 objects degraded (57.203%); 7304/38781 objects misplaced (18.834%); 4.8 MiB/s, 225 objects/s recovering 2023-10-09T04:48:33.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:33.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:33.339 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:33.407 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:34.755 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:34.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:34.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:48:34.859 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:34.913 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:35.029 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:34 smithi029 ceph-mon[223858]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 735 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 21770/35349 objects degraded (61.586%); 6769/35349 objects misplaced (19.149%); 4.8 MiB/s, 226 objects/s recovering 2023-10-09T04:48:35.030 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:34 smithi029 ceph-mon[216903]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 735 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 21770/35349 objects degraded (61.586%); 6769/35349 objects misplaced (19.149%); 4.8 MiB/s, 226 objects/s recovering 2023-10-09T04:48:35.225 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:35.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:34 smithi177 ceph-mon[195797]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 735 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 21770/35349 objects degraded (61.586%); 6769/35349 objects misplaced (19.149%); 4.8 MiB/s, 226 objects/s recovering 2023-10-09T04:48:35.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:35.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:35.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:35.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:35.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:35.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:35.677 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:35.758 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:35.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:35.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:35.890 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:35.936 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:35.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:36.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:36.100 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:36.190 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:36.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:36.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 21770/35349 objects degraded (61.586%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:36.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:36.363 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:36.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 21770/35349 objects degraded (61.586%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:36.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 21770/35349 objects degraded (61.586%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:36.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:36.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:36.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:36.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:36.649 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:36.724 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:36.802 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:36.815 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:36.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:36.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:36.958 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:37.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:48:37.183 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:37.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:37.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:37.203 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:36 smithi029 ceph-mon[216903]: pgmap v1286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 617 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 587 op/s; 21579/29646 objects degraded (72.789%); 6013/29646 objects misplaced (20.283%); 4.7 MiB/s, 192 objects/s recovering 2023-10-09T04:48:37.204 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:36 smithi029 ceph-mon[223858]: pgmap v1286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 617 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 587 op/s; 21579/29646 objects degraded (72.789%); 6013/29646 objects misplaced (20.283%); 4.7 MiB/s, 192 objects/s recovering 2023-10-09T04:48:37.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:36 smithi177 ceph-mon[195797]: pgmap v1286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 617 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 587 op/s; 21579/29646 objects degraded (72.789%); 6013/29646 objects misplaced (20.283%); 4.7 MiB/s, 192 objects/s recovering 2023-10-09T04:48:37.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:37.408 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:38.024 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:38.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:38.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:38.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:38.198 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:38.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:38.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:38.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:38.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:38.494 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:38.600 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:38.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:38.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:38.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:38.810 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:38.926 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:38.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:39.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:39.064 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:39 smithi177 ceph-mon[195797]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 559 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 597 op/s; 21199/26868 objects degraded (78.901%); 5488/26868 objects misplaced (20.426%); 4.8 MiB/s, 194 objects/s recovering 2023-10-09T04:48:39.067 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:39.111 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:39.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:39.151 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:48:39.182 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:48:39.205 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:39 smithi029 ceph-mon[216903]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 559 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 597 op/s; 21199/26868 objects degraded (78.901%); 5488/26868 objects misplaced (20.426%); 4.8 MiB/s, 194 objects/s recovering 2023-10-09T04:48:39.206 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:39 smithi029 ceph-mon[223858]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 559 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 597 op/s; 21199/26868 objects degraded (78.901%); 5488/26868 objects misplaced (20.426%); 4.8 MiB/s, 194 objects/s recovering 2023-10-09T04:48:39.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:39.289 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:40 smithi029 ceph-mon[216903]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:40.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:40 smithi029 ceph-mon[223858]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:40.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:40 smithi177 ceph-mon[195797]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:40.760 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:40.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:40.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:40.942 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:41.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:41.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:41.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:41 smithi029 ceph-mon[216903]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 547 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 538 op/s; 21199/26286 objects degraded (80.647%); 5488/26286 objects misplaced (20.878%); 4.2 MiB/s, 170 objects/s recovering 2023-10-09T04:48:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 21199/26286 objects degraded (80.647%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:41 smithi029 ceph-mon[223858]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 547 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 538 op/s; 21199/26286 objects degraded (80.647%); 5488/26286 objects misplaced (20.878%); 4.2 MiB/s, 170 objects/s recovering 2023-10-09T04:48:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 21199/26286 objects degraded (80.647%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:41.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:41.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:41 smithi177 ceph-mon[195797]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 547 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 538 op/s; 21199/26286 objects degraded (80.647%); 5488/26286 objects misplaced (20.878%); 4.2 MiB/s, 170 objects/s recovering 2023-10-09T04:48:41.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 21199/26286 objects degraded (80.647%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:41.255 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:41.364 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:41.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:41.413 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1287,"stamp":"2023-10-09T04:48:37.352306+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":585897912,"num_objects":8956,"num_object_clones":0,"num_object_copies":26868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21199,"num_objects_misplaced":5488,"num_objects_unfound":0,"num_objects_dirty":8956,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":181864,"num_write_kb":6098254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44872,"num_bytes_recovered":2645637302,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94507,"ondisk_log_size":94507,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5871880,"kb_used_data":1323616,"kb_used_omap":12,"kb_used_meta":4548019,"kb_avail":462853880,"statfs":{"total":479975178240,"available":473962373120,"internally_reserved":0,"allocated":1355382784,"data_stored":1350508166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12321,"internal_metadata":4657172447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,1,8,8,2,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-391643136,"num_objects":-5976,"num_object_clones":0,"num_object_copies":-17928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1171,"num_objects_misplaced":-1611,"num_objects_unfound":0,"num_objects_dirty":-5976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5976,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1949,"num_bytes_recovered":50659328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003561"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":75,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.826691+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:48:14.826691+0000","last_peered":"2023-10-09T04:48:14.826691+0000","last_clean":"2023-10-09T04:48:14.826691+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:48:14.826691+0000","last_undegraded":"2023-10-09T04:48:14.826691+0000","last_fullsized":"2023-10-09T04:48:14.826691+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":120,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821880+0000","last_change":"2023-10-09T04:45:45.599027+0000","last_active":"2023-10-09T04:48:14.821880+0000","last_peered":"2023-10-09T04:48:14.821880+0000","last_clean":"2023-10-09T04:48:14.821880+0000","last_became_active":"2023-10-09T04:45:45.598650+0000","last_became_peered":"2023-10-09T04:45:45.598650+0000","last_unstale":"2023-10-09T04:48:14.821880+0000","last_undegraded":"2023-10-09T04:48:14.821880+0000","last_fullsized":"2023-10-09T04:48:14.821880+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:15.414379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":324,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:18.025122+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:48:18.025122+0000","last_peered":"2023-10-09T04:48:18.025122+0000","last_clean":"2023-10-09T04:48:18.025122+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:48:18.025122+0000","last_undegraded":"2023-10-09T04:48:18.025122+0000","last_fullsized":"2023-10-09T04:48:18.025122+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811431+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:48:14.811431+0000","last_peered":"2023-10-09T04:48:14.811431+0000","last_clean":"2023-10-09T04:48:14.811431+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:48:14.811431+0000","last_undegraded":"2023-10-09T04:48:14.811431+0000","last_fullsized":"2023-10-09T04:48:14.811431+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":45,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:21.928623+0000","last_change":"2023-10-09T04:46:25.173258+0000","last_active":"2023-10-09T04:48:21.928623+0000","last_peered":"2023-10-09T04:48:21.928623+0000","last_clean":"2023-10-09T04:48:21.928623+0000","last_became_active":"2023-10-09T04:46:25.172947+0000","last_became_peered":"2023-10-09T04:46:25.172947+0000","last_unstale":"2023-10-09T04:48:21.928623+0000","last_undegraded":"2023-10-09T04:48:21.928623+0000","last_fullsized":"2023-10-09T04:48:21.928623+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:38.532832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819506+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:48:14.819506+0000","last_peered":"2023-10-09T04:48:14.819506+0000","last_clean":"2023-10-09T04:48:14.819506+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:48:14.819506+0000","last_undegraded":"2023-10-09T04:48:14.819506+0000","last_fullsized":"2023-10-09T04:48:14.819506+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":259,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819370+0000","last_change":"2023-10-09T04:46:25.155473+0000","last_active":"2023-10-09T04:48:14.819370+0000","last_peered":"2023-10-09T04:48:14.819370+0000","last_clean":"2023-10-09T04:48:14.819370+0000","last_became_active":"2023-10-09T04:46:25.155148+0000","last_became_peered":"2023-10-09T04:46:25.155148+0000","last_unstale":"2023-10-09T04:48:14.819370+0000","last_undegraded":"2023-10-09T04:48:14.819370+0000","last_fullsized":"2023-10-09T04:48:14.819370+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:38.576510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":284,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821102+0000","last_change":"2023-10-09T04:45:45.611067+0000","last_active":"2023-10-09T04:48:14.821102+0000","last_peered":"2023-10-09T04:48:14.821102+0000","last_clean":"2023-10-09T04:48:14.821102+0000","last_became_active":"2023-10-09T04:45:45.610571+0000","last_became_peered":"2023-10-09T04:45:45.610571+0000","last_unstale":"2023-10-09T04:48:14.821102+0000","last_undegraded":"2023-10-09T04:48:14.821102+0000","last_fullsized":"2023-10-09T04:48:14.821102+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:14.381921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":287,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.820078+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:48:14.820078+0000","last_peered":"2023-10-09T04:48:14.820078+0000","last_clean":"2023-10-09T04:48:14.820078+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:48:14.820078+0000","last_undegraded":"2023-10-09T04:48:14.820078+0000","last_fullsized":"2023-10-09T04:48:14.820078+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":267,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.808833+0000","last_change":"2023-10-09T04:46:25.193703+0000","last_active":"2023-10-09T04:48:14.808833+0000","last_peered":"2023-10-09T04:48:14.808833+0000","last_clean":"2023-10-09T04:48:14.808833+0000","last_became_active":"2023-10-09T04:46:25.193392+0000","last_became_peered":"2023-10-09T04:46:25.193392+0000","last_unstale":"2023-10-09T04:48:14.808833+0000","last_undegraded":"2023-10-09T04:48:14.808833+0000","last_fullsized":"2023-10-09T04:48:14.808833+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:56.086176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":353,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.807926+0000","last_change":"2023-10-09T04:46:25.479989+0000","last_active":"2023-10-09T04:48:14.807926+0000","last_peered":"2023-10-09T04:48:14.807926+0000","last_clean":"2023-10-09T04:48:14.807926+0000","last_became_active":"2023-10-09T04:46:25.381247+0000","last_became_peered":"2023-10-09T04:46:25.381247+0000","last_unstale":"2023-10-09T04:48:14.807926+0000","last_undegraded":"2023-10-09T04:48:14.807926+0000","last_fullsized":"2023-10-09T04:48:14.807926+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:12.653312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":253,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.802937+0000","last_change":"2023-10-09T04:46:25.150997+0000","last_active":"2023-10-09T04:48:14.802937+0000","last_peered":"2023-10-09T04:48:14.802937+0000","last_clean":"2023-10-09T04:48:14.802937+0000","last_became_active":"2023-10-09T04:46:25.150370+0000","last_became_peered":"2023-10-09T04:46:25.150370+0000","last_unstale":"2023-10-09T04:48:14.802937+0000","last_undegraded":"2023-10-09T04:48:14.802937+0000","last_fullsized":"2023-10-09T04:48:14.802937+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:07.231212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":105,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.820755+0000","last_change":"2023-10-09T04:44:38.813366+0000","last_active":"2023-10-09T04:48:14.820755+0000","last_peered":"2023-10-09T04:48:14.820755+0000","last_clean":"2023-10-09T04:48:14.820755+0000","last_became_active":"2023-10-09T04:44:38.777069+0000","last_became_peered":"2023-10-09T04:44:38.777069+0000","last_unstale":"2023-10-09T04:48:14.820755+0000","last_undegraded":"2023-10-09T04:48:14.820755+0000","last_fullsized":"2023-10-09T04:48:14.820755+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:39.101540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":297,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.820731+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:48:14.820731+0000","last_peered":"2023-10-09T04:48:14.820731+0000","last_clean":"2023-10-09T04:48:14.820731+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:48:14.820731+0000","last_undegraded":"2023-10-09T04:48:14.820731+0000","last_fullsized":"2023-10-09T04:48:14.820731+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":337,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.808300+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:48:14.808300+0000","last_peered":"2023-10-09T04:48:14.808300+0000","last_clean":"2023-10-09T04:48:14.808300+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:48:14.808300+0000","last_undegraded":"2023-10-09T04:48:14.808300+0000","last_fullsized":"2023-10-09T04:48:14.808300+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":120,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.810041+0000","last_change":"2023-10-09T04:45:45.607791+0000","last_active":"2023-10-09T04:48:14.810041+0000","last_peered":"2023-10-09T04:48:14.810041+0000","last_clean":"2023-10-09T04:48:14.810041+0000","last_became_active":"2023-10-09T04:45:45.607501+0000","last_became_peered":"2023-10-09T04:45:45.607501+0000","last_unstale":"2023-10-09T04:48:14.810041+0000","last_undegraded":"2023-10-09T04:48:14.810041+0000","last_fullsized":"2023-10-09T04:48:14.810041+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:05.067640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":259,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.808221+0000","last_change":"2023-10-09T04:46:25.168444+0000","last_active":"2023-10-09T04:48:14.808221+0000","last_peered":"2023-10-09T04:48:14.808221+0000","last_clean":"2023-10-09T04:48:14.808221+0000","last_became_active":"2023-10-09T04:46:25.168143+0000","last_became_peered":"2023-10-09T04:46:25.168143+0000","last_unstale":"2023-10-09T04:48:14.808221+0000","last_undegraded":"2023-10-09T04:48:14.808221+0000","last_fullsized":"2023-10-09T04:48:14.808221+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:56.166900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":306,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819269+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:48:14.819269+0000","last_peered":"2023-10-09T04:48:14.819269+0000","last_clean":"2023-10-09T04:48:14.819269+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:48:14.819269+0000","last_undegraded":"2023-10-09T04:48:14.819269+0000","last_fullsized":"2023-10-09T04:48:14.819269+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":289,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.808782+0000","last_change":"2023-10-09T04:46:25.175418+0000","last_active":"2023-10-09T04:48:14.808782+0000","last_peered":"2023-10-09T04:48:14.808782+0000","last_clean":"2023-10-09T04:48:14.808782+0000","last_became_active":"2023-10-09T04:46:25.174958+0000","last_became_peered":"2023-10-09T04:46:25.174958+0000","last_unstale":"2023-10-09T04:48:14.808782+0000","last_undegraded":"2023-10-09T04:48:14.808782+0000","last_fullsized":"2023-10-09T04:48:14.808782+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:35.362263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811503+0000","last_change":"2023-10-09T04:46:25.170147+0000","last_active":"2023-10-09T04:48:14.811503+0000","last_peered":"2023-10-09T04:48:14.811503+0000","last_clean":"2023-10-09T04:48:14.811503+0000","last_became_active":"2023-10-09T04:46:25.169835+0000","last_became_peered":"2023-10-09T04:46:25.169835+0000","last_unstale":"2023-10-09T04:48:14.811503+0000","last_undegraded":"2023-10-09T04:48:14.811503+0000","last_fullsized":"2023-10-09T04:48:14.811503+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:01.393349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":108,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819473+0000","last_change":"2023-10-09T04:46:25.271462+0000","last_active":"2023-10-09T04:48:14.819473+0000","last_peered":"2023-10-09T04:48:14.819473+0000","last_clean":"2023-10-09T04:48:14.819473+0000","last_became_active":"2023-10-09T04:46:25.166617+0000","last_became_peered":"2023-10-09T04:46:25.166617+0000","last_unstale":"2023-10-09T04:48:14.819473+0000","last_undegraded":"2023-10-09T04:48:14.819473+0000","last_fullsized":"2023-10-09T04:48:14.819473+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:30.657523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":120,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811417+0000","last_change":"2023-10-09T04:45:45.603131+0000","last_active":"2023-10-09T04:48:14.811417+0000","last_peered":"2023-10-09T04:48:14.811417+0000","last_clean":"2023-10-09T04:48:14.811417+0000","last_became_active":"2023-10-09T04:45:45.602862+0000","last_became_peered":"2023-10-09T04:45:45.602862+0000","last_unstale":"2023-10-09T04:48:14.811417+0000","last_undegraded":"2023-10-09T04:48:14.811417+0000","last_fullsized":"2023-10-09T04:48:14.811417+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:22.358682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"327'11451","reported_seq":21035,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T04:48:34.547170+0000","last_change":"2023-10-09T04:46:53.684121+0000","last_active":"2023-10-09T04:48:34.547170+0000","last_peered":"2023-10-09T04:48:34.547170+0000","last_clean":"2023-10-09T04:48:34.547170+0000","last_became_active":"2023-10-09T04:46:53.658692+0000","last_became_peered":"2023-10-09T04:46:53.658692+0000","last_unstale":"2023-10-09T04:48:34.547170+0000","last_undegraded":"2023-10-09T04:48:34.547170+0000","last_fullsized":"2023-10-09T04:48:34.547170+0000","mapping_epoch":312,"log_start":"297'5672","ondisk_log_start":"297'5672","created":284,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:44.589594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36110336,"num_objects":551,"num_object_clones":0,"num_object_copies":1653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11451,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4752,"num_bytes_recovered":242221056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":357,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811345+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:48:14.811345+0000","last_peered":"2023-10-09T04:48:14.811345+0000","last_clean":"2023-10-09T04:48:14.811345+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:48:14.811345+0000","last_undegraded":"2023-10-09T04:48:14.811345+0000","last_fullsized":"2023-10-09T04:48:14.811345+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819093+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:48:14.819093+0000","last_peered":"2023-10-09T04:48:14.819093+0000","last_clean":"2023-10-09T04:48:14.819093+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:48:14.819093+0000","last_undegraded":"2023-10-09T04:48:14.819093+0000","last_fullsized":"2023-10-09T04:48:14.819093+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"327'11446","reported_seq":20905,"reported_epoch":327,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:34.931188+0000","last_change":"2023-10-09T04:48:26.671662+0000","last_active":"2023-10-09T04:48:34.931188+0000","last_peered":"2023-10-09T04:48:34.931188+0000","last_clean":"2023-10-09T04:44:37.740580+0000","last_became_active":"2023-10-09T04:45:46.774175+0000","last_became_peered":"2023-10-09T04:45:46.774175+0000","last_unstale":"2023-10-09T04:48:34.931188+0000","last_undegraded":"2023-10-09T04:48:34.931188+0000","last_fullsized":"2023-10-09T04:48:34.931188+0000","mapping_epoch":303,"log_start":"298'6206","ondisk_log_start":"298'6206","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5240,"log_dups_size":0,"ondisk_log_size":5240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:40.805571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":568,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11446,"num_write_kb":384448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4694,"num_bytes_recovered":240713728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,3],"avail_no_missing":["7","1","3","6"],"object_location_counts":[{"shards":"1,3,7","objects":568}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":316,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811357+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:48:14.811357+0000","last_peered":"2023-10-09T04:48:14.811357+0000","last_clean":"2023-10-09T04:48:14.811357+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:48:14.811357+0000","last_undegraded":"2023-10-09T04:48:14.811357+0000","last_fullsized":"2023-10-09T04:48:14.811357+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":324,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821350+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:48:14.821350+0000","last_peered":"2023-10-09T04:48:14.821350+0000","last_clean":"2023-10-09T04:48:14.821350+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:48:14.821350+0000","last_undegraded":"2023-10-09T04:48:14.821350+0000","last_fullsized":"2023-10-09T04:48:14.821350+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"327'11255","reported_seq":11321,"reported_epoch":327,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:34.606884+0000","last_change":"2023-10-09T04:46:25.245840+0000","last_active":"2023-10-09T04:48:34.606884+0000","last_peered":"2023-10-09T04:48:34.606884+0000","last_clean":"2023-10-09T04:45:44.791779+0000","last_became_active":"2023-10-09T04:45:46.998769+0000","last_became_peered":"2023-10-09T04:45:46.998769+0000","last_unstale":"2023-10-09T04:48:34.606884+0000","last_undegraded":"2023-10-09T04:45:46.613330+0000","last_fullsized":"2023-10-09T04:45:46.574934+0000","mapping_epoch":303,"log_start":"297'5429","ondisk_log_start":"297'5429","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5826,"log_dups_size":0,"ondisk_log_size":5826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:00.584863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39911424,"num_objects":609,"num_object_clones":0,"num_object_copies":1827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5931,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11255,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":609},{"shards":"4,7","objects":-4318}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":298,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.807766+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:48:14.807766+0000","last_peered":"2023-10-09T04:48:14.807766+0000","last_clean":"2023-10-09T04:48:14.807766+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:48:14.807766+0000","last_undegraded":"2023-10-09T04:48:14.807766+0000","last_fullsized":"2023-10-09T04:48:14.807766+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":244,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.802842+0000","last_change":"2023-10-09T04:44:38.775465+0000","last_active":"2023-10-09T04:48:14.802842+0000","last_peered":"2023-10-09T04:48:14.802842+0000","last_clean":"2023-10-09T04:48:14.802842+0000","last_became_active":"2023-10-09T04:44:38.775246+0000","last_became_peered":"2023-10-09T04:44:38.775246+0000","last_unstale":"2023-10-09T04:48:14.802842+0000","last_undegraded":"2023-10-09T04:48:14.802842+0000","last_fullsized":"2023-10-09T04:48:14.802842+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.286190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.3","version":"327'11367","reported_seq":18873,"reported_epoch":327,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:48:34.976878+0000","last_change":"2023-10-09T04:46:26.476463+0000","last_active":"2023-10-09T04:48:34.976878+0000","last_peered":"2023-10-09T04:48:34.976878+0000","last_clean":"2023-10-09T04:45:44.834828+0000","last_became_active":"2023-10-09T04:46:26.386549+0000","last_became_peered":"2023-10-09T04:46:26.386549+0000","last_unstale":"2023-10-09T04:48:34.976878+0000","last_undegraded":"2023-10-09T04:46:26.372334+0000","last_fullsized":"2023-10-09T04:48:34.976878+0000","mapping_epoch":308,"log_start":"298'6180","ondisk_log_start":"298'6180","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5187,"log_dups_size":0,"ondisk_log_size":5187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:01.311399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":2249,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11367,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3718,"num_bytes_recovered":243466240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,0],"avail_no_missing":["7","0","3"],"object_location_counts":[{"shards":"0,3,7","objects":567}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":317,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.822154+0000","last_change":"2023-10-09T04:46:25.150945+0000","last_active":"2023-10-09T04:48:14.822154+0000","last_peered":"2023-10-09T04:48:14.822154+0000","last_clean":"2023-10-09T04:48:14.822154+0000","last_became_active":"2023-10-09T04:46:25.150622+0000","last_became_peered":"2023-10-09T04:46:25.150622+0000","last_unstale":"2023-10-09T04:48:14.822154+0000","last_undegraded":"2023-10-09T04:48:14.822154+0000","last_fullsized":"2023-10-09T04:48:14.822154+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:02.718023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":78,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821517+0000","last_change":"2023-10-09T04:46:25.173123+0000","last_active":"2023-10-09T04:48:14.821517+0000","last_peered":"2023-10-09T04:48:14.821517+0000","last_clean":"2023-10-09T04:48:14.821517+0000","last_became_active":"2023-10-09T04:46:25.172534+0000","last_became_peered":"2023-10-09T04:46:25.172534+0000","last_unstale":"2023-10-09T04:48:14.821517+0000","last_undegraded":"2023-10-09T04:48:14.821517+0000","last_fullsized":"2023-10-09T04:48:14.821517+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:00.581088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"327'11327","reported_seq":20711,"reported_epoch":327,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:34.960897+0000","last_change":"2023-10-09T04:47:41.808546+0000","last_active":"2023-10-09T04:48:34.960897+0000","last_peered":"2023-10-09T04:48:34.960897+0000","last_clean":"2023-10-09T04:44:37.730195+0000","last_became_active":"2023-10-09T04:45:47.970624+0000","last_became_peered":"2023-10-09T04:45:47.970624+0000","last_unstale":"2023-10-09T04:48:34.960897+0000","last_undegraded":"2023-10-09T04:48:34.960897+0000","last_fullsized":"2023-10-09T04:48:34.960897+0000","mapping_epoch":303,"log_start":"298'6136","ondisk_log_start":"298'6136","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5191,"log_dups_size":0,"ondisk_log_size":5191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:58.727121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37027840,"num_objects":565,"num_object_clones":0,"num_object_copies":1695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":565,"num_objects_unfound":0,"num_objects_dirty":565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4658,"num_bytes_recovered":305266688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,3],"avail_no_missing":["7","1","3","6"],"object_location_counts":[{"shards":"1,3,7","objects":565}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":66,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821718+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:48:14.821718+0000","last_peered":"2023-10-09T04:48:14.821718+0000","last_clean":"2023-10-09T04:48:14.821718+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:48:14.821718+0000","last_undegraded":"2023-10-09T04:48:14.821718+0000","last_fullsized":"2023-10-09T04:48:14.821718+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":95,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.822201+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:48:14.822201+0000","last_peered":"2023-10-09T04:48:14.822201+0000","last_clean":"2023-10-09T04:48:14.822201+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:48:14.822201+0000","last_undegraded":"2023-10-09T04:48:14.822201+0000","last_fullsized":"2023-10-09T04:48:14.822201+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"327'11408","reported_seq":14665,"reported_epoch":327,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:36.960028+0000","last_change":"2023-10-09T04:46:25.469213+0000","last_active":"2023-10-09T04:48:36.960028+0000","last_peered":"2023-10-09T04:48:36.960028+0000","last_clean":"2023-10-09T04:44:37.740389+0000","last_became_active":"2023-10-09T04:46:25.449027+0000","last_became_peered":"2023-10-09T04:46:25.449027+0000","last_unstale":"2023-10-09T04:48:36.960028+0000","last_undegraded":"2023-10-09T04:46:25.190986+0000","last_fullsized":"2023-10-09T04:46:25.133749+0000","mapping_epoch":307,"log_start":"296'4873","ondisk_log_start":"296'4873","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:53.663910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30801920,"num_objects":470,"num_object_clones":0,"num_object_copies":1410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11408,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1593,"num_bytes_recovered":104202240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":-3816},{"shards":"1,2,3","objects":470}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":288,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.826730+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:48:14.826730+0000","last_peered":"2023-10-09T04:48:14.826730+0000","last_clean":"2023-10-09T04:48:14.826730+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:48:14.826730+0000","last_undegraded":"2023-10-09T04:48:14.826730+0000","last_fullsized":"2023-10-09T04:48:14.826730+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":75,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821737+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:48:14.821737+0000","last_peered":"2023-10-09T04:48:14.821737+0000","last_clean":"2023-10-09T04:48:14.821737+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:48:14.821737+0000","last_undegraded":"2023-10-09T04:48:14.821737+0000","last_fullsized":"2023-10-09T04:48:14.821737+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"327'11116","reported_seq":19005,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T04:48:33.046199+0000","last_change":"2023-10-09T04:46:54.673986+0000","last_active":"2023-10-09T04:48:33.046199+0000","last_peered":"2023-10-09T04:48:33.046199+0000","last_clean":"2023-10-09T04:48:33.046199+0000","last_became_active":"2023-10-09T04:46:54.653505+0000","last_became_peered":"2023-10-09T04:46:54.653505+0000","last_unstale":"2023-10-09T04:48:33.046199+0000","last_undegraded":"2023-10-09T04:48:33.046199+0000","last_fullsized":"2023-10-09T04:48:33.046199+0000","mapping_epoch":313,"log_start":"301'6428","ondisk_log_start":"301'6428","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:19.484546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11116,"num_write_kb":377024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3907,"num_bytes_recovered":256049152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":75,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821186+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:48:14.821186+0000","last_peered":"2023-10-09T04:48:14.821186+0000","last_clean":"2023-10-09T04:48:14.821186+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:48:14.821186+0000","last_undegraded":"2023-10-09T04:48:14.821186+0000","last_fullsized":"2023-10-09T04:48:14.821186+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.829871+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:48:14.829871+0000","last_peered":"2023-10-09T04:48:14.829871+0000","last_clean":"2023-10-09T04:48:14.829871+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:48:14.829871+0000","last_undegraded":"2023-10-09T04:48:14.829871+0000","last_fullsized":"2023-10-09T04:48:14.829871+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"327'11392","reported_seq":11443,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T04:48:34.960908+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:48:34.960908+0000","last_peered":"2023-10-09T04:48:34.960908+0000","last_clean":"2023-10-09T04:48:34.960908+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:48:34.960908+0000","last_undegraded":"2023-10-09T04:48:34.960908+0000","last_fullsized":"2023-10-09T04:48:34.960908+0000","mapping_epoch":284,"log_start":"298'6196","ondisk_log_start":"298'6196","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5196,"log_dups_size":0,"ondisk_log_size":5196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38535168,"num_objects":588,"num_object_clones":0,"num_object_copies":1764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11392,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":121,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.807215+0000","last_change":"2023-10-09T04:45:45.611560+0000","last_active":"2023-10-09T04:48:14.807215+0000","last_peered":"2023-10-09T04:48:14.807215+0000","last_clean":"2023-10-09T04:48:14.807215+0000","last_became_active":"2023-10-09T04:45:45.611247+0000","last_became_peered":"2023-10-09T04:45:45.611247+0000","last_unstale":"2023-10-09T04:48:14.807215+0000","last_undegraded":"2023-10-09T04:48:14.807215+0000","last_fullsized":"2023-10-09T04:48:14.807215+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:52.196556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":273,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.822071+0000","last_change":"2023-10-09T04:45:45.604134+0000","last_active":"2023-10-09T04:48:14.822071+0000","last_peered":"2023-10-09T04:48:14.822071+0000","last_clean":"2023-10-09T04:48:14.822071+0000","last_became_active":"2023-10-09T04:45:45.603767+0000","last_became_peered":"2023-10-09T04:45:45.603767+0000","last_unstale":"2023-10-09T04:48:14.822071+0000","last_undegraded":"2023-10-09T04:48:14.822071+0000","last_fullsized":"2023-10-09T04:48:14.822071+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:14.801140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.a","version":"327'11604","reported_seq":23039,"reported_epoch":327,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:36.932741+0000","last_change":"2023-10-09T04:46:26.440123+0000","last_active":"2023-10-09T04:48:36.932741+0000","last_peered":"2023-10-09T04:48:36.932741+0000","last_clean":"2023-10-09T04:45:44.754196+0000","last_became_active":"2023-10-09T04:46:26.422562+0000","last_became_peered":"2023-10-09T04:46:26.422562+0000","last_unstale":"2023-10-09T04:48:36.932741+0000","last_undegraded":"2023-10-09T04:46:26.243542+0000","last_fullsized":"2023-10-09T04:46:26.131676+0000","mapping_epoch":308,"log_start":"296'4997","ondisk_log_start":"296'4997","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:06.608939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32768000,"num_objects":500,"num_object_clones":0,"num_object_copies":1500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4529,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11604,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3788,"num_bytes_recovered":248119296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":500}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":214,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.810684+0000","last_change":"2023-10-09T04:46:25.169137+0000","last_active":"2023-10-09T04:48:14.810684+0000","last_peered":"2023-10-09T04:48:14.810684+0000","last_clean":"2023-10-09T04:48:14.810684+0000","last_became_active":"2023-10-09T04:46:25.168834+0000","last_became_peered":"2023-10-09T04:46:25.168834+0000","last_unstale":"2023-10-09T04:48:14.810684+0000","last_undegraded":"2023-10-09T04:48:14.810684+0000","last_fullsized":"2023-10-09T04:48:14.810684+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:15.860221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":331,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819354+0000","last_change":"2023-10-09T04:45:45.824913+0000","last_active":"2023-10-09T04:48:14.819354+0000","last_peered":"2023-10-09T04:48:14.819354+0000","last_clean":"2023-10-09T04:48:14.819354+0000","last_became_active":"2023-10-09T04:45:45.824572+0000","last_became_peered":"2023-10-09T04:45:45.824572+0000","last_unstale":"2023-10-09T04:48:14.819354+0000","last_undegraded":"2023-10-09T04:48:14.819354+0000","last_fullsized":"2023-10-09T04:48:14.819354+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:36.276403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":298,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819107+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:48:14.819107+0000","last_peered":"2023-10-09T04:48:14.819107+0000","last_clean":"2023-10-09T04:48:14.819107+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:48:14.819107+0000","last_undegraded":"2023-10-09T04:48:14.819107+0000","last_fullsized":"2023-10-09T04:48:14.819107+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"327'11304","reported_seq":20907,"reported_epoch":327,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:34.526758+0000","last_change":"2023-10-09T04:46:52.203041+0000","last_active":"2023-10-09T04:48:34.526758+0000","last_peered":"2023-10-09T04:48:34.526758+0000","last_clean":"2023-10-09T04:46:24.128327+0000","last_became_active":"2023-10-09T04:46:26.494164+0000","last_became_peered":"2023-10-09T04:46:26.494164+0000","last_unstale":"2023-10-09T04:48:34.526758+0000","last_undegraded":"2023-10-09T04:46:26.365928+0000","last_fullsized":"2023-10-09T04:46:26.143463+0000","mapping_epoch":308,"log_start":"294'4696","ondisk_log_start":"294'4696","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:51.675602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":796,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11304,"num_write_kb":379840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4771,"num_bytes_recovered":226557952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"2,5","objects":-230}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":311,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819057+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:48:14.819057+0000","last_peered":"2023-10-09T04:48:14.819057+0000","last_clean":"2023-10-09T04:48:14.819057+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:48:14.819057+0000","last_undegraded":"2023-10-09T04:48:14.819057+0000","last_fullsized":"2023-10-09T04:48:14.819057+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":95,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821344+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:48:14.821344+0000","last_peered":"2023-10-09T04:48:14.821344+0000","last_clean":"2023-10-09T04:48:14.821344+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:48:14.821344+0000","last_undegraded":"2023-10-09T04:48:14.821344+0000","last_fullsized":"2023-10-09T04:48:14.821344+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":309,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.799980+0000","last_change":"2023-10-09T04:45:45.611766+0000","last_active":"2023-10-09T04:48:14.799980+0000","last_peered":"2023-10-09T04:48:14.799980+0000","last_clean":"2023-10-09T04:48:14.799980+0000","last_became_active":"2023-10-09T04:45:45.611242+0000","last_became_peered":"2023-10-09T04:45:45.611242+0000","last_unstale":"2023-10-09T04:48:14.799980+0000","last_undegraded":"2023-10-09T04:48:14.799980+0000","last_fullsized":"2023-10-09T04:48:14.799980+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:23.336238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"327'11021","reported_seq":20097,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T04:48:34.515383+0000","last_change":"2023-10-09T04:45:30.672535+0000","last_active":"2023-10-09T04:48:34.515383+0000","last_peered":"2023-10-09T04:48:34.515383+0000","last_clean":"2023-10-09T04:48:34.515383+0000","last_became_active":"2023-10-09T04:45:30.663157+0000","last_became_peered":"2023-10-09T04:45:30.663157+0000","last_unstale":"2023-10-09T04:48:34.515383+0000","last_undegraded":"2023-10-09T04:48:34.515383+0000","last_fullsized":"2023-10-09T04:48:34.515383+0000","mapping_epoch":300,"log_start":"291'4406","ondisk_log_start":"291'4406","created":284,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:31.926989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39256064,"num_objects":599,"num_object_clones":0,"num_object_copies":1797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11021,"num_write_kb":371840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4503,"num_bytes_recovered":295108608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":304,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.801252+0000","last_change":"2023-10-09T04:46:25.161784+0000","last_active":"2023-10-09T04:48:14.801252+0000","last_peered":"2023-10-09T04:48:14.801252+0000","last_clean":"2023-10-09T04:48:14.801252+0000","last_became_active":"2023-10-09T04:46:25.161498+0000","last_became_peered":"2023-10-09T04:46:25.161498+0000","last_unstale":"2023-10-09T04:48:14.801252+0000","last_undegraded":"2023-10-09T04:48:14.801252+0000","last_fullsized":"2023-10-09T04:48:14.801252+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:01.500231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":303,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.799343+0000","last_change":"2023-10-09T04:46:25.165536+0000","last_active":"2023-10-09T04:48:14.799343+0000","last_peered":"2023-10-09T04:48:14.799343+0000","last_clean":"2023-10-09T04:48:14.799343+0000","last_became_active":"2023-10-09T04:46:25.165269+0000","last_became_peered":"2023-10-09T04:46:25.165269+0000","last_unstale":"2023-10-09T04:48:14.799343+0000","last_undegraded":"2023-10-09T04:48:14.799343+0000","last_fullsized":"2023-10-09T04:48:14.799343+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:30.362732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819437+0000","last_change":"2023-10-09T04:46:25.177443+0000","last_active":"2023-10-09T04:48:14.819437+0000","last_peered":"2023-10-09T04:48:14.819437+0000","last_clean":"2023-10-09T04:48:14.819437+0000","last_became_active":"2023-10-09T04:46:25.176635+0000","last_became_peered":"2023-10-09T04:46:25.176635+0000","last_unstale":"2023-10-09T04:48:14.819437+0000","last_undegraded":"2023-10-09T04:48:14.819437+0000","last_fullsized":"2023-10-09T04:48:14.819437+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:43.367744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"327'11249","reported_seq":11300,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T04:48:34.974467+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:48:34.974467+0000","last_peered":"2023-10-09T04:48:34.974467+0000","last_clean":"2023-10-09T04:48:34.974467+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:48:34.974467+0000","last_undegraded":"2023-10-09T04:48:34.974467+0000","last_fullsized":"2023-10-09T04:48:34.974467+0000","mapping_epoch":284,"log_start":"298'6069","ondisk_log_start":"298'6069","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5180,"log_dups_size":0,"ondisk_log_size":5180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38535192,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11249,"num_write_kb":378753,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":308,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819619+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:48:14.819619+0000","last_peered":"2023-10-09T04:48:14.819619+0000","last_clean":"2023-10-09T04:48:14.819619+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:48:14.819619+0000","last_undegraded":"2023-10-09T04:48:14.819619+0000","last_fullsized":"2023-10-09T04:48:14.819619+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":755,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:14.807069+0000","last_change":"2023-10-09T04:46:26.415415+0000","last_active":"2023-10-09T04:48:14.807069+0000","last_peered":"2023-10-09T04:48:14.807069+0000","last_clean":"2023-10-09T04:46:16.698879+0000","last_became_active":"2023-10-09T04:46:26.415059+0000","last_became_peered":"2023-10-09T04:46:26.415059+0000","last_unstale":"2023-10-09T04:48:14.807069+0000","last_undegraded":"2023-10-09T04:46:26.138726+0000","last_fullsized":"2023-10-09T04:46:26.138125+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:23.417129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":257,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819143+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:48:14.819143+0000","last_peered":"2023-10-09T04:48:14.819143+0000","last_clean":"2023-10-09T04:48:14.819143+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:48:14.819143+0000","last_undegraded":"2023-10-09T04:48:14.819143+0000","last_fullsized":"2023-10-09T04:48:14.819143+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":270,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821665+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:48:14.821665+0000","last_peered":"2023-10-09T04:48:14.821665+0000","last_clean":"2023-10-09T04:48:14.821665+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:48:14.821665+0000","last_undegraded":"2023-10-09T04:48:14.821665+0000","last_fullsized":"2023-10-09T04:48:14.821665+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"327'11213","reported_seq":11272,"reported_epoch":327,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:34.616639+0000","last_change":"2023-10-09T04:46:26.351968+0000","last_active":"2023-10-09T04:48:34.616639+0000","last_peered":"2023-10-09T04:48:34.616639+0000","last_clean":"2023-10-09T04:46:24.123151+0000","last_became_active":"2023-10-09T04:46:26.320651+0000","last_became_peered":"2023-10-09T04:46:26.320651+0000","last_unstale":"2023-10-09T04:48:34.616639+0000","last_undegraded":"2023-10-09T04:48:34.616639+0000","last_fullsized":"2023-10-09T04:48:34.616639+0000","mapping_epoch":308,"log_start":"297'5376","ondisk_log_start":"297'5376","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5837,"log_dups_size":0,"ondisk_log_size":5837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:25.867280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39780352,"num_objects":607,"num_object_clones":0,"num_object_copies":1821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":607,"num_objects_unfound":0,"num_objects_dirty":607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11213,"num_write_kb":378240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":607}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":337,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.806956+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:48:14.806956+0000","last_peered":"2023-10-09T04:48:14.806956+0000","last_clean":"2023-10-09T04:48:14.806956+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:48:14.806956+0000","last_undegraded":"2023-10-09T04:48:14.806956+0000","last_fullsized":"2023-10-09T04:48:14.806956+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":95,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821277+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:48:14.821277+0000","last_peered":"2023-10-09T04:48:14.821277+0000","last_clean":"2023-10-09T04:48:14.821277+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:48:14.821277+0000","last_undegraded":"2023-10-09T04:48:14.821277+0000","last_fullsized":"2023-10-09T04:48:14.821277+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":126,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-10-09T04:48:27.911452+0000","last_change":"2023-10-09T04:46:25.174089+0000","last_active":"2023-10-09T04:48:27.911452+0000","last_peered":"2023-10-09T04:48:27.911452+0000","last_clean":"2023-10-09T04:48:27.911452+0000","last_became_active":"2023-10-09T04:46:25.173414+0000","last_became_peered":"2023-10-09T04:46:25.173414+0000","last_unstale":"2023-10-09T04:48:27.911452+0000","last_undegraded":"2023-10-09T04:48:27.911452+0000","last_fullsized":"2023-10-09T04:48:27.911452+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.877749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"327'11521","reported_seq":21198,"reported_epoch":327,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:36.950186+0000","last_change":"2023-10-09T04:48:14.871531+0000","last_active":"2023-10-09T04:48:36.950186+0000","last_peered":"2023-10-09T04:48:36.950186+0000","last_clean":"2023-10-09T04:44:37.770348+0000","last_became_active":"2023-10-09T04:45:59.017846+0000","last_became_peered":"2023-10-09T04:45:59.017846+0000","last_unstale":"2023-10-09T04:48:36.950186+0000","last_undegraded":"2023-10-09T04:48:36.950186+0000","last_fullsized":"2023-10-09T04:48:36.950186+0000","mapping_epoch":303,"log_start":"296'4903","ondisk_log_start":"296'4903","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:00.633461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30736384,"num_objects":469,"num_object_clones":0,"num_object_copies":1407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":469,"num_objects_unfound":0,"num_objects_dirty":469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11521,"num_write_kb":383680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4803,"num_bytes_recovered":243204096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,3],"avail_no_missing":["2","3","4","6"],"object_location_counts":[{"shards":"2,3,4","objects":469}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":340,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.807840+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:48:14.807840+0000","last_peered":"2023-10-09T04:48:14.807840+0000","last_clean":"2023-10-09T04:48:14.807840+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:48:14.807840+0000","last_undegraded":"2023-10-09T04:48:14.807840+0000","last_fullsized":"2023-10-09T04:48:14.807840+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":285,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.799932+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:48:14.799932+0000","last_peered":"2023-10-09T04:48:14.799932+0000","last_clean":"2023-10-09T04:48:14.799932+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:48:14.799932+0000","last_undegraded":"2023-10-09T04:48:14.799932+0000","last_fullsized":"2023-10-09T04:48:14.799932+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":181,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819769+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:48:14.819769+0000","last_peered":"2023-10-09T04:48:14.819769+0000","last_clean":"2023-10-09T04:48:14.819769+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:48:14.819769+0000","last_undegraded":"2023-10-09T04:48:14.819769+0000","last_fullsized":"2023-10-09T04:48:14.819769+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"327'11405","reported_seq":11474,"reported_epoch":327,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:36.910555+0000","last_change":"2023-10-09T04:46:25.302033+0000","last_active":"2023-10-09T04:48:36.910555+0000","last_peered":"2023-10-09T04:48:36.910555+0000","last_clean":"2023-10-09T04:45:44.820208+0000","last_became_active":"2023-10-09T04:46:25.282101+0000","last_became_peered":"2023-10-09T04:46:25.282101+0000","last_unstale":"2023-10-09T04:48:36.910555+0000","last_undegraded":"2023-10-09T04:48:36.910555+0000","last_fullsized":"2023-10-09T04:48:36.910555+0000","mapping_epoch":307,"log_start":"296'4803","ondisk_log_start":"296'4803","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6602,"log_dups_size":0,"ondisk_log_size":6602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:16.931727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33751040,"num_objects":515,"num_object_clones":0,"num_object_copies":1545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1030,"num_objects_unfound":0,"num_objects_dirty":515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11405,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,0,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":515}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":75,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811497+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:48:14.811497+0000","last_peered":"2023-10-09T04:48:14.811497+0000","last_clean":"2023-10-09T04:48:14.811497+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:48:14.811497+0000","last_undegraded":"2023-10-09T04:48:14.811497+0000","last_fullsized":"2023-10-09T04:48:14.811497+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":290,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.829789+0000","last_change":"2023-10-09T04:46:25.166210+0000","last_active":"2023-10-09T04:48:14.829789+0000","last_peered":"2023-10-09T04:48:14.829789+0000","last_clean":"2023-10-09T04:48:14.829789+0000","last_became_active":"2023-10-09T04:46:25.165948+0000","last_became_peered":"2023-10-09T04:46:25.165948+0000","last_unstale":"2023-10-09T04:48:14.829789+0000","last_undegraded":"2023-10-09T04:48:14.829789+0000","last_fullsized":"2023-10-09T04:48:14.829789+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:51.272111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":173,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.807356+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:48:14.807356+0000","last_peered":"2023-10-09T04:48:14.807356+0000","last_clean":"2023-10-09T04:48:14.807356+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:48:14.807356+0000","last_undegraded":"2023-10-09T04:48:14.807356+0000","last_fullsized":"2023-10-09T04:48:14.807356+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"327'11485","reported_seq":22415,"reported_epoch":327,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:36.900574+0000","last_change":"2023-10-09T04:46:25.450073+0000","last_active":"2023-10-09T04:48:36.900574+0000","last_peered":"2023-10-09T04:48:36.900574+0000","last_clean":"2023-10-09T04:44:37.742164+0000","last_became_active":"2023-10-09T04:45:46.956446+0000","last_became_peered":"2023-10-09T04:45:46.956446+0000","last_unstale":"2023-10-09T04:48:36.900574+0000","last_undegraded":"2023-10-09T04:46:25.447165+0000","last_fullsized":"2023-10-09T04:45:46.571293+0000","mapping_epoch":303,"log_start":"296'4903","ondisk_log_start":"296'4903","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:11.914538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32964608,"num_objects":503,"num_object_clones":0,"num_object_copies":1509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11485,"num_write_kb":383616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3619,"num_bytes_recovered":237109248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"1,2","objects":-811},{"shards":"1,2,3","objects":503}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":249,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.811259+0000","last_change":"2023-10-09T04:45:45.602634+0000","last_active":"2023-10-09T04:48:14.811259+0000","last_peered":"2023-10-09T04:48:14.811259+0000","last_clean":"2023-10-09T04:48:14.811259+0000","last_became_active":"2023-10-09T04:45:45.602317+0000","last_became_peered":"2023-10-09T04:45:45.602317+0000","last_unstale":"2023-10-09T04:48:14.811259+0000","last_undegraded":"2023-10-09T04:48:14.811259+0000","last_fullsized":"2023-10-09T04:48:14.811259+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:13.963587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":113,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.829950+0000","last_change":"2023-10-09T04:46:25.210175+0000","last_active":"2023-10-09T04:48:14.829950+0000","last_peered":"2023-10-09T04:48:14.829950+0000","last_clean":"2023-10-09T04:48:14.829950+0000","last_became_active":"2023-10-09T04:46:25.174609+0000","last_became_peered":"2023-10-09T04:46:25.174609+0000","last_unstale":"2023-10-09T04:48:14.829950+0000","last_undegraded":"2023-10-09T04:48:14.829950+0000","last_fullsized":"2023-10-09T04:48:14.829950+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:29.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821061+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:48:14.821061+0000","last_peered":"2023-10-09T04:48:14.821061+0000","last_clean":"2023-10-09T04:48:14.821061+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:48:14.821061+0000","last_undegraded":"2023-10-09T04:48:14.821061+0000","last_fullsized":"2023-10-09T04:48:14.821061+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":177,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821602+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:48:14.821602+0000","last_peered":"2023-10-09T04:48:14.821602+0000","last_clean":"2023-10-09T04:48:14.821602+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:48:14.821602+0000","last_undegraded":"2023-10-09T04:48:14.821602+0000","last_fullsized":"2023-10-09T04:48:14.821602+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":296,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.799887+0000","last_change":"2023-10-09T04:46:25.160157+0000","last_active":"2023-10-09T04:48:14.799887+0000","last_peered":"2023-10-09T04:48:14.799887+0000","last_clean":"2023-10-09T04:48:14.799887+0000","last_became_active":"2023-10-09T04:46:25.159850+0000","last_became_peered":"2023-10-09T04:46:25.159850+0000","last_unstale":"2023-10-09T04:48:14.799887+0000","last_undegraded":"2023-10-09T04:48:14.799887+0000","last_fullsized":"2023-10-09T04:48:14.799887+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:07.475665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":317,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821948+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:48:14.821948+0000","last_peered":"2023-10-09T04:48:14.821948+0000","last_clean":"2023-10-09T04:48:14.821948+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:48:14.821948+0000","last_undegraded":"2023-10-09T04:48:14.821948+0000","last_fullsized":"2023-10-09T04:48:14.821948+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821425+0000","last_change":"2023-10-09T04:45:45.609849+0000","last_active":"2023-10-09T04:48:14.821425+0000","last_peered":"2023-10-09T04:48:14.821425+0000","last_clean":"2023-10-09T04:48:14.821425+0000","last_became_active":"2023-10-09T04:45:45.609639+0000","last_became_peered":"2023-10-09T04:45:45.609639+0000","last_unstale":"2023-10-09T04:48:14.821425+0000","last_undegraded":"2023-10-09T04:48:14.821425+0000","last_fullsized":"2023-10-09T04:48:14.821425+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:26.113652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":286,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.817068+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:48:14.817068+0000","last_peered":"2023-10-09T04:48:14.817068+0000","last_clean":"2023-10-09T04:48:14.817068+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:48:14.817068+0000","last_undegraded":"2023-10-09T04:48:14.817068+0000","last_fullsized":"2023-10-09T04:48:14.817068+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.826771+0000","last_change":"2023-10-09T04:46:25.172010+0000","last_active":"2023-10-09T04:48:14.826771+0000","last_peered":"2023-10-09T04:48:14.826771+0000","last_clean":"2023-10-09T04:48:14.826771+0000","last_became_active":"2023-10-09T04:46:25.171776+0000","last_became_peered":"2023-10-09T04:46:25.171776+0000","last_unstale":"2023-10-09T04:48:14.826771+0000","last_undegraded":"2023-10-09T04:48:14.826771+0000","last_fullsized":"2023-10-09T04:48:14.826771+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:40.329129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":37,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.799724+0000","last_change":"2023-10-09T04:45:45.611105+0000","last_active":"2023-10-09T04:48:14.799724+0000","last_peered":"2023-10-09T04:48:14.799724+0000","last_clean":"2023-10-09T04:48:14.799724+0000","last_became_active":"2023-10-09T04:45:45.610758+0000","last_became_peered":"2023-10-09T04:45:45.610758+0000","last_unstale":"2023-10-09T04:48:14.799724+0000","last_undegraded":"2023-10-09T04:48:14.799724+0000","last_fullsized":"2023-10-09T04:48:14.799724+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:59.031709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":276,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:21.928661+0000","last_change":"2023-10-09T04:46:25.153824+0000","last_active":"2023-10-09T04:48:21.928661+0000","last_peered":"2023-10-09T04:48:21.928661+0000","last_clean":"2023-10-09T04:48:21.928661+0000","last_became_active":"2023-10-09T04:46:25.153497+0000","last_became_peered":"2023-10-09T04:46:25.153497+0000","last_unstale":"2023-10-09T04:48:21.928661+0000","last_undegraded":"2023-10-09T04:48:21.928661+0000","last_fullsized":"2023-10-09T04:48:21.928661+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:00.144937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":217,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.819587+0000","last_change":"2023-10-09T04:46:25.217389+0000","last_active":"2023-10-09T04:48:14.819587+0000","last_peered":"2023-10-09T04:48:14.819587+0000","last_clean":"2023-10-09T04:48:14.819587+0000","last_became_active":"2023-10-09T04:46:25.162993+0000","last_became_peered":"2023-10-09T04:46:25.162993+0000","last_unstale":"2023-10-09T04:48:14.819587+0000","last_undegraded":"2023-10-09T04:48:14.819587+0000","last_fullsized":"2023-10-09T04:48:14.819587+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:13.549717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":95,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-10-09T04:48:14.821327+0000","last_change":"2023-10-09T04:45:45.610893+0000","last_active":"2023-10-09T04:48:14.821327+0000","last_peered":"2023-10-09T04:48:14.821327+0000","last_clean":"2023-10-09T04:48:14.821327+0000","last_became_active":"2023-10-09T04:45:45.610260+0000","last_became_peered":"2023-10-09T04:45:45.610260+0000","last_unstale":"2023-10-09T04:48:14.821327+0000","last_undegraded":"2023-10-09T04:48:14.821327+0000","last_fullsized":"2023-10-09T04:48:14.821327+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:22.552117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":585302040,"num_objects":8932,"num_object_clones":0,"num_object_copies":26796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21197,"num_objects_misplaced":5488,"num_objects_unfound":0,"num_objects_dirty":8932,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181564,"num_write_kb":6095809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44807,"num_bytes_recovered":2642083840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5284638720,"data_stored":5284626504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94291,"ondisk_log_size":94291,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":11254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":17338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234500,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580272,"kb_used_data":376616,"kb_used_omap":0,"kb_used_meta":1203456,"kb_avail":92164880,"statfs":{"total":95995035648,"available":94376837120,"internally_reserved":0,"allocated":385654784,"data_stored":384929303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1232338944},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,3,5,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:40 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.1930000000000001,"15min":2.552},"min":{"1min":0.224,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.202,"5min":7.202,"15min":111.72199999999999},"last":7.8120000000000003},{"interface":"front","average":{"1min":3.5499999999999998,"5min":1.919,"15min":3.1059999999999999},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":19.152000000000001,"5min":19.152000000000001,"15min":111.76600000000001},"last":4.6719999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:47:41 2023","interfaces":[{"interface":"back","average":{"1min":8.5540000000000003,"5min":4.2640000000000002,"15min":4.1440000000000001},"min":{"1min":0.27500000000000002,"5min":0.253,"15min":0.253},"max":{"1min":128.601,"5min":128.601,"15min":264.62},"last":7.3239999999999998},{"interface":"front","average":{"1min":8.5069999999999997,"5min":4.0129999999999999,"15min":4.2560000000000002},"min":{"1min":0.29999999999999999,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":128.381,"5min":128.381,"15min":264.65899999999999},"last":12.173}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":13.303000000000001,"5min":5.2450000000000001,"15min":4.8490000000000002},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":129.10400000000001,"5min":129.10400000000001,"15min":184.72},"last":6.0289999999999999},{"interface":"front","average":{"1min":13.906000000000001,"5min":5.5090000000000003,"15min":5.234},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":129.15100000000001,"5min":129.15100000000001,"15min":184.697},"last":4.8380000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:48:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.728,"15min":4.1630000000000003},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.1500000000000004,"5min":7.8630000000000004,"15min":182.839},"last":12.134},{"interface":"front","average":{"1min":3.113,"5min":2.5129999999999999,"15min":4.7000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.866,"5min":14.866,"15min":182.678},"last":4.71}]},{"osd":4,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":1.5569999999999999,"15min":1.53},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.573,"5min":11.573,"15min":14.257999999999999},"last":4.7990000000000004},{"interface":"front","average":{"1min":2.23,"5min":1.353,"15min":1.327},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":7.9749999999999996,"5min":10.497,"15min":10.497},"last":0.374}]},{"osd":5,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.143,"15min":1.2150000000000001},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4169999999999998,"5min":8.5709999999999997,"15min":13.319000000000001},"last":12.202999999999999},{"interface":"front","average":{"1min":3.492,"5min":1.6080000000000001,"15min":1.6890000000000001},"min":{"1min":0.33600000000000002,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":12.081,"5min":12.081,"15min":18.408000000000001},"last":4.7670000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":0.98699999999999999,"15min":1.2509999999999999},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":7.4459999999999997,"5min":7.4459999999999997,"15min":11.388},"last":8.3219999999999992},{"interface":"front","average":{"1min":2.8090000000000002,"5min":1.5189999999999999,"15min":1.778},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":18.311},"last":4.7409999999999997}]}]},{"osd":6,"up_from":0,"seq":197568496134,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141223,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078804,"kb_used_data":296076,"kb_used_omap":4,"kb_used_meta":782715,"kb_avail":92666348,"statfs":{"total":95995035648,"available":94890340352,"internally_reserved":0,"allocated":303181824,"data_stored":301721275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":801501070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.448,"15min":3.4710000000000001},"min":{"1min":0.22700000000000001,"5min":0.22600000000000001,"15min":0.14799999999999999},"max":{"1min":7.4829999999999997,"5min":21.021999999999998,"15min":97.308000000000007},"last":2.9390000000000001},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.6899999999999999,"15min":3.6659999999999999},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.193},"max":{"1min":8.3580000000000005,"5min":21.280999999999999,"15min":97.055999999999997},"last":0.55100000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:48:07 2023","interfaces":[{"interface":"back","average":{"1min":16.600999999999999,"5min":5.8769999999999998,"15min":4.9039999999999999},"min":{"1min":0.50800000000000001,"5min":0.21199999999999999,"15min":0.20899999999999999},"max":{"1min":151.41900000000001,"5min":151.41900000000001,"15min":263.60500000000002},"last":2.1539999999999999},{"interface":"front","average":{"1min":16.071000000000002,"5min":5.875,"15min":5.3070000000000004},"min":{"1min":0.42199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":146.749,"5min":146.749,"15min":263.65199999999999},"last":1.883}]},{"osd":3,"last update":"Mon Oct 9 04:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":4.6420000000000003,"15min":5.6660000000000004},"min":{"1min":0.28499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.6760000000000002,"5min":379.80900000000003,"15min":379.80900000000003},"last":2.069},{"interface":"front","average":{"1min":2.2050000000000001,"5min":4.9930000000000003,"15min":6.3090000000000002},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.19700000000000001},"max":{"1min":8.4190000000000005,"5min":379.95800000000003,"15min":379.95800000000003},"last":0.51900000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":2.2069999999999999,"15min":2.5779999999999998},"min":{"1min":0.379,"5min":0.307,"15min":0.245},"max":{"1min":7.4610000000000003,"5min":19.959,"15min":170.072},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.591,"15min":2.3639999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":9.2129999999999992,"5min":11.935,"15min":169.94499999999999},"last":2.1160000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.742,"15min":2.496},"min":{"1min":0.34200000000000003,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":9.782,"5min":9.782,"15min":171.036},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.2789999999999999,"5min":2.298,"15min":2.79},"min":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.27500000000000002},"max":{"1min":7.5209999999999999,"5min":20.021000000000001,"15min":170.88},"last":0.45100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.841,"15min":2.5470000000000002},"min":{"1min":0.40799999999999997,"5min":0.35799999999999998,"15min":0.32000000000000001},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.544,"15min":2.8610000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":9.7370000000000001,"5min":20.164999999999999,"15min":132.17699999999999},"last":0.34100000000000003}]},{"osd":7,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6230000000000002,"5min":2.4129999999999998,"15min":3.339},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":8.4339999999999993,"5min":20.055,"15min":174.75999999999999},"last":1.47},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.573,"15min":2.9540000000000002},"min":{"1min":0.39800000000000002,"5min":0.38800000000000001,"15min":0.36199999999999999},"max":{"1min":8.0459999999999994,"5min":16.238,"15min":174.779},"last":6.7060000000000004}]}]},{"osd":0,"up_from":0,"seq":390842024280,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772557,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":822404,"kb_used_data":335356,"kb_used_omap":7,"kb_used_meta":487032,"kb_avail":92922748,"statfs":{"total":95995035648,"available":95152893952,"internally_reserved":0,"allocated":343404544,"data_stored":342139058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":498720849},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,0,1,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.2130000000000001,"15min":5.1769999999999996},"min":{"1min":0.253,"5min":0.183,"15min":0.183},"max":{"1min":7.7130000000000001,"5min":25.899999999999999,"15min":72.608000000000004},"last":3.4990000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.901,"15min":4.7389999999999999},"min":{"1min":0.23699999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.6379999999999999,"5min":16.550000000000001,"15min":72.632000000000005},"last":3.194}]},{"osd":1,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4550000000000001,"5min":3.8500000000000001,"15min":4.7839999999999998},"min":{"1min":0.32200000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":17.843,"5min":162.124,"15min":162.124},"last":1.1200000000000001},{"interface":"front","average":{"1min":4.0010000000000003,"5min":4.1710000000000003,"15min":5.2009999999999996},"min":{"1min":0.33300000000000002,"5min":0.223,"15min":0.223},"max":{"1min":21.329000000000001,"5min":164.68299999999999,"15min":164.68299999999999},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":6,"15min":6.9749999999999996},"min":{"1min":0.31900000000000001,"5min":0.25600000000000001,"15min":0.21199999999999999},"max":{"1min":5.8170000000000002,"5min":161.12799999999999,"15min":335.928},"last":1.3779999999999999},{"interface":"front","average":{"1min":2.9769999999999999,"5min":6.0019999999999998,"15min":7.1710000000000003},"min":{"1min":0.377,"5min":0.246,"15min":0.246},"max":{"1min":7.4580000000000002,"5min":161.386,"15min":335.68599999999998},"last":6.5869999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":2.2410000000000001,"15min":3.2429999999999999},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.315},"max":{"1min":7.6390000000000002,"5min":10.256,"15min":72.944999999999993},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.6789999999999998,"5min":2,"15min":3.2349999999999999},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9369999999999994,"5min":16.341000000000001,"15min":73.054000000000002},"last":6.508}]},{"osd":5,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":2.077,"15min":3.1379999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":12.388,"5min":12.98,"15min":72.986000000000004},"last":1.1599999999999999},{"interface":"front","average":{"1min":2.0979999999999999,"5min":2.0910000000000002,"15min":3.4470000000000001},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.32200000000000001},"max":{"1min":5.923,"5min":10.289,"15min":72.963999999999999},"last":6.4020000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":1.857,"15min":5.7329999999999997},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.33600000000000002},"max":{"1min":5.5670000000000002,"5min":16.425999999999998,"15min":135.614},"last":3.5680000000000001},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.7270000000000001,"15min":5.3230000000000004},"min":{"1min":0.39500000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.5709999999999997,"5min":10.977,"15min":130.196},"last":2.927}]},{"osd":7,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.702,"5min":2.3690000000000002,"15min":3.5390000000000001},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.7530000000000001,"5min":16.462,"15min":73.079999999999998},"last":10.579000000000001},{"interface":"front","average":{"1min":3.2650000000000001,"5min":2.347,"15min":3.1840000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.6379999999999999,"5min":10.935,"15min":73.069000000000003},"last":3.2330000000000001}]}]},{"osd":3,"up_from":0,"seq":609885356281,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019408,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379948,"kb_used_data":195428,"kb_used_omap":0,"kb_used_meta":1184512,"kb_avail":92365204,"statfs":{"total":95995035648,"available":94581968896,"internally_reserved":0,"allocated":200118272,"data_stored":199396873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1212940288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":1.4330000000000001,"15min":4.1280000000000001},"min":{"1min":0.29299999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":12.795,"5min":12.795,"15min":171.13499999999999},"last":0.248},{"interface":"front","average":{"1min":2.4199999999999999,"5min":1.716,"15min":4.1829999999999998},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":16.004000000000001,"5min":16.004000000000001,"15min":171.11799999999999},"last":7.2329999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":9.8460000000000001,"5min":5.3040000000000003,"15min":4.0389999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24099999999999999},"max":{"1min":144.65299999999999,"5min":144.65299999999999,"15min":144.65299999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":9.5960000000000001,"5min":5.0220000000000002,"15min":3.9780000000000002},"min":{"1min":0.33500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":146.50700000000001,"5min":146.50700000000001,"15min":146.50700000000001},"last":0.28399999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":16.193000000000001,"5min":6.4290000000000003,"15min":5.2519999999999998},"min":{"1min":0.47199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":74.069999999999993,"5min":98.221999999999994,"15min":195.267},"last":1.593},{"interface":"front","average":{"1min":16.573,"5min":6.5599999999999996,"15min":5.0060000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":75.219999999999999,"5min":94.219999999999999,"15min":195.15799999999999},"last":7.5430000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:48:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":4.1449999999999996,"15min":5.4080000000000004},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":16.082999999999998,"5min":141.334,"15min":357.125},"last":7.2670000000000003},{"interface":"front","average":{"1min":3.8730000000000002,"5min":4.2380000000000004,"15min":5.3609999999999998},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":12.090999999999999,"5min":141.18299999999999,"15min":359.02800000000002},"last":0.63}]},{"osd":5,"last update":"Mon Oct 9 04:48:25 2023","interfaces":[{"interface":"back","average":{"1min":3.6259999999999999,"5min":1.931,"15min":1.8600000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":12.053000000000001,"5min":24.234999999999999,"15min":107.476},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.653,"5min":1.5529999999999999,"15min":1.7909999999999999},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.24099999999999999},"max":{"1min":18.785,"5min":18.785,"15min":107.414},"last":7.1769999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:47:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.5409999999999999,"15min":1.8320000000000001},"min":{"1min":0.40799999999999997,"5min":0.29799999999999999,"15min":0.29599999999999999},"max":{"1min":12.122999999999999,"5min":22.920999999999999,"15min":65.75},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.2270000000000001,"15min":1.7430000000000001},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.095000000000001,"5min":15.095000000000001,"15min":65.849999999999994},"last":1.6240000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.403,"5min":1.8580000000000001,"15min":1.5880000000000001},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":16.109000000000002,"5min":16.109000000000002,"15min":16.109000000000002},"last":7.298},{"interface":"front","average":{"1min":3.7069999999999999,"5min":2.2799999999999998,"15min":1.7390000000000001},"min":{"1min":0.45000000000000001,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":12.013999999999999,"5min":22.858000000000001,"15min":22.858000000000001},"last":2.254}]}]},{"osd":5,"up_from":38,"seq":163208757772,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1010452,"kb_used_data":120140,"kb_used_omap":0,"kb_used_meta":890304,"kb_avail":92734700,"statfs":{"total":95995035648,"available":94960332800,"internally_reserved":0,"allocated":123023360,"data_stored":122321657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911671296},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5139999999999998,"5min":3.7269999999999999,"15min":5.0339999999999998},"min":{"1min":0.32700000000000001,"5min":0.26100000000000001,"15min":0.22700000000000001},"max":{"1min":9.0749999999999993,"5min":163.70699999999999,"15min":210.85499999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":1.8260000000000001,"5min":3.629,"15min":5.1070000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.249},"max":{"1min":7.0599999999999996,"5min":163.756,"15min":210.23500000000001},"last":3.5790000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:47:50 2023","interfaces":[{"interface":"back","average":{"1min":7.2080000000000002,"5min":7.7009999999999996,"15min":5.4790000000000001},"min":{"1min":0.42899999999999999,"5min":0.318,"15min":0.252},"max":{"1min":98.331000000000003,"5min":211.85400000000001,"15min":211.85400000000001},"last":3.2690000000000001},{"interface":"front","average":{"1min":7.9880000000000004,"5min":7.6689999999999996,"15min":5.7080000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":98.358999999999995,"5min":211.435,"15min":211.435},"last":2.8490000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:48:03 2023","interfaces":[{"interface":"back","average":{"1min":9.3610000000000007,"5min":5.8159999999999998,"15min":5.4619999999999997},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":82.561999999999998,"5min":82.561999999999998,"15min":90.763999999999996},"last":5.0869999999999997},{"interface":"front","average":{"1min":10.454000000000001,"5min":5.8280000000000003,"15min":5.5670000000000002},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":83.082999999999998,"5min":83.082999999999998,"15min":91.340000000000003},"last":1.5600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.766,"15min":5.1589999999999998},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":7.585,"5min":46.497,"15min":268.63299999999998},"last":3.605},{"interface":"front","average":{"1min":1.7070000000000001,"5min":2.0539999999999998,"15min":4.7560000000000002},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.9450000000000003,"5min":46.125999999999998,"15min":268.92500000000001},"last":7.2469999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.3320000000000001,"15min":2.5299999999999998},"min":{"1min":0.34100000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5650000000000004,"5min":10.268000000000001,"15min":361.09399999999999},"last":3.2370000000000001},{"interface":"front","average":{"1min":3.1970000000000001,"5min":1.4650000000000001,"15min":2.8860000000000001},"min":{"1min":0.33200000000000002,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":12.609999999999999,"5min":12.609999999999999,"15min":361.71300000000002},"last":3.3370000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.208,"15min":1.2849999999999999},"min":{"1min":0.378,"5min":0.28499999999999998,"15min":0.23100000000000001},"max":{"1min":3.8490000000000002,"5min":13.162000000000001,"15min":13.162000000000001},"last":3.472},{"interface":"front","average":{"1min":1.925,"5min":1.1220000000000001,"15min":1.3879999999999999},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":7.3040000000000003,"5min":7.3040000000000003,"15min":13.231999999999999},"last":3.548}]},{"osd":7,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":2.5249999999999999,"15min":1.764},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":7.1029999999999998,"5min":68.411000000000001,"15min":68.411000000000001},"last":3.5150000000000001},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.754,"15min":1.786},"min":{"1min":0.434,"5min":0.38300000000000001,"15min":0.30599999999999999},"max":{"1min":8.4499999999999993,"5min":68.435000000000002,"15min":68.435000000000002},"last":3.4390000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":788856832,"data_stored":788856832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":295243776,"data_stored":295239704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":322965504,"data_stored":322961432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1538981888,"data_stored":1538981888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":190971904,"data_stored":190971904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":116916224,"data_stored":116916224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1654390784,"data_stored":1654390784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":376311808,"data_stored":376307736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:48:41.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T04:48:41.475 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:41.526 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:41.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:41.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:41.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:41.829 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:42.035 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:42.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:42.142 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:42.182 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:42.252 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:42.279 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:42.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:42.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:42.455 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:42.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:42.522 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:48:42.522 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:48:42.522 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:48:42.522 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:48:42.522 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:48:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:42.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:42.583 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:42.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:42.699 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:42.821 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:48:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:48:42] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-10-09T04:48:43.058 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:48:43.091 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:43 smithi029 ceph-mon[223858]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 423 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 20882/20349 objects degraded (102.619%); 4693/20349 objects misplaced (23.063%); 5.1 MiB/s, 219 objects/s recovering 2023-10-09T04:48:43.091 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1437088504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:48:43.091 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:43 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:48:43.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:43 smithi029 ceph-mon[216903]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 423 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 20882/20349 objects degraded (102.619%); 4693/20349 objects misplaced (23.063%); 5.1 MiB/s, 219 objects/s recovering 2023-10-09T04:48:43.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1437088504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:48:43.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:43 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:48:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:43 smithi177 ceph-mon[195797]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 423 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 20882/20349 objects degraded (102.619%); 4693/20349 objects misplaced (23.063%); 5.1 MiB/s, 219 objects/s recovering 2023-10-09T04:48:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1437088504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:48:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:43 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:48:44.376 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: osdmap e328: 8 total, 8 up, 5 in 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: osdmap e328: 8 total, 8 up, 5 in 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.384 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:44.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:44.485 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: osdmap e328: 8 total, 8 up, 5 in 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: failed to encode map e328 with expected crc 2023-10-09T04:48:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:44.565 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:44.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:44.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:44.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:48:44.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:44.847 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:45.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:45 smithi029 ceph-mon[216903]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 348 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 20474/16770 objects degraded (122.087%); 4084/16770 objects misplaced (24.353%); 4.9 MiB/s, 176 objects/s recovering 2023-10-09T04:48:45.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:45 smithi029 ceph-mon[223858]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 348 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 20474/16770 objects degraded (122.087%); 4084/16770 objects misplaced (24.353%); 4.9 MiB/s, 176 objects/s recovering 2023-10-09T04:48:45.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:48:45.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:45 smithi177 ceph-mon[195797]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 348 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 20474/16770 objects degraded (122.087%); 4084/16770 objects misplaced (24.353%); 4.9 MiB/s, 176 objects/s recovering 2023-10-09T04:48:45.794 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:45.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:45.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:45.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:48:45.930 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:46.080 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:48:46.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:46.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:48:46.184 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: osdmap e329: 8 total, 8 up, 5 in 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.243 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 20474/16770 objects degraded (122.087%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: osdmap e329: 8 total, 8 up, 5 in 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.245 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.246 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 20474/16770 objects degraded (122.087%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:46.248 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:46.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: osdmap e329: 8 total, 8 up, 5 in 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:46.534 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 20474/16770 objects degraded (122.087%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: pgmap v1293: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 226 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 18403/10890 objects degraded (168.990%); 3510/10890 objects misplaced (32.231%); 4.7 MiB/s, 220 objects/s recovering 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: failed to encode map e329 with expected crc 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: osdmap e330: 8 total, 8 up, 5 in 2023-10-09T04:48:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[223858]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: pgmap v1293: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 226 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 18403/10890 objects degraded (168.990%); 3510/10890 objects misplaced (32.231%); 4.7 MiB/s, 220 objects/s recovering 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: failed to encode map e329 with expected crc 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: osdmap e330: 8 total, 8 up, 5 in 2023-10-09T04:48:47.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:47 smithi029 ceph-mon[216903]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: pgmap v1293: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 226 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 665 op/s; 18403/10890 objects degraded (168.990%); 3510/10890 objects misplaced (32.231%); 4.7 MiB/s, 220 objects/s recovering 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: failed to encode map e329 with expected crc 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: osdmap e330: 8 total, 8 up, 5 in 2023-10-09T04:48:47.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:47 smithi177 ceph-mon[195797]: failed to encode map e330 with expected crc 2023-10-09T04:48:47.550 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:47.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:47.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:47.735 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:47.984 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:47.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:47.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:48.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:48.208 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:48.230 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:48.445 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:48.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:48.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:48.637 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:48.996 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:49.030 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:49.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:49.145 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:48:49.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:49.199 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:48:49.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:49 smithi029 ceph-mon[223858]: pgmap v1295: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 175 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 661 op/s; 18073/8433 objects degraded (214.313%); 3016/8433 objects misplaced (35.764%); 5.8 MiB/s, 207 objects/s recovering 2023-10-09T04:48:49.217 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:49 smithi029 ceph-mon[216903]: pgmap v1295: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 175 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 661 op/s; 18073/8433 objects degraded (214.313%); 3016/8433 objects misplaced (35.764%); 5.8 MiB/s, 207 objects/s recovering 2023-10-09T04:48:49.226 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:49.264 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:48:49.294 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:48:49.294 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18073/8433 objects degraded (214.313%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1865,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":330,"num_osds":8,"num_up_osds":8,"osd_up_since":1696826206,"num_in_osds":5,"osd_in_since":1696826784,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":77},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":2811,"data_bytes":183179192,"bytes_used":5793472512,"bytes_avail":474181705728,"bytes_total":479975178240,"degraded_objects":18073,"degraded_total":8433,"degraded_ratio":2.1431281868848573,"misplaced_objects":3016,"misplaced_total":8433,"misplaced_ratio":0.35764259456895531,"recovering_objects_per_sec":207,"recovering_bytes_per_sec":6114131,"recovering_keys_per_sec":0,"num_objects_recovered":1243,"num_bytes_recovered":36700160,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":661},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":29,"modified":"2023-10-09T04:48:37.352463+0000","services":{}},"progress_events":{"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true},"5769ab26-d645-4ed3-9a6d-e6e40a9fc14f":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 36s)","progress":0.86516851186752319,"add_to_ceph_s":true}}} 2023-10-09T04:48:49.307 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:48:49.337 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:48:49.381 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:48:49.418 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:48:49.460 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:48:49.478 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.478 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.478 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.478 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.479 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.479 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.479 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.479 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:48:49.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:49 smithi177 ceph-mon[195797]: pgmap v1295: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 175 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 661 op/s; 18073/8433 objects degraded (214.313%); 3016/8433 objects misplaced (35.764%); 5.8 MiB/s, 207 objects/s recovering 2023-10-09T04:48:50.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 95247 objects 2023-10-09T04:48:50.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :215.06 2023-10-09T04:48:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3653320809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3395602295' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:48:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3653320809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3395602295' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:48:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3653320809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:48:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3395602295' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T04:48:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:51 smithi029 ceph-mon[223858]: pgmap v1296: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 164 MiB data, 5.3 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 498 op/s; 18073/7911 objects degraded (228.454%); 3016/7911 objects misplaced (38.124%); 3.5 MiB/s, 164 objects/s recovering 2023-10-09T04:48:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 18073/7911 objects degraded (228.454%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:51 smithi029 ceph-mon[216903]: pgmap v1296: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 164 MiB data, 5.3 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 498 op/s; 18073/7911 objects degraded (228.454%); 3016/7911 objects misplaced (38.124%); 3.5 MiB/s, 164 objects/s recovering 2023-10-09T04:48:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 18073/7911 objects degraded (228.454%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:51 smithi177 ceph-mon[195797]: pgmap v1296: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 164 MiB data, 5.3 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 498 op/s; 18073/7911 objects degraded (228.454%); 3016/7911 objects misplaced (38.124%); 3.5 MiB/s, 164 objects/s recovering 2023-10-09T04:48:51.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 18073/7911 objects degraded (228.454%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:52.356 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T04:48:52.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:48:52.876 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:52.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:52.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:48:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:48:52] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2023-10-09T04:48:52.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:53.008 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:53.360 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:53 smithi029 ceph-mon[216903]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 59 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 17926/2868 objects degraded (625.035%); 2153/2868 objects misplaced (75.070%); 5.4 MiB/s, 171 objects/s recovering 2023-10-09T04:48:53.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:53 smithi029 ceph-mon[223858]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 59 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 17926/2868 objects degraded (625.035%); 2153/2868 objects misplaced (75.070%); 5.4 MiB/s, 171 objects/s recovering 2023-10-09T04:48:53.362 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:53.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:53.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:53.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:53 smithi177 ceph-mon[195797]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 59 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 579 op/s; 17926/2868 objects degraded (625.035%); 2153/2868 objects misplaced (75.070%); 5.4 MiB/s, 171 objects/s recovering 2023-10-09T04:48:53.494 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:53.538 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:53.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:53.620 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:48:53.620 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:48:53.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:53.677 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:53.773 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:54.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:48:54.685 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:54.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:54.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:54.818 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:54.920 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:54.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:54.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:48:55.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:55.050 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:55.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:55.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:55.120 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:48:55.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:55.239 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:55.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:55 smithi029 ceph-mon[216903]: pgmap v1298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 12 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 411 op/s; 17664/639 objects degraded (2764.319%); 1757/639 objects misplaced (274.961%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:48:55.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1679915805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:55.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:55 smithi029 ceph-mon[223858]: pgmap v1298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 12 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 411 op/s; 17664/639 objects degraded (2764.319%); 1757/639 objects misplaced (274.961%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:48:55.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1679915805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:55 smithi177 ceph-mon[195797]: pgmap v1298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 12 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 411 op/s; 17664/639 objects degraded (2764.319%); 1757/639 objects misplaced (274.961%); 4.7 MiB/s, 75 objects/s recovering 2023-10-09T04:48:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1679915805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:48:56.023 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:56.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:56.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:56.157 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:56.204 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-10-09T04:48:56.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-10-09T04:48:56.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:48:56.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[223858]: failed to encode map e331 with expected crc 2023-10-09T04:48:56.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1679915805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:56.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[223858]: osdmap e331: 8 total, 8 up, 5 in 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[223858]: failed to encode map e331 with expected crc 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 17664/639 objects degraded (2764.319%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[216903]: failed to encode map e331 with expected crc 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1679915805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[216903]: osdmap e331: 8 total, 8 up, 5 in 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[216903]: failed to encode map e331 with expected crc 2023-10-09T04:48:56.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 17664/639 objects degraded (2764.319%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:56 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:48:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:56 smithi177 ceph-mon[195797]: failed to encode map e331 with expected crc 2023-10-09T04:48:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1679915805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:48:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:56 smithi177 ceph-mon[195797]: osdmap e331: 8 total, 8 up, 5 in 2023-10-09T04:48:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:56 smithi177 ceph-mon[195797]: failed to encode map e331 with expected crc 2023-10-09T04:48:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 17664/639 objects degraded (2764.319%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:48:56.738 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:56.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:56.782 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:56.843 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:48:56.878 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:57.050 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:48:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:57.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:48:57.110 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:57.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:57.191 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:57.248 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:57.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:57.354 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:57.380 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:57 smithi177 ceph-mon[195797]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 4.4 MiB/s, 119 objects/s recovering 2023-10-09T04:48:57.383 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:57.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:57 smithi029 ceph-mon[216903]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 4.4 MiB/s, 119 objects/s recovering 2023-10-09T04:48:57.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:57 smithi029 ceph-mon[223858]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 388 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 4.4 MiB/s, 119 objects/s recovering 2023-10-09T04:48:57.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:57.432 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:48:57.434 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:48:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:57.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:57.457 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:48:57.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:48:57.561 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:57.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:48:57.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:57.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:48:57.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:48:57.739 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:58.439 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:48:58.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:58.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:48:58.543 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:58.568 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:58.611 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:58.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:58.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:48:58.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:58.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:48:58.995 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1301,"stamp":"2023-10-09T04:48:57.358804+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17650,"num_objects_misplaced":1682,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":190796,"num_write_kb":6098254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47748,"num_bytes_recovered":2727491766,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94739,"ondisk_log_size":94739,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5230800,"kb_used_data":17960,"kb_used_omap":12,"kb_used_meta":5212787,"kb_avail":463494960,"statfs":{"total":479975178240,"available":474618839040,"internally_reserved":0,"allocated":18391040,"data_stored":13447160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12321,"internal_metadata":5337894879},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-182583320,"num_objects":-2787,"num_object_clones":0,"num_object_copies":-8361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-423,"num_objects_misplaced":-1334,"num_objects_unfound":0,"num_objects_dirty":-2787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2787,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":947,"num_bytes_recovered":31719424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002732"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":78,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.089799+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:48:45.089799+0000","last_peered":"2023-10-09T04:48:45.089799+0000","last_clean":"2023-10-09T04:48:45.089799+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:48:45.089799+0000","last_undegraded":"2023-10-09T04:48:45.089799+0000","last_fullsized":"2023-10-09T04:48:45.089799+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":123,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080754+0000","last_change":"2023-10-09T04:45:45.599027+0000","last_active":"2023-10-09T04:48:45.080754+0000","last_peered":"2023-10-09T04:48:45.080754+0000","last_clean":"2023-10-09T04:48:45.080754+0000","last_became_active":"2023-10-09T04:45:45.598650+0000","last_became_peered":"2023-10-09T04:45:45.598650+0000","last_unstale":"2023-10-09T04:48:45.080754+0000","last_undegraded":"2023-10-09T04:48:45.080754+0000","last_fullsized":"2023-10-09T04:48:45.080754+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:15.414379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":327,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.061128+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:48:45.061128+0000","last_peered":"2023-10-09T04:48:45.061128+0000","last_clean":"2023-10-09T04:48:45.061128+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:48:45.061128+0000","last_undegraded":"2023-10-09T04:48:45.061128+0000","last_fullsized":"2023-10-09T04:48:45.061128+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":263,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.087616+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:48:45.087616+0000","last_peered":"2023-10-09T04:48:45.087616+0000","last_clean":"2023-10-09T04:48:45.087616+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:48:45.087616+0000","last_undegraded":"2023-10-09T04:48:45.087616+0000","last_fullsized":"2023-10-09T04:48:45.087616+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":49,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.080844+0000","last_change":"2023-10-09T04:46:25.173258+0000","last_active":"2023-10-09T04:48:46.080844+0000","last_peered":"2023-10-09T04:48:46.080844+0000","last_clean":"2023-10-09T04:48:46.080844+0000","last_became_active":"2023-10-09T04:46:25.172947+0000","last_became_peered":"2023-10-09T04:46:25.172947+0000","last_unstale":"2023-10-09T04:48:46.080844+0000","last_undegraded":"2023-10-09T04:48:46.080844+0000","last_fullsized":"2023-10-09T04:48:46.080844+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:38.532832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":130,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080129+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:48:45.080129+0000","last_peered":"2023-10-09T04:48:45.080129+0000","last_clean":"2023-10-09T04:48:45.080129+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:48:45.080129+0000","last_undegraded":"2023-10-09T04:48:45.080129+0000","last_fullsized":"2023-10-09T04:48:45.080129+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.079995+0000","last_change":"2023-10-09T04:46:25.155473+0000","last_active":"2023-10-09T04:48:45.079995+0000","last_peered":"2023-10-09T04:48:45.079995+0000","last_clean":"2023-10-09T04:48:45.079995+0000","last_became_active":"2023-10-09T04:46:25.155148+0000","last_became_peered":"2023-10-09T04:46:25.155148+0000","last_unstale":"2023-10-09T04:48:45.079995+0000","last_undegraded":"2023-10-09T04:48:45.079995+0000","last_fullsized":"2023-10-09T04:48:45.079995+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:38.576510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":287,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.076609+0000","last_change":"2023-10-09T04:45:45.611067+0000","last_active":"2023-10-09T04:48:45.076609+0000","last_peered":"2023-10-09T04:48:45.076609+0000","last_clean":"2023-10-09T04:48:45.076609+0000","last_became_active":"2023-10-09T04:45:45.610571+0000","last_became_peered":"2023-10-09T04:45:45.610571+0000","last_unstale":"2023-10-09T04:48:45.076609+0000","last_undegraded":"2023-10-09T04:48:45.076609+0000","last_fullsized":"2023-10-09T04:48:45.076609+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:14.381921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":290,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.077927+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:48:45.077927+0000","last_peered":"2023-10-09T04:48:45.077927+0000","last_clean":"2023-10-09T04:48:45.077927+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:48:45.077927+0000","last_undegraded":"2023-10-09T04:48:45.077927+0000","last_fullsized":"2023-10-09T04:48:45.077927+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":270,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.088899+0000","last_change":"2023-10-09T04:46:25.193703+0000","last_active":"2023-10-09T04:48:45.088899+0000","last_peered":"2023-10-09T04:48:45.088899+0000","last_clean":"2023-10-09T04:48:45.088899+0000","last_became_active":"2023-10-09T04:46:25.193392+0000","last_became_peered":"2023-10-09T04:46:25.193392+0000","last_unstale":"2023-10-09T04:48:45.088899+0000","last_undegraded":"2023-10-09T04:48:45.088899+0000","last_fullsized":"2023-10-09T04:48:45.088899+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:56.086176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":356,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083992+0000","last_change":"2023-10-09T04:46:25.479989+0000","last_active":"2023-10-09T04:48:45.083992+0000","last_peered":"2023-10-09T04:48:45.083992+0000","last_clean":"2023-10-09T04:48:45.083992+0000","last_became_active":"2023-10-09T04:46:25.381247+0000","last_became_peered":"2023-10-09T04:46:25.381247+0000","last_unstale":"2023-10-09T04:48:45.083992+0000","last_undegraded":"2023-10-09T04:48:45.083992+0000","last_fullsized":"2023-10-09T04:48:45.083992+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:12.653312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":257,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.105786+0000","last_change":"2023-10-09T04:46:25.150997+0000","last_active":"2023-10-09T04:48:46.105786+0000","last_peered":"2023-10-09T04:48:46.105786+0000","last_clean":"2023-10-09T04:48:46.105786+0000","last_became_active":"2023-10-09T04:46:25.150370+0000","last_became_peered":"2023-10-09T04:46:25.150370+0000","last_unstale":"2023-10-09T04:48:46.105786+0000","last_undegraded":"2023-10-09T04:48:46.105786+0000","last_fullsized":"2023-10-09T04:48:46.105786+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:07.231212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":108,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080555+0000","last_change":"2023-10-09T04:44:38.813366+0000","last_active":"2023-10-09T04:48:45.080555+0000","last_peered":"2023-10-09T04:48:45.080555+0000","last_clean":"2023-10-09T04:48:45.080555+0000","last_became_active":"2023-10-09T04:44:38.777069+0000","last_became_peered":"2023-10-09T04:44:38.777069+0000","last_unstale":"2023-10-09T04:48:45.080555+0000","last_undegraded":"2023-10-09T04:48:45.080555+0000","last_fullsized":"2023-10-09T04:48:45.080555+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:39.101540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":300,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080802+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:48:45.080802+0000","last_peered":"2023-10-09T04:48:45.080802+0000","last_clean":"2023-10-09T04:48:45.080802+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:48:45.080802+0000","last_undegraded":"2023-10-09T04:48:45.080802+0000","last_fullsized":"2023-10-09T04:48:45.080802+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":340,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.090422+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:48:45.090422+0000","last_peered":"2023-10-09T04:48:45.090422+0000","last_clean":"2023-10-09T04:48:45.090422+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:48:45.090422+0000","last_undegraded":"2023-10-09T04:48:45.090422+0000","last_fullsized":"2023-10-09T04:48:45.090422+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083063+0000","last_change":"2023-10-09T04:45:45.607791+0000","last_active":"2023-10-09T04:48:45.083063+0000","last_peered":"2023-10-09T04:48:45.083063+0000","last_clean":"2023-10-09T04:48:45.083063+0000","last_became_active":"2023-10-09T04:45:45.607501+0000","last_became_peered":"2023-10-09T04:45:45.607501+0000","last_unstale":"2023-10-09T04:48:45.083063+0000","last_undegraded":"2023-10-09T04:48:45.083063+0000","last_fullsized":"2023-10-09T04:48:45.083063+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:05.067640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":262,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083829+0000","last_change":"2023-10-09T04:46:25.168444+0000","last_active":"2023-10-09T04:48:45.083829+0000","last_peered":"2023-10-09T04:48:45.083829+0000","last_clean":"2023-10-09T04:48:45.083829+0000","last_became_active":"2023-10-09T04:46:25.168143+0000","last_became_peered":"2023-10-09T04:46:25.168143+0000","last_unstale":"2023-10-09T04:48:45.083829+0000","last_undegraded":"2023-10-09T04:48:45.083829+0000","last_fullsized":"2023-10-09T04:48:45.083829+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:56.166900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074927+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:48:45.074927+0000","last_peered":"2023-10-09T04:48:45.074927+0000","last_clean":"2023-10-09T04:48:45.074927+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:48:45.074927+0000","last_undegraded":"2023-10-09T04:48:45.074927+0000","last_fullsized":"2023-10-09T04:48:45.074927+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":292,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.088825+0000","last_change":"2023-10-09T04:46:25.175418+0000","last_active":"2023-10-09T04:48:45.088825+0000","last_peered":"2023-10-09T04:48:45.088825+0000","last_clean":"2023-10-09T04:48:45.088825+0000","last_became_active":"2023-10-09T04:46:25.174958+0000","last_became_peered":"2023-10-09T04:46:25.174958+0000","last_unstale":"2023-10-09T04:48:45.088825+0000","last_undegraded":"2023-10-09T04:48:45.088825+0000","last_fullsized":"2023-10-09T04:48:45.088825+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:35.362263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":150,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.087698+0000","last_change":"2023-10-09T04:46:25.170147+0000","last_active":"2023-10-09T04:48:45.087698+0000","last_peered":"2023-10-09T04:48:45.087698+0000","last_clean":"2023-10-09T04:48:45.087698+0000","last_became_active":"2023-10-09T04:46:25.169835+0000","last_became_peered":"2023-10-09T04:46:25.169835+0000","last_unstale":"2023-10-09T04:48:45.087698+0000","last_undegraded":"2023-10-09T04:48:45.087698+0000","last_fullsized":"2023-10-09T04:48:45.087698+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:01.393349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074694+0000","last_change":"2023-10-09T04:46:25.271462+0000","last_active":"2023-10-09T04:48:45.074694+0000","last_peered":"2023-10-09T04:48:45.074694+0000","last_clean":"2023-10-09T04:48:45.074694+0000","last_became_active":"2023-10-09T04:46:25.166617+0000","last_became_peered":"2023-10-09T04:46:25.166617+0000","last_unstale":"2023-10-09T04:48:45.074694+0000","last_undegraded":"2023-10-09T04:48:45.074694+0000","last_fullsized":"2023-10-09T04:48:45.074694+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:30.657523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":123,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083076+0000","last_change":"2023-10-09T04:45:45.603131+0000","last_active":"2023-10-09T04:48:45.083076+0000","last_peered":"2023-10-09T04:48:45.083076+0000","last_clean":"2023-10-09T04:48:45.083076+0000","last_became_active":"2023-10-09T04:45:45.602862+0000","last_became_peered":"2023-10-09T04:45:45.602862+0000","last_unstale":"2023-10-09T04:48:45.083076+0000","last_undegraded":"2023-10-09T04:48:45.083076+0000","last_fullsized":"2023-10-09T04:48:45.083076+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:22.358682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"330'12002","reported_seq":21589,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.165812+0000","last_change":"2023-10-09T04:46:53.684121+0000","last_active":"2023-10-09T04:48:50.165812+0000","last_peered":"2023-10-09T04:48:50.165812+0000","last_clean":"2023-10-09T04:48:50.165812+0000","last_became_active":"2023-10-09T04:46:53.658692+0000","last_became_peered":"2023-10-09T04:46:53.658692+0000","last_unstale":"2023-10-09T04:48:50.165812+0000","last_undegraded":"2023-10-09T04:48:50.165812+0000","last_fullsized":"2023-10-09T04:48:50.165812+0000","mapping_epoch":312,"log_start":"298'6172","ondisk_log_start":"298'6172","created":284,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5830,"log_dups_size":0,"ondisk_log_size":5830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:44.589594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12002,"num_write_kb":384064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4752,"num_bytes_recovered":242221056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":360,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.082988+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:48:45.082988+0000","last_peered":"2023-10-09T04:48:45.082988+0000","last_clean":"2023-10-09T04:48:45.082988+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:48:45.082988+0000","last_undegraded":"2023-10-09T04:48:45.082988+0000","last_fullsized":"2023-10-09T04:48:45.082988+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":282,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074587+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:48:45.074587+0000","last_peered":"2023-10-09T04:48:45.074587+0000","last_clean":"2023-10-09T04:48:45.074587+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:48:45.074587+0000","last_undegraded":"2023-10-09T04:48:45.074587+0000","last_fullsized":"2023-10-09T04:48:45.074587+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"330'12014","reported_seq":21476,"reported_epoch":330,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:50.173984+0000","last_change":"2023-10-09T04:48:26.671662+0000","last_active":"2023-10-09T04:48:50.173984+0000","last_peered":"2023-10-09T04:48:50.173984+0000","last_clean":"2023-10-09T04:44:37.740580+0000","last_became_active":"2023-10-09T04:45:46.774175+0000","last_became_peered":"2023-10-09T04:45:46.774175+0000","last_unstale":"2023-10-09T04:48:50.173984+0000","last_undegraded":"2023-10-09T04:48:50.173984+0000","last_fullsized":"2023-10-09T04:48:50.173984+0000","mapping_epoch":303,"log_start":"301'6806","ondisk_log_start":"301'6806","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5208,"log_dups_size":0,"ondisk_log_size":5208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:40.805571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12014,"num_write_kb":384448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4694,"num_bytes_recovered":240713728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,3],"avail_no_missing":["7","1","3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":319,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.087541+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:48:45.087541+0000","last_peered":"2023-10-09T04:48:45.087541+0000","last_clean":"2023-10-09T04:48:45.087541+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:48:45.087541+0000","last_undegraded":"2023-10-09T04:48:45.087541+0000","last_fullsized":"2023-10-09T04:48:45.087541+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":327,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080602+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:48:45.080602+0000","last_peered":"2023-10-09T04:48:45.080602+0000","last_clean":"2023-10-09T04:48:45.080602+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:48:45.080602+0000","last_undegraded":"2023-10-09T04:48:45.080602+0000","last_fullsized":"2023-10-09T04:48:45.080602+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"330'11864","reported_seq":13224,"reported_epoch":330,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:54.653714+0000","last_change":"2023-10-09T04:48:44.347260+0000","last_active":"2023-10-09T04:48:54.653714+0000","last_peered":"2023-10-09T04:48:54.653714+0000","last_clean":"2023-10-09T04:45:44.791779+0000","last_became_active":"2023-10-09T04:45:46.998769+0000","last_became_peered":"2023-10-09T04:45:46.998769+0000","last_unstale":"2023-10-09T04:48:54.653714+0000","last_undegraded":"2023-10-09T04:45:46.613330+0000","last_fullsized":"2023-10-09T04:45:46.574934+0000","mapping_epoch":303,"log_start":"298'6029","ondisk_log_start":"298'6029","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5835,"log_dups_size":0,"ondisk_log_size":5835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:12:00.584863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11864,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":646,"num_bytes_recovered":11993088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"4,7","objects":-4745}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":301,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083832+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:48:45.083832+0000","last_peered":"2023-10-09T04:48:45.083832+0000","last_clean":"2023-10-09T04:48:45.083832+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:48:45.083832+0000","last_undegraded":"2023-10-09T04:48:45.083832+0000","last_fullsized":"2023-10-09T04:48:45.083832+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.105737+0000","last_change":"2023-10-09T04:44:38.775465+0000","last_active":"2023-10-09T04:48:46.105737+0000","last_peered":"2023-10-09T04:48:46.105737+0000","last_clean":"2023-10-09T04:48:46.105737+0000","last_became_active":"2023-10-09T04:44:38.775246+0000","last_became_peered":"2023-10-09T04:44:38.775246+0000","last_unstale":"2023-10-09T04:48:46.105737+0000","last_undegraded":"2023-10-09T04:48:46.105737+0000","last_fullsized":"2023-10-09T04:48:46.105737+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.286190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.3","version":"330'11934","reported_seq":20577,"reported_epoch":330,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:48:50.213563+0000","last_change":"2023-10-09T04:46:26.476463+0000","last_active":"2023-10-09T04:48:50.213563+0000","last_peered":"2023-10-09T04:48:50.213563+0000","last_clean":"2023-10-09T04:45:44.834828+0000","last_became_active":"2023-10-09T04:46:26.386549+0000","last_became_peered":"2023-10-09T04:46:26.386549+0000","last_unstale":"2023-10-09T04:48:50.213563+0000","last_undegraded":"2023-10-09T04:46:26.372334+0000","last_fullsized":"2023-10-09T04:48:50.213563+0000","mapping_epoch":308,"log_start":"301'6680","ondisk_log_start":"301'6680","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5254,"log_dups_size":0,"ondisk_log_size":5254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:01.311399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1682,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11934,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4285,"num_bytes_recovered":280625152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,0],"avail_no_missing":["7","0","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":320,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.079774+0000","last_change":"2023-10-09T04:46:25.150945+0000","last_active":"2023-10-09T04:48:45.079774+0000","last_peered":"2023-10-09T04:48:45.079774+0000","last_clean":"2023-10-09T04:48:45.079774+0000","last_became_active":"2023-10-09T04:46:25.150622+0000","last_became_peered":"2023-10-09T04:46:25.150622+0000","last_unstale":"2023-10-09T04:48:45.079774+0000","last_undegraded":"2023-10-09T04:48:45.079774+0000","last_fullsized":"2023-10-09T04:48:45.079774+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:02.718023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080588+0000","last_change":"2023-10-09T04:46:25.173123+0000","last_active":"2023-10-09T04:48:45.080588+0000","last_peered":"2023-10-09T04:48:45.080588+0000","last_clean":"2023-10-09T04:48:45.080588+0000","last_became_active":"2023-10-09T04:46:25.172534+0000","last_became_peered":"2023-10-09T04:46:25.172534+0000","last_unstale":"2023-10-09T04:48:45.080588+0000","last_undegraded":"2023-10-09T04:48:45.080588+0000","last_fullsized":"2023-10-09T04:48:45.080588+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:00.581088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.4","version":"330'11892","reported_seq":21279,"reported_epoch":330,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:50.149772+0000","last_change":"2023-10-09T04:47:41.808546+0000","last_active":"2023-10-09T04:48:50.149772+0000","last_peered":"2023-10-09T04:48:50.149772+0000","last_clean":"2023-10-09T04:44:37.730195+0000","last_became_active":"2023-10-09T04:45:47.970624+0000","last_became_peered":"2023-10-09T04:45:47.970624+0000","last_unstale":"2023-10-09T04:48:50.149772+0000","last_undegraded":"2023-10-09T04:48:50.149772+0000","last_fullsized":"2023-10-09T04:48:50.149772+0000","mapping_epoch":303,"log_start":"301'6636","ondisk_log_start":"301'6636","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5256,"log_dups_size":0,"ondisk_log_size":5256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:55:58.727121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11892,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4658,"num_bytes_recovered":305266688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,3],"avail_no_missing":["7","1","3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":69,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080851+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:48:45.080851+0000","last_peered":"2023-10-09T04:48:45.080851+0000","last_clean":"2023-10-09T04:48:45.080851+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:48:45.080851+0000","last_undegraded":"2023-10-09T04:48:45.080851+0000","last_fullsized":"2023-10-09T04:48:45.080851+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080727+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:48:45.080727+0000","last_peered":"2023-10-09T04:48:45.080727+0000","last_clean":"2023-10-09T04:48:45.080727+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:48:45.080727+0000","last_undegraded":"2023-10-09T04:48:45.080727+0000","last_fullsized":"2023-10-09T04:48:45.080727+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"330'11878","reported_seq":15142,"reported_epoch":331,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:55.719043+0000","last_change":"2023-10-09T04:46:25.469213+0000","last_active":"2023-10-09T04:48:55.719043+0000","last_peered":"2023-10-09T04:48:55.719043+0000","last_clean":"2023-10-09T04:44:37.740389+0000","last_became_active":"2023-10-09T04:46:25.449027+0000","last_became_peered":"2023-10-09T04:46:25.449027+0000","last_unstale":"2023-10-09T04:48:55.719043+0000","last_undegraded":"2023-10-09T04:46:25.190986+0000","last_fullsized":"2023-10-09T04:46:25.133749+0000","mapping_epoch":307,"log_start":"297'5273","ondisk_log_start":"297'5273","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:23:53.663910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5938,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11878,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1593,"num_bytes_recovered":104202240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":-4286}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":291,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.089881+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:48:45.089881+0000","last_peered":"2023-10-09T04:48:45.089881+0000","last_clean":"2023-10-09T04:48:45.089881+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:48:45.089881+0000","last_undegraded":"2023-10-09T04:48:45.089881+0000","last_fullsized":"2023-10-09T04:48:45.089881+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080810+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:48:45.080810+0000","last_peered":"2023-10-09T04:48:45.080810+0000","last_clean":"2023-10-09T04:48:45.080810+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:48:45.080810+0000","last_undegraded":"2023-10-09T04:48:45.080810+0000","last_fullsized":"2023-10-09T04:48:45.080810+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"330'11782","reported_seq":19675,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.211699+0000","last_change":"2023-10-09T04:46:54.673986+0000","last_active":"2023-10-09T04:48:50.211699+0000","last_peered":"2023-10-09T04:48:50.211699+0000","last_clean":"2023-10-09T04:48:50.211699+0000","last_became_active":"2023-10-09T04:46:54.653505+0000","last_became_peered":"2023-10-09T04:46:54.653505+0000","last_unstale":"2023-10-09T04:48:50.211699+0000","last_undegraded":"2023-10-09T04:48:50.211699+0000","last_fullsized":"2023-10-09T04:48:50.211699+0000","mapping_epoch":313,"log_start":"305'7128","ondisk_log_start":"305'7128","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":4654,"log_dups_size":0,"ondisk_log_size":4654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:19.484546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11782,"num_write_kb":377024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3907,"num_bytes_recovered":256049152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080488+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:48:45.080488+0000","last_peered":"2023-10-09T04:48:45.080488+0000","last_clean":"2023-10-09T04:48:45.080488+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:48:45.080488+0000","last_undegraded":"2023-10-09T04:48:45.080488+0000","last_fullsized":"2023-10-09T04:48:45.080488+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":310,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080982+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:48:45.080982+0000","last_peered":"2023-10-09T04:48:45.080982+0000","last_clean":"2023-10-09T04:48:45.080982+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:48:45.080982+0000","last_undegraded":"2023-10-09T04:48:45.080982+0000","last_fullsized":"2023-10-09T04:48:45.080982+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"330'11980","reported_seq":12034,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.198120+0000","last_change":"2023-10-09T04:43:34.550405+0000","last_active":"2023-10-09T04:48:50.198120+0000","last_peered":"2023-10-09T04:48:50.198120+0000","last_clean":"2023-10-09T04:48:50.198120+0000","last_became_active":"2023-10-09T04:43:34.550067+0000","last_became_peered":"2023-10-09T04:43:34.550067+0000","last_unstale":"2023-10-09T04:48:50.198120+0000","last_undegraded":"2023-10-09T04:48:50.198120+0000","last_fullsized":"2023-10-09T04:48:50.198120+0000","mapping_epoch":284,"log_start":"301'6796","ondisk_log_start":"301'6796","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5184,"log_dups_size":0,"ondisk_log_size":5184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:07:07.437511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11980,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.082822+0000","last_change":"2023-10-09T04:45:45.611560+0000","last_active":"2023-10-09T04:48:45.082822+0000","last_peered":"2023-10-09T04:48:45.082822+0000","last_clean":"2023-10-09T04:48:45.082822+0000","last_became_active":"2023-10-09T04:45:45.611247+0000","last_became_peered":"2023-10-09T04:45:45.611247+0000","last_unstale":"2023-10-09T04:48:45.082822+0000","last_undegraded":"2023-10-09T04:48:45.082822+0000","last_fullsized":"2023-10-09T04:48:45.082822+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:52.196556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":276,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.081351+0000","last_change":"2023-10-09T04:45:45.604134+0000","last_active":"2023-10-09T04:48:45.081351+0000","last_peered":"2023-10-09T04:48:45.081351+0000","last_clean":"2023-10-09T04:48:45.081351+0000","last_became_active":"2023-10-09T04:45:45.603767+0000","last_became_peered":"2023-10-09T04:45:45.603767+0000","last_unstale":"2023-10-09T04:48:45.081351+0000","last_undegraded":"2023-10-09T04:48:45.081351+0000","last_fullsized":"2023-10-09T04:48:45.081351+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:14.801140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.a","version":"330'12104","reported_seq":25042,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:50.213495+0000","last_change":"2023-10-09T04:46:26.440123+0000","last_active":"2023-10-09T04:48:50.213495+0000","last_peered":"2023-10-09T04:48:50.213495+0000","last_clean":"2023-10-09T04:45:44.754196+0000","last_became_active":"2023-10-09T04:46:26.422562+0000","last_became_peered":"2023-10-09T04:46:26.422562+0000","last_unstale":"2023-10-09T04:48:50.213495+0000","last_undegraded":"2023-10-09T04:46:26.243542+0000","last_fullsized":"2023-10-09T04:46:26.131676+0000","mapping_epoch":308,"log_start":"297'5497","ondisk_log_start":"297'5497","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:44:06.608939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3529,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12104,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4288,"num_bytes_recovered":280887296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":217,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083207+0000","last_change":"2023-10-09T04:46:25.169137+0000","last_active":"2023-10-09T04:48:45.083207+0000","last_peered":"2023-10-09T04:48:45.083207+0000","last_clean":"2023-10-09T04:48:45.083207+0000","last_became_active":"2023-10-09T04:46:25.168834+0000","last_became_peered":"2023-10-09T04:46:25.168834+0000","last_unstale":"2023-10-09T04:48:45.083207+0000","last_undegraded":"2023-10-09T04:48:45.083207+0000","last_fullsized":"2023-10-09T04:48:45.083207+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:15.860221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":334,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074997+0000","last_change":"2023-10-09T04:45:45.824913+0000","last_active":"2023-10-09T04:48:45.074997+0000","last_peered":"2023-10-09T04:48:45.074997+0000","last_clean":"2023-10-09T04:48:45.074997+0000","last_became_active":"2023-10-09T04:45:45.824572+0000","last_became_peered":"2023-10-09T04:45:45.824572+0000","last_unstale":"2023-10-09T04:48:45.074997+0000","last_undegraded":"2023-10-09T04:48:45.074997+0000","last_fullsized":"2023-10-09T04:48:45.074997+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:36.276403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":301,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074626+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:48:45.074626+0000","last_peered":"2023-10-09T04:48:45.074626+0000","last_clean":"2023-10-09T04:48:45.074626+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:48:45.074626+0000","last_undegraded":"2023-10-09T04:48:45.074626+0000","last_fullsized":"2023-10-09T04:48:45.074626+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"330'11870","reported_seq":23816,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.147765+0000","last_change":"2023-10-09T04:48:46.138723+0000","last_active":"2023-10-09T04:48:50.147765+0000","last_peered":"2023-10-09T04:48:50.147765+0000","last_clean":"2023-10-09T04:48:50.147765+0000","last_became_active":"2023-10-09T04:48:46.105833+0000","last_became_peered":"2023-10-09T04:48:46.105833+0000","last_unstale":"2023-10-09T04:48:50.147765+0000","last_undegraded":"2023-10-09T04:48:50.147765+0000","last_fullsized":"2023-10-09T04:48:50.147765+0000","mapping_epoch":329,"log_start":"297'5296","ondisk_log_start":"297'5296","created":284,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:58:07.304984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11870,"num_write_kb":379840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5935,"num_bytes_recovered":226557952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":314,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074495+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:48:45.074495+0000","last_peered":"2023-10-09T04:48:45.074495+0000","last_clean":"2023-10-09T04:48:45.074495+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:48:45.074495+0000","last_undegraded":"2023-10-09T04:48:45.074495+0000","last_fullsized":"2023-10-09T04:48:45.074495+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080386+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:48:45.080386+0000","last_peered":"2023-10-09T04:48:45.080386+0000","last_clean":"2023-10-09T04:48:45.080386+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:48:45.080386+0000","last_undegraded":"2023-10-09T04:48:45.080386+0000","last_fullsized":"2023-10-09T04:48:45.080386+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":313,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.081106+0000","last_change":"2023-10-09T04:45:45.611766+0000","last_active":"2023-10-09T04:48:46.081106+0000","last_peered":"2023-10-09T04:48:46.081106+0000","last_clean":"2023-10-09T04:48:46.081106+0000","last_became_active":"2023-10-09T04:45:45.611242+0000","last_became_peered":"2023-10-09T04:45:45.611242+0000","last_unstale":"2023-10-09T04:48:46.081106+0000","last_undegraded":"2023-10-09T04:48:46.081106+0000","last_fullsized":"2023-10-09T04:48:46.081106+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:23.336238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"330'11620","reported_seq":20700,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.125506+0000","last_change":"2023-10-09T04:45:30.672535+0000","last_active":"2023-10-09T04:48:50.125506+0000","last_peered":"2023-10-09T04:48:50.125506+0000","last_clean":"2023-10-09T04:48:50.125506+0000","last_became_active":"2023-10-09T04:45:30.663157+0000","last_became_peered":"2023-10-09T04:45:30.663157+0000","last_unstale":"2023-10-09T04:48:50.125506+0000","last_undegraded":"2023-10-09T04:48:50.125506+0000","last_fullsized":"2023-10-09T04:48:50.125506+0000","mapping_epoch":300,"log_start":"297'5006","ondisk_log_start":"297'5006","created":284,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:27:31.926989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11620,"num_write_kb":371840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4503,"num_bytes_recovered":295108608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":308,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.080843+0000","last_change":"2023-10-09T04:46:25.161784+0000","last_active":"2023-10-09T04:48:46.080843+0000","last_peered":"2023-10-09T04:48:46.080843+0000","last_clean":"2023-10-09T04:48:46.080843+0000","last_became_active":"2023-10-09T04:46:25.161498+0000","last_became_peered":"2023-10-09T04:46:25.161498+0000","last_unstale":"2023-10-09T04:48:46.080843+0000","last_undegraded":"2023-10-09T04:48:46.080843+0000","last_fullsized":"2023-10-09T04:48:46.080843+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:01.500231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":307,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.080530+0000","last_change":"2023-10-09T04:46:25.165536+0000","last_active":"2023-10-09T04:48:46.080530+0000","last_peered":"2023-10-09T04:48:46.080530+0000","last_clean":"2023-10-09T04:48:46.080530+0000","last_became_active":"2023-10-09T04:46:25.165269+0000","last_became_peered":"2023-10-09T04:46:25.165269+0000","last_unstale":"2023-10-09T04:48:46.080530+0000","last_undegraded":"2023-10-09T04:48:46.080530+0000","last_fullsized":"2023-10-09T04:48:46.080530+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:30.362732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":212,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080062+0000","last_change":"2023-10-09T04:46:25.177443+0000","last_active":"2023-10-09T04:48:45.080062+0000","last_peered":"2023-10-09T04:48:45.080062+0000","last_clean":"2023-10-09T04:48:45.080062+0000","last_became_active":"2023-10-09T04:46:25.176635+0000","last_became_peered":"2023-10-09T04:46:25.176635+0000","last_unstale":"2023-10-09T04:48:45.080062+0000","last_undegraded":"2023-10-09T04:48:45.080062+0000","last_fullsized":"2023-10-09T04:48:45.080062+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:43.367744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"330'11838","reported_seq":11893,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:50.211290+0000","last_change":"2023-10-09T04:43:34.550344+0000","last_active":"2023-10-09T04:48:50.211290+0000","last_peered":"2023-10-09T04:48:50.211290+0000","last_clean":"2023-10-09T04:48:50.211290+0000","last_became_active":"2023-10-09T04:43:34.550066+0000","last_became_peered":"2023-10-09T04:43:34.550066+0000","last_unstale":"2023-10-09T04:48:50.211290+0000","last_undegraded":"2023-10-09T04:48:50.211290+0000","last_fullsized":"2023-10-09T04:48:50.211290+0000","mapping_epoch":284,"log_start":"301'6569","ondisk_log_start":"301'6569","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5269,"log_dups_size":0,"ondisk_log_size":5269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:50:37.030437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11838,"num_write_kb":378753,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":311,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.075440+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:48:45.075440+0000","last_peered":"2023-10-09T04:48:45.075440+0000","last_clean":"2023-10-09T04:48:45.075440+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:48:45.075440+0000","last_undegraded":"2023-10-09T04:48:45.075440+0000","last_fullsized":"2023-10-09T04:48:45.075440+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":758,"reported_epoch":329,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:45.082671+0000","last_change":"2023-10-09T04:46:26.415415+0000","last_active":"2023-10-09T04:48:45.082671+0000","last_peered":"2023-10-09T04:48:45.082671+0000","last_clean":"2023-10-09T04:46:16.698879+0000","last_became_active":"2023-10-09T04:46:26.415059+0000","last_became_peered":"2023-10-09T04:46:26.415059+0000","last_unstale":"2023-10-09T04:48:45.082671+0000","last_undegraded":"2023-10-09T04:46:26.138726+0000","last_fullsized":"2023-10-09T04:46:26.138125+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:23.417129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":260,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.074718+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:48:45.074718+0000","last_peered":"2023-10-09T04:48:45.074718+0000","last_clean":"2023-10-09T04:48:45.074718+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:48:45.074718+0000","last_undegraded":"2023-10-09T04:48:45.074718+0000","last_fullsized":"2023-10-09T04:48:45.074718+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":273,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080619+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:48:45.080619+0000","last_peered":"2023-10-09T04:48:45.080619+0000","last_clean":"2023-10-09T04:48:45.080619+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:48:45.080619+0000","last_undegraded":"2023-10-09T04:48:45.080619+0000","last_fullsized":"2023-10-09T04:48:45.080619+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"330'11820","reported_seq":11884,"reported_epoch":330,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:50.202224+0000","last_change":"2023-10-09T04:46:26.351968+0000","last_active":"2023-10-09T04:48:50.202224+0000","last_peered":"2023-10-09T04:48:50.202224+0000","last_clean":"2023-10-09T04:46:24.123151+0000","last_became_active":"2023-10-09T04:46:26.320651+0000","last_became_peered":"2023-10-09T04:46:26.320651+0000","last_unstale":"2023-10-09T04:48:50.202224+0000","last_undegraded":"2023-10-09T04:48:50.202224+0000","last_fullsized":"2023-10-09T04:48:50.202224+0000","mapping_epoch":308,"log_start":"298'5976","ondisk_log_start":"298'5976","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":5844,"log_dups_size":0,"ondisk_log_size":5844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:34:25.867280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11820,"num_write_kb":378240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,0],"avail_no_missing":["4","0","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":340,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.082540+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:48:45.082540+0000","last_peered":"2023-10-09T04:48:45.082540+0000","last_clean":"2023-10-09T04:48:45.082540+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:48:45.082540+0000","last_undegraded":"2023-10-09T04:48:45.082540+0000","last_fullsized":"2023-10-09T04:48:45.082540+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080787+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:48:45.080787+0000","last_peered":"2023-10-09T04:48:45.080787+0000","last_clean":"2023-10-09T04:48:45.080787+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:48:45.080787+0000","last_undegraded":"2023-10-09T04:48:45.080787+0000","last_fullsized":"2023-10-09T04:48:45.080787+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":130,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.105526+0000","last_change":"2023-10-09T04:46:25.174089+0000","last_active":"2023-10-09T04:48:46.105526+0000","last_peered":"2023-10-09T04:48:46.105526+0000","last_clean":"2023-10-09T04:48:46.105526+0000","last_became_active":"2023-10-09T04:46:25.173414+0000","last_became_peered":"2023-10-09T04:46:25.173414+0000","last_unstale":"2023-10-09T04:48:46.105526+0000","last_undegraded":"2023-10-09T04:48:46.105526+0000","last_fullsized":"2023-10-09T04:48:46.105526+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.877749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.b","version":"330'11990","reported_seq":21671,"reported_epoch":330,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:50.186176+0000","last_change":"2023-10-09T04:48:14.871531+0000","last_active":"2023-10-09T04:48:50.186176+0000","last_peered":"2023-10-09T04:48:50.186176+0000","last_clean":"2023-10-09T04:44:37.770348+0000","last_became_active":"2023-10-09T04:45:59.017846+0000","last_became_peered":"2023-10-09T04:45:59.017846+0000","last_unstale":"2023-10-09T04:48:50.186176+0000","last_undegraded":"2023-10-09T04:48:50.186176+0000","last_fullsized":"2023-10-09T04:48:50.186176+0000","mapping_epoch":303,"log_start":"297'5403","ondisk_log_start":"297'5403","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:00.633461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11990,"num_write_kb":383680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4803,"num_bytes_recovered":243204096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,3],"avail_no_missing":["2","1","3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":343,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083912+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:48:45.083912+0000","last_peered":"2023-10-09T04:48:45.083912+0000","last_clean":"2023-10-09T04:48:45.083912+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:48:45.083912+0000","last_undegraded":"2023-10-09T04:48:45.083912+0000","last_fullsized":"2023-10-09T04:48:45.083912+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":289,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.081029+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:48:46.081029+0000","last_peered":"2023-10-09T04:48:46.081029+0000","last_clean":"2023-10-09T04:48:46.081029+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:48:46.081029+0000","last_undegraded":"2023-10-09T04:48:46.081029+0000","last_fullsized":"2023-10-09T04:48:46.081029+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":184,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.075477+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:48:45.075477+0000","last_peered":"2023-10-09T04:48:45.075477+0000","last_clean":"2023-10-09T04:48:45.075477+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:48:45.075477+0000","last_undegraded":"2023-10-09T04:48:45.075477+0000","last_fullsized":"2023-10-09T04:48:45.075477+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"330'11920","reported_seq":11993,"reported_epoch":330,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T04:48:50.205125+0000","last_change":"2023-10-09T04:46:25.302033+0000","last_active":"2023-10-09T04:48:50.205125+0000","last_peered":"2023-10-09T04:48:50.205125+0000","last_clean":"2023-10-09T04:45:44.820208+0000","last_became_active":"2023-10-09T04:46:25.282101+0000","last_became_peered":"2023-10-09T04:46:25.282101+0000","last_unstale":"2023-10-09T04:48:50.205125+0000","last_undegraded":"2023-10-09T04:48:50.205125+0000","last_fullsized":"2023-10-09T04:48:50.205125+0000","mapping_epoch":307,"log_start":"297'5303","ondisk_log_start":"297'5303","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:34:16.931727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11920,"num_write_kb":381440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,0,3],"avail_no_missing":["2","0","3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":78,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.083161+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:48:45.083161+0000","last_peered":"2023-10-09T04:48:45.083161+0000","last_clean":"2023-10-09T04:48:45.083161+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:48:45.083161+0000","last_undegraded":"2023-10-09T04:48:45.083161+0000","last_fullsized":"2023-10-09T04:48:45.083161+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":293,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080939+0000","last_change":"2023-10-09T04:46:25.166210+0000","last_active":"2023-10-09T04:48:45.080939+0000","last_peered":"2023-10-09T04:48:45.080939+0000","last_clean":"2023-10-09T04:48:45.080939+0000","last_became_active":"2023-10-09T04:46:25.165948+0000","last_became_peered":"2023-10-09T04:46:25.165948+0000","last_unstale":"2023-10-09T04:48:45.080939+0000","last_undegraded":"2023-10-09T04:48:45.080939+0000","last_fullsized":"2023-10-09T04:48:45.080939+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:51.272111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":176,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.082978+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:48:45.082978+0000","last_peered":"2023-10-09T04:48:45.082978+0000","last_clean":"2023-10-09T04:48:45.082978+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:48:45.082978+0000","last_undegraded":"2023-10-09T04:48:45.082978+0000","last_fullsized":"2023-10-09T04:48:45.082978+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"330'11988","reported_seq":22922,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:48:50.204387+0000","last_change":"2023-10-09T04:46:25.450073+0000","last_active":"2023-10-09T04:48:50.204387+0000","last_peered":"2023-10-09T04:48:50.204387+0000","last_clean":"2023-10-09T04:44:37.742164+0000","last_became_active":"2023-10-09T04:45:46.956446+0000","last_became_peered":"2023-10-09T04:45:46.956446+0000","last_unstale":"2023-10-09T04:48:50.204387+0000","last_undegraded":"2023-10-09T04:46:25.447165+0000","last_fullsized":"2023-10-09T04:45:46.571293+0000","mapping_epoch":303,"log_start":"297'5403","ondisk_log_start":"297'5403","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:43:33.532834+0000","last_clean_scrub_stamp":"2023-10-09T04:43:33.532834+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:11.914538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11988,"num_write_kb":383616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3619,"num_bytes_recovered":237109248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"1,2","objects":-1314}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":252,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.082896+0000","last_change":"2023-10-09T04:45:45.602634+0000","last_active":"2023-10-09T04:48:45.082896+0000","last_peered":"2023-10-09T04:48:45.082896+0000","last_clean":"2023-10-09T04:48:45.082896+0000","last_became_active":"2023-10-09T04:45:45.602317+0000","last_became_peered":"2023-10-09T04:45:45.602317+0000","last_unstale":"2023-10-09T04:48:45.082896+0000","last_undegraded":"2023-10-09T04:48:45.082896+0000","last_fullsized":"2023-10-09T04:48:45.082896+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:13.963587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.081068+0000","last_change":"2023-10-09T04:46:25.210175+0000","last_active":"2023-10-09T04:48:45.081068+0000","last_peered":"2023-10-09T04:48:45.081068+0000","last_clean":"2023-10-09T04:48:45.081068+0000","last_became_active":"2023-10-09T04:46:25.174609+0000","last_became_peered":"2023-10-09T04:46:25.174609+0000","last_unstale":"2023-10-09T04:48:45.081068+0000","last_undegraded":"2023-10-09T04:48:45.081068+0000","last_fullsized":"2023-10-09T04:48:45.081068+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:29.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080318+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:48:45.080318+0000","last_peered":"2023-10-09T04:48:45.080318+0000","last_clean":"2023-10-09T04:48:45.080318+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:48:45.080318+0000","last_undegraded":"2023-10-09T04:48:45.080318+0000","last_fullsized":"2023-10-09T04:48:45.080318+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":180,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080752+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:48:45.080752+0000","last_peered":"2023-10-09T04:48:45.080752+0000","last_clean":"2023-10-09T04:48:45.080752+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:48:45.080752+0000","last_undegraded":"2023-10-09T04:48:45.080752+0000","last_fullsized":"2023-10-09T04:48:45.080752+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":300,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.080932+0000","last_change":"2023-10-09T04:46:25.160157+0000","last_active":"2023-10-09T04:48:46.080932+0000","last_peered":"2023-10-09T04:48:46.080932+0000","last_clean":"2023-10-09T04:48:46.080932+0000","last_became_active":"2023-10-09T04:46:25.159850+0000","last_became_peered":"2023-10-09T04:46:25.159850+0000","last_unstale":"2023-10-09T04:48:46.080932+0000","last_undegraded":"2023-10-09T04:48:46.080932+0000","last_fullsized":"2023-10-09T04:48:46.080932+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:07.475665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":320,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.079604+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:48:45.079604+0000","last_peered":"2023-10-09T04:48:45.079604+0000","last_clean":"2023-10-09T04:48:45.079604+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:48:45.079604+0000","last_undegraded":"2023-10-09T04:48:45.079604+0000","last_fullsized":"2023-10-09T04:48:45.079604+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":88,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080430+0000","last_change":"2023-10-09T04:45:45.609849+0000","last_active":"2023-10-09T04:48:45.080430+0000","last_peered":"2023-10-09T04:48:45.080430+0000","last_clean":"2023-10-09T04:48:45.080430+0000","last_became_active":"2023-10-09T04:45:45.609639+0000","last_became_peered":"2023-10-09T04:45:45.609639+0000","last_unstale":"2023-10-09T04:48:45.080430+0000","last_undegraded":"2023-10-09T04:48:45.080430+0000","last_fullsized":"2023-10-09T04:48:45.080430+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:26.113652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":289,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.075079+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:48:45.075079+0000","last_peered":"2023-10-09T04:48:45.075079+0000","last_clean":"2023-10-09T04:48:45.075079+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:48:45.075079+0000","last_undegraded":"2023-10-09T04:48:45.075079+0000","last_fullsized":"2023-10-09T04:48:45.075079+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.089956+0000","last_change":"2023-10-09T04:46:25.172010+0000","last_active":"2023-10-09T04:48:45.089956+0000","last_peered":"2023-10-09T04:48:45.089956+0000","last_clean":"2023-10-09T04:48:45.089956+0000","last_became_active":"2023-10-09T04:46:25.171776+0000","last_became_peered":"2023-10-09T04:46:25.171776+0000","last_unstale":"2023-10-09T04:48:45.089956+0000","last_undegraded":"2023-10-09T04:48:45.089956+0000","last_fullsized":"2023-10-09T04:48:45.089956+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:40.329129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":41,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.105598+0000","last_change":"2023-10-09T04:45:45.611105+0000","last_active":"2023-10-09T04:48:46.105598+0000","last_peered":"2023-10-09T04:48:46.105598+0000","last_clean":"2023-10-09T04:48:46.105598+0000","last_became_active":"2023-10-09T04:45:45.610758+0000","last_became_peered":"2023-10-09T04:45:45.610758+0000","last_unstale":"2023-10-09T04:48:46.105598+0000","last_undegraded":"2023-10-09T04:48:46.105598+0000","last_fullsized":"2023-10-09T04:48:46.105598+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:59.031709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":280,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-10-09T04:48:46.080748+0000","last_change":"2023-10-09T04:46:25.153824+0000","last_active":"2023-10-09T04:48:46.080748+0000","last_peered":"2023-10-09T04:48:46.080748+0000","last_clean":"2023-10-09T04:48:46.080748+0000","last_became_active":"2023-10-09T04:46:25.153497+0000","last_became_peered":"2023-10-09T04:46:25.153497+0000","last_unstale":"2023-10-09T04:48:46.080748+0000","last_undegraded":"2023-10-09T04:48:46.080748+0000","last_fullsized":"2023-10-09T04:48:46.080748+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:00.144937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":220,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.080199+0000","last_change":"2023-10-09T04:46:25.217389+0000","last_active":"2023-10-09T04:48:45.080199+0000","last_peered":"2023-10-09T04:48:45.080199+0000","last_clean":"2023-10-09T04:48:45.080199+0000","last_became_active":"2023-10-09T04:46:25.162993+0000","last_became_peered":"2023-10-09T04:46:25.162993+0000","last_unstale":"2023-10-09T04:48:45.080199+0000","last_undegraded":"2023-10-09T04:48:45.080199+0000","last_fullsized":"2023-10-09T04:48:45.080199+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:13.549717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-10-09T04:48:45.076700+0000","last_change":"2023-10-09T04:45:45.610893+0000","last_active":"2023-10-09T04:48:45.076700+0000","last_peered":"2023-10-09T04:48:45.076700+0000","last_clean":"2023-10-09T04:48:45.076700+0000","last_became_active":"2023-10-09T04:45:45.610260+0000","last_became_peered":"2023-10-09T04:45:45.610260+0000","last_unstale":"2023-10-09T04:48:45.076700+0000","last_undegraded":"2023-10-09T04:48:45.076700+0000","last_fullsized":"2023-10-09T04:48:45.076700+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:22.552117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17648,"num_objects_misplaced":1682,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190496,"num_write_kb":6095809,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47683,"num_bytes_recovered":2723938304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3601465344,"data_stored":3601465344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94523,"ondisk_log_size":94523,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":11254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":17338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234504,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521104,"kb_used_data":3656,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":92224048,"statfs":{"total":95995035648,"available":94437425152,"internally_reserved":0,"allocated":3743744,"data_stored":3010577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.4359999999999999,"15min":2.5329999999999999},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":14.970000000000001,"5min":14.970000000000001,"15min":111.72199999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.754,"5min":2.1749999999999998,"15min":3.0499999999999998},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":6.1689999999999996,"5min":19.152000000000001,"15min":111.76600000000001},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":4.6619999999999999,"15min":4.1500000000000004},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":14.896000000000001,"5min":128.601,"15min":264.62},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.4329999999999998,"5min":4.4050000000000002,"15min":4.2160000000000002},"min":{"1min":0.27900000000000003,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":14.933999999999999,"5min":128.381,"15min":264.65899999999999},"last":0.434}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":13.303000000000001,"5min":5.2450000000000001,"15min":4.8490000000000002},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":129.10400000000001,"5min":129.10400000000001,"15min":184.72},"last":0.53400000000000003},{"interface":"front","average":{"1min":13.906000000000001,"5min":5.5090000000000003,"15min":5.234},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":129.15100000000001,"5min":129.15100000000001,"15min":184.697},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:48:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.728,"15min":4.1630000000000003},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.1500000000000004,"5min":7.8630000000000004,"15min":182.839},"last":0.495},{"interface":"front","average":{"1min":3.113,"5min":2.5129999999999999,"15min":4.7000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.866,"5min":14.866,"15min":182.678},"last":0.44400000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":1.5569999999999999,"15min":1.53},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.573,"5min":11.573,"15min":14.257999999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.23,"5min":1.353,"15min":1.327},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":7.9749999999999996,"5min":10.497,"15min":10.497},"last":0.36799999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.143,"15min":1.2150000000000001},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4169999999999998,"5min":8.5709999999999997,"15min":13.319000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":3.492,"5min":1.6080000000000001,"15min":1.6890000000000001},"min":{"1min":0.33600000000000002,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":12.081,"5min":12.081,"15min":18.408000000000001},"last":0.38400000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":0.98699999999999999,"15min":1.2509999999999999},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":7.4459999999999997,"5min":7.4459999999999997,"15min":11.388},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.8090000000000002,"5min":1.5189999999999999,"15min":1.778},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":18.311},"last":0.46700000000000003}]}]},{"osd":6,"up_from":0,"seq":197568496138,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141227,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":812992,"kb_used_data":4472,"kb_used_omap":4,"kb_used_meta":808507,"kb_avail":92932160,"statfs":{"total":95995035648,"available":95162531840,"internally_reserved":0,"allocated":4579328,"data_stored":3099835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":827912078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.448,"15min":3.4710000000000001},"min":{"1min":0.22700000000000001,"5min":0.22600000000000001,"15min":0.14799999999999999},"max":{"1min":7.4829999999999997,"5min":21.021999999999998,"15min":97.308000000000007},"last":5.7800000000000002},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.6899999999999999,"15min":3.6659999999999999},"min":{"1min":0.27600000000000002,"5min":0.252,"15min":0.193},"max":{"1min":8.3580000000000005,"5min":21.280999999999999,"15min":97.055999999999997},"last":4.1200000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:48:07 2023","interfaces":[{"interface":"back","average":{"1min":16.600999999999999,"5min":5.8769999999999998,"15min":4.9039999999999999},"min":{"1min":0.50800000000000001,"5min":0.21199999999999999,"15min":0.20899999999999999},"max":{"1min":151.41900000000001,"5min":151.41900000000001,"15min":263.60500000000002},"last":5.8170000000000002},{"interface":"front","average":{"1min":16.071000000000002,"5min":5.875,"15min":5.3070000000000004},"min":{"1min":0.42199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":146.749,"5min":146.749,"15min":263.65199999999999},"last":4.0979999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:48:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.946,"15min":5.6550000000000002},"min":{"1min":0.26500000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.0709999999999997,"5min":11.68,"15min":379.80900000000003},"last":5.5880000000000001},{"interface":"front","average":{"1min":2.508,"5min":2.3340000000000001,"15min":6.3159999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.246},"max":{"1min":12.615,"5min":18.285,"15min":379.95800000000003},"last":4.0510000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":2.2069999999999999,"15min":2.5779999999999998},"min":{"1min":0.379,"5min":0.307,"15min":0.245},"max":{"1min":7.4610000000000003,"5min":19.959,"15min":170.072},"last":4.4059999999999997},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.591,"15min":2.3639999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":9.2129999999999992,"5min":11.935,"15min":169.94499999999999},"last":5.6289999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.742,"15min":2.496},"min":{"1min":0.34200000000000003,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":9.782,"5min":9.782,"15min":171.036},"last":4.0119999999999996},{"interface":"front","average":{"1min":2.2789999999999999,"5min":2.298,"15min":2.79},"min":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.27500000000000002},"max":{"1min":7.5209999999999999,"5min":20.021000000000001,"15min":170.88},"last":4.173}]},{"osd":6,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.841,"15min":2.5470000000000002},"min":{"1min":0.40799999999999997,"5min":0.35799999999999998,"15min":0.32000000000000001},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":132.36099999999999},"last":4.0759999999999996},{"interface":"front","average":{"1min":3.1749999999999998,"5min":2.544,"15min":2.8610000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":9.7370000000000001,"5min":20.164999999999999,"15min":132.17699999999999},"last":4.218}]},{"osd":7,"last update":"Mon Oct 9 04:48:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6230000000000002,"5min":2.4129999999999998,"15min":3.339},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":8.4339999999999993,"5min":20.055,"15min":174.75999999999999},"last":7.8970000000000002},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.573,"15min":2.9540000000000002},"min":{"1min":0.39800000000000002,"5min":0.38800000000000001,"15min":0.36199999999999999},"max":{"1min":8.0459999999999994,"5min":16.238,"15min":174.779},"last":5.5469999999999997}]}]},{"osd":0,"up_from":0,"seq":390842024284,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772561,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":512888,"kb_used_data":3696,"kb_used_omap":7,"kb_used_meta":509176,"kb_avail":93232264,"statfs":{"total":95995035648,"available":95469838336,"internally_reserved":0,"allocated":3784704,"data_stored":2500102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":521396305},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7890000000000001,"5min":2.2130000000000001,"15min":5.1769999999999996},"min":{"1min":0.253,"5min":0.183,"15min":0.183},"max":{"1min":7.7130000000000001,"5min":25.899999999999999,"15min":72.608000000000004},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.901,"15min":4.7389999999999999},"min":{"1min":0.23699999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":6.6379999999999999,"5min":16.550000000000001,"15min":72.632000000000005},"last":0.36599999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":4.4550000000000001,"5min":3.8500000000000001,"15min":4.7839999999999998},"min":{"1min":0.32200000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":17.843,"5min":162.124,"15min":162.124},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.0010000000000003,"5min":4.1710000000000003,"15min":5.2009999999999996},"min":{"1min":0.33300000000000002,"5min":0.223,"15min":0.223},"max":{"1min":21.329000000000001,"5min":164.68299999999999,"15min":164.68299999999999},"last":0.316}]},{"osd":3,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":6,"15min":6.9749999999999996},"min":{"1min":0.31900000000000001,"5min":0.25600000000000001,"15min":0.21199999999999999},"max":{"1min":5.8170000000000002,"5min":161.12799999999999,"15min":335.928},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.9769999999999999,"5min":6.0019999999999998,"15min":7.1710000000000003},"min":{"1min":0.377,"5min":0.246,"15min":0.246},"max":{"1min":7.4580000000000002,"5min":161.386,"15min":335.68599999999998},"last":0.27700000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":2.2410000000000001,"15min":3.2429999999999999},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.315},"max":{"1min":7.6390000000000002,"5min":10.256,"15min":72.944999999999993},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.6789999999999998,"5min":2,"15min":3.2349999999999999},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.9369999999999994,"5min":16.341000000000001,"15min":73.054000000000002},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7360000000000002,"5min":2.077,"15min":3.1379999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":12.388,"5min":12.98,"15min":72.986000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.0979999999999999,"5min":2.0910000000000002,"15min":3.4470000000000001},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.32200000000000001},"max":{"1min":5.923,"5min":10.289,"15min":72.963999999999999},"last":0.39700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":1.857,"15min":5.7329999999999997},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.33600000000000002},"max":{"1min":5.5670000000000002,"5min":16.425999999999998,"15min":135.614},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.7270000000000001,"15min":5.3230000000000004},"min":{"1min":0.39500000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.5709999999999997,"5min":10.977,"15min":130.196},"last":0.58199999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:48:05 2023","interfaces":[{"interface":"back","average":{"1min":3.702,"5min":2.3690000000000002,"15min":3.5390000000000001},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.7530000000000001,"5min":16.462,"15min":73.079999999999998},"last":0.46400000000000002},{"interface":"front","average":{"1min":3.2650000000000001,"5min":2.347,"15min":3.1840000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.6379999999999999,"5min":10.935,"15min":73.069000000000003},"last":0.55900000000000005}]}]},{"osd":3,"up_from":0,"seq":609885356285,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019412,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494544,"kb_used_data":3080,"kb_used_omap":0,"kb_used_meta":1491456,"kb_avail":92250608,"statfs":{"total":95995035648,"available":94464622592,"internally_reserved":0,"allocated":3153920,"data_stored":2420763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1527250944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:47:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7569999999999999,"5min":1.4330000000000001,"15min":4.1280000000000001},"min":{"1min":0.29299999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":12.795,"5min":12.795,"15min":171.13499999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.4199999999999999,"5min":1.716,"15min":4.1829999999999998},"min":{"1min":0.28799999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":16.004000000000001,"5min":16.004000000000001,"15min":171.11799999999999},"last":0.36099999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:48:53 2023","interfaces":[{"interface":"back","average":{"1min":22.486999999999998,"5min":9.3059999999999992,"15min":5.2519999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":385.78199999999998,"5min":385.78199999999998,"15min":385.78199999999998},"last":0.27000000000000002},{"interface":"front","average":{"1min":23.029,"5min":9.0790000000000006,"15min":5.2350000000000003},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":386.09399999999999,"5min":386.09399999999999,"15min":386.09399999999999},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":16.193000000000001,"5min":6.4290000000000003,"15min":5.2519999999999998},"min":{"1min":0.47199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":74.069999999999993,"5min":98.221999999999994,"15min":195.267},"last":0.40500000000000003},{"interface":"front","average":{"1min":16.573,"5min":6.5599999999999996,"15min":5.0060000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":75.219999999999999,"5min":94.219999999999999,"15min":195.15799999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:48:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":4.1449999999999996,"15min":5.4080000000000004},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":16.082999999999998,"5min":141.334,"15min":357.125},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.8730000000000002,"5min":4.2380000000000004,"15min":5.3609999999999998},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":12.090999999999999,"5min":141.18299999999999,"15min":359.02800000000002},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:48:25 2023","interfaces":[{"interface":"back","average":{"1min":3.6259999999999999,"5min":1.931,"15min":1.8600000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":12.053000000000001,"5min":24.234999999999999,"15min":107.476},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.653,"5min":1.5529999999999999,"15min":1.7909999999999999},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.24099999999999999},"max":{"1min":18.785,"5min":18.785,"15min":107.414},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:47:55 2023","interfaces":[{"interface":"back","average":{"1min":2.2549999999999999,"5min":1.5409999999999999,"15min":1.8320000000000001},"min":{"1min":0.40799999999999997,"5min":0.29799999999999999,"15min":0.29599999999999999},"max":{"1min":12.122999999999999,"5min":22.920999999999999,"15min":65.75},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.0110000000000001,"5min":1.2270000000000001,"15min":1.7430000000000001},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.095000000000001,"5min":15.095000000000001,"15min":65.849999999999994},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.403,"5min":1.8580000000000001,"15min":1.5880000000000001},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":16.109000000000002,"5min":16.109000000000002,"15min":16.109000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":3.7069999999999999,"5min":2.2799999999999998,"15min":1.7390000000000001},"min":{"1min":0.45000000000000001,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":12.013999999999999,"5min":22.858000000000001,"15min":22.858000000000001},"last":0.435}]}]},{"osd":5,"up_from":38,"seq":163208757776,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889272,"kb_used_data":3056,"kb_used_omap":0,"kb_used_meta":886208,"kb_avail":92855880,"statfs":{"total":95995035648,"available":95084421120,"internally_reserved":0,"allocated":3129344,"data_stored":2415883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907476992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":3.8980000000000001,"15min":4.7910000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22700000000000001},"max":{"1min":4.367,"5min":163.70699999999999,"15min":210.85499999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.4419999999999999,"5min":3.8119999999999998,"15min":5.0449999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.2800000000000002,"5min":163.756,"15min":210.23500000000001},"last":0.61399999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:48:53 2023","interfaces":[{"interface":"back","average":{"1min":7.9909999999999997,"5min":9.1769999999999996,"15min":5.8499999999999996},"min":{"1min":0.35599999999999998,"5min":0.318,"15min":0.252},"max":{"1min":128.596,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":8.2149999999999999,"5min":9.2070000000000007,"15min":6.0060000000000002},"min":{"1min":0.30199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":128.67699999999999,"5min":211.435,"15min":211.435},"last":0.59999999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:48:03 2023","interfaces":[{"interface":"back","average":{"1min":9.3610000000000007,"5min":5.8159999999999998,"15min":5.4619999999999997},"min":{"1min":0.31900000000000001,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":82.561999999999998,"5min":82.561999999999998,"15min":90.763999999999996},"last":0.66200000000000003},{"interface":"front","average":{"1min":10.454000000000001,"5min":5.8280000000000003,"15min":5.5670000000000002},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":83.082999999999998,"5min":83.082999999999998,"15min":91.340000000000003},"last":0.47599999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.766,"15min":5.1589999999999998},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":7.585,"5min":46.497,"15min":268.63299999999998},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.7070000000000001,"5min":2.0539999999999998,"15min":4.7560000000000002},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.9450000000000003,"5min":46.125999999999998,"15min":268.92500000000001},"last":0.625}]},{"osd":4,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.3320000000000001,"15min":2.5299999999999998},"min":{"1min":0.34100000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5650000000000004,"5min":10.268000000000001,"15min":361.09399999999999},"last":0.37},{"interface":"front","average":{"1min":3.1970000000000001,"5min":1.4650000000000001,"15min":2.8860000000000001},"min":{"1min":0.33200000000000002,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":12.609999999999999,"5min":12.609999999999999,"15min":361.71300000000002},"last":0.308}]},{"osd":6,"last update":"Mon Oct 9 04:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.208,"15min":1.2849999999999999},"min":{"1min":0.378,"5min":0.28499999999999998,"15min":0.23100000000000001},"max":{"1min":3.8490000000000002,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.925,"5min":1.1220000000000001,"15min":1.3879999999999999},"min":{"1min":0.314,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":7.3040000000000003,"5min":7.3040000000000003,"15min":13.231999999999999},"last":0.38700000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":2.5249999999999999,"15min":1.764},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":7.1029999999999998,"5min":68.411000000000001,"15min":68.411000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.754,"15min":1.786},"min":{"1min":0.434,"5min":0.38300000000000001,"15min":0.30599999999999999},"max":{"1min":8.4499999999999993,"5min":68.435000000000002,"15min":68.435000000000002},"last":0.41199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":609222656,"data_stored":609222656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1377173504,"data_stored":1377173504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1615069184,"data_stored":1615069184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[216903]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 278 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 3.0 MiB/s, 94 objects/s recovering 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[216903]: from='client.36459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[223858]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 278 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 3.0 MiB/s, 94 objects/s recovering 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[223858]: from='client.36459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:59.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:48:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:59.399 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:48:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:59.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:48:59.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:48:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:59 smithi177 ceph-mon[195797]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 278 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 3.0 MiB/s, 94 objects/s recovering 2023-10-09T04:48:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:59 smithi177 ceph-mon[195797]: from='client.36459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:48:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:48:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:48:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:48:59.532 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:59.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:48:59.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:59.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:48:59.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:48:59.710 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:59.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:48:59.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:59.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:48:59.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:48:59.894 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:48:59.937 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:48:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:48:59.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:00.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:00.072 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:00.115 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:00.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:00.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:00.224 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:00.256 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:00.467 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:00.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:00.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:00.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:00.598 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:01.005 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:01.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:01.109 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.136 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:01.181 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:01.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:01.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:01.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:01.315 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:01 smithi029 ceph-mon[223858]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 261 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 3.0 MiB/s, 94 objects/s recovering 2023-10-09T04:49:01.316 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 17650/72 objects degraded (24513.889%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:49:01.316 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:01 smithi029 ceph-mon[216903]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 261 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 3.0 MiB/s, 94 objects/s recovering 2023-10-09T04:49:01.316 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 17650/72 objects degraded (24513.889%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:49:01.319 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:01.425 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:01.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:01 smithi177 ceph-mon[195797]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 261 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 3.0 MiB/s, 94 objects/s recovering 2023-10-09T04:49:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 17650/72 objects degraded (24513.889%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:49:01.531 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:01.557 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:01.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:01.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:01.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:01.668 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:49:01.668 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:49:01.683 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:49:01.686 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:49:01.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:01.708 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:01.710 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:49:01.730 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:49:01.734 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:01.752 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:49:01.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:49:01.774 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.774 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.775 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.775 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.775 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.775 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.775 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.775 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:49:01.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:01.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:01.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:01.920 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:02.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:49:02.043 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:49:02.044 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:49:02.044 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:49:02.044 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:49:02.181 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_7' removed 2023-10-09T04:49:02.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2599947556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:02.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1572479945' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:49:02.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:49:02.354 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2599947556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:02.354 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1572479945' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:49:02.354 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:49:02.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2599947556' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1572479945' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:49:02.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:49:02.681 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:02.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:02.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:02.814 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:02.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:02.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:02.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:02.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:02.991 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:49:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:49:02] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-10-09T04:49:02.995 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: pgmap v1303: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 93 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 985 KiB/s, 115 objects/s recovering 2023-10-09T04:49:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:49:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: osdmap e332: 8 total, 8 up, 5 in 2023-10-09T04:49:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[223858]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: pgmap v1303: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 93 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 985 KiB/s, 115 objects/s recovering 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: osdmap e332: 8 total, 8 up, 5 in 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:03 smithi029 ceph-mon[216903]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: pgmap v1303: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 93 op/s; 17650/72 objects degraded (24513.889%); 1682/72 objects misplaced (2336.111%); 985 KiB/s, 115 objects/s recovering 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: osdmap e332: 8 total, 8 up, 5 in 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: failed to encode map e332 with expected crc 2023-10-09T04:49:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:03 smithi177 ceph-mon[195797]: failed to encode map e332 with expected crc 2023-10-09T04:49:04.121 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:04.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:04.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[216903]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[216903]: osdmap e333: 8 total, 8 up, 5 in 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[216903]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[216903]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[216903]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[223858]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[223858]: osdmap e333: 8 total, 8 up, 5 in 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[223858]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[223858]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:04 smithi029 ceph-mon[223858]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:49:04.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:04.256 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:04.306 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:04.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:04.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:04.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:04.450 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:04 smithi177 ceph-mon[195797]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:04 smithi177 ceph-mon[195797]: osdmap e333: 8 total, 8 up, 5 in 2023-10-09T04:49:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:04 smithi177 ceph-mon[195797]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:04 smithi177 ceph-mon[195797]: failed to encode map e333 with expected crc 2023-10-09T04:49:04.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:04 smithi177 ceph-mon[195797]: failed to encode map e333 with expected crc 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: pgmap v1306: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: osdmap e334: 8 total, 8 up, 5 in 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: pgmap v1306: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: osdmap e334: 8 total, 8 up, 5 in 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:05 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: pgmap v1306: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 2/72 objects degraded (2.778%) 2023-10-09T04:49:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: osdmap e334: 8 total, 8 up, 5 in 2023-10-09T04:49:05.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:05 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:05.791 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:05.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:05.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:05.895 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:05.926 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:06 smithi029 ceph-mon[216903]: failed to encode map e334 with expected crc 2023-10-09T04:49:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:49:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:06 smithi029 ceph-mon[223858]: failed to encode map e334 with expected crc 2023-10-09T04:49:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:49:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:06 smithi177 ceph-mon[195797]: failed to encode map e334 with expected crc 2023-10-09T04:49:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:49:06.636 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:49:06.636 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:49:06.637 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-10-09T04:49:06.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-10-09T04:49:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[216903]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:49:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:49:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[223858]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:49:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:49:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:07.509 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:07.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:07 smithi177 ceph-mon[195797]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:07 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded, 1 pg undersized) 2023-10-09T04:49:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:07 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:49:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:49:07.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:07.659 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:07.884 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:07.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:07.927 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:07.929 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:07.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:08.015 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:08.064 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:08.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:08.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:08.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:08.196 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:08.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:08.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:08.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:08.393 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:08.488 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:09.151 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:09.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:09.159 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:49:09.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:09.285 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:09.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:09.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:09.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:09 smithi029 ceph-mon[223858]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1070485109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:09 smithi029 ceph-mon[216903]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1070485109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:09.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:09.463 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:09 smithi177 ceph-mon[195797]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1070485109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:09.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:49:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[216903]: failed to encode map e335 with expected crc 2023-10-09T04:49:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:49:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[216903]: osdmap e335: 8 total, 8 up, 5 in 2023-10-09T04:49:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[223858]: failed to encode map e335 with expected crc 2023-10-09T04:49:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:49:10.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:10 smithi029 ceph-mon[223858]: osdmap e335: 8 total, 8 up, 5 in 2023-10-09T04:49:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:10 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:10 smithi177 ceph-mon[195797]: failed to encode map e335 with expected crc 2023-10-09T04:49:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:49:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:10 smithi177 ceph-mon[195797]: osdmap e335: 8 total, 8 up, 5 in 2023-10-09T04:49:10.715 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:10.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:10.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:10.853 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:11.060 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:11.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:11.164 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:11.194 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:11.241 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: failed to encode map e335 with expected crc 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: failed to encode map e335 with expected crc 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.243 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: failed to encode map e335 with expected crc 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: failed to encode map e335 with expected crc 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.244 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:11 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:11.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:11.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:11.373 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:11.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2023-10-09T04:49:11.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: failed to encode map e335 with expected crc 2023-10-09T04:49:11.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: failed to encode map e335 with expected crc 2023-10-09T04:49:11.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.453 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:11.453 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:11 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:12.385 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:12.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:12.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:12.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:12.537 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:49:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:49:12] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-10-09T04:49:13.339 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:13.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:13.445 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:13.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:13 smithi029 ceph-mon[216903]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:49:13.472 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:13 smithi029 ceph-mon[223858]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:49:13.474 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:13 smithi177 ceph-mon[195797]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-09T04:49:13.479 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:13.521 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:49:13.522 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:49:13.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:13.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:13.631 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:13.663 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:13.875 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:49:13.875 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:49:13.875 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:49:13.875 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:49:13.876 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:49:13.949 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:14.667 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:14.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:14.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[216903]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[216903]: from='client.46379 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[223858]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[223858]: from='client.46379 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:14.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:14.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:14.801 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:14.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:49:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:14 smithi177 ceph-mon[195797]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-10-09T04:49:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:14 smithi177 ceph-mon[195797]: from='client.46379 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:49:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:15.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:49:15.433 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1313,"stamp":"2023-10-09T04:49:13.362914+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":4143830,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":5239804,"kb_used_data":18772,"kb_used_omap":12,"kb_used_meta":5220979,"kb_avail":463485956,"statfs":{"total":479975178240,"available":474609618944,"internally_reserved":0,"allocated":19222528,"data_stored":14205799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12321,"internal_metadata":5346283487},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.220945"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":82,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302092+0000","last_change":"2023-10-09T04:41:02.608433+0000","last_active":"2023-10-09T04:49:03.302092+0000","last_peered":"2023-10-09T04:49:03.302092+0000","last_clean":"2023-10-09T04:49:03.302092+0000","last_became_active":"2023-10-09T04:41:02.605704+0000","last_became_peered":"2023-10-09T04:41:02.605704+0000","last_unstale":"2023-10-09T04:49:03.302092+0000","last_undegraded":"2023-10-09T04:49:03.302092+0000","last_fullsized":"2023-10-09T04:49:03.302092+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:05:55.693940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":127,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302850+0000","last_change":"2023-10-09T04:45:45.599027+0000","last_active":"2023-10-09T04:49:03.302850+0000","last_peered":"2023-10-09T04:49:03.302850+0000","last_clean":"2023-10-09T04:49:03.302850+0000","last_became_active":"2023-10-09T04:45:45.598650+0000","last_became_peered":"2023-10-09T04:45:45.598650+0000","last_unstale":"2023-10-09T04:49:03.302850+0000","last_undegraded":"2023-10-09T04:49:03.302850+0000","last_fullsized":"2023-10-09T04:49:03.302850+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:15.414379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":332,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144485+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:49:04.144485+0000","last_peered":"2023-10-09T04:49:04.144485+0000","last_clean":"2023-10-09T04:49:04.144485+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:49:04.144485+0000","last_undegraded":"2023-10-09T04:49:04.144485+0000","last_fullsized":"2023-10-09T04:49:04.144485+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":267,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.144152+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:49:03.144152+0000","last_peered":"2023-10-09T04:49:03.144152+0000","last_clean":"2023-10-09T04:49:03.144152+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:49:03.144152+0000","last_undegraded":"2023-10-09T04:49:03.144152+0000","last_fullsized":"2023-10-09T04:49:03.144152+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144092+0000","last_change":"2023-10-09T04:46:25.173258+0000","last_active":"2023-10-09T04:49:04.144092+0000","last_peered":"2023-10-09T04:49:04.144092+0000","last_clean":"2023-10-09T04:49:04.144092+0000","last_became_active":"2023-10-09T04:46:25.172947+0000","last_became_peered":"2023-10-09T04:46:25.172947+0000","last_unstale":"2023-10-09T04:49:04.144092+0000","last_undegraded":"2023-10-09T04:49:04.144092+0000","last_fullsized":"2023-10-09T04:49:04.144092+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:54:38.532832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":135,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477921+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:49:04.477921+0000","last_peered":"2023-10-09T04:49:04.477921+0000","last_clean":"2023-10-09T04:49:04.477921+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:49:04.477921+0000","last_undegraded":"2023-10-09T04:49:04.477921+0000","last_fullsized":"2023-10-09T04:49:04.477921+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":267,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477852+0000","last_change":"2023-10-09T04:46:25.155473+0000","last_active":"2023-10-09T04:49:04.477852+0000","last_peered":"2023-10-09T04:49:04.477852+0000","last_clean":"2023-10-09T04:49:04.477852+0000","last_became_active":"2023-10-09T04:46:25.155148+0000","last_became_peered":"2023-10-09T04:46:25.155148+0000","last_unstale":"2023-10-09T04:49:04.477852+0000","last_undegraded":"2023-10-09T04:49:04.477852+0000","last_fullsized":"2023-10-09T04:49:04.477852+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:38.576510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":291,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.301890+0000","last_change":"2023-10-09T04:45:45.611067+0000","last_active":"2023-10-09T04:49:03.301890+0000","last_peered":"2023-10-09T04:49:03.301890+0000","last_clean":"2023-10-09T04:49:03.301890+0000","last_became_active":"2023-10-09T04:45:45.610571+0000","last_became_peered":"2023-10-09T04:45:45.610571+0000","last_unstale":"2023-10-09T04:49:03.301890+0000","last_undegraded":"2023-10-09T04:49:03.301890+0000","last_fullsized":"2023-10-09T04:49:03.301890+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:14.381921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":295,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.476918+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:49:04.476918+0000","last_peered":"2023-10-09T04:49:04.476918+0000","last_clean":"2023-10-09T04:49:04.476918+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:49:04.476918+0000","last_undegraded":"2023-10-09T04:49:04.476918+0000","last_fullsized":"2023-10-09T04:49:04.476918+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":274,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143910+0000","last_change":"2023-10-09T04:46:25.193703+0000","last_active":"2023-10-09T04:49:03.143910+0000","last_peered":"2023-10-09T04:49:03.143910+0000","last_clean":"2023-10-09T04:49:03.143910+0000","last_became_active":"2023-10-09T04:46:25.193392+0000","last_became_peered":"2023-10-09T04:46:25.193392+0000","last_unstale":"2023-10-09T04:49:03.143910+0000","last_undegraded":"2023-10-09T04:49:03.143910+0000","last_fullsized":"2023-10-09T04:49:03.143910+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:53:56.086176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":360,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143833+0000","last_change":"2023-10-09T04:46:25.479989+0000","last_active":"2023-10-09T04:49:03.143833+0000","last_peered":"2023-10-09T04:49:03.143833+0000","last_clean":"2023-10-09T04:49:03.143833+0000","last_became_active":"2023-10-09T04:46:25.381247+0000","last_became_peered":"2023-10-09T04:46:25.381247+0000","last_unstale":"2023-10-09T04:49:03.143833+0000","last_undegraded":"2023-10-09T04:49:03.143833+0000","last_fullsized":"2023-10-09T04:49:03.143833+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:19:12.653312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":261,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.143794+0000","last_change":"2023-10-09T04:46:25.150997+0000","last_active":"2023-10-09T04:49:04.143794+0000","last_peered":"2023-10-09T04:49:04.143794+0000","last_clean":"2023-10-09T04:49:04.143794+0000","last_became_active":"2023-10-09T04:46:25.150370+0000","last_became_peered":"2023-10-09T04:46:25.150370+0000","last_unstale":"2023-10-09T04:49:04.143794+0000","last_undegraded":"2023-10-09T04:49:04.143794+0000","last_fullsized":"2023-10-09T04:49:04.143794+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:40:07.231212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":112,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302456+0000","last_change":"2023-10-09T04:44:38.813366+0000","last_active":"2023-10-09T04:49:03.302456+0000","last_peered":"2023-10-09T04:49:03.302456+0000","last_clean":"2023-10-09T04:49:03.302456+0000","last_became_active":"2023-10-09T04:44:38.777069+0000","last_became_peered":"2023-10-09T04:44:38.777069+0000","last_unstale":"2023-10-09T04:49:03.302456+0000","last_undegraded":"2023-10-09T04:49:03.302456+0000","last_fullsized":"2023-10-09T04:49:03.302456+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:33:39.101540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":304,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302536+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:49:03.302536+0000","last_peered":"2023-10-09T04:49:03.302536+0000","last_clean":"2023-10-09T04:49:03.302536+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:49:03.302536+0000","last_undegraded":"2023-10-09T04:49:03.302536+0000","last_fullsized":"2023-10-09T04:49:03.302536+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":344,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143711+0000","last_change":"2023-10-09T04:41:02.620545+0000","last_active":"2023-10-09T04:49:03.143711+0000","last_peered":"2023-10-09T04:49:03.143711+0000","last_clean":"2023-10-09T04:49:03.143711+0000","last_became_active":"2023-10-09T04:41:02.609478+0000","last_became_peered":"2023-10-09T04:41:02.609478+0000","last_unstale":"2023-10-09T04:49:03.143711+0000","last_undegraded":"2023-10-09T04:49:03.143711+0000","last_fullsized":"2023-10-09T04:49:03.143711+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:26.624662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143473+0000","last_change":"2023-10-09T04:45:45.607791+0000","last_active":"2023-10-09T04:49:03.143473+0000","last_peered":"2023-10-09T04:49:03.143473+0000","last_clean":"2023-10-09T04:49:03.143473+0000","last_became_active":"2023-10-09T04:45:45.607501+0000","last_became_peered":"2023-10-09T04:45:45.607501+0000","last_unstale":"2023-10-09T04:49:03.143473+0000","last_undegraded":"2023-10-09T04:49:03.143473+0000","last_fullsized":"2023-10-09T04:49:03.143473+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:05.067640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":266,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143475+0000","last_change":"2023-10-09T04:46:25.168444+0000","last_active":"2023-10-09T04:49:03.143475+0000","last_peered":"2023-10-09T04:49:03.143475+0000","last_clean":"2023-10-09T04:49:03.143475+0000","last_became_active":"2023-10-09T04:46:25.168143+0000","last_became_peered":"2023-10-09T04:46:25.168143+0000","last_unstale":"2023-10-09T04:49:03.143475+0000","last_undegraded":"2023-10-09T04:49:03.143475+0000","last_fullsized":"2023-10-09T04:49:03.143475+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:22:56.166900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":314,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477768+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:49:04.477768+0000","last_peered":"2023-10-09T04:49:04.477768+0000","last_clean":"2023-10-09T04:49:04.477768+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:49:04.477768+0000","last_undegraded":"2023-10-09T04:49:04.477768+0000","last_fullsized":"2023-10-09T04:49:04.477768+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":296,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143818+0000","last_change":"2023-10-09T04:46:25.175418+0000","last_active":"2023-10-09T04:49:03.143818+0000","last_peered":"2023-10-09T04:49:03.143818+0000","last_clean":"2023-10-09T04:49:03.143818+0000","last_became_active":"2023-10-09T04:46:25.174958+0000","last_became_peered":"2023-10-09T04:46:25.174958+0000","last_unstale":"2023-10-09T04:49:03.143818+0000","last_undegraded":"2023-10-09T04:49:03.143818+0000","last_fullsized":"2023-10-09T04:49:03.143818+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:12:35.362263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.144154+0000","last_change":"2023-10-09T04:46:25.170147+0000","last_active":"2023-10-09T04:49:03.144154+0000","last_peered":"2023-10-09T04:49:03.144154+0000","last_clean":"2023-10-09T04:49:03.144154+0000","last_became_active":"2023-10-09T04:46:25.169835+0000","last_became_peered":"2023-10-09T04:46:25.169835+0000","last_unstale":"2023-10-09T04:49:03.144154+0000","last_undegraded":"2023-10-09T04:49:03.144154+0000","last_fullsized":"2023-10-09T04:49:03.144154+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:01.393349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":116,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477232+0000","last_change":"2023-10-09T04:46:25.271462+0000","last_active":"2023-10-09T04:49:04.477232+0000","last_peered":"2023-10-09T04:49:04.477232+0000","last_clean":"2023-10-09T04:49:04.477232+0000","last_became_active":"2023-10-09T04:46:25.166617+0000","last_became_peered":"2023-10-09T04:46:25.166617+0000","last_unstale":"2023-10-09T04:49:04.477232+0000","last_undegraded":"2023-10-09T04:49:04.477232+0000","last_fullsized":"2023-10-09T04:49:04.477232+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:27:30.657523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":127,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.144451+0000","last_change":"2023-10-09T04:45:45.603131+0000","last_active":"2023-10-09T04:49:03.144451+0000","last_peered":"2023-10-09T04:49:03.144451+0000","last_clean":"2023-10-09T04:49:03.144451+0000","last_became_active":"2023-10-09T04:45:45.602862+0000","last_became_peered":"2023-10-09T04:45:45.602862+0000","last_unstale":"2023-10-09T04:49:03.144451+0000","last_undegraded":"2023-10-09T04:49:03.144451+0000","last_fullsized":"2023-10-09T04:49:03.144451+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:20:22.358682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":364,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.144440+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:49:03.144440+0000","last_peered":"2023-10-09T04:49:03.144440+0000","last_clean":"2023-10-09T04:49:03.144440+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:49:03.144440+0000","last_undegraded":"2023-10-09T04:49:03.144440+0000","last_fullsized":"2023-10-09T04:49:03.144440+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":287,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477154+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:49:04.477154+0000","last_peered":"2023-10-09T04:49:04.477154+0000","last_clean":"2023-10-09T04:49:04.477154+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:49:04.477154+0000","last_undegraded":"2023-10-09T04:49:04.477154+0000","last_fullsized":"2023-10-09T04:49:04.477154+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":323,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143876+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:49:03.143876+0000","last_peered":"2023-10-09T04:49:03.143876+0000","last_clean":"2023-10-09T04:49:03.143876+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:49:03.143876+0000","last_undegraded":"2023-10-09T04:49:03.143876+0000","last_fullsized":"2023-10-09T04:49:03.143876+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":331,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302163+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:49:03.302163+0000","last_peered":"2023-10-09T04:49:03.302163+0000","last_clean":"2023-10-09T04:49:03.302163+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:49:03.302163+0000","last_undegraded":"2023-10-09T04:49:03.302163+0000","last_fullsized":"2023-10-09T04:49:03.302163+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":305,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143639+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:49:03.143639+0000","last_peered":"2023-10-09T04:49:03.143639+0000","last_clean":"2023-10-09T04:49:03.143639+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:49:03.143639+0000","last_undegraded":"2023-10-09T04:49:03.143639+0000","last_fullsized":"2023-10-09T04:49:03.143639+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":253,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:06.976916+0000","last_change":"2023-10-09T04:44:38.775465+0000","last_active":"2023-10-09T04:49:06.976916+0000","last_peered":"2023-10-09T04:49:06.976916+0000","last_clean":"2023-10-09T04:49:06.976916+0000","last_became_active":"2023-10-09T04:44:38.775246+0000","last_became_peered":"2023-10-09T04:44:38.775246+0000","last_unstale":"2023-10-09T04:49:06.976916+0000","last_undegraded":"2023-10-09T04:49:06.976916+0000","last_fullsized":"2023-10-09T04:49:06.976916+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:39.286190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":325,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477649+0000","last_change":"2023-10-09T04:46:25.150945+0000","last_active":"2023-10-09T04:49:04.477649+0000","last_peered":"2023-10-09T04:49:04.477649+0000","last_clean":"2023-10-09T04:49:04.477649+0000","last_became_active":"2023-10-09T04:46:25.150622+0000","last_became_peered":"2023-10-09T04:46:25.150622+0000","last_unstale":"2023-10-09T04:49:04.477649+0000","last_undegraded":"2023-10-09T04:49:04.477649+0000","last_fullsized":"2023-10-09T04:49:04.477649+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:02.718023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302461+0000","last_change":"2023-10-09T04:46:25.173123+0000","last_active":"2023-10-09T04:49:03.302461+0000","last_peered":"2023-10-09T04:49:03.302461+0000","last_clean":"2023-10-09T04:49:03.302461+0000","last_became_active":"2023-10-09T04:46:25.172534+0000","last_became_peered":"2023-10-09T04:46:25.172534+0000","last_unstale":"2023-10-09T04:49:03.302461+0000","last_undegraded":"2023-10-09T04:49:03.302461+0000","last_fullsized":"2023-10-09T04:49:03.302461+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:00.581088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":73,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302586+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:49:03.302586+0000","last_peered":"2023-10-09T04:49:03.302586+0000","last_clean":"2023-10-09T04:49:03.302586+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:49:03.302586+0000","last_undegraded":"2023-10-09T04:49:03.302586+0000","last_fullsized":"2023-10-09T04:49:03.302586+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":102,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302510+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:49:03.302510+0000","last_peered":"2023-10-09T04:49:03.302510+0000","last_clean":"2023-10-09T04:49:03.302510+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:49:03.302510+0000","last_undegraded":"2023-10-09T04:49:03.302510+0000","last_fullsized":"2023-10-09T04:49:03.302510+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":295,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302140+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:49:03.302140+0000","last_peered":"2023-10-09T04:49:03.302140+0000","last_clean":"2023-10-09T04:49:03.302140+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:49:03.302140+0000","last_undegraded":"2023-10-09T04:49:03.302140+0000","last_fullsized":"2023-10-09T04:49:03.302140+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":82,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302726+0000","last_change":"2023-10-09T04:41:02.610801+0000","last_active":"2023-10-09T04:49:03.302726+0000","last_peered":"2023-10-09T04:49:03.302726+0000","last_clean":"2023-10-09T04:49:03.302726+0000","last_became_active":"2023-10-09T04:41:02.606870+0000","last_became_peered":"2023-10-09T04:41:02.606870+0000","last_unstale":"2023-10-09T04:49:03.302726+0000","last_undegraded":"2023-10-09T04:49:03.302726+0000","last_fullsized":"2023-10-09T04:49:03.302726+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:56:19.966812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":82,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302017+0000","last_change":"2023-10-09T04:41:02.608644+0000","last_active":"2023-10-09T04:49:03.302017+0000","last_peered":"2023-10-09T04:49:03.302017+0000","last_clean":"2023-10-09T04:49:03.302017+0000","last_became_active":"2023-10-09T04:41:02.608252+0000","last_became_peered":"2023-10-09T04:41:02.608252+0000","last_unstale":"2023-10-09T04:49:03.302017+0000","last_undegraded":"2023-10-09T04:49:03.302017+0000","last_fullsized":"2023-10-09T04:49:03.302017+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:11.678595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":314,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302485+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:49:03.302485+0000","last_peered":"2023-10-09T04:49:03.302485+0000","last_clean":"2023-10-09T04:49:03.302485+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:49:03.302485+0000","last_undegraded":"2023-10-09T04:49:03.302485+0000","last_fullsized":"2023-10-09T04:49:03.302485+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.145093+0000","last_change":"2023-10-09T04:45:45.611560+0000","last_active":"2023-10-09T04:49:03.145093+0000","last_peered":"2023-10-09T04:49:03.145093+0000","last_clean":"2023-10-09T04:49:03.145093+0000","last_became_active":"2023-10-09T04:45:45.611247+0000","last_became_peered":"2023-10-09T04:45:45.611247+0000","last_unstale":"2023-10-09T04:49:03.145093+0000","last_undegraded":"2023-10-09T04:49:03.145093+0000","last_fullsized":"2023-10-09T04:49:03.145093+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:52.196556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":280,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302607+0000","last_change":"2023-10-09T04:45:45.604134+0000","last_active":"2023-10-09T04:49:03.302607+0000","last_peered":"2023-10-09T04:49:03.302607+0000","last_clean":"2023-10-09T04:49:03.302607+0000","last_became_active":"2023-10-09T04:45:45.603767+0000","last_became_peered":"2023-10-09T04:45:45.603767+0000","last_unstale":"2023-10-09T04:49:03.302607+0000","last_undegraded":"2023-10-09T04:49:03.302607+0000","last_fullsized":"2023-10-09T04:49:03.302607+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:14.801140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":221,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143941+0000","last_change":"2023-10-09T04:46:25.169137+0000","last_active":"2023-10-09T04:49:03.143941+0000","last_peered":"2023-10-09T04:49:03.143941+0000","last_clean":"2023-10-09T04:49:03.143941+0000","last_became_active":"2023-10-09T04:46:25.168834+0000","last_became_peered":"2023-10-09T04:46:25.168834+0000","last_unstale":"2023-10-09T04:49:03.143941+0000","last_undegraded":"2023-10-09T04:49:03.143941+0000","last_fullsized":"2023-10-09T04:49:03.143941+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:15.860221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":339,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477806+0000","last_change":"2023-10-09T04:45:45.824913+0000","last_active":"2023-10-09T04:49:04.477806+0000","last_peered":"2023-10-09T04:49:04.477806+0000","last_clean":"2023-10-09T04:49:04.477806+0000","last_became_active":"2023-10-09T04:45:45.824572+0000","last_became_peered":"2023-10-09T04:45:45.824572+0000","last_unstale":"2023-10-09T04:49:04.477806+0000","last_undegraded":"2023-10-09T04:49:04.477806+0000","last_fullsized":"2023-10-09T04:49:04.477806+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:38:36.276403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":306,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477417+0000","last_change":"2023-10-09T04:41:02.607551+0000","last_active":"2023-10-09T04:49:04.477417+0000","last_peered":"2023-10-09T04:49:04.477417+0000","last_clean":"2023-10-09T04:49:04.477417+0000","last_became_active":"2023-10-09T04:41:02.607315+0000","last_became_peered":"2023-10-09T04:41:02.607315+0000","last_unstale":"2023-10-09T04:49:04.477417+0000","last_undegraded":"2023-10-09T04:49:04.477417+0000","last_fullsized":"2023-10-09T04:49:04.477417+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:09:11.918923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":319,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477319+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:49:04.477319+0000","last_peered":"2023-10-09T04:49:04.477319+0000","last_clean":"2023-10-09T04:49:04.477319+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:49:04.477319+0000","last_undegraded":"2023-10-09T04:49:04.477319+0000","last_fullsized":"2023-10-09T04:49:04.477319+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":102,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302157+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:49:03.302157+0000","last_peered":"2023-10-09T04:49:03.302157+0000","last_clean":"2023-10-09T04:49:03.302157+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:49:03.302157+0000","last_undegraded":"2023-10-09T04:49:03.302157+0000","last_fullsized":"2023-10-09T04:49:03.302157+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":317,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144164+0000","last_change":"2023-10-09T04:45:45.611766+0000","last_active":"2023-10-09T04:49:04.144164+0000","last_peered":"2023-10-09T04:49:04.144164+0000","last_clean":"2023-10-09T04:49:04.144164+0000","last_became_active":"2023-10-09T04:45:45.611242+0000","last_became_peered":"2023-10-09T04:45:45.611242+0000","last_unstale":"2023-10-09T04:49:04.144164+0000","last_undegraded":"2023-10-09T04:49:04.144164+0000","last_fullsized":"2023-10-09T04:49:04.144164+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:23.336238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":312,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144799+0000","last_change":"2023-10-09T04:46:25.161784+0000","last_active":"2023-10-09T04:49:04.144799+0000","last_peered":"2023-10-09T04:49:04.144799+0000","last_clean":"2023-10-09T04:49:04.144799+0000","last_became_active":"2023-10-09T04:46:25.161498+0000","last_became_peered":"2023-10-09T04:46:25.161498+0000","last_unstale":"2023-10-09T04:49:04.144799+0000","last_undegraded":"2023-10-09T04:49:04.144799+0000","last_fullsized":"2023-10-09T04:49:04.144799+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:13:01.500231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":311,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.143867+0000","last_change":"2023-10-09T04:46:25.165536+0000","last_active":"2023-10-09T04:49:04.143867+0000","last_peered":"2023-10-09T04:49:04.143867+0000","last_clean":"2023-10-09T04:49:04.143867+0000","last_became_active":"2023-10-09T04:46:25.165269+0000","last_became_peered":"2023-10-09T04:46:25.165269+0000","last_unstale":"2023-10-09T04:49:04.143867+0000","last_undegraded":"2023-10-09T04:49:04.143867+0000","last_fullsized":"2023-10-09T04:49:04.143867+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:30.362732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":217,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477896+0000","last_change":"2023-10-09T04:46:25.177443+0000","last_active":"2023-10-09T04:49:04.477896+0000","last_peered":"2023-10-09T04:49:04.477896+0000","last_clean":"2023-10-09T04:49:04.477896+0000","last_became_active":"2023-10-09T04:46:25.176635+0000","last_became_peered":"2023-10-09T04:46:25.176635+0000","last_unstale":"2023-10-09T04:49:04.477896+0000","last_undegraded":"2023-10-09T04:49:04.477896+0000","last_fullsized":"2023-10-09T04:49:04.477896+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:43.367744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":316,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477561+0000","last_change":"2023-10-09T04:36:47.548057+0000","last_active":"2023-10-09T04:49:04.477561+0000","last_peered":"2023-10-09T04:49:04.477561+0000","last_clean":"2023-10-09T04:49:04.477561+0000","last_became_active":"2023-10-09T04:36:47.547779+0000","last_became_peered":"2023-10-09T04:36:47.547779+0000","last_unstale":"2023-10-09T04:49:04.477561+0000","last_undegraded":"2023-10-09T04:49:04.477561+0000","last_fullsized":"2023-10-09T04:49:04.477561+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":222,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:40.979646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":778,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.152163+0000","last_change":"2023-10-09T04:49:04.152163+0000","last_active":"2023-10-09T04:49:04.152163+0000","last_peered":"2023-10-09T04:49:04.152163+0000","last_clean":"2023-10-09T04:49:04.152163+0000","last_became_active":"2023-10-09T04:49:04.151740+0000","last_became_peered":"2023-10-09T04:49:04.151740+0000","last_unstale":"2023-10-09T04:49:04.152163+0000","last_undegraded":"2023-10-09T04:49:04.152163+0000","last_fullsized":"2023-10-09T04:49:04.152163+0000","mapping_epoch":333,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:06:49.103232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":265,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477492+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:49:04.477492+0000","last_peered":"2023-10-09T04:49:04.477492+0000","last_clean":"2023-10-09T04:49:04.477492+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:49:04.477492+0000","last_undegraded":"2023-10-09T04:49:04.477492+0000","last_fullsized":"2023-10-09T04:49:04.477492+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":277,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302759+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:49:03.302759+0000","last_peered":"2023-10-09T04:49:03.302759+0000","last_clean":"2023-10-09T04:49:03.302759+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:49:03.302759+0000","last_undegraded":"2023-10-09T04:49:03.302759+0000","last_fullsized":"2023-10-09T04:49:03.302759+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":344,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.145022+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:49:03.145022+0000","last_peered":"2023-10-09T04:49:03.145022+0000","last_clean":"2023-10-09T04:49:03.145022+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:49:03.145022+0000","last_undegraded":"2023-10-09T04:49:03.145022+0000","last_fullsized":"2023-10-09T04:49:03.145022+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":102,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302210+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:49:03.302210+0000","last_peered":"2023-10-09T04:49:03.302210+0000","last_clean":"2023-10-09T04:49:03.302210+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:49:03.302210+0000","last_undegraded":"2023-10-09T04:49:03.302210+0000","last_fullsized":"2023-10-09T04:49:03.302210+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":134,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144497+0000","last_change":"2023-10-09T04:46:25.174089+0000","last_active":"2023-10-09T04:49:04.144497+0000","last_peered":"2023-10-09T04:49:04.144497+0000","last_clean":"2023-10-09T04:49:04.144497+0000","last_became_active":"2023-10-09T04:46:25.173414+0000","last_became_peered":"2023-10-09T04:46:25.173414+0000","last_unstale":"2023-10-09T04:49:04.144497+0000","last_undegraded":"2023-10-09T04:49:04.144497+0000","last_fullsized":"2023-10-09T04:49:04.144497+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:00.877749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":347,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.143723+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:49:03.143723+0000","last_peered":"2023-10-09T04:49:03.143723+0000","last_clean":"2023-10-09T04:49:03.143723+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:49:03.143723+0000","last_undegraded":"2023-10-09T04:49:03.143723+0000","last_fullsized":"2023-10-09T04:49:03.143723+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144037+0000","last_change":"2023-10-09T04:39:40.400975+0000","last_active":"2023-10-09T04:49:04.144037+0000","last_peered":"2023-10-09T04:49:04.144037+0000","last_clean":"2023-10-09T04:49:04.144037+0000","last_became_active":"2023-10-09T04:39:40.400247+0000","last_became_peered":"2023-10-09T04:39:40.400247+0000","last_unstale":"2023-10-09T04:49:04.144037+0000","last_undegraded":"2023-10-09T04:49:04.144037+0000","last_fullsized":"2023-10-09T04:49:04.144037+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:15:58.915936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":189,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477680+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:49:04.477680+0000","last_peered":"2023-10-09T04:49:04.477680+0000","last_clean":"2023-10-09T04:49:04.477680+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:49:04.477680+0000","last_undegraded":"2023-10-09T04:49:04.477680+0000","last_fullsized":"2023-10-09T04:49:04.477680+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":82,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.144549+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:49:03.144549+0000","last_peered":"2023-10-09T04:49:03.144549+0000","last_clean":"2023-10-09T04:49:03.144549+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:49:03.144549+0000","last_undegraded":"2023-10-09T04:49:03.144549+0000","last_fullsized":"2023-10-09T04:49:03.144549+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":297,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302445+0000","last_change":"2023-10-09T04:46:25.166210+0000","last_active":"2023-10-09T04:49:03.302445+0000","last_peered":"2023-10-09T04:49:03.302445+0000","last_clean":"2023-10-09T04:49:03.302445+0000","last_became_active":"2023-10-09T04:46:25.165948+0000","last_became_peered":"2023-10-09T04:46:25.165948+0000","last_unstale":"2023-10-09T04:49:03.302445+0000","last_undegraded":"2023-10-09T04:49:03.302445+0000","last_fullsized":"2023-10-09T04:49:03.302445+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:51.272111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":180,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.145161+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:49:03.145161+0000","last_peered":"2023-10-09T04:49:03.145161+0000","last_clean":"2023-10-09T04:49:03.145161+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:49:03.145161+0000","last_undegraded":"2023-10-09T04:49:03.145161+0000","last_fullsized":"2023-10-09T04:49:03.145161+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":256,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.144359+0000","last_change":"2023-10-09T04:45:45.602634+0000","last_active":"2023-10-09T04:49:03.144359+0000","last_peered":"2023-10-09T04:49:03.144359+0000","last_clean":"2023-10-09T04:49:03.144359+0000","last_became_active":"2023-10-09T04:45:45.602317+0000","last_became_peered":"2023-10-09T04:45:45.602317+0000","last_unstale":"2023-10-09T04:49:03.144359+0000","last_undegraded":"2023-10-09T04:49:03.144359+0000","last_fullsized":"2023-10-09T04:49:03.144359+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:13.963587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":120,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302574+0000","last_change":"2023-10-09T04:46:25.210175+0000","last_active":"2023-10-09T04:49:03.302574+0000","last_peered":"2023-10-09T04:49:03.302574+0000","last_clean":"2023-10-09T04:49:03.302574+0000","last_became_active":"2023-10-09T04:46:25.174609+0000","last_became_peered":"2023-10-09T04:46:25.174609+0000","last_unstale":"2023-10-09T04:49:03.302574+0000","last_undegraded":"2023-10-09T04:49:03.302574+0000","last_fullsized":"2023-10-09T04:49:03.302574+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:09:29.833868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":102,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.301937+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:49:03.301937+0000","last_peered":"2023-10-09T04:49:03.301937+0000","last_clean":"2023-10-09T04:49:03.301937+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:49:03.301937+0000","last_undegraded":"2023-10-09T04:49:03.301937+0000","last_fullsized":"2023-10-09T04:49:03.301937+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":184,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302430+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:49:03.302430+0000","last_peered":"2023-10-09T04:49:03.302430+0000","last_clean":"2023-10-09T04:49:03.302430+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:49:03.302430+0000","last_undegraded":"2023-10-09T04:49:03.302430+0000","last_fullsized":"2023-10-09T04:49:03.302430+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":304,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.143947+0000","last_change":"2023-10-09T04:46:25.160157+0000","last_active":"2023-10-09T04:49:04.143947+0000","last_peered":"2023-10-09T04:49:04.143947+0000","last_clean":"2023-10-09T04:49:04.143947+0000","last_became_active":"2023-10-09T04:46:25.159850+0000","last_became_peered":"2023-10-09T04:46:25.159850+0000","last_unstale":"2023-10-09T04:49:04.143947+0000","last_undegraded":"2023-10-09T04:49:04.143947+0000","last_fullsized":"2023-10-09T04:49:04.143947+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:55:07.475665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":325,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477640+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:49:04.477640+0000","last_peered":"2023-10-09T04:49:04.477640+0000","last_clean":"2023-10-09T04:49:04.477640+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:49:04.477640+0000","last_undegraded":"2023-10-09T04:49:04.477640+0000","last_fullsized":"2023-10-09T04:49:04.477640+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":92,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302257+0000","last_change":"2023-10-09T04:45:45.609849+0000","last_active":"2023-10-09T04:49:03.302257+0000","last_peered":"2023-10-09T04:49:03.302257+0000","last_clean":"2023-10-09T04:49:03.302257+0000","last_became_active":"2023-10-09T04:45:45.609639+0000","last_became_peered":"2023-10-09T04:45:45.609639+0000","last_unstale":"2023-10-09T04:49:03.302257+0000","last_undegraded":"2023-10-09T04:49:03.302257+0000","last_fullsized":"2023-10-09T04:49:03.302257+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:26.113652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":294,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.477889+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:49:04.477889+0000","last_peered":"2023-10-09T04:49:04.477889+0000","last_clean":"2023-10-09T04:49:04.477889+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:49:04.477889+0000","last_undegraded":"2023-10-09T04:49:04.477889+0000","last_fullsized":"2023-10-09T04:49:04.477889+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":102,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.302217+0000","last_change":"2023-10-09T04:46:25.172010+0000","last_active":"2023-10-09T04:49:03.302217+0000","last_peered":"2023-10-09T04:49:03.302217+0000","last_clean":"2023-10-09T04:49:03.302217+0000","last_became_active":"2023-10-09T04:46:25.171776+0000","last_became_peered":"2023-10-09T04:46:25.171776+0000","last_unstale":"2023-10-09T04:49:03.302217+0000","last_undegraded":"2023-10-09T04:49:03.302217+0000","last_fullsized":"2023-10-09T04:49:03.302217+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:40.329129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":45,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144587+0000","last_change":"2023-10-09T04:45:45.611105+0000","last_active":"2023-10-09T04:49:04.144587+0000","last_peered":"2023-10-09T04:49:04.144587+0000","last_clean":"2023-10-09T04:49:04.144587+0000","last_became_active":"2023-10-09T04:45:45.610758+0000","last_became_peered":"2023-10-09T04:45:45.610758+0000","last_unstale":"2023-10-09T04:49:04.144587+0000","last_undegraded":"2023-10-09T04:49:04.144587+0000","last_fullsized":"2023-10-09T04:49:04.144587+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:13:59.031709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":284,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.144748+0000","last_change":"2023-10-09T04:46:25.153824+0000","last_active":"2023-10-09T04:49:04.144748+0000","last_peered":"2023-10-09T04:49:04.144748+0000","last_clean":"2023-10-09T04:49:04.144748+0000","last_became_active":"2023-10-09T04:46:25.153497+0000","last_became_peered":"2023-10-09T04:46:25.153497+0000","last_unstale":"2023-10-09T04:49:04.144748+0000","last_undegraded":"2023-10-09T04:49:04.144748+0000","last_fullsized":"2023-10-09T04:49:04.144748+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:10:00.144937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":225,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-10-09T04:49:04.478003+0000","last_change":"2023-10-09T04:46:25.217389+0000","last_active":"2023-10-09T04:49:04.478003+0000","last_peered":"2023-10-09T04:49:04.478003+0000","last_clean":"2023-10-09T04:49:04.478003+0000","last_became_active":"2023-10-09T04:46:25.162993+0000","last_became_peered":"2023-10-09T04:46:25.162993+0000","last_unstale":"2023-10-09T04:49:04.478003+0000","last_undegraded":"2023-10-09T04:49:04.478003+0000","last_fullsized":"2023-10-09T04:49:04.478003+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:13.549717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":102,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-10-09T04:49:03.301989+0000","last_change":"2023-10-09T04:45:45.610893+0000","last_active":"2023-10-09T04:49:03.301989+0000","last_peered":"2023-10-09T04:49:03.301989+0000","last_clean":"2023-10-09T04:49:03.301989+0000","last_became_active":"2023-10-09T04:45:45.610260+0000","last_became_peered":"2023-10-09T04:45:45.610260+0000","last_unstale":"2023-10-09T04:49:03.301989+0000","last_undegraded":"2023-10-09T04:49:03.301989+0000","last_fullsized":"2023-10-09T04:49:03.301989+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:22.552117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":11254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":17338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234507,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521144,"kb_used_data":3696,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":92224008,"statfs":{"total":95995035648,"available":94437384192,"internally_reserved":0,"allocated":3784704,"data_stored":3042283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.4359999999999999,"15min":2.5329999999999999},"min":{"1min":0.245,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":14.970000000000001,"5min":14.970000000000001,"15min":111.72199999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.754,"5min":2.1749999999999998,"15min":3.0499999999999998},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":6.1689999999999996,"5min":19.152000000000001,"15min":111.76600000000001},"last":0.502}]},{"osd":1,"last update":"Mon Oct 9 04:48:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":4.6619999999999999,"15min":4.1500000000000004},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.253},"max":{"1min":14.896000000000001,"5min":128.601,"15min":264.62},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.4329999999999998,"5min":4.4050000000000002,"15min":4.2160000000000002},"min":{"1min":0.27900000000000003,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":14.933999999999999,"5min":128.381,"15min":264.65899999999999},"last":0.54700000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":13.303000000000001,"5min":5.2450000000000001,"15min":4.8490000000000002},"min":{"1min":0.29499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":129.10400000000001,"5min":129.10400000000001,"15min":184.72},"last":0.85999999999999999},{"interface":"front","average":{"1min":13.906000000000001,"5min":5.5090000000000003,"15min":5.234},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":129.15100000000001,"5min":129.15100000000001,"15min":184.697},"last":0.70099999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:48:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.728,"15min":4.1630000000000003},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.1500000000000004,"5min":7.8630000000000004,"15min":182.839},"last":0.92300000000000004},{"interface":"front","average":{"1min":3.113,"5min":2.5129999999999999,"15min":4.7000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":14.866,"5min":14.866,"15min":182.678},"last":0.81999999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.399,"5min":1.5569999999999999,"15min":1.53},"min":{"1min":0.24399999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.573,"5min":11.573,"15min":14.257999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.23,"5min":1.353,"15min":1.327},"min":{"1min":0.317,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":7.9749999999999996,"5min":10.497,"15min":10.497},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.143,"15min":1.2150000000000001},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4169999999999998,"5min":8.5709999999999997,"15min":13.319000000000001},"last":1.01},{"interface":"front","average":{"1min":3.492,"5min":1.6080000000000001,"15min":1.6890000000000001},"min":{"1min":0.33600000000000002,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":12.081,"5min":12.081,"15min":18.408000000000001},"last":0.88600000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":0.98699999999999999,"15min":1.2509999999999999},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":7.4459999999999997,"5min":7.4459999999999997,"15min":11.388},"last":0.97099999999999997},{"interface":"front","average":{"1min":2.8090000000000002,"5min":1.5189999999999999,"15min":1.778},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":11.680999999999999,"5min":11.680999999999999,"15min":18.311},"last":0.94199999999999995}]}]},{"osd":6,"up_from":0,"seq":197568496141,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141231,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":821252,"kb_used_data":4540,"kb_used_omap":4,"kb_used_meta":816699,"kb_avail":92923900,"statfs":{"total":95995035648,"available":95154073600,"internally_reserved":0,"allocated":4648960,"data_stored":3141102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":836300686},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":1.833,"15min":3.077},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.14799999999999999},"max":{"1min":9.0229999999999997,"5min":21.021999999999998,"15min":93.171999999999997},"last":0.436},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.925,"15min":3.1589999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.193},"max":{"1min":8.8190000000000008,"5min":21.280999999999999,"15min":93.147000000000006},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:49:08 2023","interfaces":[{"interface":"back","average":{"1min":3.4049999999999998,"5min":6.4340000000000002,"15min":4.9550000000000001},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.20899999999999999},"max":{"1min":21.114999999999998,"5min":151.41900000000001,"15min":263.60500000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.1269999999999998,"5min":6.3680000000000003,"15min":5.319},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.995999999999999,"5min":146.749,"15min":263.65199999999999},"last":0.41999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:48:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.946,"15min":5.6550000000000002},"min":{"1min":0.26500000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.0709999999999997,"5min":11.68,"15min":379.80900000000003},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.508,"5min":2.3340000000000001,"15min":6.3159999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.246},"max":{"1min":12.615,"5min":18.285,"15min":379.95800000000003},"last":0.34000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":2.4609999999999999,"15min":2.6640000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":11.928000000000001,"5min":19.959,"15min":170.072},"last":0.55600000000000005},{"interface":"front","average":{"1min":3.0939999999999999,"5min":1.998,"15min":2.4929999999999999},"min":{"1min":0.35399999999999998,"5min":0.31900000000000001,"15min":0.28799999999999998},"max":{"1min":9.1010000000000009,"5min":11.598000000000001,"15min":169.94499999999999},"last":0.627}]},{"osd":5,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.1080000000000001,"15min":2.444},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":8.7639999999999993,"5min":9.782,"15min":171.036},"last":0.57299999999999995},{"interface":"front","average":{"1min":2.7970000000000002,"5min":2.7000000000000002,"15min":2.8029999999999999},"min":{"1min":0.372,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":11.667999999999999,"5min":20.021000000000001,"15min":170.88},"last":0.66100000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.0619999999999998,"15min":2.4780000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":8.7040000000000006,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.1240000000000001,"5min":2.8029999999999999,"15min":2.8660000000000001},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":10.006,"5min":20.164999999999999,"15min":132.17699999999999},"last":0.72999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:49:00 2023","interfaces":[{"interface":"back","average":{"1min":4.1399999999999997,"5min":3.0750000000000002,"15min":3.496},"min":{"1min":0.40100000000000002,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":14.976000000000001,"5min":20.055,"15min":174.75999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":4.2699999999999996,"5min":2.3069999999999999,"15min":3.1360000000000001},"min":{"1min":0.42199999999999999,"5min":0.38800000000000001,"15min":0.377},"max":{"1min":15.066000000000001,"5min":16.238,"15min":174.779},"last":0.54600000000000004}]}]},{"osd":0,"up_from":0,"seq":390842024287,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772564,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513512,"kb_used_data":4320,"kb_used_omap":7,"kb_used_meta":509176,"kb_avail":93231640,"statfs":{"total":95995035648,"available":95469199360,"internally_reserved":0,"allocated":4423680,"data_stored":3122356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":521396305},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4740000000000002,"5min":2.6309999999999998,"15min":4.6459999999999999},"min":{"1min":0.214,"5min":0.188,"15min":0.183},"max":{"1min":11.769,"5min":25.899999999999999,"15min":72.608000000000004},"last":0.27800000000000002},{"interface":"front","average":{"1min":1.373,"5min":2.073,"15min":4.1680000000000001},"min":{"1min":0.24399999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.007,"5min":16.550000000000001,"15min":72.632000000000005},"last":0.42499999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":9.3989999999999991,"5min":5.5960000000000001,"15min":5.056},"min":{"1min":0.22,"5min":0.22,"15min":0.217},"max":{"1min":170.172,"5min":170.172,"15min":170.172},"last":0.39900000000000002},{"interface":"front","average":{"1min":9.8070000000000004,"5min":6.0490000000000004,"15min":5.4649999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":170.511,"5min":170.511,"15min":170.511},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.25,"5min":5.4240000000000004,"15min":6.7649999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.21199999999999999},"max":{"1min":8.6449999999999996,"5min":161.12799999999999,"15min":335.928},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.0750000000000002,"5min":5.3940000000000001,"15min":6.9580000000000002},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":9.0410000000000004,"5min":161.386,"15min":335.68599999999998},"last":0.36799999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.52,"15min":3.0539999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.315},"max":{"1min":8.7029999999999994,"5min":10.256,"15min":72.944999999999993},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.657,"5min":2.4209999999999998,"15min":3.0609999999999999},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.8680000000000003,"5min":16.341000000000001,"15min":73.054000000000002},"last":0.501}]},{"osd":5,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":2.2719999999999998,"15min":2.952},"min":{"1min":0.38900000000000001,"5min":0.33000000000000002,"15min":0.32600000000000001},"max":{"1min":9.9000000000000004,"5min":12.98,"15min":72.986000000000004},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.29,"5min":2.4329999999999998,"15min":3.254},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.32200000000000001},"max":{"1min":8.8369999999999997,"5min":10.289,"15min":72.963999999999999},"last":0.625}]},{"osd":6,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":2.2869999999999999,"15min":5.0789999999999997},"min":{"1min":0.38200000000000001,"5min":0.374,"15min":0.33600000000000002},"max":{"1min":9.7720000000000002,"5min":16.425999999999998,"15min":135.614},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.04,"15min":4.7030000000000003},"min":{"1min":0.439,"5min":0.35199999999999998,"15min":0.34699999999999998},"max":{"1min":9.9429999999999996,"5min":10.977,"15min":130.196},"last":0.61099999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":2.766,"15min":3.355},"min":{"1min":0.46400000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":10.579000000000001,"5min":16.462,"15min":73.079999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.1019999999999999,"5min":2.6000000000000001,"15min":3.0129999999999999},"min":{"1min":0.48499999999999999,"5min":0.373,"15min":0.373},"max":{"1min":9.9740000000000002,"5min":10.935,"15min":73.069000000000003},"last":0.63900000000000001}]}]},{"osd":3,"up_from":0,"seq":609885356289,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019415,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494584,"kb_used_data":3120,"kb_used_omap":0,"kb_used_meta":1491456,"kb_avail":92250568,"statfs":{"total":95995035648,"available":94464581632,"internally_reserved":0,"allocated":3194880,"data_stored":2452469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1527250944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5779999999999998,"5min":1.845,"15min":4.0599999999999996},"min":{"1min":0.248,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":11.987,"5min":12.795,"15min":171.13499999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.641,"5min":1.9319999999999999,"15min":4.0300000000000002},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.2329999999999997,"5min":16.004000000000001,"15min":171.11799999999999},"last":0.46999999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:48:53 2023","interfaces":[{"interface":"back","average":{"1min":22.486999999999998,"5min":9.3059999999999992,"15min":5.2519999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":385.78199999999998,"5min":385.78199999999998,"15min":385.78199999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":23.029,"5min":9.0790000000000006,"15min":5.2350000000000003},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":386.09399999999999,"5min":386.09399999999999,"15min":386.09399999999999},"last":0.56100000000000005}]},{"osd":2,"last update":"Mon Oct 9 04:48:11 2023","interfaces":[{"interface":"back","average":{"1min":16.193000000000001,"5min":6.4290000000000003,"15min":5.2519999999999998},"min":{"1min":0.47199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":74.069999999999993,"5min":98.221999999999994,"15min":195.267},"last":0.58499999999999996},{"interface":"front","average":{"1min":16.573,"5min":6.5599999999999996,"15min":5.0060000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":75.219999999999999,"5min":94.219999999999999,"15min":195.15799999999999},"last":0.504}]},{"osd":3,"last update":"Mon Oct 9 04:48:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9129999999999998,"5min":4.1449999999999996,"15min":5.4080000000000004},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":16.082999999999998,"5min":141.334,"15min":357.125},"last":0.61699999999999999},{"interface":"front","average":{"1min":3.8730000000000002,"5min":4.2380000000000004,"15min":5.3609999999999998},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":12.090999999999999,"5min":141.18299999999999,"15min":359.02800000000002},"last":0.54300000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:48:25 2023","interfaces":[{"interface":"back","average":{"1min":3.6259999999999999,"5min":1.931,"15min":1.8600000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":12.053000000000001,"5min":24.234999999999999,"15min":107.476},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.653,"5min":1.5529999999999999,"15min":1.7909999999999999},"min":{"1min":0.39200000000000002,"5min":0.28599999999999998,"15min":0.24099999999999999},"max":{"1min":18.785,"5min":18.785,"15min":107.414},"last":0.35599999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:48:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.8680000000000001,"15min":1.8540000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29599999999999999},"max":{"1min":11.619,"5min":22.920999999999999,"15min":65.75},"last":0.313},{"interface":"front","average":{"1min":2.6040000000000001,"5min":1.6279999999999999,"15min":1.7649999999999999},"min":{"1min":0.38,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.039,"5min":15.095000000000001,"15min":65.849999999999994},"last":0.33600000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:48:17 2023","interfaces":[{"interface":"back","average":{"1min":3.403,"5min":1.8580000000000001,"15min":1.5880000000000001},"min":{"1min":0.38700000000000001,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":16.109000000000002,"5min":16.109000000000002,"15min":16.109000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.7069999999999999,"5min":2.2799999999999998,"15min":1.7390000000000001},"min":{"1min":0.45000000000000001,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":12.013999999999999,"5min":22.858000000000001,"15min":22.858000000000001},"last":0.40100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757779,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889312,"kb_used_data":3096,"kb_used_omap":0,"kb_used_meta":886208,"kb_avail":92855840,"statfs":{"total":95995035648,"available":95084380160,"internally_reserved":0,"allocated":3170304,"data_stored":2447589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907476992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:48:53 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":3.8980000000000001,"15min":4.7910000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22700000000000001},"max":{"1min":4.367,"5min":163.70699999999999,"15min":210.85499999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.4419999999999999,"5min":3.8119999999999998,"15min":5.0449999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.2800000000000002,"5min":163.756,"15min":210.23500000000001},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:48:53 2023","interfaces":[{"interface":"back","average":{"1min":7.9909999999999997,"5min":9.1769999999999996,"15min":5.8499999999999996},"min":{"1min":0.35599999999999998,"5min":0.318,"15min":0.252},"max":{"1min":128.596,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":8.2149999999999999,"5min":9.2070000000000007,"15min":6.0060000000000002},"min":{"1min":0.30199999999999999,"5min":0.255,"15min":0.255},"max":{"1min":128.67699999999999,"5min":211.435,"15min":211.435},"last":0.46899999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:49:05 2023","interfaces":[{"interface":"back","average":{"1min":6.109,"5min":6.0869999999999997,"15min":5.3819999999999997},"min":{"1min":0.35399999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":86.488,"5min":86.488,"15min":90.763999999999996},"last":0.54100000000000004},{"interface":"front","average":{"1min":5.5720000000000001,"5min":5.9649999999999999,"15min":5.4420000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":86.510999999999996,"5min":86.510999999999996,"15min":91.340000000000003},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.766,"15min":5.1589999999999998},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":7.585,"5min":46.497,"15min":268.63299999999998},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.7070000000000001,"5min":2.0539999999999998,"15min":4.7560000000000002},"min":{"1min":0.45700000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.9450000000000003,"5min":46.125999999999998,"15min":268.92500000000001},"last":0.69899999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.3320000000000001,"15min":2.5299999999999998},"min":{"1min":0.34100000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.5650000000000004,"5min":10.268000000000001,"15min":361.09399999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":3.1970000000000001,"5min":1.4650000000000001,"15min":2.8860000000000001},"min":{"1min":0.33200000000000002,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":12.609999999999999,"5min":12.609999999999999,"15min":361.71300000000002},"last":0.65100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.353,"15min":1.2709999999999999},"min":{"1min":0.32800000000000001,"5min":0.28499999999999998,"15min":0.23100000000000001},"max":{"1min":4.1340000000000003,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.236,"15min":1.3560000000000001},"min":{"1min":0.36799999999999999,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":3.5609999999999999,"5min":7.3040000000000003,"15min":13.231999999999999},"last":0.67900000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:48:21 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":2.5249999999999999,"15min":1.764},"min":{"1min":0.35099999999999998,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":7.1029999999999998,"5min":68.411000000000001,"15min":68.411000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.754,"15min":1.786},"min":{"1min":0.434,"5min":0.38300000000000001,"15min":0.30599999999999999},"max":{"1min":8.4499999999999993,"5min":68.435000000000002,"15min":68.435000000000002},"last":0.78500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:49:15.438 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:49:15.439 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:49:15.439 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 4, 5, 7] 2023-10-09T04:49:15.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T04:49:15.606 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:15.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:15.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:15.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:15.653 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:15.653 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:15.681 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:15.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:15.748 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:15.797 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:15.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:15.902 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:15.954 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:16.278 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:16.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:16.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:16.415 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:16.440 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:16.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:16.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:16.571 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:16.600 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:16.644 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:16.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:16.750 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:16.759 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_8' created 2023-10-09T04:49:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:16 smithi177 ceph-mon[195797]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:16.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1167760319' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T04:49:16.783 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:16 smithi029 ceph-mon[216903]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1167760319' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T04:49:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:16 smithi029 ceph-mon[223858]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1167760319' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-09T04:49:17.640 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:17.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:17.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:17.775 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:17.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:17.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[216903]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[216903]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[216903]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1167760319' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T04:49:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[216903]: osdmap e336: 8 total, 8 up, 5 in 2023-10-09T04:49:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[216903]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[223858]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[223858]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[223858]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1167760319' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T04:49:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[223858]: osdmap e336: 8 total, 8 up, 5 in 2023-10-09T04:49:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:17 smithi029 ceph-mon[223858]: failed to encode map e336 with expected crc 2023-10-09T04:49:17.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:17.956 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:18.009 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:18.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:17 smithi177 ceph-mon[195797]: failed to encode map e336 with expected crc 2023-10-09T04:49:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:17 smithi177 ceph-mon[195797]: failed to encode map e336 with expected crc 2023-10-09T04:49:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:17 smithi177 ceph-mon[195797]: failed to encode map e336 with expected crc 2023-10-09T04:49:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1167760319' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-09T04:49:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:17 smithi177 ceph-mon[195797]: osdmap e336: 8 total, 8 up, 5 in 2023-10-09T04:49:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:17 smithi177 ceph-mon[195797]: failed to encode map e336 with expected crc 2023-10-09T04:49:18.113 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:18.142 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:18.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:18.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:18.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:18.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:18.319 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:18.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:18.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:18.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:18.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:18.495 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[216903]: pgmap v1316: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[216903]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[216903]: osdmap e337: 8 total, 8 up, 5 in 2023-10-09T04:49:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[216903]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[216903]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[216903]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.698 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:18.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:18.801 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:18.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:18.830 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-10-09T04:49:18.839 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[223858]: pgmap v1316: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[223858]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[223858]: osdmap e337: 8 total, 8 up, 5 in 2023-10-09T04:49:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[223858]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[223858]: failed to encode map e337 with expected crc 2023-10-09T04:49:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:18 smithi029 ceph-mon[223858]: failed to encode map e337 with expected crc 2023-10-09T04:49:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:18 smithi177 ceph-mon[195797]: pgmap v1316: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:18 smithi177 ceph-mon[195797]: failed to encode map e337 with expected crc 2023-10-09T04:49:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:18 smithi177 ceph-mon[195797]: osdmap e337: 8 total, 8 up, 5 in 2023-10-09T04:49:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:18 smithi177 ceph-mon[195797]: failed to encode map e337 with expected crc 2023-10-09T04:49:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:18 smithi177 ceph-mon[195797]: failed to encode map e337 with expected crc 2023-10-09T04:49:19.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:18 smithi177 ceph-mon[195797]: failed to encode map e337 with expected crc 2023-10-09T04:49:19.916 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.917 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:49:19.918 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:49:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:19 smithi177 ceph-mon[195797]: osdmap e338: 8 total, 8 up, 5 in 2023-10-09T04:49:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:19 smithi177 ceph-mon[195797]: failed to encode map e338 with expected crc 2023-10-09T04:49:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1109205231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:19 smithi029 ceph-mon[216903]: osdmap e338: 8 total, 8 up, 5 in 2023-10-09T04:49:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:19 smithi029 ceph-mon[216903]: failed to encode map e338 with expected crc 2023-10-09T04:49:20.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1109205231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:20.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:19 smithi029 ceph-mon[223858]: osdmap e338: 8 total, 8 up, 5 in 2023-10-09T04:49:20.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:19 smithi029 ceph-mon[223858]: failed to encode map e338 with expected crc 2023-10-09T04:49:20.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1109205231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:20.495 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:20.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:20.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:20.645 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:20.689 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:49:20.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:20.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:20.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:20.831 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:20 smithi177 ceph-mon[195797]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:20 smithi177 ceph-mon[195797]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:20 smithi177 ceph-mon[195797]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:20 smithi177 ceph-mon[195797]: pgmap v1319: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/362394679' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:20 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[216903]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[216903]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[216903]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[216903]: pgmap v1319: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/362394679' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[223858]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[223858]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[223858]: failed to encode map e338 with expected crc 2023-10-09T04:49:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[223858]: pgmap v1319: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/362394679' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:20 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:21.429 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:21.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:21.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:21.561 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:21 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:21 smithi177 ceph-mon[195797]: osdmap e339: 8 total, 8 up, 5 in 2023-10-09T04:49:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:21 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:21 smithi029 ceph-mon[216903]: osdmap e339: 8 total, 8 up, 5 in 2023-10-09T04:49:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:21 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:21 smithi029 ceph-mon[223858]: osdmap e339: 8 total, 8 up, 5 in 2023-10-09T04:49:22.311 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:22.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:22.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:22.401 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:22.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:22.451 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:22.499 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:22.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:22.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:22.606 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:22.643 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:22 smithi177 ceph-mon[195797]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:22 smithi177 ceph-mon[195797]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:22 smithi177 ceph-mon[195797]: pgmap v1321: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:22 smithi177 ceph-mon[195797]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:22 smithi177 ceph-mon[195797]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.831 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:49:22.831 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:49:22.836 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:22.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[216903]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[216903]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[216903]: pgmap v1321: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:22.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[216903]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[216903]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[223858]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[223858]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[223858]: pgmap v1321: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:22.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[223858]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:22 smithi029 ceph-mon[223858]: failed to encode map e339 with expected crc 2023-10-09T04:49:22.839 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:49:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:49:22] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2023-10-09T04:49:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:22.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:22.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:22.972 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:23.422 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:23.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:23.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:23.558 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:23.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:23 smithi029 conmon[216878]: 2023-10-09T04:49:23.636+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e339 definitely_dead 0 2023-10-09T04:49:23.862 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:23.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:23.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:23.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:23.995 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3013613360' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T04:49:23.995 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:23 smithi177 ceph-mon[195797]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T04:49:23.998 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:24.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:24.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:24.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:24.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3013613360' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T04:49:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:23 smithi029 ceph-mon[216903]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T04:49:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3013613360' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T04:49:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:23 smithi029 ceph-mon[223858]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T04:49:24.182 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:24.232 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:24.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:24.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:49:24.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:24.400 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:24.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:49:24.449 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.2. 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3013613360' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: osdmap e340: 8 total, 8 up, 4 in 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.723 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[216903]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3013613360' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: osdmap e340: 8 total, 8 up, 4 in 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.724 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:24 smithi029 ceph-mon[223858]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.814 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:24.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:24.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:24.947 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:24.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:24.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 5.0 GiB used, 442 GiB / 447 GiB avail 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3013613360' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: osdmap e340: 8 total, 8 up, 4 in 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:24 smithi177 ceph-mon[195797]: failed to encode map e340 with expected crc 2023-10-09T04:49:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:24.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:25.099 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:25.129 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: failed to encode map e341 with expected crc 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: osdmap e341: 8 total, 8 up, 4 in 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: failed to encode map e341 with expected crc 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: failed to encode map e341 with expected crc 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: failed to encode map e341 with expected crc 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: failed to encode map e341 with expected crc 2023-10-09T04:49:25.710 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[223858]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: osdmap e341: 8 total, 8 up, 4 in 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:25 smithi177 ceph-mon[195797]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: osdmap e341: 8 total, 8 up, 4 in 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:25 smithi029 ceph-mon[216903]: failed to encode map e341 with expected crc 2023-10-09T04:49:26.178 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:26.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:26.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:49:26.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:26.342 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:26.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:26.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:26.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:26.502 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:26.522 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:26.529 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:26.539 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:26.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:26.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:26.638 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:26.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:26.705 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:26.717 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-10-09T04:49:26.751 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:26.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:26.832 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: pgmap v1325: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 20 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:49:26.832 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T04:49:26.832 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: failed to encode map e342 with expected crc 2023-10-09T04:49:26.832 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: failed to encode map e342 with expected crc 2023-10-09T04:49:26.833 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: osdmap e342: 8 total, 8 up, 4 in 2023-10-09T04:49:26.833 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2792726116' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:49:26.833 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:49:26.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:26.883 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:26.959 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: pgmap v1325: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 20 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: failed to encode map e342 with expected crc 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: failed to encode map e342 with expected crc 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: osdmap e342: 8 total, 8 up, 4 in 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2792726116' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: pgmap v1325: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 20 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-09T04:49:26.960 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: failed to encode map e342 with expected crc 2023-10-09T04:49:26.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: failed to encode map e342 with expected crc 2023-10-09T04:49:26.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: osdmap e342: 8 total, 8 up, 4 in 2023-10-09T04:49:26.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2792726116' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:49:26.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:49:26.982 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:26.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:27.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:27.113 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:27.840 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:27.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:27.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e343 with expected crc 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: osdmap e343: 8 total, 8 up, 4 in 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e343 with expected crc 2023-10-09T04:49:27.950 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[216903]: failed to encode map e343 with expected crc 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e343 with expected crc 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: osdmap e343: 8 total, 8 up, 4 in 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e343 with expected crc 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e342 with expected crc 2023-10-09T04:49:27.952 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:27 smithi029 ceph-mon[223858]: failed to encode map e343 with expected crc 2023-10-09T04:49:27.978 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e342 with expected crc 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e342 with expected crc 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e342 with expected crc 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e343 with expected crc 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: osdmap e343: 8 total, 8 up, 4 in 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e343 with expected crc 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:28.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e342 with expected crc 2023-10-09T04:49:28.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:27 smithi177 ceph-mon[195797]: failed to encode map e343 with expected crc 2023-10-09T04:49:28.547 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:49:28.547 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:49:28.580 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:28.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:28.684 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:28.713 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:28.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:28.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:28.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:28.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: pgmap v1328: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 30 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T04:49:28.886 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: osdmap e344: 8 total, 8 up, 4 in 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:28.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: pgmap v1328: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 30 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: osdmap e344: 8 total, 8 up, 4 in 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: failed to encode map e344 with expected crc 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:28.888 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:28.889 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:28.953 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-10-09T04:49:28.956 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:49:28.956 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:49:29.003 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:49:29.003 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:49:29.003 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_495570 2023-10-09T04:49:29.003 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:49:29.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:49:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: pgmap v1328: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 30 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T04:49:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: osdmap e344: 8 total, 8 up, 4 in 2023-10-09T04:49:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: failed to encode map e344 with expected crc 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: failed to encode map e344 with expected crc 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: failed to encode map e344 with expected crc 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: failed to encode map e344 with expected crc 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: failed to encode map e344 with expected crc 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:29.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:49:29.414 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:29.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:29.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:29.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:29.562 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:29.625 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:29.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:29.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:29.796 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:29.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:29.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:29.973 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:30.002 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:49:30.003 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 2776 2760 172.491 172.5 0.00641077 0.00577465 2023-10-09T04:49:30.025 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:49:30.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:30.048 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:49:30.071 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:49:30.099 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:30.100 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:49:30.137 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:49:30.164 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:49:30.190 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:49:30.192 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.192 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.192 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.192 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.193 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.193 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.193 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.193 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:49:30.255 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:30.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:30.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:30.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:30.363 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2499757240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:30.391 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:30.391 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2499757240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:30.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:30.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2499757240' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:30.397 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:30.459 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:30.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:30.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:30.598 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:30.652 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:30.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:30.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:30.755 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:30.785 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:30.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:30.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:30.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:30.960 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:31.001 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:31.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 4590 4574 142.924 113.375 0.00149618 0.00585836 2023-10-09T04:49:31.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:31.049 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7] out_osds: [6, 3, 0, 2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:49:31.049 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:49:31.049 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:49:31.049 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-09T04:49:31.049 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T04:49:31.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:31.140 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:31.217 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:49:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:31 smithi177 ceph-mon[195797]: pgmap v1330: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 26 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T04:49:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/596414459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:49:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:31 smithi029 ceph-mon[216903]: pgmap v1330: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 26 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T04:49:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/596414459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:49:31.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:31 smithi029 ceph-mon[223858]: pgmap v1330: 89 pgs: 1 remapped+peering, 28 peering, 60 active+clean; 582 KiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 26 B/s, 2 keys/s, 0 objects/s recovering 2023-10-09T04:49:31.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/596414459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:49:31.600 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:31.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:31.705 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:31.731 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:31.773 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:31.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:31.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:31.901 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:31.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:31.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:31.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:32.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 6618 6602 137.528 126.75 0.0043976 0.00726142 2023-10-09T04:49:32.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:32.114 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:32.337 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:32.339 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:32.339 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[216903]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.339 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/596414459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[216903]: osdmap e345: 8 total, 8 up, 4 in 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[216903]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[216903]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[223858]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/596414459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[223858]: osdmap e345: 8 total, 8 up, 4 in 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[223858]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:32 smithi029 ceph-mon[223858]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:32.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:32.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:32.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:32 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:32.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:32 smithi177 ceph-mon[195797]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/596414459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:49:32.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:32 smithi177 ceph-mon[195797]: osdmap e345: 8 total, 8 up, 4 in 2023-10-09T04:49:32.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:32 smithi177 ceph-mon[195797]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.490 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:32 smithi177 ceph-mon[195797]: failed to encode map e345 with expected crc 2023-10-09T04:49:32.496 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:32.554 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:32.607 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:32.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:32.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:32.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:32.904 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:32.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:32.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:33.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 9089 9073 141.752 154.438 0.00564175 0.00704719 2023-10-09T04:49:33.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:33.026 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:49:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:49:32] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.43.0" 2023-10-09T04:49:33.043 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:33.255 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:33.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:33.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:33.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:33.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:33 smithi029 ceph-mon[223858]: pgmap v1332: 89 pgs: 89 active+clean; 128 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 23 MiB/s wr, 361 op/s; 103 KiB/s, 2 objects/s recovering 2023-10-09T04:49:33.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:33 smithi029 ceph-mon[223858]: failed to encode map e345 with expected crc 2023-10-09T04:49:33.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:33 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T04:49:33.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:33 smithi029 ceph-mon[216903]: pgmap v1332: 89 pgs: 89 active+clean; 128 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 23 MiB/s wr, 361 op/s; 103 KiB/s, 2 objects/s recovering 2023-10-09T04:49:33.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:33 smithi029 ceph-mon[216903]: failed to encode map e345 with expected crc 2023-10-09T04:49:33.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:33 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T04:49:33.424 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:33.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:33 smithi177 ceph-mon[195797]: pgmap v1332: 89 pgs: 89 active+clean; 128 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 23 MiB/s wr, 361 op/s; 103 KiB/s, 2 objects/s recovering 2023-10-09T04:49:33.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:33 smithi177 ceph-mon[195797]: failed to encode map e345 with expected crc 2023-10-09T04:49:33.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:33 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2023-10-09T04:49:33.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:33.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:33.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:33.952 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:34.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 10730 10714 133.913 102.562 0.00628995 0.00745998 2023-10-09T04:49:34.095 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:49:34.214 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:34.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:34.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:34.358 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:34.427 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:34.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:34.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:49:34.474 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:49:34.474 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:49:34.475 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:49:34.475 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:49:34.475 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:49:34.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:34.572 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:34.644 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:34.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:34.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:34.749 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:34.759 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:34.813 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:34.930 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:34.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:35.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 12886 12870 134.05 134.75 0.00574067 0.00745034 2023-10-09T04:49:35.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:35.087 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:35.149 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:35.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:35.232 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T04:49:35.247 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:35.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:35.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:35 smithi029 ceph-mon[216903]: pgmap v1333: 89 pgs: 89 active+clean; 128 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 19 MiB/s wr, 307 op/s; 87 KiB/s, 1 objects/s recovering 2023-10-09T04:49:35.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2912276676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:35.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:35 smithi029 ceph-mon[223858]: pgmap v1333: 89 pgs: 89 active+clean; 128 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 19 MiB/s wr, 307 op/s; 87 KiB/s, 1 objects/s recovering 2023-10-09T04:49:35.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2912276676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:35.302 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:35.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:35 smithi177 ceph-mon[195797]: pgmap v1333: 89 pgs: 89 active+clean; 128 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 19 MiB/s wr, 307 op/s; 87 KiB/s, 1 objects/s recovering 2023-10-09T04:49:35.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2912276676' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:35.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:49:36.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 13515 13499 120.515 39.3125 0.0021343 0.00742259 2023-10-09T04:49:36.463 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:36.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2912276676' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:49:36.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:36 smithi177 ceph-mon[195797]: osdmap e346: 8 total, 8 up, 4 in 2023-10-09T04:49:36.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:36 smithi177 ceph-mon[195797]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:36 smithi177 ceph-mon[195797]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:36 smithi177 ceph-mon[195797]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2912276676' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:49:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[223858]: osdmap e346: 8 total, 8 up, 4 in 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[223858]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[223858]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[223858]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2912276676' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[216903]: osdmap e346: 8 total, 8 up, 4 in 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[216903]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[216903]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:36 smithi029 ceph-mon[216903]: failed to encode map e346 with expected crc 2023-10-09T04:49:36.809 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:36.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:36.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:36.948 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:37.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 15122 15106 118.004 100.438 0.00769962 0.00846716 2023-10-09T04:49:37.072 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:37.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:37.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:37.213 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:37.260 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:37.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:37.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:37.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[216903]: pgmap v1335: 89 pgs: 89 active+clean; 767 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 100 MiB/s wr, 1.60k op/s; 76 KiB/s, 1 objects/s recovering 2023-10-09T04:49:37.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[216903]: osdmap e347: 8 total, 8 up, 4 in 2023-10-09T04:49:37.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[216903]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[216903]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[216903]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[216903]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[223858]: pgmap v1335: 89 pgs: 89 active+clean; 767 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 100 MiB/s wr, 1.60k op/s; 76 KiB/s, 1 objects/s recovering 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[223858]: osdmap e347: 8 total, 8 up, 4 in 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[223858]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[223858]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[223858]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:37 smithi029 ceph-mon[223858]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.409 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:37.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:37 smithi177 ceph-mon[195797]: pgmap v1335: 89 pgs: 89 active+clean; 767 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 100 MiB/s wr, 1.60k op/s; 76 KiB/s, 1 objects/s recovering 2023-10-09T04:49:37.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:37 smithi177 ceph-mon[195797]: osdmap e347: 8 total, 8 up, 4 in 2023-10-09T04:49:37.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:37 smithi177 ceph-mon[195797]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:37 smithi177 ceph-mon[195797]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:37 smithi177 ceph-mon[195797]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:37 smithi177 ceph-mon[195797]: failed to encode map e347 with expected crc 2023-10-09T04:49:37.649 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:49:37.672 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:49:37.697 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:49:37.721 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:49:37.755 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:49:37.774 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:37.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:37.789 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:49:37.820 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:49:37.844 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:49:37.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:37.909 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:38.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 16831 16815 116.76 106.812 0.00531174 0.00855895 2023-10-09T04:49:38.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:38.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:38.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:38.303 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:38.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3161519900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3161519900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:38.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3161519900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:39.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 18363 18347 114.658 95.75 0.00487966 0.00871489 2023-10-09T04:49:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:39 smithi029 ceph-mon[216903]: pgmap v1337: 89 pgs: 89 active+clean; 767 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T04:49:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:39 smithi029 ceph-mon[223858]: pgmap v1337: 89 pgs: 89 active+clean; 767 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T04:49:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:39.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:39 smithi177 ceph-mon[195797]: pgmap v1337: 89 pgs: 89 active+clean; 767 MiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-09T04:49:39.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:39.543 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:49:39.544 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:49:39.954 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:39.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:39.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:40.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 20344 20328 115.489 123.812 0.00872197 0.00865351 2023-10-09T04:49:40.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:40.095 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:40.153 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:40.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:40.257 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:40.322 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:40.441 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:40.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:40.551 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:40.589 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:40.647 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:40.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:40.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:40.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:40.783 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:40.837 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:40.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:40.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:40.972 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:41.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 22322 22306 116.167 123.625 0.00624793 0.00860165 2023-10-09T04:49:41.028 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:41.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:41.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:41.187 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:41.350 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:41.360 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:49:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:41.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:41.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:41 smithi177 ceph-mon[195797]: pgmap v1338: 89 pgs: 89 active+clean; 824 MiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:41.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2350993893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:41.452 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:41 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:41.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:41.549 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:41.661 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:49:41.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:41 smithi029 ceph-mon[216903]: pgmap v1338: 89 pgs: 89 active+clean; 824 MiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:41.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2350993893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:41.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:41 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:41 smithi029 ceph-mon[223858]: pgmap v1338: 89 pgs: 89 active+clean; 824 MiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2350993893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:41 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:49:41.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:41.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:41.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:41.801 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:41.872 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:41.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:41.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:41.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:42.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 23961 23945 115.11 102.438 0.00983495 0.00868181 2023-10-09T04:49:42.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:49:42.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:42.096 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7] out_osds: [6, 3, 0, 2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:49:42.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T04:49:42.105 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:42.142 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:42.210 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:42.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:42.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: osdmap e348: 8 total, 8 up, 4 in 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:42 smithi177 ceph-mon[195797]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:42.355 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: osdmap e348: 8 total, 8 up, 4 in 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[216903]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: osdmap e348: 8 total, 8 up, 4 in 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: failed to encode map e348 with expected crc 2023-10-09T04:49:42.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:42 smithi029 ceph-mon[223858]: failed to encode map e348 with expected crc 2023-10-09T04:49:43.004 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 25713 25697 114.709 109.5 0.0113543 0.00871337 2023-10-09T04:49:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:49:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:49:42] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.43.0" 2023-10-09T04:49:43.210 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:43.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:43.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:43.374 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:43.432 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:43.492 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:43.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:43.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:43.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:43 smithi177 ceph-mon[195797]: pgmap v1339: 89 pgs: 89 active+clean; 1.2 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 134 MiB/s wr, 2.15k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:43.569 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:43.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:43.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:43 smithi029 ceph-mon[216903]: pgmap v1339: 89 pgs: 89 active+clean; 1.2 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 134 MiB/s wr, 2.15k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:43.627 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:43 smithi029 ceph-mon[223858]: pgmap v1339: 89 pgs: 89 active+clean; 1.2 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 134 MiB/s wr, 2.15k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:43.630 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:43.692 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:43.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:43.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:43.873 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:44.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 27421 27405 114.178 106.75 0.00996217 0.00875229 2023-10-09T04:49:44.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:44.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:44.115 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:44.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:44 smithi029 ceph-mon[216903]: pgmap v1341: 89 pgs: 89 active+clean; 1.3 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:44 smithi029 ceph-mon[223858]: pgmap v1341: 89 pgs: 89 active+clean; 1.3 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:44.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:44 smithi177 ceph-mon[195797]: pgmap v1341: 89 pgs: 89 active+clean; 1.3 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s; 16 KiB/s, 0 objects/s recovering 2023-10-09T04:49:44.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:44.952 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:45.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 29095 29079 113.58 104.625 0.00907831 0.0087999 2023-10-09T04:49:45.315 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:49:45.316 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:49:45.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:49:45.378 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:45.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:45.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:45.556 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:45.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:45.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:45.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:45.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:45.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:45.915 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:45.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:46.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 30448 30432 111.872 84.5625 0.00363664 0.00886956 2023-10-09T04:49:46.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:46.044 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:46.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:46.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:46.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:46.189 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:46.244 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:46.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:46.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:46.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:46.429 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:46.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:46.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:46.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:46.575 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:46.610 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:46.660 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:46 smithi177 ceph-mon[195797]: pgmap v1342: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 108 MiB/s wr, 1.73k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T04:49:46.773 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:46.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:46.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:46.878 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:46.915 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:46 smithi029 ceph-mon[216903]: pgmap v1342: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 108 MiB/s wr, 1.73k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T04:49:46.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:46 smithi029 ceph-mon[223858]: pgmap v1342: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 108 MiB/s wr, 1.73k op/s; 14 KiB/s, 0 objects/s recovering 2023-10-09T04:49:46.919 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:46.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:49:46.992 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:47.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:47.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 31373 31357 108.869 57.8125 0.0110386 0.00918053 2023-10-09T04:49:47.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:47.170 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:47.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:47.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:47.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:47.429 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:47.432 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:47.475 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:47.550 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:49:47.574 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:49:47.590 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:47.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:47.601 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:49:47.626 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:49:47.649 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:49:47.662 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:47.678 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T04:49:47.681 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:49:47.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:49:47.711 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:49:47.735 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:47.760 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.762 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.763 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:49:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3596081505' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/264790626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3596081505' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/264790626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:47.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3596081505' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:47.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T04:49:47.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/264790626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:48.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 32678 32662 107.431 81.5625 0.00549803 0.00930288 2023-10-09T04:49:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[223858]: pgmap v1343: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 99 MiB/s wr, 1.58k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T04:49:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[223858]: failed to encode map e349 with expected crc 2023-10-09T04:49:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:49:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[223858]: osdmap e349: 8 total, 8 up, 4 in 2023-10-09T04:49:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[223858]: failed to encode map e349 with expected crc 2023-10-09T04:49:48.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[223858]: failed to encode map e349 with expected crc 2023-10-09T04:49:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[216903]: pgmap v1343: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 99 MiB/s wr, 1.58k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T04:49:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[216903]: failed to encode map e349 with expected crc 2023-10-09T04:49:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:49:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[216903]: osdmap e349: 8 total, 8 up, 4 in 2023-10-09T04:49:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[216903]: failed to encode map e349 with expected crc 2023-10-09T04:49:48.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:48 smithi029 ceph-mon[216903]: failed to encode map e349 with expected crc 2023-10-09T04:49:49.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:49:49.005607+0000 min lat: 0.0013453 max lat: 1.02767 avg lat: 0.00943978 2023-10-09T04:49:49.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:49:49.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 33798 33782 105.559 70 0.00211897 0.00943978 2023-10-09T04:49:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:48 smithi177 ceph-mon[195797]: pgmap v1343: 89 pgs: 89 active+clean; 1.7 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 99 MiB/s wr, 1.58k op/s; 13 KiB/s, 0 objects/s recovering 2023-10-09T04:49:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:48 smithi177 ceph-mon[195797]: failed to encode map e349 with expected crc 2023-10-09T04:49:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T04:49:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:48 smithi177 ceph-mon[195797]: osdmap e349: 8 total, 8 up, 4 in 2023-10-09T04:49:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:48 smithi177 ceph-mon[195797]: failed to encode map e349 with expected crc 2023-10-09T04:49:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:48 smithi177 ceph-mon[195797]: failed to encode map e349 with expected crc 2023-10-09T04:49:49.467 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T04:49:49.506 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:49:49.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:49.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:49:49.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:49.638 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:49.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:49.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:49.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:49.880 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:50.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 34372 34356 102.241 35.875 0.00382139 0.00971696 2023-10-09T04:49:50.359 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:49:50.359 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:49:50.359 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:49:50.359 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:49:50.359 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:49:50.674 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:50 smithi177 ceph-mon[195797]: pgmap v1345: 89 pgs: 89 active+clean; 1.8 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:49:50.763 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:50.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:50.867 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:50.912 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:50.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:50.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:51.004 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:51.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 35519 35503 100.851 71.6875 0.00877291 0.00985896 2023-10-09T04:49:51.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:51.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:50 smithi029 ceph-mon[216903]: pgmap v1345: 89 pgs: 89 active+clean; 1.8 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:49:51.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:50 smithi029 ceph-mon[223858]: pgmap v1345: 89 pgs: 89 active+clean; 1.8 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-09T04:49:51.091 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:51.360 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:51.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:51.464 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:51.510 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:51.544 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:49:51.569 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:51.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:51.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:51.753 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:51.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:51.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:52.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:52.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 36766 36750 99.8549 77.9375 0.0123764 0.0100083 2023-10-09T04:49:52.032 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:52.084 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:52.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:52.162 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:52.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:52.214 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:52.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:52.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:52.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:52.461 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:52.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:52.699 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:49:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:52.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:52.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:52 smithi177 ceph-mon[195797]: pgmap v1346: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T04:49:52.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3708435836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:52.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:52.834 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:52.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:49:52.940 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:49:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:49:52] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T04:49:52.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:52 smithi029 ceph-mon[223858]: pgmap v1346: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T04:49:52.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3708435836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:52.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:52 smithi029 ceph-mon[216903]: pgmap v1346: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-10-09T04:49:52.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3708435836' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:49:53.005 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 37816 37800 98.4284 65.625 0.00699286 0.0101555 2023-10-09T04:49:53.051 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:53.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:53.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:53.188 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:53.762 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:53.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:53.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:53.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:49:53.900 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3708435836' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: osdmap e350: 8 total, 8 up, 4 in 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[216903]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3708435836' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: osdmap e350: 8 total, 8 up, 4 in 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: failed to encode map e350 with expected crc 2023-10-09T04:49:53.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:53 smithi029 ceph-mon[223858]: failed to encode map e350 with expected crc 2023-10-09T04:49:54.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 38824 38808 97.0111 63 0.0112681 0.0103024 2023-10-09T04:49:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:49:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: failed to encode map e350 with expected crc 2023-10-09T04:49:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3708435836' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:49:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: osdmap e350: 8 total, 8 up, 4 in 2023-10-09T04:49:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: failed to encode map e350 with expected crc 2023-10-09T04:49:54.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: failed to encode map e350 with expected crc 2023-10-09T04:49:54.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:53 smithi177 ceph-mon[195797]: failed to encode map e350 with expected crc 2023-10-09T04:49:54.130 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:49:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:54.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:49:54.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:54.269 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:54.329 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:54.382 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:54.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:54.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:54.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:54.558 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:54.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:54.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:54.706 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:49:54.706 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:49:54.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.45869.0:145940 11.7 11:ed08b1b0:::benchmark_data_smithi029_392508_object45384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e261)", 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:41:01.860794+0000", 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 532.840663221, 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90692854300000003, 2023-10-09T04:49:54.707 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.45869", 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 145940 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.708 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860794+0000", 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860794+0000", 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860792+0000", 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860800+0000", 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7330000000000003e-06 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860818+0000", 2023-10-09T04:49:54.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8258999999999999e-05 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860826+0000", 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7470000000000005e-06 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860892+0000", 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5526999999999998e-05 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:01.860904+0000", 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2452999999999999e-05 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.757260+0000", 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89635559200000003 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.757432+0000", 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017239499999999999 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.757601+0000", 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016914299999999999 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.763593+0000", 2023-10-09T04:49:54.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059920989999999999 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.766613+0000", 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030199979999999999 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.766746+0000", 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000133084 2023-10-09T04:49:54.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.766789+0000", 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2710000000000003e-05 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:41:02.767722+0000", 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00093332400000000002 2023-10-09T04:49:54.715 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122037 12.c 12:351c053a:::benchmark_data_smithi029_442458_object26787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.129316+0000", 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 210.57214137099999, 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.432926562, 2023-10-09T04:49:54.716 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122037 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.717 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129316+0000", 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129316+0000", 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129314+0000", 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129320+0000", 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5210000000000002e-06 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129331+0000", 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1483e-05 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129337+0000", 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2429999999999996e-06 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129388+0000", 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1019000000000001e-05 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129534+0000", 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000146379 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.129661+0000", 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126556 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.132910+0000", 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032498010000000001 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.146328+0000", 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013417584 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159582+0000", 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013253811000000001 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159593+0000", 2023-10-09T04:49:54.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0739e-05 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.149117+0000", 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98952476700000003 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.149129+0000", 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2018000000000001e-05 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.524766+0000", 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3756364050000001 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.524902+0000", 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013640399999999999 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.562127+0000", 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.037225095999999999 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.562242+0000", 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000114868 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.726 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122044 12.c 12:3399bf74:::benchmark_data_smithi029_442458_object26794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.149660+0000", 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 210.55179673800001, 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4611223089999998, 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122044 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149660+0000", 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149660+0000", 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149659+0000", 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149664+0000", 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3359999999999997e-06 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149677+0000", 2023-10-09T04:49:54.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2931e-05 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149685+0000", 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1640000000000001e-06 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149735+0000", 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9997000000000003e-05 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.149890+0000", 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015476199999999999 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.150010+0000", 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120862 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159617+0000", 2023-10-09T04:49:54.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096066729999999996 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.159623+0000", 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1959999999999996e-06 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.149166+0000", 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98954217300000002 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.149176+0000", 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0448000000000001e-05 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528665+0000", 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3794887229999999 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528815+0000", 2023-10-09T04:49:54.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000150085 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.610710+0000", 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081895361999999999 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.610783+0000", 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2428999999999994e-05 2023-10-09T04:49:54.734 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122082 12.c 12:37278ac7:::benchmark_data_smithi029_442458_object26832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:25.485919+0000", 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 209.215538439, 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.118586764, 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.735 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122082 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.485919+0000", 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.485919+0000", 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.485917+0000", 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.485928+0000", 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0903999999999999e-05 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.485951+0000", 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3654000000000001e-05 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.485961+0000", 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8030000000000008e-06 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.486029+0000", 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7744999999999996e-05 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.486045+0000", 2023-10-09T04:49:54.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6492999999999999e-05 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528849+0000", 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0428038399999999 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528994+0000", 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000144518 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.529640+0000", 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00064675099999999999 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.550066+0000", 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020425322999999999 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.553973+0000", 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039075159999999998 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.600501+0000", 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.046528093 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.600577+0000", 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6116000000000001e-05 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.604505+0000", 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039279079999999999 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187694 12.c 12:314f2354:::benchmark_data_smithi029_442458_object92444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.109140+0000", 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.592317309999999, 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0346555909999999, 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187694 2023-10-09T04:49:54.744 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109140+0000", 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109140+0000", 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.745 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109138+0000", 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109145+0000", 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4239999999999998e-06 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109158+0000", 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3264999999999999e-05 2023-10-09T04:49:54.746 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109163+0000", 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8679999999999998e-06 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109239+0000", 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5788e-05 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.747 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109249+0000", 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0169e-05 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.135498+0000", 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0262491389999999 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.135590+0000", 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2022999999999996e-05 2023-10-09T04:49:54.748 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.135691+0000", 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010091599999999999 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138806+0000", 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031145320000000002 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.749 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143694+0000", 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004888674 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143736+0000", 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1980000000000001e-05 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143769+0000", 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3108000000000002e-05 2023-10-09T04:49:54.750 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143795+0000", 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5899999999999999e-05 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187700 12.c 12:3cbc49fb:::benchmark_data_smithi029_442458_object92450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.122655+0000", 2023-10-09T04:49:54.751 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.578801763000001, 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0213057059999999, 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187700 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122655+0000", 2023-10-09T04:49:54.752 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122655+0000", 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122653+0000", 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.753 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.754 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122662+0000", 2023-10-09T04:49:54.754 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7339999999999997e-06 2023-10-09T04:49:54.754 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.754 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122679+0000", 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6849000000000001e-05 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122686+0000", 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5719999999999999e-06 2023-10-09T04:49:54.755 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122746+0000", 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0888999999999999e-05 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122761+0000", 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4823e-05 2023-10-09T04:49:54.756 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138823+0000", 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.016062021 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138896+0000", 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2527000000000005e-05 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.757 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138981+0000", 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5233000000000002e-05 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143831+0000", 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048499040000000004 2023-10-09T04:49:54.758 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143848+0000", 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6872e-05 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143909+0000", 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1285999999999996e-05 2023-10-09T04:49:54.759 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143935+0000", 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6083000000000001e-05 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143961+0000", 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5778e-05 2023-10-09T04:49:54.760 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187715 12.c 12:36b39d4a:::benchmark_data_smithi029_442458_object92465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.135898+0000", 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.565559381, 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.014459099, 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.761 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187715 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135898+0000", 2023-10-09T04:49:54.762 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135898+0000", 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135896+0000", 2023-10-09T04:49:54.763 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135905+0000", 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7730000000000005e-06 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135921+0000", 2023-10-09T04:49:54.764 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6716000000000001e-05 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135929+0000", 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.362e-06 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135983+0000", 2023-10-09T04:49:54.765 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4255999999999997e-05 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135993+0000", 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8509999999999994e-06 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.141569+0000", 2023-10-09T04:49:54.766 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0055756890000001 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.141650+0000", 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1264e-05 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.141717+0000", 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6793999999999998e-05 2023-10-09T04:49:54.767 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144008+0000", 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022912869999999999 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.146873+0000", 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002865126 2023-10-09T04:49:54.768 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150256+0000", 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033833079999999998 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150312+0000", 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5896000000000002e-05 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.769 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150357+0000", 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4393e-05 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187724 12.c 12:3953830f:::benchmark_data_smithi029_442458_object92474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.770 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.152662+0000", 2023-10-09T04:49:54.771 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.548794728000004, 2023-10-09T04:49:54.771 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0087711610000001, 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187724 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.773 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152662+0000", 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152662+0000", 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.774 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152660+0000", 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152671+0000", 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0864e-05 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.775 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152687+0000", 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6359999999999999e-05 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152694+0000", 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6409999999999996e-06 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.776 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152755+0000", 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0846e-05 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152768+0000", 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3054000000000001e-05 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.777 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144036+0000", 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99126849699999997 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144124+0000", 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7875999999999997e-05 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.778 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144251+0000", 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012712699999999999 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150409+0000", 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061577849999999998 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.779 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150416+0000", 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1930000000000004e-06 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161346+0000", 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010929179000000001 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.780 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161391+0000", 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5664999999999997e-05 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161433+0000", 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2184000000000001e-05 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.781 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187752 12.c 12:33babd0f:::benchmark_data_smithi029_442458_object92502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.196820+0000", 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.504636778999995, 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.964821496, 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.782 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187752 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196820+0000", 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.783 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196820+0000", 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196819+0000", 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.784 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196827+0000", 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6640000000000004e-06 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196843+0000", 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5953000000000001e-05 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.785 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196851+0000", 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0090000000000007e-06 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196921+0000", 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9896999999999998e-05 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.786 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196936+0000", 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4489e-05 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.146966+0000", 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95003068899999998 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.147089+0000", 2023-10-09T04:49:54.787 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000122676 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.147221+0000", 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013162199999999999 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150454+0000", 2023-10-09T04:49:54.788 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003233194 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153417+0000", 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029634829999999998 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161534+0000", 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081165960000000002 2023-10-09T04:49:54.789 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161583+0000", 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9410999999999998e-05 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161642+0000", 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8391999999999999e-05 2023-10-09T04:49:54.790 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187760 12.c 12:31175804:::benchmark_data_smithi029_442458_object92510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.207903+0000", 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.493554435999997, 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95389128000000001, 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.791 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187760 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207903+0000", 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.792 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207903+0000", 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207901+0000", 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.793 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207909+0000", 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9100000000000005e-06 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207930+0000", 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0650999999999999e-05 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.794 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207938+0000", 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4989999999999994e-06 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.208018+0000", 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9079000000000006e-05 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.795 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.208030+0000", 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2801e-05 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150486+0000", 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94245597299999995 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.796 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150580+0000", 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3826999999999995e-05 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150697+0000", 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116768 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.797 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161318+0000", 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010620983000000001 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161443+0000", 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000125341 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161726+0000", 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028248599999999998 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161771+0000", 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5303999999999999e-05 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.799 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161794+0000", 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2821000000000001e-05 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187771 12.c 12:3ad3988e:::benchmark_data_smithi029_442458_object92521:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.225905+0000", 2023-10-09T04:49:54.800 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.475551992000007, 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94938117399999999, 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187771 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.801 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225905+0000", 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225905+0000", 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.802 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225904+0000", 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225911+0000", 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.199e-06 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225925+0000", 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.383e-05 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225931+0000", 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5119999999999997e-06 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225996+0000", 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5029000000000004e-05 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.226005+0000", 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4179999999999997e-06 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153442+0000", 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92743711900000003 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153541+0000", 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8963999999999996e-05 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.806 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153611+0000", 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0160999999999995e-05 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161449+0000", 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078379420000000005 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161495+0000", 2023-10-09T04:49:54.807 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6381999999999997e-05 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175187+0000", 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013692205000000001 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175241+0000", 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3088999999999999e-05 2023-10-09T04:49:54.808 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175286+0000", 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5627000000000002e-05 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.809 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187781 12.c 12:3456af2d:::benchmark_data_smithi029_442458_object92531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.245437+0000", 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.456019917000006, 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93002534800000003, 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187781 2023-10-09T04:49:54.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245437+0000", 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245437+0000", 2023-10-09T04:49:54.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245436+0000", 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245442+0000", 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0769999999999996e-06 2023-10-09T04:49:54.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245453+0000", 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1226000000000001e-05 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245458+0000", 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0050000000000004e-06 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245518+0000", 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0186999999999998e-05 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.245530+0000", 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1548e-05 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.157015+0000", 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91148486699999998 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.157125+0000", 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109753 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.157671+0000", 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00054609700000000005 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161681+0000", 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040099790000000003 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.164592+0000", 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029111900000000001 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175383+0000", 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010790867000000001 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175427+0000", 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4075000000000001e-05 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175462+0000", 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5607e-05 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.818 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187787 12.c 12:31fd324d:::benchmark_data_smithi029_442458_object92537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.253507+0000", 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.447950000999995, 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92204995700000003, 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.819 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187787 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253507+0000", 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253507+0000", 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253505+0000", 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253513+0000", 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2960000000000001e-06 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253529+0000", 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6532e-05 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253536+0000", 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7789999999999998e-06 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253596+0000", 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9481999999999997e-05 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.253606+0000", 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.023e-05 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161823+0000", 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90821681899999995 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161910+0000", 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7837000000000006e-05 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161984+0000", 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3453000000000003e-05 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175123+0000", 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013138963 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175300+0000", 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017679499999999999 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.826 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175494+0000", 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019473 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175523+0000", 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9054e-05 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175557+0000", 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3494999999999997e-05 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187798 12.c 12:325007c3:::benchmark_data_smithi029_442458_object92548:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.284929+0000", 2023-10-09T04:49:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.416527861999995, 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89072974800000004, 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187798 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.284929+0000", 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.284929+0000", 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.284928+0000", 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.284934+0000", 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3409999999999999e-06 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.284947+0000", 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3098999999999999e-05 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.284952+0000", 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9309999999999999e-06 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.285011+0000", 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9372e-05 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.285022+0000", 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0365e-05 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.164641+0000", 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.87961912499999995 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.164762+0000", 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000121259 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.164892+0000", 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000130333 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175151+0000", 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010258376 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.835 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175309+0000", 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015778499999999999 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175596+0000", 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028718199999999998 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175623+0000", 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6798999999999999e-05 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175659+0000", 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6242999999999998e-05 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187861 12.c 12:34968a7d:::benchmark_data_smithi029_442458_object92611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.394304+0000", 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.307153213999996, 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79891335399999996, 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.838 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187861 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394304+0000", 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.839 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394304+0000", 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394303+0000", 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394309+0000", 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4640000000000003e-06 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394322+0000", 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2826000000000001e-05 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394328+0000", 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6670000000000001e-06 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394387+0000", 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9735000000000003e-05 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.394398+0000", 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0295000000000001e-05 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.168157+0000", 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.77375899400000003 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.843 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.168277+0000", 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120226 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.168404+0000", 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000127019 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.844 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175317+0000", 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069127019999999997 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175343+0000", 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6787999999999998e-05 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.193129+0000", 2023-10-09T04:49:54.845 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017785266000000001 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.193178+0000", 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9922000000000001e-05 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.193217+0000", 2023-10-09T04:49:54.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8637000000000003e-05 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187908 12.c 12:3d418517:::benchmark_data_smithi029_442458_object92658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.486333+0000", 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.215123707000004, 2023-10-09T04:49:54.847 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.706987213, 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187908 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.848 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486333+0000", 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486333+0000", 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486332+0000", 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486339+0000", 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6510000000000001e-06 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486352+0000", 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2872e-05 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486359+0000", 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4269999999999996e-06 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.851 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486419+0000", 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0387000000000003e-05 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.486428+0000", 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3349999999999996e-06 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.171590+0000", 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68516160999999998 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.171705+0000", 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000115482 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:49:54.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.171817+0000", 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011176899999999999 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175474+0000", 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036572050000000002 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.186401+0000", 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010926938000000001 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.193254+0000", 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068523070000000002 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.193289+0000", 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5126999999999998e-05 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.193321+0000", 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1789999999999999e-05 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.856 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:33788 13.8 13:1198b971:::benchmark_data_smithi029_495570_object33787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:49:48.940136+0000", 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 5.7613211629999999, 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39140339299999999, 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:49:54.857 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 33788 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.940136+0000", 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.940136+0000", 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.940134+0000", 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941613+0000", 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014783610000000001 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941634+0000", 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0874999999999999e-05 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941642+0000", 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2700000000000004e-06 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941745+0000", 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103523 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941856+0000", 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110827 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942037+0000", 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018027399999999999 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944821+0000", 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027848980000000001 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944833+0000", 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1589e-05 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.331444+0000", 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38661129700000002 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.331494+0000", 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9428999999999997e-05 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.331539+0000", 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5439000000000003e-05 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:33790 13.8 13:16cd0853:::benchmark_data_smithi029_495570_object33789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:49:48.941656+0000", 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 5.7598012409999999, 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39098203599999998, 2023-10-09T04:49:54.865 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 33790 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941656+0000", 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941656+0000", 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941655+0000", 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941692+0000", 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7580999999999998e-05 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941704+0000", 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1348e-05 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.941709+0000", 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5570000000000003e-06 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.943327+0000", 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001617486 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.943436+0000", 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109121 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.943571+0000", 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013471300000000001 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.945302+0000", 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017316160000000001 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.945457+0000", 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015479799999999999 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.332524+0000", 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38706712900000001 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.332568+0000", 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4221999999999998e-05 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.332638+0000", 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9480999999999994e-05 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:33793 13.2 13:45f32538:::benchmark_data_smithi029_495570_object33792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:49:48.942135+0000", 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 5.7593215730000002, 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38674726999999998, 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:49:54.874 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 33793 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942135+0000", 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942135+0000", 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942135+0000", 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942200+0000", 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5408999999999997e-05 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942214+0000", 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3869999999999999e-05 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942220+0000", 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.071e-06 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942269+0000", 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9230999999999997e-05 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942393+0000", 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123616 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.942516+0000", 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123213 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.945846+0000", 2023-10-09T04:49:54.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033299739999999999 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.947677+0000", 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001830641 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.328780+0000", 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38110354200000002 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.328838+0000", 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7525000000000002e-05 2023-10-09T04:49:54.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.328883+0000", 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4953000000000001e-05 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:33795 13.2 13:42267ff8:::benchmark_data_smithi029_495570_object33794:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T04:49:54.883 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:49:48.944100+0000", 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 5.7573565420000001, 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.388433625, 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 33795 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.884 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944100+0000", 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944100+0000", 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:49:54.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944099+0000", 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944378+0000", 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000279289 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944396+0000", 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7978e-05 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944402+0000", 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0460000000000002e-06 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944446+0000", 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3439999999999997e-05 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944566+0000", 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0001205 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944702+0000", 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013602800000000001 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.946084+0000", 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001381735 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.947815+0000", 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001731015 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.332448+0000", 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38463237500000003 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.332496+0000", 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8529000000000002e-05 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.332534+0000", 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7996e-05 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.891 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.892 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.892 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:49:54.892 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:49:54.892 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:49:54.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:54 smithi029 ceph-mon[216903]: pgmap v1348: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T04:49:54.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:54 smithi029 ceph-mon[223858]: pgmap v1348: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T04:49:54.949 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:54 smithi177 ceph-mon[195797]: pgmap v1348: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-09T04:49:54.957 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:55.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 40099 40083 96.3446 79.6875 0.0113517 0.0103753 2023-10-09T04:49:55.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:55.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:55.179 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:55.208 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:55.249 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:55.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:55.353 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:55.377 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:55.420 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:55.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:55.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:55.589 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:55.710 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T04:49:55.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/947106042' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:49:55.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:55 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:49:55.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:55.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:55.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:55.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:49:55.942 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:49:55.973 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/947106042' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:49:55.973 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:55 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:49:55.973 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/947106042' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:49:55.973 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:55 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T04:49:56.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 40869 40853 94.5585 48.125 0.00207548 0.010557 2023-10-09T04:49:56.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:49:56.057 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:56.547 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:49:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:56.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:49:56.623 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T04:49:56.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:56.664 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:49:56.728 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:56.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:56.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:56.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:56.848 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: pgmap v1349: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: osdmap e351: 8 total, 8 up, 4 in 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[216903]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: pgmap v1349: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: osdmap e351: 8 total, 8 up, 4 in 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:56 smithi029 ceph-mon[223858]: failed to encode map e351 with expected crc 2023-10-09T04:49:56.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:56.998 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:57.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 42007 41991 93.7214 71.125 0.0113316 0.010664 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: pgmap v1349: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: failed to encode map e351 with expected crc 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: osdmap e351: 8 total, 8 up, 4 in 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: failed to encode map e351 with expected crc 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: failed to encode map e351 with expected crc 2023-10-09T04:49:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:56 smithi177 ceph-mon[195797]: failed to encode map e351 with expected crc 2023-10-09T04:49:57.212 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:49:57.237 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:49:57.256 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:57.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:57.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:57.261 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:49:57.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:49:57.316 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:49:57.350 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:49:57.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:49:57.387 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:49:57.396 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:57.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:49:57.413 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.413 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.413 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.413 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.413 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.414 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.414 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.414 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:49:57.470 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:49:57.513 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:49:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:57.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:49:57.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:49:57.688 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:57.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:49:57.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:57.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:49:57.860 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:57.930 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:58.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 43230 43214 93.1251 76.4375 0.00615265 0.0107251 2023-10-09T04:49:58.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:58.008 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/647379940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:58.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:58.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:58.148 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/647379940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/647379940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:49:58.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:58.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:49:58.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:58.318 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:58.359 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:58.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:58.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:58.540 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:58.713 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:58 smithi177 ceph-mon[195797]: pgmap v1351: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:49:58.713 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:58.713 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:49:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:58.796 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:58.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:58.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:49:58.937 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:58.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:49:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:58.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:49:59.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 43884 43868 91.3832 40.875 0.00393414 0.0108814 2023-10-09T04:49:59.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:59.161 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:58 smithi029 ceph-mon[216903]: pgmap v1351: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:49:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:49:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:58 smithi029 ceph-mon[223858]: pgmap v1351: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-09T04:49:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:49:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:49:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:49:59.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:59.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:59.435 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:49:59.435 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:49:59.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:49:59.553 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:59.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:49:59.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:59.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:49:59.837 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:49:59.906 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:49:59.980 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:49:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:49:59.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:00.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 45009 44993 90.7032 70.3125 0.0123846 0.0110052 2023-10-09T04:50:00.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:00.116 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:00.641 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:50:00.769 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:00.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:00.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:00.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:00 smithi177 ceph-mon[195797]: pgmap v1352: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-09T04:50:00.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:00 smithi177 ceph-mon[195797]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T04:50:00.961 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:00 smithi177 ceph-mon[195797]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:50:00.962 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:01.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 45616 45600 89.0542 37.9375 0.015602 0.0112236 2023-10-09T04:50:01.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:01.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:01.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:01.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:00 smithi029 ceph-mon[216903]: pgmap v1352: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-09T04:50:01.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:00 smithi029 ceph-mon[216903]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T04:50:01.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:00 smithi029 ceph-mon[216903]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:50:01.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:00 smithi029 ceph-mon[223858]: pgmap v1352: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-09T04:50:01.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:00 smithi029 ceph-mon[223858]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-09T04:50:01.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:00 smithi029 ceph-mon[223858]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-09T04:50:01.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:01.238 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:01.299 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:01.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:01.397 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:50:01.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:01.438 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:01.596 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:01.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:01.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:50:01.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:01.744 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:01.746 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:50:01.816 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:01.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:01.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:01.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:01.956 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3450963597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:01.958 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3450963597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:01.997 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3450963597' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:02.004 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:02.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 46785 46769 88.5695 73.0625 0.0215386 0.0112861 2023-10-09T04:50:02.179 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:02.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:02.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:02.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:02.310 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:02.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:02.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:02.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:02.492 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:02.533 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:02.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:02.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:02.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:02.672 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:02.749 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: pgmap v1353: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: failed to encode map e352 with expected crc 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: failed to encode map e352 with expected crc 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3450963597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: osdmap e352: 8 total, 8 up, 4 in 2023-10-09T04:50:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: failed to encode map e352 with expected crc 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/207540488' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: pgmap v1353: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: failed to encode map e352 with expected crc 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: failed to encode map e352 with expected crc 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3450963597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: osdmap e352: 8 total, 8 up, 4 in 2023-10-09T04:50:02.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: failed to encode map e352 with expected crc 2023-10-09T04:50:02.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/207540488' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:02.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:02.923 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:50:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:50:02] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-10-09T04:50:03.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 47543 47527 87.3577 47.375 0.00897921 0.0114426 2023-10-09T04:50:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: pgmap v1353: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: failed to encode map e352 with expected crc 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: failed to encode map e352 with expected crc 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3450963597' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: osdmap e352: 8 total, 8 up, 4 in 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: failed to encode map e352 with expected crc 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/207540488' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:03.303 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:03.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:03.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:03.441 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:03.662 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:03.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:03.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T04:50:03.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:50:03.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:03 smithi177 ceph-mon[195797]: osdmap e353: 8 total, 8 up, 4 in 2023-10-09T04:50:03.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:03 smithi177 ceph-mon[195797]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:03 smithi177 ceph-mon[195797]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.761 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:03 smithi177 ceph-mon[195797]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:03.801 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:03.831 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[216903]: osdmap e353: 8 total, 8 up, 4 in 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[216903]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[216903]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[216903]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[223858]: osdmap e353: 8 total, 8 up, 4 in 2023-10-09T04:50:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[223858]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[223858]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:03 smithi029 ceph-mon[223858]: failed to encode map e353 with expected crc 2023-10-09T04:50:03.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:03.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:04.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 48408 48392 86.4063 54.0625 0.0128972 0.0115677 2023-10-09T04:50:04.095 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:04.186 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:04.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:04.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:04.357 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:50:04.357 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:50:04.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:04.452 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:04.508 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:04.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:04.612 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:04.641 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:04.686 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:04.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:04.789 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:04.816 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:04.860 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:04.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:04.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:04.991 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:05.006 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 49198 49182 85.3776 49.375 0.00337351 0.0116866 2023-10-09T04:50:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:04 smithi177 ceph-mon[195797]: pgmap v1356: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 47 MiB/s wr, 759 op/s 2023-10-09T04:50:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:04 smithi029 ceph-mon[216903]: pgmap v1356: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 47 MiB/s wr, 759 op/s 2023-10-09T04:50:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:04 smithi029 ceph-mon[223858]: pgmap v1356: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 47 MiB/s wr, 759 op/s 2023-10-09T04:50:05.390 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:05.397 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:05.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:05.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:05.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:50:05.506 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:05.540 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:50:05.591 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:05.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:05.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:05.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:05.929 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:06.007 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 50190 50174 84.7457 62 0.011538 0.0117944 2023-10-09T04:50:06.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:06.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:06.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:06.199 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:06.301 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:50:06.325 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:50:06.346 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:50:06.367 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:06.368 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:50:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:06.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:06.384 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:50:06.415 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:50:06.439 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:50:06.466 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:50:06.472 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:06.496 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:50:06.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.498 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.499 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:50:06.506 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:06.560 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:06.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:06.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:06.695 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:06.751 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:06.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:06.855 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:06.884 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:06.935 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:06.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:07.007 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 51318 51302 84.3706 70.5 0.0161549 0.0118476 2023-10-09T04:50:07.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:06 smithi177 ceph-mon[195797]: pgmap v1357: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T04:50:07.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4015447719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:07.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:07.116 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:06 smithi029 ceph-mon[216903]: pgmap v1357: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T04:50:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4015447719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:06 smithi029 ceph-mon[223858]: pgmap v1357: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-09T04:50:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4015447719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:07.250 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:07.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:07.355 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:07.426 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:07.581 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:07.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:07.690 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:07.717 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:07.946 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:07.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:08.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 52063 52047 83.3988 46.5625 0.00942159 0.0119853 2023-10-09T04:50:08.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:08.084 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:08.152 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:08.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:08.186 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:08.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:08.292 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:08.406 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:08.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:08.511 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:08.546 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:08.658 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:08.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:08.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:08.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:08.843 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:08 smithi029 ceph-mon[216903]: pgmap v1358: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:50:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:08 smithi029 ceph-mon[223858]: pgmap v1358: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:50:08.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:08.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:08.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:09.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:50:09.008525+0000 min lat: 0.0013453 max lat: 1.02767 avg lat: 0.0120343 2023-10-09T04:50:09.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:50:09.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 53164 53148 83.034 68.8125 0.00426552 0.0120343 2023-10-09T04:50:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:08 smithi177 ceph-mon[195797]: pgmap v1358: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-09T04:50:09.078 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:09.106 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:09.153 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:09.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:09.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:09.197 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:50:09.197 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:50:09.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:50:09.197 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:50:09.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e309/303 cost 1049576)", 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:27.799837+0000", 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.396548032, 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043397239999999997, 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799837+0000", 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799837+0000", 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799836+0000", 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799854+0000", 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8139e-05 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799876+0000", 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1835e-05 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.799884+0000", 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5089999999999993e-06 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.802185+0000", 2023-10-09T04:50:09.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023006260000000001 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:27.804177+0000", 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00199204 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e310/303 cost 1049576)", 2023-10-09T04:50:09.202 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:43.438763+0000", 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 205.75762247099999, 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065351089999999999, 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438763+0000", 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438763+0000", 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438761+0000", 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438778+0000", 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7088e-05 2023-10-09T04:50:09.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438801+0000", 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2906999999999999e-05 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438809+0000", 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5750000000000003e-06 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.438888+0000", 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9142999999999998e-05 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:43.445298+0000", 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006409756 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e311/303 cost 1049576)", 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:50.840780+0000", 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 198.355605134, 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081103370000000004, 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840780+0000", 2023-10-09T04:50:09.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840780+0000", 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840779+0000", 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840797+0000", 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8349999999999999e-05 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840818+0000", 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0732000000000001e-05 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840825+0000", 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4839999999999998e-06 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.840901+0000", 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5360999999999999e-05 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:50.848890+0000", 2023-10-09T04:50:09.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079897200000000005 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e312/303 cost 1049576)", 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:52.885160+0000", 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 196.31122523299999, 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008749649, 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885160+0000", 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885160+0000", 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885158+0000", 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885175+0000", 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6892999999999999e-05 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885197+0000", 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1460000000000001e-05 2023-10-09T04:50:09.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885205+0000", 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1810000000000001e-06 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.885284+0000", 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8854000000000001e-05 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.893909+0000", 2023-10-09T04:50:09.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086257069999999998 2023-10-09T04:50:09.215 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.215 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.216 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.216 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e317/303 cost 1049576)", 2023-10-09T04:50:09.216 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:12.404238+0000", 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 176.79214717799999, 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049423480000000001, 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404238+0000", 2023-10-09T04:50:09.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404238+0000", 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404236+0000", 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404257+0000", 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0886999999999998e-05 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404281+0000", 2023-10-09T04:50:09.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3643e-05 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404291+0000", 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7790000000000007e-06 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.404893+0000", 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00060200200000000003 2023-10-09T04:50:09.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:12.409180+0000", 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042874560000000003 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e317/303 cost 1049576)", 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:14.744656+0000", 2023-10-09T04:50:09.221 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 174.451728578, 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041294390000000004, 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744656+0000", 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744656+0000", 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744655+0000", 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744690+0000", 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5431e-05 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744712+0000", 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.207e-05 2023-10-09T04:50:09.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744721+0000", 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2210000000000003e-06 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.744807+0000", 2023-10-09T04:50:09.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5366999999999998e-05 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:14.748786+0000", 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039791009999999996 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e319/303 cost 1049576)", 2023-10-09T04:50:09.226 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:28.742384+0000", 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 160.45400115800001, 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065389030000000004, 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.742384+0000", 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.742384+0000", 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.742382+0000", 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.742411+0000", 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8524999999999999e-05 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.742429+0000", 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8695e-05 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.742437+0000", 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9899999999999997e-06 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.743350+0000", 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00091234599999999997 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:28.748923+0000", 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055729309999999997 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.231 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e320/303 cost 1049576)", 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:29.812866+0000", 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 159.38351927900001, 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051244890000000003, 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812866+0000", 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812866+0000", 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812865+0000", 2023-10-09T04:50:09.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812884+0000", 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9159000000000001e-05 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812905+0000", 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1526999999999999e-05 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812914+0000", 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5480000000000002e-06 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.812965+0000", 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1527999999999999e-05 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:29.817990+0000", 2023-10-09T04:50:09.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005024962 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e320/303 cost 1049576)", 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:30.032063+0000", 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 159.16432191600001, 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042492520000000002, 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.236 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032063+0000", 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032063+0000", 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032061+0000", 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032082+0000", 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0531999999999999e-05 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032106+0000", 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4068000000000001e-05 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032115+0000", 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1730000000000001e-06 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.032192+0000", 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7257999999999998e-05 2023-10-09T04:50:09.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.036312+0000", 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041202859999999999 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e320/303 cost 1049576)", 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:30.620671+0000", 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 158.57571367200001, 2023-10-09T04:50:09.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004233545, 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620671+0000", 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620671+0000", 2023-10-09T04:50:09.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620670+0000", 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620690+0000", 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.997e-05 2023-10-09T04:50:09.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620713+0000", 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3119e-05 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620722+0000", 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7979999999999995e-06 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.620776+0000", 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4039999999999998e-05 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:30.624905+0000", 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004129321 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.244 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e320/303 cost 1049576)", 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:32.781290+0000", 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 156.41509495599999, 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039683310000000003, 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781290+0000", 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781290+0000", 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781288+0000", 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781305+0000", 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6991999999999999e-05 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781328+0000", 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2263999999999999e-05 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781336+0000", 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6629999999999999e-06 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.781430+0000", 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3740999999999996e-05 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:32.785258+0000", 2023-10-09T04:50:09.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038282640000000001 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:36.019477+0000", 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 153.176908242, 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046591430000000001, 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.249 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019477+0000", 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019477+0000", 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019475+0000", 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019492+0000", 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6603999999999999e-05 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019512+0000", 2023-10-09T04:50:09.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0545000000000001e-05 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019521+0000", 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.019573+0000", 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2432000000000002e-05 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.024136+0000", 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045629879999999996 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.253 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:36.323418+0000", 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.87296652800001, 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043539820000000002, 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323418+0000", 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323418+0000", 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323417+0000", 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323434+0000", 2023-10-09T04:50:09.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6875e-05 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323454+0000", 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0211e-05 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323462+0000", 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.79e-06 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.323529+0000", 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6706999999999997e-05 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.327772+0000", 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042439310000000003 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:36.431660+0000", 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.76472469399999, 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0077380690000000002, 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431660+0000", 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431660+0000", 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431659+0000", 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431677+0000", 2023-10-09T04:50:09.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8138000000000002e-05 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431697+0000", 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0180999999999999e-05 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431706+0000", 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7490000000000004e-06 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.431776+0000", 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0574000000000006e-05 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:36.439398+0000", 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076220979999999999 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:37.005043+0000", 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.19134195399999, 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045117109999999998, 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005043+0000", 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005043+0000", 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005042+0000", 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005071+0000", 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9116e-05 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005091+0000", 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9505e-05 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005099+0000", 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3410000000000003e-06 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.005162+0000", 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3097999999999998e-05 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.009555+0000", 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043927719999999996 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:37.107068+0000", 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.08931655999999, 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0089942809999999998, 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.107068+0000", 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.107068+0000", 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.107067+0000", 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.107084+0000", 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7376000000000001e-05 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.107106+0000", 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1132999999999999e-05 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.107114+0000", 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1289999999999996e-06 2023-10-09T04:50:09.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.115504+0000", 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083898859999999992 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.116063+0000", 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055909600000000003 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:37.135677+0000", 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 152.060707917, 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044196310000000003, 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.135677+0000", 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.135677+0000", 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.135676+0000", 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.135692+0000", 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6790999999999999e-05 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.135718+0000", 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5388000000000002e-05 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.135726+0000", 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6539999999999999e-06 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.137687+0000", 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001961122 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:37.140097+0000", 2023-10-09T04:50:09.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024101050000000001 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:38.641247+0000", 2023-10-09T04:50:09.274 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.55513764299999, 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0074848060000000001, 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641247+0000", 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641247+0000", 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641246+0000", 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641263+0000", 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7227999999999998e-05 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641285+0000", 2023-10-09T04:50:09.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1138e-05 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641293+0000", 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0830000000000004e-06 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.641342+0000", 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9295999999999997e-05 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.648732+0000", 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.007390209 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:38.817211+0000", 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.37917418699999, 2023-10-09T04:50:09.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045728339999999996, 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817211+0000", 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817211+0000", 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817209+0000", 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817228+0000", 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9253e-05 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817249+0000", 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0956999999999999e-05 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817257+0000", 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9689999999999994e-06 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.817320+0000", 2023-10-09T04:50:09.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2688999999999996e-05 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:38.821784+0000", 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044635589999999998 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPull(12.4 e321/303 cost 1049576)", 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:47:39.528816+0000", 2023-10-09T04:50:09.284 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 149.667568786, 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059479670000000002, 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528816+0000", 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:09.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528816+0000", 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528815+0000", 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528833+0000", 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8148999999999999e-05 2023-10-09T04:50:09.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528855+0000", 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1753999999999999e-05 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528864+0000", 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9639999999999992e-06 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.528939+0000", 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5208000000000002e-05 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:47:39.534764+0000", 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058251839999999997 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.288 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:09.289 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:09.289 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:50:09.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:09.333 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:09.389 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:09.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:09.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:09.561 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:09.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:09.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:09.779 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T04:50:09.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:09.825 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:09.867 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:09.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:09.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:10.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 53775 53759 81.94 38.1875 0.00733629 0.0121984 2023-10-09T04:50:10.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2768904926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:50:10.040 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:10.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:50:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2768904926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:50:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2768904926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T04:50:10.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:50:10.407 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:10.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:10.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:10.544 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:10.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:10.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:10.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:10.730 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:10.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:10 smithi177 ceph-mon[195797]: pgmap v1359: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 42 MiB/s wr, 667 op/s 2023-10-09T04:50:10.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:10 smithi177 ceph-mon[195797]: failed to encode map e354 with expected crc 2023-10-09T04:50:11.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 54668 54652 81.3179 55.8125 0.00992583 0.0122917 2023-10-09T04:50:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2768904926' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:50:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:10 smithi177 ceph-mon[195797]: osdmap e354: 8 total, 8 up, 4 in 2023-10-09T04:50:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3040994708' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:11.050 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:50:11.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[223858]: pgmap v1359: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 42 MiB/s wr, 667 op/s 2023-10-09T04:50:11.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[223858]: failed to encode map e354 with expected crc 2023-10-09T04:50:11.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2768904926' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:50:11.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[223858]: osdmap e354: 8 total, 8 up, 4 in 2023-10-09T04:50:11.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3040994708' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:11.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[216903]: pgmap v1359: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 42 MiB/s wr, 667 op/s 2023-10-09T04:50:11.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[216903]: failed to encode map e354 with expected crc 2023-10-09T04:50:11.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2768904926' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T04:50:11.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[216903]: osdmap e354: 8 total, 8 up, 4 in 2023-10-09T04:50:11.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3040994708' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:11.129 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:11.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:11.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:11.139 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T04:50:11.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:50:11.224 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:11.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:11.279 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:11.470 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:11.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:11.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:11.663 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:11.675 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:50:11.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:11.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:11.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:11.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:12.001 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:11 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:12.001 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3040994708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:50:12.001 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:11 smithi177 ceph-mon[195797]: osdmap e355: 8 total, 8 up, 4 in 2023-10-09T04:50:12.006 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:12.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 55710 55694 80.9412 65.125 0.0201131 0.0123486 2023-10-09T04:50:12.061 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:11 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:12.062 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3040994708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:50:12.062 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:11 smithi029 ceph-mon[216903]: osdmap e355: 8 total, 8 up, 4 in 2023-10-09T04:50:12.062 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:11 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:12.062 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3040994708' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:50:12.062 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:11 smithi029 ceph-mon[223858]: osdmap e355: 8 total, 8 up, 4 in 2023-10-09T04:50:12.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:12.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:12.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:12.266 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:12.308 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:12.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:12.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:12.443 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:12.590 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:12.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:12.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:12.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:12.759 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:12.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:12.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:12.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:12.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:13.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 56336 56320 79.9908 39.125 0.0308903 0.0124954 2023-10-09T04:50:13.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: pgmap v1361: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:50:13.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.015 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:12 smithi177 ceph-mon[195797]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.034 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:50:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:50:12] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-10-09T04:50:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: pgmap v1361: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:50:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[223858]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: pgmap v1361: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:12 smithi029 ceph-mon[216903]: failed to encode map e355 with expected crc 2023-10-09T04:50:13.204 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:13.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:13.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:13.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:13.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:13.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:13.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:13.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:13.522 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:50:13.522 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:50:13.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.522 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:56721 13.f 13:fc326bf0:::benchmark_data_smithi029_495570_object56720:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.500870+0000", 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.020597021, 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.02062143, 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 56721 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.523 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.500870+0000", 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.500870+0000", 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.500869+0000", 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.500940+0000", 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0735000000000006e-05 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.500951+0000", 2023-10-09T04:50:13.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1653e-05 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.500958+0000", 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1990000000000003e-06 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.503329+0000", 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023710879999999999 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.503460+0000", 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013124499999999999 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.503613+0000", 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015263899999999999 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:13.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.509800+0000", 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061870680000000004 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.511426+0000", 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016259149999999999 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:56723 13.6 e355/340 13:6facae70:::benchmark_data_smithi029_495570_object56722:head v 355'3564, mlcod=355'3561)", 2023-10-09T04:50:13.528 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.512108+0000", 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0093589109999999993, 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.009442064, 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512108+0000", 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512108+0000", 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512107+0000", 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512126+0000", 2023-10-09T04:50:13.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8961e-05 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512137+0000", 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0787e-05 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512142+0000", 2023-10-09T04:50:13.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6439999999999999e-06 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512182+0000", 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0519999999999998e-05 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512203+0000", 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1005999999999998e-05 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.532 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:56726 13.1 e355/340 13:8b6d5736:::benchmark_data_smithi029_495570_object56725:head v 355'3552, mlcod=355'3549)", 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.512166+0000", 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0093008190000000001, 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0094294429999999992, 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512166+0000", 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512166+0000", 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512166+0000", 2023-10-09T04:50:13.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512185+0000", 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9171999999999999e-05 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512191+0000", 2023-10-09T04:50:13.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3629999999999999e-06 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.512195+0000", 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7799999999999998e-06 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513479+0000", 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012841129999999999 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513502+0000", 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3244e-05 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:56728 13.c e355/346 13:3f4702b2:::benchmark_data_smithi029_495570_object56727:head v 355'3610, mlcod=355'3607)", 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.513817+0000", 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0076497709999999997, 2023-10-09T04:50:13.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078347320000000005, 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513817+0000", 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513817+0000", 2023-10-09T04:50:13.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513817+0000", 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513869+0000", 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2172999999999997e-05 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513883+0000", 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4365999999999999e-05 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.513888+0000", 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2140000000000004e-06 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.514057+0000", 2023-10-09T04:50:13.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016890300000000001 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.514075+0000", 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7842e-05 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:56730 13.c e355/346 13:3b4fa178:::benchmark_data_smithi029_495570_object56729:head v 355'3611, mlcod=355'3607)", 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.515555+0000", 2023-10-09T04:50:13.541 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0059115449999999998, 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061582950000000003, 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.515555+0000", 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.515555+0000", 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.515555+0000", 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.515594+0000", 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9675999999999998e-05 2023-10-09T04:50:13.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.515603+0000", 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8009999999999993e-06 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.515608+0000", 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5889999999999996e-06 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.516128+0000", 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00051986999999999997 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.516144+0000", 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6426000000000002e-05 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:56729 13.1 e355/340 13:83a1a107:::benchmark_data_smithi029_495570_object56728:head v 355'3553, mlcod=355'3550)", 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.517794+0000", 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0036725, 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039795389999999998, 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517794+0000", 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517794+0000", 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517794+0000", 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517818+0000", 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4321999999999999e-05 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517829+0000", 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0738e-05 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517834+0000", 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4720000000000003e-06 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517891+0000", 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6918e-05 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.517909+0000", 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7740999999999998e-05 2023-10-09T04:50:13.549 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:56733 13.e 13:7ba40c9c:::benchmark_data_smithi029_495570_object56732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:50:13.519194+0000", 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0022726399999999998, 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026504039999999999, 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:13.550 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 56733 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519194+0000", 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519194+0000", 2023-10-09T04:50:13.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519193+0000", 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519252+0000", 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9218999999999997e-05 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519272+0000", 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0497999999999999e-05 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519280+0000", 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:13.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.519903+0000", 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062351599999999996 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.520025+0000", 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012134900000000001 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:50:13.520152+0000", 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012766700000000001 2023-10-09T04:50:13.554 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.555 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:13.555 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.555 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:13.555 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:50:13.555 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 7 2023-10-09T04:50:13.555 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:50:13.578 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:13.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:13.722 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:13.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:13.865 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:13.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:13.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:13.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:14.004 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:14.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 57169 57153 79.3701 52.0625 0.0243779 0.0125941 2023-10-09T04:50:14.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:14.638 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:50:14.877 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:50:14.877 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:50:14.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:50:14.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:50:14.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:50:14.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:50:14.878 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:50:14.903 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:50:14.905 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:50:15.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 57799 57783 78.5006 39.375 0.00871402 0.0127336 2023-10-09T04:50:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:14 smithi177 ceph-mon[195797]: pgmap v1363: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 42 MiB/s wr, 674 op/s 2023-10-09T04:50:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2747436753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:14 smithi029 ceph-mon[216903]: pgmap v1363: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 42 MiB/s wr, 674 op/s 2023-10-09T04:50:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2747436753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:14 smithi029 ceph-mon[223858]: pgmap v1363: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 42 MiB/s wr, 674 op/s 2023-10-09T04:50:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2747436753' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:15.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:15.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:15.400 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:50:15.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:15.504 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:15.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:50:15.977 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:15.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:16.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 58836 58820 78.2093 64.8125 0.0114137 0.0127809 2023-10-09T04:50:16.081 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:16.112 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:16.205 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:16.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:16.313 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:16.355 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:16.518 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:16.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:16.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:16.558 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:16.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:16.660 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:16.703 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:16.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:16.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:16.814 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:50:16.815 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:50:16.833 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:16.876 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:16.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:16.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:16.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:17.008 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 59589 59573 77.5603 47.0625 0.00734369 0.0128883 2023-10-09T04:50:17.014 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:17.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:16 smithi177 ceph-mon[195797]: pgmap v1364: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:50:17.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:16 smithi029 ceph-mon[216903]: pgmap v1364: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:50:17.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:16 smithi029 ceph-mon[223858]: pgmap v1364: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-09T04:50:17.228 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:17.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:17.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:17.365 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:17.673 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:17.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:17.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:17.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:17.824 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:17.826 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:50:17.826 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:50:17.874 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:17 smithi177 ceph-mon[195797]: from='client.36618 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:18.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 60277 60261 76.855 43 0.00796026 0.01299 2023-10-09T04:50:18.080 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:18.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:17 smithi029 ceph-mon[223858]: from='client.36618 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:17 smithi029 ceph-mon[216903]: from='client.36618 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:18.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:18.214 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:18.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:50:18.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:18.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:18.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:18.361 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:18.389 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:18.674 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:18.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:18.752 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1364,"stamp":"2023-10-09T04:50:15.377895+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3766163360,"num_objects":57482,"num_object_clones":0,"num_object_copies":172446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57482,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":57758,"num_write_kb":3679757,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":4870682,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57674,"ondisk_log_size":57674,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":16035672,"kb_used_data":10981240,"kb_used_omap":4,"kb_used_meta":5054331,"kb_avail":358944936,"statfs":{"total":383980142592,"available":367559614464,"internally_reserved":0,"allocated":11244789760,"data_stored":11240958207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":5175635854},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,17,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":564133888,"num_objects":8608,"num_object_clones":0,"num_object_copies":25824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8608,"num_write_kb":550912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001809"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":113,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780020+0000","last_change":"2023-10-09T04:49:24.722690+0000","last_active":"2023-10-09T04:50:09.780020+0000","last_peered":"2023-10-09T04:50:09.780020+0000","last_clean":"2023-10-09T04:50:09.780020+0000","last_became_active":"2023-10-09T04:49:24.722128+0000","last_became_peered":"2023-10-09T04:49:24.722128+0000","last_unstale":"2023-10-09T04:50:09.780020+0000","last_undegraded":"2023-10-09T04:50:09.780020+0000","last_fullsized":"2023-10-09T04:50:09.780020+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:52.116095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":107,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775146+0000","last_change":"2023-10-09T04:49:36.252176+0000","last_active":"2023-10-09T04:50:09.775146+0000","last_peered":"2023-10-09T04:50:09.775146+0000","last_clean":"2023-10-09T04:50:09.775146+0000","last_became_active":"2023-10-09T04:49:36.251166+0000","last_became_peered":"2023-10-09T04:49:36.251166+0000","last_unstale":"2023-10-09T04:50:09.775146+0000","last_undegraded":"2023-10-09T04:50:09.775146+0000","last_fullsized":"2023-10-09T04:50:09.775146+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:40.935795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":352,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774182+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:50:09.774182+0000","last_peered":"2023-10-09T04:50:09.774182+0000","last_clean":"2023-10-09T04:50:09.774182+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:50:09.774182+0000","last_undegraded":"2023-10-09T04:50:09.774182+0000","last_fullsized":"2023-10-09T04:50:09.774182+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":288,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.974170+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:50:09.974170+0000","last_peered":"2023-10-09T04:50:09.974170+0000","last_clean":"2023-10-09T04:50:09.974170+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:50:09.974170+0000","last_undegraded":"2023-10-09T04:50:09.974170+0000","last_fullsized":"2023-10-09T04:50:09.974170+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":149,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774157+0000","last_change":"2023-10-09T04:49:36.253446+0000","last_active":"2023-10-09T04:50:09.774157+0000","last_peered":"2023-10-09T04:50:09.774157+0000","last_clean":"2023-10-09T04:50:09.774157+0000","last_became_active":"2023-10-09T04:49:36.252717+0000","last_became_peered":"2023-10-09T04:49:36.252717+0000","last_unstale":"2023-10-09T04:50:09.774157+0000","last_undegraded":"2023-10-09T04:50:09.774157+0000","last_fullsized":"2023-10-09T04:50:09.774157+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:30.302532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":155,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.983086+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:50:09.983086+0000","last_peered":"2023-10-09T04:50:09.983086+0000","last_clean":"2023-10-09T04:50:09.983086+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:50:09.983086+0000","last_undegraded":"2023-10-09T04:50:09.983086+0000","last_fullsized":"2023-10-09T04:50:09.983086+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":297,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982896+0000","last_change":"2023-10-09T04:49:24.727794+0000","last_active":"2023-10-09T04:50:09.982896+0000","last_peered":"2023-10-09T04:50:09.982896+0000","last_clean":"2023-10-09T04:50:09.982896+0000","last_became_active":"2023-10-09T04:49:24.726934+0000","last_became_peered":"2023-10-09T04:49:24.726934+0000","last_unstale":"2023-10-09T04:50:09.982896+0000","last_undegraded":"2023-10-09T04:50:09.982896+0000","last_fullsized":"2023-10-09T04:50:09.982896+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:43.043505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":271,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774249+0000","last_change":"2023-10-09T04:49:36.252722+0000","last_active":"2023-10-09T04:50:09.774249+0000","last_peered":"2023-10-09T04:50:09.774249+0000","last_clean":"2023-10-09T04:50:09.774249+0000","last_became_active":"2023-10-09T04:49:36.251598+0000","last_became_peered":"2023-10-09T04:49:36.251598+0000","last_unstale":"2023-10-09T04:50:09.774249+0000","last_undegraded":"2023-10-09T04:50:09.774249+0000","last_fullsized":"2023-10-09T04:50:09.774249+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.458714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":315,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981701+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:50:09.981701+0000","last_peered":"2023-10-09T04:50:09.981701+0000","last_clean":"2023-10-09T04:50:09.981701+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:50:09.981701+0000","last_undegraded":"2023-10-09T04:50:09.981701+0000","last_fullsized":"2023-10-09T04:50:09.981701+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":305,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.972489+0000","last_change":"2023-10-09T04:49:24.728045+0000","last_active":"2023-10-09T04:50:09.972489+0000","last_peered":"2023-10-09T04:50:09.972489+0000","last_clean":"2023-10-09T04:50:09.972489+0000","last_became_active":"2023-10-09T04:49:24.727278+0000","last_became_peered":"2023-10-09T04:49:24.727278+0000","last_unstale":"2023-10-09T04:50:09.972489+0000","last_undegraded":"2023-10-09T04:50:09.972489+0000","last_fullsized":"2023-10-09T04:50:09.972489+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:17.700648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":397,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973382+0000","last_change":"2023-10-09T04:49:24.751066+0000","last_active":"2023-10-09T04:50:09.973382+0000","last_peered":"2023-10-09T04:50:09.973382+0000","last_clean":"2023-10-09T04:50:09.973382+0000","last_became_active":"2023-10-09T04:49:24.731080+0000","last_became_peered":"2023-10-09T04:49:24.731080+0000","last_unstale":"2023-10-09T04:50:09.973382+0000","last_undegraded":"2023-10-09T04:50:09.973382+0000","last_fullsized":"2023-10-09T04:50:09.973382+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:47.656154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":250,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775600+0000","last_change":"2023-10-09T04:49:36.255264+0000","last_active":"2023-10-09T04:50:09.775600+0000","last_peered":"2023-10-09T04:50:09.775600+0000","last_clean":"2023-10-09T04:50:09.775600+0000","last_became_active":"2023-10-09T04:49:36.254631+0000","last_became_peered":"2023-10-09T04:49:36.254631+0000","last_unstale":"2023-10-09T04:50:09.775600+0000","last_undegraded":"2023-10-09T04:50:09.775600+0000","last_fullsized":"2023-10-09T04:50:09.775600+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:26.380768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":153,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780369+0000","last_change":"2023-10-09T04:49:24.739093+0000","last_active":"2023-10-09T04:50:09.780369+0000","last_peered":"2023-10-09T04:50:09.780369+0000","last_clean":"2023-10-09T04:50:09.780369+0000","last_became_active":"2023-10-09T04:49:24.723103+0000","last_became_peered":"2023-10-09T04:49:24.723103+0000","last_unstale":"2023-10-09T04:50:09.780369+0000","last_undegraded":"2023-10-09T04:50:09.780369+0000","last_fullsized":"2023-10-09T04:50:09.780369+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:41.664075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":325,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780344+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:50:09.780344+0000","last_peered":"2023-10-09T04:50:09.780344+0000","last_clean":"2023-10-09T04:50:09.780344+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:50:09.780344+0000","last_undegraded":"2023-10-09T04:50:09.780344+0000","last_fullsized":"2023-10-09T04:50:09.780344+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":381,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.972162+0000","last_change":"2023-10-09T04:49:24.748611+0000","last_active":"2023-10-09T04:50:09.972162+0000","last_peered":"2023-10-09T04:50:09.972162+0000","last_clean":"2023-10-09T04:50:09.972162+0000","last_became_active":"2023-10-09T04:49:24.730507+0000","last_became_peered":"2023-10-09T04:49:24.730507+0000","last_unstale":"2023-10-09T04:50:09.972162+0000","last_undegraded":"2023-10-09T04:50:09.972162+0000","last_fullsized":"2023-10-09T04:50:09.972162+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:22.219826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":155,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.776214+0000","last_change":"2023-10-09T04:49:36.255342+0000","last_active":"2023-10-09T04:50:09.776214+0000","last_peered":"2023-10-09T04:50:09.776214+0000","last_clean":"2023-10-09T04:50:09.776214+0000","last_became_active":"2023-10-09T04:49:36.254372+0000","last_became_peered":"2023-10-09T04:49:36.254372+0000","last_unstale":"2023-10-09T04:50:09.776214+0000","last_undegraded":"2023-10-09T04:50:09.776214+0000","last_fullsized":"2023-10-09T04:50:09.776214+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:37.972615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":297,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.972786+0000","last_change":"2023-10-09T04:49:24.731694+0000","last_active":"2023-10-09T04:50:09.972786+0000","last_peered":"2023-10-09T04:50:09.972786+0000","last_clean":"2023-10-09T04:50:09.972786+0000","last_became_active":"2023-10-09T04:49:24.731338+0000","last_became_peered":"2023-10-09T04:49:24.731338+0000","last_unstale":"2023-10-09T04:50:09.972786+0000","last_undegraded":"2023-10-09T04:50:09.972786+0000","last_fullsized":"2023-10-09T04:50:09.972786+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:45.968733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981248+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:50:09.981248+0000","last_peered":"2023-10-09T04:50:09.981248+0000","last_clean":"2023-10-09T04:50:09.981248+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:50:09.981248+0000","last_undegraded":"2023-10-09T04:50:09.981248+0000","last_fullsized":"2023-10-09T04:50:09.981248+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":327,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.972417+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:09.972417+0000","last_peered":"2023-10-09T04:50:09.972417+0000","last_clean":"2023-10-09T04:50:09.972417+0000","last_became_active":"2023-10-09T04:49:24.727752+0000","last_became_peered":"2023-10-09T04:49:24.727752+0000","last_unstale":"2023-10-09T04:50:09.972417+0000","last_undegraded":"2023-10-09T04:50:09.972417+0000","last_fullsized":"2023-10-09T04:50:09.972417+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:37.448051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":185,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.974252+0000","last_change":"2023-10-09T04:49:24.734384+0000","last_active":"2023-10-09T04:50:09.974252+0000","last_peered":"2023-10-09T04:50:09.974252+0000","last_clean":"2023-10-09T04:50:09.974252+0000","last_became_active":"2023-10-09T04:49:24.733948+0000","last_became_peered":"2023-10-09T04:49:24.733948+0000","last_unstale":"2023-10-09T04:50:09.974252+0000","last_undegraded":"2023-10-09T04:50:09.974252+0000","last_fullsized":"2023-10-09T04:50:09.974252+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:55.081133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":156,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982161+0000","last_change":"2023-10-09T04:49:24.743137+0000","last_active":"2023-10-09T04:50:09.982161+0000","last_peered":"2023-10-09T04:50:09.982161+0000","last_clean":"2023-10-09T04:50:09.982161+0000","last_became_active":"2023-10-09T04:49:24.726930+0000","last_became_peered":"2023-10-09T04:49:24.726930+0000","last_unstale":"2023-10-09T04:50:09.982161+0000","last_undegraded":"2023-10-09T04:50:09.982161+0000","last_fullsized":"2023-10-09T04:50:09.982161+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:38.154496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973617+0000","last_change":"2023-10-09T04:49:24.731691+0000","last_active":"2023-10-09T04:50:09.973617+0000","last_peered":"2023-10-09T04:50:09.973617+0000","last_clean":"2023-10-09T04:50:09.973617+0000","last_became_active":"2023-10-09T04:49:24.731374+0000","last_became_peered":"2023-10-09T04:49:24.731374+0000","last_unstale":"2023-10-09T04:50:09.973617+0000","last_undegraded":"2023-10-09T04:50:09.973617+0000","last_fullsized":"2023-10-09T04:50:09.973617+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:39.511857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"355'3590","reported_seq":3618,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.530825+0000","last_change":"2023-10-09T04:49:24.722035+0000","last_active":"2023-10-09T04:50:14.530825+0000","last_peered":"2023-10-09T04:50:14.530825+0000","last_clean":"2023-10-09T04:50:14.530825+0000","last_became_active":"2023-10-09T04:49:24.721093+0000","last_became_peered":"2023-10-09T04:49:24.721093+0000","last_unstale":"2023-10-09T04:50:14.530825+0000","last_undegraded":"2023-10-09T04:50:14.530825+0000","last_fullsized":"2023-10-09T04:50:14.530825+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3590,"log_dups_size":0,"ondisk_log_size":3590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:16.988764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235274240,"num_objects":3590,"num_object_clones":0,"num_object_copies":10770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3590,"num_write_kb":229760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":385,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973319+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:50:09.973319+0000","last_peered":"2023-10-09T04:50:09.973319+0000","last_clean":"2023-10-09T04:50:09.973319+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:50:09.973319+0000","last_undegraded":"2023-10-09T04:50:09.973319+0000","last_fullsized":"2023-10-09T04:50:09.973319+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":307,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981899+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:50:09.981899+0000","last_peered":"2023-10-09T04:50:09.981899+0000","last_clean":"2023-10-09T04:50:09.981899+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:50:09.981899+0000","last_undegraded":"2023-10-09T04:50:09.981899+0000","last_fullsized":"2023-10-09T04:50:09.981899+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"355'3608","reported_seq":3643,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:15.238382+0000","last_change":"2023-10-09T04:49:24.729511+0000","last_active":"2023-10-09T04:50:15.238382+0000","last_peered":"2023-10-09T04:50:15.238382+0000","last_clean":"2023-10-09T04:50:15.238382+0000","last_became_active":"2023-10-09T04:49:24.728925+0000","last_became_peered":"2023-10-09T04:49:24.728925+0000","last_unstale":"2023-10-09T04:50:15.238382+0000","last_undegraded":"2023-10-09T04:50:15.238382+0000","last_fullsized":"2023-10-09T04:50:15.238382+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3608,"log_dups_size":0,"ondisk_log_size":3608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.606555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236453888,"num_objects":3608,"num_object_clones":0,"num_object_copies":10824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3608,"num_write_kb":230912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":344,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.974088+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:50:09.974088+0000","last_peered":"2023-10-09T04:50:09.974088+0000","last_clean":"2023-10-09T04:50:09.974088+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:50:09.974088+0000","last_undegraded":"2023-10-09T04:50:09.974088+0000","last_fullsized":"2023-10-09T04:50:09.974088+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":352,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780136+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:50:09.780136+0000","last_peered":"2023-10-09T04:50:09.780136+0000","last_clean":"2023-10-09T04:50:09.780136+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:50:09.780136+0000","last_undegraded":"2023-10-09T04:50:09.780136+0000","last_fullsized":"2023-10-09T04:50:09.780136+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"355'3583","reported_seq":3610,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.557690+0000","last_change":"2023-10-09T04:49:17.684070+0000","last_active":"2023-10-09T04:50:14.557690+0000","last_peered":"2023-10-09T04:50:14.557690+0000","last_clean":"2023-10-09T04:50:14.557690+0000","last_became_active":"2023-10-09T04:49:17.683772+0000","last_became_peered":"2023-10-09T04:49:17.683772+0000","last_unstale":"2023-10-09T04:50:14.557690+0000","last_undegraded":"2023-10-09T04:50:14.557690+0000","last_fullsized":"2023-10-09T04:50:14.557690+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3583,"log_dups_size":0,"ondisk_log_size":3583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:47.529254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234815488,"num_objects":3583,"num_object_clones":0,"num_object_copies":10749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3583,"num_write_kb":229312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":326,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973227+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:50:09.973227+0000","last_peered":"2023-10-09T04:50:09.973227+0000","last_clean":"2023-10-09T04:50:09.973227+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:50:09.973227+0000","last_undegraded":"2023-10-09T04:50:09.973227+0000","last_fullsized":"2023-10-09T04:50:09.973227+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":223,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775238+0000","last_change":"2023-10-09T04:49:36.255047+0000","last_active":"2023-10-09T04:50:09.775238+0000","last_peered":"2023-10-09T04:50:09.775238+0000","last_clean":"2023-10-09T04:50:09.775238+0000","last_became_active":"2023-10-09T04:49:36.254324+0000","last_became_peered":"2023-10-09T04:49:36.254324+0000","last_unstale":"2023-10-09T04:50:09.775238+0000","last_undegraded":"2023-10-09T04:50:09.775238+0000","last_fullsized":"2023-10-09T04:50:09.775238+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:55.547119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"355'3604","reported_seq":3631,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.532443+0000","last_change":"2023-10-09T04:49:17.682749+0000","last_active":"2023-10-09T04:50:14.532443+0000","last_peered":"2023-10-09T04:50:14.532443+0000","last_clean":"2023-10-09T04:50:14.532443+0000","last_became_active":"2023-10-09T04:49:17.682418+0000","last_became_peered":"2023-10-09T04:49:17.682418+0000","last_unstale":"2023-10-09T04:50:14.532443+0000","last_undegraded":"2023-10-09T04:50:14.532443+0000","last_fullsized":"2023-10-09T04:50:14.532443+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3604,"log_dups_size":0,"ondisk_log_size":3604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:33.471194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236191744,"num_objects":3604,"num_object_clones":0,"num_object_copies":10812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3604,"num_write_kb":230656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":355,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982138+0000","last_change":"2023-10-09T04:49:24.728445+0000","last_active":"2023-10-09T04:50:09.982138+0000","last_peered":"2023-10-09T04:50:09.982138+0000","last_clean":"2023-10-09T04:50:09.982138+0000","last_became_active":"2023-10-09T04:49:24.728130+0000","last_became_peered":"2023-10-09T04:49:24.728130+0000","last_unstale":"2023-10-09T04:50:09.982138+0000","last_undegraded":"2023-10-09T04:50:09.982138+0000","last_fullsized":"2023-10-09T04:50:09.982138+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:04.777758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":70,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774375+0000","last_change":"2023-10-09T04:49:36.253080+0000","last_active":"2023-10-09T04:50:09.774375+0000","last_peered":"2023-10-09T04:50:09.774375+0000","last_clean":"2023-10-09T04:50:09.774375+0000","last_became_active":"2023-10-09T04:49:36.252682+0000","last_became_peered":"2023-10-09T04:49:36.252682+0000","last_unstale":"2023-10-09T04:50:09.774375+0000","last_undegraded":"2023-10-09T04:50:09.774375+0000","last_fullsized":"2023-10-09T04:50:09.774375+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:31.712210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.5","version":"355'3507","reported_seq":3534,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.540873+0000","last_change":"2023-10-09T04:49:17.681809+0000","last_active":"2023-10-09T04:50:14.540873+0000","last_peered":"2023-10-09T04:50:14.540873+0000","last_clean":"2023-10-09T04:50:14.540873+0000","last_became_active":"2023-10-09T04:49:17.681477+0000","last_became_peered":"2023-10-09T04:49:17.681477+0000","last_unstale":"2023-10-09T04:50:14.540873+0000","last_undegraded":"2023-10-09T04:50:14.540873+0000","last_fullsized":"2023-10-09T04:50:14.540873+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3507,"log_dups_size":0,"ondisk_log_size":3507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:07.783547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229834752,"num_objects":3507,"num_object_clones":0,"num_object_copies":10521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3507,"num_write_kb":224448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":94,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780390+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:50:09.780390+0000","last_peered":"2023-10-09T04:50:09.780390+0000","last_clean":"2023-10-09T04:50:09.780390+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:50:09.780390+0000","last_undegraded":"2023-10-09T04:50:09.780390+0000","last_fullsized":"2023-10-09T04:50:09.780390+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780422+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:50:09.780422+0000","last_peered":"2023-10-09T04:50:09.780422+0000","last_clean":"2023-10-09T04:50:09.780422+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:50:09.780422+0000","last_undegraded":"2023-10-09T04:50:09.780422+0000","last_fullsized":"2023-10-09T04:50:09.780422+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"355'3446","reported_seq":3484,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:13.095828+0000","last_change":"2023-10-09T04:49:36.269590+0000","last_active":"2023-10-09T04:50:13.095828+0000","last_peered":"2023-10-09T04:50:13.095828+0000","last_clean":"2023-10-09T04:50:13.095828+0000","last_became_active":"2023-10-09T04:49:36.252739+0000","last_became_peered":"2023-10-09T04:49:36.252739+0000","last_unstale":"2023-10-09T04:50:13.095828+0000","last_undegraded":"2023-10-09T04:50:13.095828+0000","last_fullsized":"2023-10-09T04:50:13.095828+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3446,"log_dups_size":0,"ondisk_log_size":3446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:26.767295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225837056,"num_objects":3446,"num_object_clones":0,"num_object_copies":10338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3446,"num_write_kb":220544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":316,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780070+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:50:09.780070+0000","last_peered":"2023-10-09T04:50:09.780070+0000","last_clean":"2023-10-09T04:50:09.780070+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:50:09.780070+0000","last_undegraded":"2023-10-09T04:50:09.780070+0000","last_fullsized":"2023-10-09T04:50:09.780070+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":113,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780543+0000","last_change":"2023-10-09T04:49:24.712121+0000","last_active":"2023-10-09T04:50:09.780543+0000","last_peered":"2023-10-09T04:50:09.780543+0000","last_clean":"2023-10-09T04:50:09.780543+0000","last_became_active":"2023-10-09T04:49:24.711716+0000","last_became_peered":"2023-10-09T04:49:24.711716+0000","last_unstale":"2023-10-09T04:50:09.780543+0000","last_undegraded":"2023-10-09T04:50:09.780543+0000","last_fullsized":"2023-10-09T04:50:09.780543+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:37.141689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"355'3569","reported_seq":3596,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:15.240069+0000","last_change":"2023-10-09T04:49:17.686417+0000","last_active":"2023-10-09T04:50:15.240069+0000","last_peered":"2023-10-09T04:50:15.240069+0000","last_clean":"2023-10-09T04:50:15.240069+0000","last_became_active":"2023-10-09T04:49:17.686126+0000","last_became_peered":"2023-10-09T04:49:17.686126+0000","last_unstale":"2023-10-09T04:50:15.240069+0000","last_undegraded":"2023-10-09T04:50:15.240069+0000","last_fullsized":"2023-10-09T04:50:15.240069+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3569,"log_dups_size":0,"ondisk_log_size":3569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:58.636951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233897984,"num_objects":3569,"num_object_clones":0,"num_object_copies":10707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3569,"num_write_kb":228416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780052+0000","last_change":"2023-10-09T04:49:24.721842+0000","last_active":"2023-10-09T04:50:09.780052+0000","last_peered":"2023-10-09T04:50:09.780052+0000","last_clean":"2023-10-09T04:50:09.780052+0000","last_became_active":"2023-10-09T04:49:24.720666+0000","last_became_peered":"2023-10-09T04:49:24.720666+0000","last_unstale":"2023-10-09T04:50:09.780052+0000","last_undegraded":"2023-10-09T04:50:09.780052+0000","last_fullsized":"2023-10-09T04:50:09.780052+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:16.572332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":335,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780540+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:50:09.780540+0000","last_peered":"2023-10-09T04:50:09.780540+0000","last_clean":"2023-10-09T04:50:09.780540+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:50:09.780540+0000","last_undegraded":"2023-10-09T04:50:09.780540+0000","last_fullsized":"2023-10-09T04:50:09.780540+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"355'3624","reported_seq":3661,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.551037+0000","last_change":"2023-10-09T04:49:24.721346+0000","last_active":"2023-10-09T04:50:14.551037+0000","last_peered":"2023-10-09T04:50:14.551037+0000","last_clean":"2023-10-09T04:50:14.551037+0000","last_became_active":"2023-10-09T04:49:24.720955+0000","last_became_peered":"2023-10-09T04:49:24.720955+0000","last_unstale":"2023-10-09T04:50:14.551037+0000","last_undegraded":"2023-10-09T04:50:14.551037+0000","last_fullsized":"2023-10-09T04:50:14.551037+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3624,"log_dups_size":0,"ondisk_log_size":3624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:58:55.342602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237502464,"num_objects":3624,"num_object_clones":0,"num_object_copies":10872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3624,"num_write_kb":231936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":108,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774574+0000","last_change":"2023-10-09T04:49:36.253172+0000","last_active":"2023-10-09T04:50:09.774574+0000","last_peered":"2023-10-09T04:50:09.774574+0000","last_clean":"2023-10-09T04:50:09.774574+0000","last_became_active":"2023-10-09T04:49:36.252175+0000","last_became_peered":"2023-10-09T04:49:36.252175+0000","last_unstale":"2023-10-09T04:50:09.774574+0000","last_undegraded":"2023-10-09T04:50:09.774574+0000","last_fullsized":"2023-10-09T04:50:09.774574+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:38.269236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":260,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.776361+0000","last_change":"2023-10-09T04:49:36.251118+0000","last_active":"2023-10-09T04:50:09.776361+0000","last_peered":"2023-10-09T04:50:09.776361+0000","last_clean":"2023-10-09T04:50:09.776361+0000","last_became_active":"2023-10-09T04:49:36.250803+0000","last_became_peered":"2023-10-09T04:49:36.250803+0000","last_unstale":"2023-10-09T04:50:09.776361+0000","last_undegraded":"2023-10-09T04:50:09.776361+0000","last_fullsized":"2023-10-09T04:50:09.776361+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:38.765478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"355'3613","reported_seq":3650,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.539745+0000","last_change":"2023-10-09T04:49:24.727773+0000","last_active":"2023-10-09T04:50:14.539745+0000","last_peered":"2023-10-09T04:50:14.539745+0000","last_clean":"2023-10-09T04:50:14.539745+0000","last_became_active":"2023-10-09T04:49:24.727411+0000","last_became_peered":"2023-10-09T04:49:24.727411+0000","last_unstale":"2023-10-09T04:50:14.539745+0000","last_undegraded":"2023-10-09T04:50:14.539745+0000","last_fullsized":"2023-10-09T04:50:14.539745+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3613,"log_dups_size":0,"ondisk_log_size":3613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:21.626311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236781568,"num_objects":3613,"num_object_clones":0,"num_object_copies":10839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3613,"num_write_kb":231232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":249,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.776134+0000","last_change":"2023-10-09T04:49:36.250912+0000","last_active":"2023-10-09T04:50:09.776134+0000","last_peered":"2023-10-09T04:50:09.776134+0000","last_clean":"2023-10-09T04:50:09.776134+0000","last_became_active":"2023-10-09T04:49:36.250530+0000","last_became_peered":"2023-10-09T04:49:36.250530+0000","last_unstale":"2023-10-09T04:50:09.776134+0000","last_undegraded":"2023-10-09T04:50:09.776134+0000","last_fullsized":"2023-10-09T04:50:09.776134+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:26.975687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":369,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981170+0000","last_change":"2023-10-09T04:49:24.727377+0000","last_active":"2023-10-09T04:50:09.981170+0000","last_peered":"2023-10-09T04:50:09.981170+0000","last_clean":"2023-10-09T04:50:09.981170+0000","last_became_active":"2023-10-09T04:49:24.724195+0000","last_became_peered":"2023-10-09T04:49:24.724195+0000","last_unstale":"2023-10-09T04:50:09.981170+0000","last_undegraded":"2023-10-09T04:50:09.981170+0000","last_fullsized":"2023-10-09T04:50:09.981170+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:46.895156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":244,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775606+0000","last_change":"2023-10-09T04:49:36.254992+0000","last_active":"2023-10-09T04:50:09.775606+0000","last_peered":"2023-10-09T04:50:09.775606+0000","last_clean":"2023-10-09T04:50:09.775606+0000","last_became_active":"2023-10-09T04:49:36.254288+0000","last_became_peered":"2023-10-09T04:49:36.254288+0000","last_unstale":"2023-10-09T04:50:09.775606+0000","last_undegraded":"2023-10-09T04:50:09.775606+0000","last_fullsized":"2023-10-09T04:50:09.775606+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:13.355684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"355'3651","reported_seq":3677,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.521251+0000","last_change":"2023-10-09T04:49:17.684748+0000","last_active":"2023-10-09T04:50:14.521251+0000","last_peered":"2023-10-09T04:50:14.521251+0000","last_clean":"2023-10-09T04:50:14.521251+0000","last_became_active":"2023-10-09T04:49:17.684346+0000","last_became_peered":"2023-10-09T04:49:17.684346+0000","last_unstale":"2023-10-09T04:50:14.521251+0000","last_undegraded":"2023-10-09T04:50:14.521251+0000","last_fullsized":"2023-10-09T04:50:14.521251+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3651,"log_dups_size":0,"ondisk_log_size":3651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:39.897521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239271936,"num_objects":3651,"num_object_clones":0,"num_object_copies":10953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3651,"num_write_kb":233664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":339,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.984931+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:50:09.984931+0000","last_peered":"2023-10-09T04:50:09.984931+0000","last_clean":"2023-10-09T04:50:09.984931+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:50:09.984931+0000","last_undegraded":"2023-10-09T04:50:09.984931+0000","last_fullsized":"2023-10-09T04:50:09.984931+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780032+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:50:09.780032+0000","last_peered":"2023-10-09T04:50:09.780032+0000","last_clean":"2023-10-09T04:50:09.780032+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:50:09.780032+0000","last_undegraded":"2023-10-09T04:50:09.780032+0000","last_fullsized":"2023-10-09T04:50:09.780032+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":298,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.776055+0000","last_change":"2023-10-09T04:49:36.254929+0000","last_active":"2023-10-09T04:50:09.776055+0000","last_peered":"2023-10-09T04:50:09.776055+0000","last_clean":"2023-10-09T04:50:09.776055+0000","last_became_active":"2023-10-09T04:49:36.252868+0000","last_became_peered":"2023-10-09T04:49:36.252868+0000","last_unstale":"2023-10-09T04:50:09.776055+0000","last_undegraded":"2023-10-09T04:50:09.776055+0000","last_fullsized":"2023-10-09T04:50:09.776055+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:26.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"355'3702","reported_seq":3728,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:15.251885+0000","last_change":"2023-10-09T04:49:17.686557+0000","last_active":"2023-10-09T04:50:15.251885+0000","last_peered":"2023-10-09T04:50:15.251885+0000","last_clean":"2023-10-09T04:50:15.251885+0000","last_became_active":"2023-10-09T04:49:17.686236+0000","last_became_peered":"2023-10-09T04:49:17.686236+0000","last_unstale":"2023-10-09T04:50:15.251885+0000","last_undegraded":"2023-10-09T04:50:15.251885+0000","last_fullsized":"2023-10-09T04:50:15.251885+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3702,"log_dups_size":0,"ondisk_log_size":3702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:02.935353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242614272,"num_objects":3702,"num_object_clones":0,"num_object_copies":11106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3702,"num_write_kb":236928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":299,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774350+0000","last_change":"2023-10-09T04:49:36.250480+0000","last_active":"2023-10-09T04:50:09.774350+0000","last_peered":"2023-10-09T04:50:09.774350+0000","last_clean":"2023-10-09T04:50:09.774350+0000","last_became_active":"2023-10-09T04:49:36.250048+0000","last_became_peered":"2023-10-09T04:49:36.250048+0000","last_unstale":"2023-10-09T04:50:09.774350+0000","last_undegraded":"2023-10-09T04:50:09.774350+0000","last_fullsized":"2023-10-09T04:50:09.774350+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:21.602101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":298,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.774337+0000","last_change":"2023-10-09T04:49:36.253118+0000","last_active":"2023-10-09T04:50:09.774337+0000","last_peered":"2023-10-09T04:50:09.774337+0000","last_clean":"2023-10-09T04:50:09.774337+0000","last_became_active":"2023-10-09T04:49:36.252036+0000","last_became_peered":"2023-10-09T04:49:36.252036+0000","last_unstale":"2023-10-09T04:50:09.774337+0000","last_undegraded":"2023-10-09T04:50:09.774337+0000","last_fullsized":"2023-10-09T04:50:09.774337+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.648245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":240,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.776276+0000","last_change":"2023-10-09T04:49:36.252116+0000","last_active":"2023-10-09T04:50:09.776276+0000","last_peered":"2023-10-09T04:50:09.776276+0000","last_clean":"2023-10-09T04:50:09.776276+0000","last_became_active":"2023-10-09T04:49:36.251630+0000","last_became_peered":"2023-10-09T04:49:36.251630+0000","last_unstale":"2023-10-09T04:50:09.776276+0000","last_undegraded":"2023-10-09T04:50:09.776276+0000","last_fullsized":"2023-10-09T04:50:09.776276+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:48.681072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"355'3601","reported_seq":3650,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:13.292847+0000","last_change":"2023-10-09T04:49:36.276163+0000","last_active":"2023-10-09T04:50:13.292847+0000","last_peered":"2023-10-09T04:50:13.292847+0000","last_clean":"2023-10-09T04:50:13.292847+0000","last_became_active":"2023-10-09T04:49:36.253314+0000","last_became_peered":"2023-10-09T04:49:36.253314+0000","last_unstale":"2023-10-09T04:50:13.292847+0000","last_undegraded":"2023-10-09T04:50:13.292847+0000","last_fullsized":"2023-10-09T04:50:13.292847+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3601,"log_dups_size":0,"ondisk_log_size":3601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:44.334582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235995136,"num_objects":3601,"num_object_clones":0,"num_object_copies":10803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3601,"num_write_kb":230464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":346,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.984029+0000","last_change":"2023-10-09T04:49:24.727573+0000","last_active":"2023-10-09T04:50:09.984029+0000","last_peered":"2023-10-09T04:50:09.984029+0000","last_clean":"2023-10-09T04:50:09.984029+0000","last_became_active":"2023-10-09T04:49:24.727151+0000","last_became_peered":"2023-10-09T04:49:24.727151+0000","last_unstale":"2023-10-09T04:50:09.984029+0000","last_undegraded":"2023-10-09T04:50:09.984029+0000","last_fullsized":"2023-10-09T04:50:09.984029+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:53.790164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":825,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973344+0000","last_change":"2023-10-09T04:49:27.734937+0000","last_active":"2023-10-09T04:50:09.973344+0000","last_peered":"2023-10-09T04:50:09.973344+0000","last_clean":"2023-10-09T04:50:09.973344+0000","last_became_active":"2023-10-09T04:49:27.734587+0000","last_became_peered":"2023-10-09T04:49:27.734587+0000","last_unstale":"2023-10-09T04:50:09.973344+0000","last_undegraded":"2023-10-09T04:50:09.973344+0000","last_fullsized":"2023-10-09T04:50:09.973344+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:38.000046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":285,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.985119+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:50:09.985119+0000","last_peered":"2023-10-09T04:50:09.985119+0000","last_clean":"2023-10-09T04:50:09.985119+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:50:09.985119+0000","last_undegraded":"2023-10-09T04:50:09.985119+0000","last_fullsized":"2023-10-09T04:50:09.985119+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":298,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780511+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:50:09.780511+0000","last_peered":"2023-10-09T04:50:09.780511+0000","last_clean":"2023-10-09T04:50:09.780511+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:50:09.780511+0000","last_undegraded":"2023-10-09T04:50:09.780511+0000","last_fullsized":"2023-10-09T04:50:09.780511+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"355'3597","reported_seq":3634,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:15.213505+0000","last_change":"2023-10-09T04:49:24.733991+0000","last_active":"2023-10-09T04:50:15.213505+0000","last_peered":"2023-10-09T04:50:15.213505+0000","last_clean":"2023-10-09T04:50:15.213505+0000","last_became_active":"2023-10-09T04:49:24.733661+0000","last_became_peered":"2023-10-09T04:49:24.733661+0000","last_unstale":"2023-10-09T04:50:15.213505+0000","last_undegraded":"2023-10-09T04:50:15.213505+0000","last_fullsized":"2023-10-09T04:50:15.213505+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3597,"log_dups_size":0,"ondisk_log_size":3597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:22.030285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235732992,"num_objects":3597,"num_object_clones":0,"num_object_copies":10791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3597,"num_write_kb":230208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":365,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973291+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:50:09.973291+0000","last_peered":"2023-10-09T04:50:09.973291+0000","last_clean":"2023-10-09T04:50:09.973291+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:50:09.973291+0000","last_undegraded":"2023-10-09T04:50:09.973291+0000","last_fullsized":"2023-10-09T04:50:09.973291+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780155+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:50:09.780155+0000","last_peered":"2023-10-09T04:50:09.780155+0000","last_clean":"2023-10-09T04:50:09.780155+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:50:09.780155+0000","last_undegraded":"2023-10-09T04:50:09.780155+0000","last_fullsized":"2023-10-09T04:50:09.780155+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":125,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.776133+0000","last_change":"2023-10-09T04:49:36.252316+0000","last_active":"2023-10-09T04:50:09.776133+0000","last_peered":"2023-10-09T04:50:09.776133+0000","last_clean":"2023-10-09T04:50:09.776133+0000","last_became_active":"2023-10-09T04:49:36.251771+0000","last_became_peered":"2023-10-09T04:49:36.251771+0000","last_unstale":"2023-10-09T04:50:09.776133+0000","last_undegraded":"2023-10-09T04:50:09.776133+0000","last_fullsized":"2023-10-09T04:50:09.776133+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:08.690791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"355'3622","reported_seq":3657,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:13.305616+0000","last_change":"2023-10-09T04:49:36.254761+0000","last_active":"2023-10-09T04:50:13.305616+0000","last_peered":"2023-10-09T04:50:13.305616+0000","last_clean":"2023-10-09T04:50:13.305616+0000","last_became_active":"2023-10-09T04:49:36.250489+0000","last_became_peered":"2023-10-09T04:49:36.250489+0000","last_unstale":"2023-10-09T04:50:13.305616+0000","last_undegraded":"2023-10-09T04:50:13.305616+0000","last_fullsized":"2023-10-09T04:50:13.305616+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3622,"log_dups_size":0,"ondisk_log_size":3622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:19.664434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237371392,"num_objects":3622,"num_object_clones":0,"num_object_copies":10866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3622,"num_write_kb":231808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":368,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973299+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:50:09.973299+0000","last_peered":"2023-10-09T04:50:09.973299+0000","last_clean":"2023-10-09T04:50:09.973299+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:50:09.973299+0000","last_undegraded":"2023-10-09T04:50:09.973299+0000","last_fullsized":"2023-10-09T04:50:09.973299+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":251,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775983+0000","last_change":"2023-10-09T04:49:36.254730+0000","last_active":"2023-10-09T04:50:09.775983+0000","last_peered":"2023-10-09T04:50:09.775983+0000","last_clean":"2023-10-09T04:50:09.775983+0000","last_became_active":"2023-10-09T04:49:36.252242+0000","last_became_peered":"2023-10-09T04:49:36.252242+0000","last_unstale":"2023-10-09T04:50:09.775983+0000","last_undegraded":"2023-10-09T04:50:09.775983+0000","last_fullsized":"2023-10-09T04:50:09.775983+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.106203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982167+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:50:09.982167+0000","last_peered":"2023-10-09T04:50:09.982167+0000","last_clean":"2023-10-09T04:50:09.982167+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:50:09.982167+0000","last_undegraded":"2023-10-09T04:50:09.982167+0000","last_fullsized":"2023-10-09T04:50:09.982167+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"355'3567","reported_seq":3604,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.513843+0000","last_change":"2023-10-09T04:49:24.718431+0000","last_active":"2023-10-09T04:50:14.513843+0000","last_peered":"2023-10-09T04:50:14.513843+0000","last_clean":"2023-10-09T04:50:14.513843+0000","last_became_active":"2023-10-09T04:49:24.718108+0000","last_became_peered":"2023-10-09T04:49:24.718108+0000","last_unstale":"2023-10-09T04:50:14.513843+0000","last_undegraded":"2023-10-09T04:50:14.513843+0000","last_fullsized":"2023-10-09T04:50:14.513843+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3567,"log_dups_size":0,"ondisk_log_size":3567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.309575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233766912,"num_objects":3567,"num_object_clones":0,"num_object_copies":10701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3567,"num_write_kb":228288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":103,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973734+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:50:09.973734+0000","last_peered":"2023-10-09T04:50:09.973734+0000","last_clean":"2023-10-09T04:50:09.973734+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:50:09.973734+0000","last_undegraded":"2023-10-09T04:50:09.973734+0000","last_fullsized":"2023-10-09T04:50:09.973734+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":325,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775826+0000","last_change":"2023-10-09T04:49:36.251899+0000","last_active":"2023-10-09T04:50:09.775826+0000","last_peered":"2023-10-09T04:50:09.775826+0000","last_clean":"2023-10-09T04:50:09.775826+0000","last_became_active":"2023-10-09T04:49:36.251283+0000","last_became_peered":"2023-10-09T04:49:36.251283+0000","last_unstale":"2023-10-09T04:50:09.775826+0000","last_undegraded":"2023-10-09T04:50:09.775826+0000","last_fullsized":"2023-10-09T04:50:09.775826+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:52.505049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":201,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973428+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:50:09.973428+0000","last_peered":"2023-10-09T04:50:09.973428+0000","last_clean":"2023-10-09T04:50:09.973428+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:50:09.973428+0000","last_undegraded":"2023-10-09T04:50:09.973428+0000","last_fullsized":"2023-10-09T04:50:09.973428+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"355'3574","reported_seq":3602,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:14.520461+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:14.520461+0000","last_peered":"2023-10-09T04:50:14.520461+0000","last_clean":"2023-10-09T04:50:14.520461+0000","last_became_active":"2023-10-09T04:49:24.727688+0000","last_became_peered":"2023-10-09T04:49:24.727688+0000","last_unstale":"2023-10-09T04:50:14.520461+0000","last_undegraded":"2023-10-09T04:50:14.520461+0000","last_fullsized":"2023-10-09T04:50:14.520461+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3574,"log_dups_size":0,"ondisk_log_size":3574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:00.374022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234225664,"num_objects":3574,"num_object_clones":0,"num_object_copies":10722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3574,"num_write_kb":228736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":287,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.973232+0000","last_change":"2023-10-09T04:49:24.730477+0000","last_active":"2023-10-09T04:50:09.973232+0000","last_peered":"2023-10-09T04:50:09.973232+0000","last_clean":"2023-10-09T04:50:09.973232+0000","last_became_active":"2023-10-09T04:49:24.730115+0000","last_became_peered":"2023-10-09T04:49:24.730115+0000","last_unstale":"2023-10-09T04:50:09.973232+0000","last_undegraded":"2023-10-09T04:50:09.973232+0000","last_fullsized":"2023-10-09T04:50:09.973232+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:49.730424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":124,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775977+0000","last_change":"2023-10-09T04:49:36.250949+0000","last_active":"2023-10-09T04:50:09.775977+0000","last_peered":"2023-10-09T04:50:09.775977+0000","last_clean":"2023-10-09T04:50:09.775977+0000","last_became_active":"2023-10-09T04:49:36.250519+0000","last_became_peered":"2023-10-09T04:49:36.250519+0000","last_unstale":"2023-10-09T04:50:09.775977+0000","last_undegraded":"2023-10-09T04:50:09.775977+0000","last_fullsized":"2023-10-09T04:50:09.775977+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:51.821472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.779986+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:50:09.779986+0000","last_peered":"2023-10-09T04:50:09.779986+0000","last_clean":"2023-10-09T04:50:09.779986+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:50:09.779986+0000","last_undegraded":"2023-10-09T04:50:09.779986+0000","last_fullsized":"2023-10-09T04:50:09.779986+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":205,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780263+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:50:09.780263+0000","last_peered":"2023-10-09T04:50:09.780263+0000","last_clean":"2023-10-09T04:50:09.780263+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:50:09.780263+0000","last_undegraded":"2023-10-09T04:50:09.780263+0000","last_fullsized":"2023-10-09T04:50:09.780263+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":291,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775935+0000","last_change":"2023-10-09T04:49:36.254629+0000","last_active":"2023-10-09T04:50:09.775935+0000","last_peered":"2023-10-09T04:50:09.775935+0000","last_clean":"2023-10-09T04:50:09.775935+0000","last_became_active":"2023-10-09T04:49:36.251774+0000","last_became_peered":"2023-10-09T04:49:36.251774+0000","last_unstale":"2023-10-09T04:50:09.775935+0000","last_undegraded":"2023-10-09T04:50:09.775935+0000","last_fullsized":"2023-10-09T04:50:09.775935+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:24.682458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":345,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982303+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:50:09.982303+0000","last_peered":"2023-10-09T04:50:09.982303+0000","last_clean":"2023-10-09T04:50:09.982303+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:50:09.982303+0000","last_undegraded":"2023-10-09T04:50:09.982303+0000","last_fullsized":"2023-10-09T04:50:09.982303+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780177+0000","last_change":"2023-10-09T04:49:24.721368+0000","last_active":"2023-10-09T04:50:09.780177+0000","last_peered":"2023-10-09T04:50:09.780177+0000","last_clean":"2023-10-09T04:50:09.780177+0000","last_became_active":"2023-10-09T04:49:24.721024+0000","last_became_peered":"2023-10-09T04:49:24.721024+0000","last_unstale":"2023-10-09T04:50:09.780177+0000","last_undegraded":"2023-10-09T04:50:09.780177+0000","last_fullsized":"2023-10-09T04:50:09.780177+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:57.736030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":314,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981090+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:50:09.981090+0000","last_peered":"2023-10-09T04:50:09.981090+0000","last_clean":"2023-10-09T04:50:09.981090+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:50:09.981090+0000","last_undegraded":"2023-10-09T04:50:09.981090+0000","last_fullsized":"2023-10-09T04:50:09.981090+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":133,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780114+0000","last_change":"2023-10-09T04:49:24.711760+0000","last_active":"2023-10-09T04:50:09.780114+0000","last_peered":"2023-10-09T04:50:09.780114+0000","last_clean":"2023-10-09T04:50:09.780114+0000","last_became_active":"2023-10-09T04:49:24.711383+0000","last_became_peered":"2023-10-09T04:49:24.711383+0000","last_unstale":"2023-10-09T04:50:09.780114+0000","last_undegraded":"2023-10-09T04:50:09.780114+0000","last_fullsized":"2023-10-09T04:50:09.780114+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:04.370272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":308,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.775664+0000","last_change":"2023-10-09T04:49:24.725553+0000","last_active":"2023-10-09T04:50:09.775664+0000","last_peered":"2023-10-09T04:50:09.775664+0000","last_clean":"2023-10-09T04:50:09.775664+0000","last_became_active":"2023-10-09T04:49:24.725179+0000","last_became_peered":"2023-10-09T04:49:24.725179+0000","last_unstale":"2023-10-09T04:50:09.775664+0000","last_undegraded":"2023-10-09T04:50:09.775664+0000","last_fullsized":"2023-10-09T04:50:09.775664+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.045784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.980974+0000","last_change":"2023-10-09T04:49:24.718316+0000","last_active":"2023-10-09T04:50:09.980974+0000","last_peered":"2023-10-09T04:50:09.980974+0000","last_clean":"2023-10-09T04:50:09.980974+0000","last_became_active":"2023-10-09T04:49:24.717948+0000","last_became_peered":"2023-10-09T04:49:24.717948+0000","last_unstale":"2023-10-09T04:50:09.980974+0000","last_undegraded":"2023-10-09T04:50:09.980974+0000","last_fullsized":"2023-10-09T04:50:09.980974+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:48.454201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":261,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.983175+0000","last_change":"2023-10-09T04:49:24.745757+0000","last_active":"2023-10-09T04:50:09.983175+0000","last_peered":"2023-10-09T04:50:09.983175+0000","last_clean":"2023-10-09T04:50:09.983175+0000","last_became_active":"2023-10-09T04:49:24.727010+0000","last_became_peered":"2023-10-09T04:49:24.727010+0000","last_unstale":"2023-10-09T04:50:09.983175+0000","last_undegraded":"2023-10-09T04:50:09.983175+0000","last_fullsized":"2023-10-09T04:50:09.983175+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:51.654447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.779919+0000","last_change":"2023-10-09T04:49:24.710860+0000","last_active":"2023-10-09T04:50:09.779919+0000","last_peered":"2023-10-09T04:50:09.779919+0000","last_clean":"2023-10-09T04:50:09.779919+0000","last_became_active":"2023-10-09T04:49:24.710496+0000","last_became_peered":"2023-10-09T04:49:24.710496+0000","last_unstale":"2023-10-09T04:50:09.779919+0000","last_undegraded":"2023-10-09T04:50:09.779919+0000","last_fullsized":"2023-10-09T04:50:09.779919+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:24.901415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3765567488,"num_objects":57458,"num_object_clones":0,"num_object_copies":172374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":57458,"num_write_kb":3677312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11291787264,"data_stored":11291787264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57458,"ondisk_log_size":57458,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16666,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234520,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4296612,"kb_used_data":2783260,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":89448540,"statfs":{"total":95995035648,"available":91595304960,"internally_reserved":0,"allocated":2850058240,"data_stored":2849307275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.4039999999999999,"15min":2.0910000000000002},"min":{"1min":0.31,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":1.141,"5min":14.970000000000001,"15min":68.495999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.048,"15min":2.6200000000000001},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.249},"max":{"1min":2.1800000000000002,"5min":19.152000000000001,"15min":68.540000000000006},"last":0.56799999999999995}]},{"osd":1,"last update":"Mon Oct 9 04:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":4.5730000000000004,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":1.179,"5min":128.601,"15min":264.62},"last":74.759},{"interface":"front","average":{"1min":0.60799999999999998,"5min":4.2990000000000004,"15min":4.2069999999999999},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":1.264,"5min":128.381,"15min":264.65899999999999},"last":74.727000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:49:16 2023","interfaces":[{"interface":"back","average":{"1min":8.0559999999999992,"5min":6.6950000000000003,"15min":5.266},"min":{"1min":0.34999999999999998,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":117.126,"5min":129.10400000000001,"15min":184.72},"last":0.52200000000000002},{"interface":"front","average":{"1min":7.6420000000000003,"5min":6.8559999999999999,"15min":5.5899999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28100000000000003},"max":{"1min":117.10899999999999,"5min":129.15100000000001,"15min":184.697},"last":0.50600000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:49:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.9790000000000001,"15min":3.7010000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.29499999999999998},"max":{"1min":12.324,"5min":12.324,"15min":182.839},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.1319999999999999,"5min":2.4950000000000001,"15min":4.1040000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.7400000000000002,"5min":14.866,"15min":182.678},"last":0.54100000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:49:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":1.901,"15min":1.595},"min":{"1min":0.27700000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.811999999999999,"5min":14.811999999999999,"15min":14.811999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.752,"5min":1.6020000000000001,"15min":1.3480000000000001},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":13.523,"5min":13.523,"15min":13.523},"last":0.35799999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":1.369,"15min":1.2669999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.202999999999999,"5min":12.202999999999999,"15min":13.319000000000001},"last":0.38},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.7869999999999999,"15min":1.708},"min":{"1min":0.33400000000000002,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":5.1769999999999996,"5min":12.081,"15min":18.408000000000001},"last":0.432}]},{"osd":6,"last update":"Mon Oct 9 04:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.159,"15min":1.2509999999999999},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":10.047000000000001,"5min":10.047000000000001,"15min":11.388},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.3380000000000001,"5min":1.649,"15min":1.76},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.30099999999999999},"max":{"1min":4.9550000000000001,"5min":11.680999999999999,"15min":18.311},"last":0.39700000000000002}]}]},{"osd":6,"up_from":0,"seq":197568496154,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141243,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3099576,"kb_used_data":2241008,"kb_used_omap":4,"kb_used_meta":858555,"kb_avail":90645576,"statfs":{"total":95995035648,"available":92821069824,"internally_reserved":0,"allocated":2294792192,"data_stored":2293201846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":879161230},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.484,"15min":2.9529999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.14799999999999999},"max":{"1min":1.78,"5min":11.653,"15min":93.171999999999997},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.613,"15min":3.0219999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.193},"max":{"1min":2.1739999999999999,"5min":10.765000000000001,"15min":93.147000000000006},"last":0.42299999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":5.0199999999999996,"15min":4.8129999999999997},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":1.3899999999999999,"5min":151.41900000000001,"15min":263.60500000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":5.0780000000000003,"15min":5.1609999999999996},"min":{"1min":0.29599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.4299999999999999,"5min":146.749,"15min":263.65199999999999},"last":0.375}]},{"osd":3,"last update":"Mon Oct 9 04:49:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.893,"15min":5.5119999999999996},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":5.5880000000000001,"5min":11.68,"15min":379.80900000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":2.343,"15min":6.085},"min":{"1min":0.34000000000000002,"5min":0.248,"15min":0.246},"max":{"1min":4.0510000000000002,"5min":18.285,"15min":379.95800000000003},"last":0.44600000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.371,"15min":2.657},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":4.1109999999999998,"5min":19.959,"15min":170.072},"last":65.036000000000001},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.96,"15min":2.484},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":2.7040000000000002,"5min":11.598000000000001,"15min":169.94499999999999},"last":62.585000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":2.0600000000000001,"15min":2.4319999999999999},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":3.2170000000000001,"5min":9.782,"15min":171.036},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.71999999999999997,"5min":2.4849999999999999,"15min":2.762},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":1.407,"5min":20.021000000000001,"15min":170.88},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.954,"15min":2.4159999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":2.786,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":2.5739999999999998,"15min":2.7989999999999999},"min":{"1min":0.45800000000000002,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":2.802,"5min":20.164999999999999,"15min":132.17699999999999},"last":0.436}]},{"osd":7,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":2.8460000000000001,"15min":3.4540000000000002},"min":{"1min":0.438,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":3.2280000000000002,"5min":20.055,"15min":174.75999999999999},"last":59.972000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.254,"15min":3.1160000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.377},"max":{"1min":3.2109999999999999,"5min":16.238,"15min":174.779},"last":60.011000000000003}]}]},{"osd":0,"up_from":0,"seq":390842024300,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":0,"seq":949187772576,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":609885356301,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019428,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4697636,"kb_used_data":3202076,"kb_used_omap":0,"kb_used_meta":1495552,"kb_avail":89047516,"statfs":{"total":95995035648,"available":91184656384,"internally_reserved":0,"allocated":3278925824,"data_stored":3278174859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1531445248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,14],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.7410000000000001,"15min":4.0309999999999997},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":1.0529999999999999,"5min":12.795,"15min":171.13499999999999},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.9350000000000001,"15min":4.0229999999999997},"min":{"1min":0.40999999999999998,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":3.4009999999999998,"5min":16.004000000000001,"15min":171.11799999999999},"last":0.85299999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:49:53 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":9.6300000000000008,"15min":5.351},"min":{"1min":0.45200000000000001,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":37.944000000000003,"5min":385.78199999999998,"15min":385.78199999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.4700000000000002,"5min":9.2650000000000006,"15min":5.3330000000000002},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":37.408999999999999,"5min":386.09399999999999,"15min":386.09399999999999},"last":0.78500000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:50:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":11.032999999999999,"15min":6.5179999999999998},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.131,"5min":440.75599999999997,"15min":440.75599999999997},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":11.414,"15min":6.367},"min":{"1min":0.374,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":3.4399999999999999,"5min":440.61900000000003,"15min":440.61900000000003},"last":0.91200000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":4.2069999999999999,"15min":5.4000000000000004},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":7.2670000000000003,"5min":141.334,"15min":357.125},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.306,"5min":4.242,"15min":5.3310000000000004},"min":{"1min":0.44500000000000001,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":6.1319999999999997,"5min":141.18299999999999,"15min":359.02800000000002},"last":0.50700000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.994,"15min":1.8440000000000001},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":6.1040000000000001,"5min":24.234999999999999,"15min":107.476},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.054,"5min":1.6359999999999999,"15min":1.802},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":7.1769999999999996,"5min":18.785,"15min":107.414},"last":4.3579999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:49:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.9690000000000001,"15min":1.9019999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.29599999999999999},"max":{"1min":5.9349999999999996,"5min":22.920999999999999,"15min":65.75},"last":1.03},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.5329999999999999,"15min":1.7430000000000001},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":1.2669999999999999,"5min":15.095000000000001,"15min":65.849999999999994},"last":1.2130000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":2.0369999999999999,"15min":1.5489999999999999},"min":{"1min":0.34599999999999997,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":7.298,"5min":16.109000000000002,"15min":16.109000000000002},"last":1.3600000000000001},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.444,"15min":1.7529999999999999},"min":{"1min":0.36899999999999999,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":8.0570000000000004,"5min":22.858000000000001,"15min":22.858000000000001},"last":1.0569999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757792,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3941848,"kb_used_data":2754896,"kb_used_omap":0,"kb_used_meta":1186880,"kb_avail":89803304,"statfs":{"total":95995035648,"available":91958583296,"internally_reserved":0,"allocated":2821013504,"data_stored":2820274227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1215365120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.851,"15min":4.798},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23100000000000001},"max":{"1min":8.7309999999999999,"5min":9.6890000000000001,"15min":210.85499999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.6950000000000001,"15min":5.0359999999999996},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.5489999999999999,"5min":8.4749999999999996,"15min":210.23500000000001},"last":0.372}]},{"osd":1,"last update":"Mon Oct 9 04:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":8.3260000000000005,"15min":5.8259999999999996},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.252},"max":{"1min":1.0900000000000001,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.2110000000000001,"5min":8.5410000000000004,"15min":6.0119999999999996},"min":{"1min":0.33200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":8.6620000000000008,"5min":211.435,"15min":211.435},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":5.5179999999999998,"15min":4.9299999999999997},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":86.488,"15min":90.763999999999996},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.385,"5min":5.5199999999999996,"15min":5.0369999999999999},"min":{"1min":0.34100000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.7590000000000003,"5min":86.510999999999996,"15min":91.340000000000003},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:49:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":2.778,"15min":5.1289999999999996},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":3.605,"5min":46.497,"15min":268.63299999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.2909999999999999,"5min":2.133,"15min":4.7380000000000004},"min":{"1min":0.40600000000000003,"5min":0.38400000000000001,"15min":0.29799999999999999},"max":{"1min":7.2469999999999999,"5min":46.125999999999998,"15min":268.92500000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:49:26 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.3700000000000001,"15min":2.5299999999999998},"min":{"1min":0.314,"5min":0.307,"15min":0.23599999999999999},"max":{"1min":4.1799999999999997,"5min":10.268000000000001,"15min":361.09399999999999},"last":116.054},{"interface":"front","average":{"1min":1.03,"5min":1.544,"15min":2.8380000000000001},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":4.4269999999999996,"5min":12.609999999999999,"15min":361.71300000000002},"last":118.47}]},{"osd":6,"last update":"Mon Oct 9 04:50:02 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.242,"15min":1.228},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.23100000000000001},"max":{"1min":3.5579999999999998,"5min":10.295999999999999,"15min":13.162000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.262,"15min":1.321},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.28000000000000003},"max":{"1min":3.6259999999999999,"5min":7.3040000000000003,"15min":13.231999999999999},"last":0.437}]},{"osd":7,"last update":"Mon Oct 9 04:49:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.5449999999999999,"15min":1.704},"min":{"1min":0.34399999999999997,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":3.5150000000000001,"5min":68.411000000000001,"15min":68.411000000000001},"last":110.521},{"interface":"front","average":{"1min":1.2509999999999999,"5min":2.8679999999999999,"15min":1.7609999999999999},"min":{"1min":0.41199999999999998,"5min":0.38300000000000001,"15min":0.30599999999999999},"max":{"1min":4.157,"5min":68.435000000000002,"15min":68.435000000000002},"last":110.746}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2315124736,"data_stored":2315124736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3294035968,"data_stored":3294035968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2818113536,"data_stored":2818113536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2864513024,"data_stored":2864513024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:50:18.759 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:50:18.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:18.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:50:18.853 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:18 smithi029 ceph-mon[216903]: pgmap v1365: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T04:50:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:18 smithi029 ceph-mon[223858]: pgmap v1365: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T04:50:18.992 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:19.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 60919 60903 76.1203 40.125 0.0125312 0.0131319 2023-10-09T04:50:19.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:18 smithi177 ceph-mon[195797]: pgmap v1365: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 60 MiB/s wr, 962 op/s 2023-10-09T04:50:19.079 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:19.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:19.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:19.254 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:19.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:19.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:19.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:19.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:19.541 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:19.592 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:50:19.808 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:19.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:19.848 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:50:19.912 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:19.940 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:19.985 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:19.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1829811710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:19.987 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1829811710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:19.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:20.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 61731 61715 75.6227 50.75 0.00693024 0.0131756 2023-10-09T04:50:20.089 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:20.119 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:20.126 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1829811710' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:20.171 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:20.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:20.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:20.305 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:20.620 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:20.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:20.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:20.752 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[216903]: pgmap v1366: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 38 MiB/s wr, 603 op/s 2023-10-09T04:50:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:50:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:50:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1829811710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[216903]: osdmap e356: 8 total, 8 up, 4 in 2023-10-09T04:50:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[223858]: pgmap v1366: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 38 MiB/s wr, 603 op/s 2023-10-09T04:50:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:50:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:50:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1829811710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:20 smithi029 ceph-mon[223858]: osdmap e356: 8 total, 8 up, 4 in 2023-10-09T04:50:20.970 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:21.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 62403 62387 74.9761 42 0.178149 0.0133143 2023-10-09T04:50:21.179 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:21.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:21.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:20 smithi177 ceph-mon[195797]: pgmap v1366: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 38 MiB/s wr, 603 op/s 2023-10-09T04:50:21.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:20 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:50:21.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:20 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:50:21.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1829811710' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:21.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:20 smithi177 ceph-mon[195797]: osdmap e356: 8 total, 8 up, 4 in 2023-10-09T04:50:21.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:21.317 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:21.475 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:21.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:21.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:21.579 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:21.670 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:21.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:21.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:21.961 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:50:21.982 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:50:22.005 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:50:22.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 63320 63304 74.6428 57.3125 0.0194697 0.0133922 2023-10-09T04:50:22.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:22.027 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:50:22.054 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:22.057 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:50:22.094 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:50:22.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:21 smithi029 ceph-mon[216903]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:21 smithi029 ceph-mon[216903]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:21 smithi029 ceph-mon[216903]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:21 smithi029 ceph-mon[223858]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:21 smithi029 ceph-mon[223858]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:21 smithi029 ceph-mon[223858]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.122 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:50:22.144 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:50:22.145 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.145 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.145 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.145 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.146 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.146 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.146 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.146 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:50:22.161 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:22.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:22.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:22.188 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111555 12.3 12:c25dcaa0:::benchmark_data_smithi029_442458_object16305:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:44.814094+0000", 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 277.36843124199999, 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1358333539999999, 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:22.189 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111555 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814094+0000", 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814094+0000", 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814093+0000", 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814100+0000", 2023-10-09T04:50:22.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.601e-06 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814114+0000", 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4347e-05 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814121+0000", 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2660000000000004e-06 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814230+0000", 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108893 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814326+0000", 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5826e-05 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.814415+0000", 2023-10-09T04:50:22.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8688000000000002e-05 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.818142+0000", 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037275960000000001 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.829376+0000", 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011233724 2023-10-09T04:50:22.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.850506+0000", 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021129972 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.850520+0000", 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4561e-05 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596806+0000", 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.746286067 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596817+0000", 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0074e-05 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.918273+0000", 2023-10-09T04:50:22.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.321456746 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.918434+0000", 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016048700000000001 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.949791+0000", 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.031357260999999997 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.949927+0000", 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000136296 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111567 12.3 12:c852941b:::benchmark_data_smithi029_442458_object16317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:44.839187+0000", 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 277.34333822000002, 2023-10-09T04:50:22.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1219626819999999, 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111567 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839187+0000", 2023-10-09T04:50:22.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839187+0000", 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839186+0000", 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:22.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839193+0000", 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0539999999999997e-06 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839207+0000", 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.366e-05 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839214+0000", 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4510000000000003e-06 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839257+0000", 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2626000000000002e-05 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839371+0000", 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113804 2023-10-09T04:50:22.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.839447+0000", 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6570999999999996e-05 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.850546+0000", 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011098943 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.850552+0000", 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7039999999999999e-06 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596843+0000", 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.74629092399999997 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596850+0000", 2023-10-09T04:50:22.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8090000000000004e-06 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.918480+0000", 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3216300190000001 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.918576+0000", 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6677999999999998e-05 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.961044+0000", 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.042468071000000003 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.961150+0000", 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105379 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111570 12.3 12:cb1ed368:::benchmark_data_smithi029_442458_object16320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:44.851574+0000", 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 277.33095155699999, 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1038120039999999, 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111570 2023-10-09T04:50:22.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851574+0000", 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851574+0000", 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851572+0000", 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851583+0000", 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1029999999999999e-05 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851596+0000", 2023-10-09T04:50:22.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2719e-05 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851603+0000", 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0879999999999997e-06 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851698+0000", 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4945000000000007e-05 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.851707+0000", 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9539999999999993e-06 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596870+0000", 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.74516307999999998 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596877+0000", 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7610000000000002e-06 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.918638+0000", 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.321761508 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.918787+0000", 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014869699999999999 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.919341+0000", 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00055375200000000002 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.933994+0000", 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014653006999999999 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.940781+0000", 2023-10-09T04:50:22.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006787403 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.951909+0000", 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011127682 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.951969+0000", 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0517999999999999e-05 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.955386+0000", 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034163879999999998 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111577 12.3 12:c9cc3669:::benchmark_data_smithi029_442458_object16327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:44.871616+0000", 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 277.310909079, 2023-10-09T04:50:22.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0916641180000002, 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111577 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871616+0000", 2023-10-09T04:50:22.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871616+0000", 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871615+0000", 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871621+0000", 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6730000000000003e-06 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871641+0000", 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9021999999999998e-05 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871646+0000", 2023-10-09T04:50:22.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.04e-06 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871693+0000", 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7518999999999997e-05 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.871702+0000", 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8750000000000006e-06 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596898+0000", 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72519582500000002 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596904+0000", 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9379999999999997e-06 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.924223+0000", 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3273195150000001 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.924348+0000", 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000124678 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.924471+0000", 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012298699999999999 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.937138+0000", 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012666785999999999 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.940820+0000", 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036827880000000002 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.961172+0000", 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020351891 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.961206+0000", 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3872999999999999e-05 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.963280+0000", 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020740469999999999 2023-10-09T04:50:22.223 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111581 12.3 12:cbe1490d:::benchmark_data_smithi029_442458_object16331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:44.897280+0000", 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 277.28524554000001, 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0688240449999999, 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.224 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111581 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897280+0000", 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897280+0000", 2023-10-09T04:50:22.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897278+0000", 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897290+0000", 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1299e-05 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897307+0000", 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7895000000000001e-05 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897314+0000", 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5300000000000002e-06 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897381+0000", 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6977999999999995e-05 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:44.897394+0000", 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2593e-05 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596920+0000", 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.69952656099999999 2023-10-09T04:50:22.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.596926+0000", 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.913e-06 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.927289+0000", 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.330362748 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.927465+0000", 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017623500000000001 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.927661+0000", 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000195625 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.937150+0000", 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0094889580000000005 2023-10-09T04:50:22.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.940861+0000", 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037111430000000001 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.963352+0000", 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022490936999999999 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.963419+0000", 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6975000000000002e-05 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.966104+0000", 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00268505 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.232 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111575 12.4 12:2fec8e56:::benchmark_data_smithi029_442458_object16325:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e303)", 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:45.575375+0000", 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 276.60714974000001, 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0685818249999999, 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111575 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575375+0000", 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575375+0000", 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575374+0000", 2023-10-09T04:50:22.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575381+0000", 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3350000000000001e-06 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575398+0000", 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6744000000000001e-05 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575406+0000", 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8050000000000007e-06 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575474+0000", 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8491000000000005e-05 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.575486+0000", 2023-10-09T04:50:22.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1092e-05 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.597820+0000", 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022334052 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:45.597829+0000", 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2469999999999997e-06 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.223365+0000", 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6255360570000001 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.226174+0000", 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028091240000000001 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.229838+0000", 2023-10-09T04:50:22.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036642150000000002 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.635277+0000", 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40543923900000001 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.635513+0000", 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023571700000000001 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.636300+0000", 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00078727099999999998 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.637092+0000", 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00079181200000000003 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.638253+0000", 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011607340000000001 2023-10-09T04:50:22.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.643957+0000", 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057042990000000003 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111596 12.4 12:250254b6:::benchmark_data_smithi029_442458_object16346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:46.964981+0000", 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.21754455299998, 2023-10-09T04:50:22.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7139895329999999, 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111596 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.964981+0000", 2023-10-09T04:50:22.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.964981+0000", 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.964979+0000", 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.964986+0000", 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2830000000000004e-06 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.965007+0000", 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0324000000000001e-05 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:46.965014+0000", 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4789999999999999e-06 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.644700+0000", 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6796858449999998 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.647420+0000", 2023-10-09T04:50:22.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002720156 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.650940+0000", 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035201799999999999 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.669271+0000", 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018330576000000001 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.669507+0000", 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000236386 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.670279+0000", 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000771442 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.670981+0000", 2023-10-09T04:50:22.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00070247699999999996 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.672402+0000", 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014206780000000001 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.678970+0000", 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065683670000000003 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.248 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111604 12.4 12:24ba002f:::benchmark_data_smithi029_442458_object16354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:47.011767+0000", 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.170758466, 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6768685460000001, 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.249 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111604 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011767+0000", 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011767+0000", 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011765+0000", 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011776+0000", 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0443e-05 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011793+0000", 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7427000000000001e-05 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011800+0000", 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.973e-06 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.656136+0000", 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.644335551 2023-10-09T04:50:22.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.658815+0000", 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026794359999999999 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.663161+0000", 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043456170000000004 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.679992+0000", 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016830937000000001 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.680277+0000", 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028532699999999999 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.681061+0000", 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00078419500000000001 2023-10-09T04:50:22.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.681930+0000", 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00086918599999999998 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.683389+0000", 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014591610000000001 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.688635+0000", 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052459059999999998 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111605 12.4 12:2ad61585:::benchmark_data_smithi029_442458_object16355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.256 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:47.011808+0000", 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.17071763500002, 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7105047259999999, 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111605 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011808+0000", 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011808+0000", 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011807+0000", 2023-10-09T04:50:22.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011810+0000", 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5689999999999999e-06 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011813+0000", 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5360000000000001e-06 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.011816+0000", 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.993e-06 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.689087+0000", 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.677270126 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.692446+0000", 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033597100000000001 2023-10-09T04:50:22.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.696233+0000", 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037872159999999999 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.712870+0000", 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016636613000000001 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.713594+0000", 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00072387500000000002 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.713832+0000", 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023795500000000001 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.714789+0000", 2023-10-09T04:50:22.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00095743100000000002 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.715894+0000", 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011049969999999999 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.722312+0000", 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064179900000000002 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.263 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111609 12.4 12:28cfc755:::benchmark_data_smithi029_442458_object16359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:47.028426+0000", 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.15409915499998, 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.703140992, 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.264 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111609 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.028426+0000", 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.028426+0000", 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.028425+0000", 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.028432+0000", 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5000000000000002e-06 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.028447+0000", 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.446e-05 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.028453+0000", 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6869999999999997e-06 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.701084+0000", 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.672630514 2023-10-09T04:50:22.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.703883+0000", 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002799359 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.707562+0000", 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036786480000000001 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.714062+0000", 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065001049999999999 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.723109+0000", 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0090465400000000005 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.723842+0000", 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00073364999999999997 2023-10-09T04:50:22.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.724551+0000", 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00070850100000000005 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.725709+0000", 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011579019999999999 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.731567+0000", 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058585119999999997 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111630 12.4 12:2ca65aff:::benchmark_data_smithi029_442458_object16380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:47.093242+0000", 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.08928328399998, 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6736700629999999, 2023-10-09T04:50:22.271 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111630 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.093242+0000", 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.093242+0000", 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.093241+0000", 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.093247+0000", 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.801e-06 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.093258+0000", 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1299e-05 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.093265+0000", 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5980000000000002e-06 2023-10-09T04:50:22.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.731939+0000", 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.638674907 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.735161+0000", 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032214790000000002 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.738859+0000", 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036981539999999999 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.757147+0000", 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018288294 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.757840+0000", 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00069284399999999997 2023-10-09T04:50:22.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.758071+0000", 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023099000000000001 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.759301+0000", 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001229316 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.760729+0000", 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001428374 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.766912+0000", 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061831159999999998 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111635 12.4 12:2dda9ff3:::benchmark_data_smithi029_442458_object16385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.278 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:47.114222+0000", 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.06830284300003, 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6625357009999999, 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111635 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.114222+0000", 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.114222+0000", 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.114221+0000", 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.114227+0000", 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8529999999999997e-06 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.114238+0000", 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1211e-05 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.114243+0000", 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.215e-06 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.744579+0000", 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6303355229999998 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.747490+0000", 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029113250000000002 2023-10-09T04:50:22.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.751203+0000", 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037125629999999999 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.758301+0000", 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0070982010000000002 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.767821+0000", 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095205800000000007 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.768742+0000", 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00092108200000000004 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.769498+0000", 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00075585300000000002 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.770681+0000", 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011831649999999999 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.776758+0000", 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060766939999999997 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:111644 12.4 12:21e88c4c:::benchmark_data_smithi029_442458_object16394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:45:47.166070+0000", 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 275.01645553600002, 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6477540670000002, 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 111644 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.286 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.166070+0000", 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.166070+0000", 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.166068+0000", 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.166076+0000", 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.166091+0000", 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4783000000000001e-05 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:47.166096+0000", 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3290000000000001e-06 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.777063+0000", 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6109671539999999 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.780361+0000", 2023-10-09T04:50:22.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032976469999999999 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,4", 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.784966+0000", 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046054160000000002 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.803447+0000", 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018480376999999999 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.804327+0000", 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00088016899999999998 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.804604+0000", 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027726699999999999 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.805572+0000", 2023-10-09T04:50:22.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00096728299999999999 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.807167+0000", 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015951419999999999 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:45:48.813824+0000", 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006657072 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.292 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122045 12.3 12:c4441d2d:::benchmark_data_smithi029_442458_object26795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.152020+0000", 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 238.03050490300001, 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3768657260000001, 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.293 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122045 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152020+0000", 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152020+0000", 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152018+0000", 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152026+0000", 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8820000000000001e-06 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152039+0000", 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2585999999999999e-05 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152046+0000", 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6719999999999998e-06 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152234+0000", 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018830799999999999 2023-10-09T04:50:22.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152407+0000", 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000173435 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.152575+0000", 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000167541 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.160623+0000", 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080485299999999999 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.174008+0000", 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013384121000000001 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.174019+0000", 2023-10-09T04:50:22.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1799e-05 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.148206+0000", 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97418687199999998 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.148216+0000", 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9520000000000006e-06 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.408764+0000", 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.26054756 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.408881+0000", 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116814 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528777+0000", 2023-10-09T04:50:22.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.119896927 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528886+0000", 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108577 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122081 12.3 12:c20dcfa8:::benchmark_data_smithi029_442458_object26831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-10-09T04:50:22.301 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:25.304312+0000", 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 236.87821293600001, 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.251359748, 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122081 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304312+0000", 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304312+0000", 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304311+0000", 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:22.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304317+0000", 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5310000000000001e-06 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304406+0000", 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9635000000000005e-05 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.304 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304411+0000", 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8300000000000003e-06 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304483+0000", 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1779999999999994e-05 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.304497+0000", 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3974e-05 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.408977+0000", 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.10447955 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.409589+0000", 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00061213700000000001 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.487411+0000", 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.077822775999999996 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.487516+0000", 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010419799999999999 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.488196+0000", 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00068001400000000005 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.519548+0000", 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.031352338 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528953+0000", 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0094048780000000002 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.528988+0000", 2023-10-09T04:50:22.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.468e-05 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.552089+0000", 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023101403 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.552144+0000", 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5068000000000002e-05 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.555672+0000", 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035280369999999999 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122086 12.3 12:c3bf1047:::benchmark_data_smithi029_442458_object26836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:25.627260+0000", 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 236.55526540899999, 2023-10-09T04:50:22.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93389076100000001, 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122086 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627260+0000", 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.312 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.312 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627260+0000", 2023-10-09T04:50:22.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:22.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627259+0000", 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627263+0000", 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.89e-06 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627270+0000", 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.579e-06 2023-10-09T04:50:22.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627274+0000", 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7759999999999999e-06 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627330+0000", 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6957999999999999e-05 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:50:22.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.627341+0000", 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.006e-05 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.409622+0000", 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.78228175200000005 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.410198+0000", 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057598600000000001 2023-10-09T04:50:22.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.509332+0000", 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.099133712999999998 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.509437+0000", 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010505699999999999 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4,5", 2023-10-09T04:50:22.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.509596+0000", 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015935799999999999 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.522898+0000", 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013301386 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.529021+0000", 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061227929999999996 2023-10-09T04:50:22.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.529060+0000", 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9638000000000002e-05 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.557593+0000", 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028532445 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.557658+0000", 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4920999999999997e-05 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.561151+0000", 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003493014 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.320 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(12.4 311/303 [PushOp(12:22fa9735:::benchmark_data_smithi029_442458_object50436:head, version: 289'3157, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:22fa9735:::benchmark_data_smithi029_442458_object50436:head@289'3157, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:48.090785+0000", 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 214.09173976899999, 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.69916073999999995, 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090785+0000", 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090785+0000", 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090785+0000", 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967296 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090841+0000", 2023-10-09T04:50:22.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6017999999999998e-05 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090857+0000", 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5349000000000002e-05 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090865+0000", 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1799999999999996e-06 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.778520+0000", 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68765535300000002 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.778758+0000", 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000237402 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.789946+0000", 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011188673999999999 2023-10-09T04:50:22.324 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(12.4 311/303 [PushOp(12:209ecbed:::benchmark_data_smithi029_442458_object50442:head, version: 289'3159, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:209ecbed:::benchmark_data_smithi029_442458_object50442:head@289'3159, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:48.090877+0000", 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 214.091647768, 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72271680500000002, 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090877+0000", 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090877+0000", 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090877+0000", 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967296 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090910+0000", 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2772999999999999e-05 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090920+0000", 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0173999999999999e-05 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.090927+0000", 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2699999999999999e-06 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.800041+0000", 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70911317799999996 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.800295+0000", 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025405399999999998 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:48.813594+0000", 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013299577 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(12.4 311/303 [PushOp(12:2a333262:::benchmark_data_smithi029_442458_object51061:head, version: 289'3192, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:2a333262:::benchmark_data_smithi029_442458_object51061:head@289'3192, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:50:22.329 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:51.014822+0000", 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 211.16770328300001, 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55045109599999997, 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.014822+0000", 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.014822+0000", 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.014821+0000", 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.014854+0000", 2023-10-09T04:50:22.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3243e-05 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.014870+0000", 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6028999999999998e-05 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.014879+0000", 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.553986+0000", 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.539107789 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.554237+0000", 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025056900000000002 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:51.565273+0000", 2023-10-09T04:50:22.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011036110999999999 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:33794 13.e 13:735d498e:::benchmark_data_smithi029_495570_object33793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e349)", 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:49:48.944206+0000", 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 33.238318831999997, 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40090359800000003, 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:50:22.334 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 33794 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944206+0000", 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.335 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944206+0000", 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944205+0000", 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944231+0000", 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6067999999999998e-05 2023-10-09T04:50:22.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944243+0000", 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1800999999999999e-05 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:48.944249+0000", 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0809999999999999e-06 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.329215+0000", 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38496537199999997 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.329357+0000", 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000141951 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.329515+0000", 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015808400000000001 2023-10-09T04:50:22.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.333857+0000", 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043424040000000002 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.340326+0000", 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006469168 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.345014+0000", 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046873280000000002 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.345069+0000", 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5469e-05 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:49:49.345110+0000", 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1001000000000003e-05 2023-10-09T04:50:22.340 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.341 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.341 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.341 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:50:22.341 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:50:22.341 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:50:22.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:21 smithi177 ceph-mon[195797]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:21 smithi177 ceph-mon[195797]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.341 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:21 smithi177 ceph-mon[195797]: failed to encode map e356 with expected crc 2023-10-09T04:50:22.343 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:22.400 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:22.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:22.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:22.531 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:23.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 64027 64011 74.0787 44.1875 0.0193997 0.0134942 2023-10-09T04:50:23.078 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:50:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:50:22] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-10-09T04:50:23.078 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:22 smithi029 ceph-mon[216903]: pgmap v1368: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T04:50:23.078 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3101702929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:23.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:22 smithi029 ceph-mon[223858]: pgmap v1368: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T04:50:23.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3101702929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:23.091 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:23.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:23.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:23.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:22 smithi177 ceph-mon[195797]: pgmap v1368: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-09T04:50:23.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3101702929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:23.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:50:23.223 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:23.311 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:50:23.862 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:23.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:23.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:24.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 64972 64956 73.8056 59.0625 0.016564 0.013545 2023-10-09T04:50:24.024 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:24.051 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:24.069 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:24.197 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:24.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:24.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:24.358 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:24.635 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:24.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:24.688 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:50:24.690 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:50:24.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:24.780 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:25.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 65989 65973 73.6226 63.5625 0.0104448 0.0135777 2023-10-09T04:50:25.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:24 smithi029 ceph-mon[216903]: pgmap v1369: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:50:25.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:24 smithi029 ceph-mon[223858]: pgmap v1369: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:50:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:24 smithi177 ceph-mon[195797]: pgmap v1369: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-09T04:50:25.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:50:26.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 66763 66747 73.1796 48.375 0.0268221 0.0136607 2023-10-09T04:50:26.106 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:26.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:25 smithi029 ceph-mon[216903]: from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:25 smithi029 ceph-mon[223858]: from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:26.212 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:26.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:25 smithi177 ceph-mon[195797]: from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:27.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 67739 67723 72.9695 61 0.00334654 0.0136893 2023-10-09T04:50:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:26 smithi029 ceph-mon[216903]: pgmap v1370: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T04:50:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:26 smithi029 ceph-mon[223858]: pgmap v1370: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T04:50:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:26 smithi177 ceph-mon[195797]: pgmap v1370: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T04:50:27.370 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:27.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:27.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:27.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:27.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:27.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:27.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:27.687 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1369,"stamp":"2023-10-09T04:50:23.379751+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4070774688,"num_objects":62130,"num_object_clones":0,"num_object_copies":186390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62130,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":62406,"num_write_kb":3977229,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":4870682,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62322,"ondisk_log_size":62322,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":16933412,"kb_used_data":11883140,"kb_used_omap":4,"kb_used_meta":5050235,"kb_avail":358047196,"statfs":{"total":383980142592,"available":366640328704,"internally_reserved":0,"allocated":12168335360,"data_stored":12164499750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":5171441550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":515047424,"num_objects":7859,"num_object_clones":0,"num_object_copies":23577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7859,"num_write_kb":502976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002282"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":113,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780020+0000","last_change":"2023-10-09T04:49:24.722690+0000","last_active":"2023-10-09T04:50:09.780020+0000","last_peered":"2023-10-09T04:50:09.780020+0000","last_clean":"2023-10-09T04:50:09.780020+0000","last_became_active":"2023-10-09T04:49:24.722128+0000","last_became_peered":"2023-10-09T04:49:24.722128+0000","last_unstale":"2023-10-09T04:50:09.780020+0000","last_undegraded":"2023-10-09T04:50:09.780020+0000","last_fullsized":"2023-10-09T04:50:09.780020+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:52.116095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.671381+0000","last_change":"2023-10-09T04:49:36.252176+0000","last_active":"2023-10-09T04:50:11.671381+0000","last_peered":"2023-10-09T04:50:11.671381+0000","last_clean":"2023-10-09T04:50:11.671381+0000","last_became_active":"2023-10-09T04:49:36.251166+0000","last_became_peered":"2023-10-09T04:49:36.251166+0000","last_unstale":"2023-10-09T04:50:11.671381+0000","last_undegraded":"2023-10-09T04:50:11.671381+0000","last_fullsized":"2023-10-09T04:50:11.671381+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:40.935795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":353,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670178+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:50:11.670178+0000","last_peered":"2023-10-09T04:50:11.670178+0000","last_clean":"2023-10-09T04:50:11.670178+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:50:11.670178+0000","last_undegraded":"2023-10-09T04:50:11.670178+0000","last_fullsized":"2023-10-09T04:50:11.670178+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":289,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.701999+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:50:11.701999+0000","last_peered":"2023-10-09T04:50:11.701999+0000","last_clean":"2023-10-09T04:50:11.701999+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:50:11.701999+0000","last_undegraded":"2023-10-09T04:50:11.701999+0000","last_fullsized":"2023-10-09T04:50:11.701999+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":150,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670178+0000","last_change":"2023-10-09T04:49:36.253446+0000","last_active":"2023-10-09T04:50:11.670178+0000","last_peered":"2023-10-09T04:50:11.670178+0000","last_clean":"2023-10-09T04:50:11.670178+0000","last_became_active":"2023-10-09T04:49:36.252717+0000","last_became_peered":"2023-10-09T04:49:36.252717+0000","last_unstale":"2023-10-09T04:50:11.670178+0000","last_undegraded":"2023-10-09T04:50:11.670178+0000","last_fullsized":"2023-10-09T04:50:11.670178+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:30.302532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":155,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.983086+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:50:09.983086+0000","last_peered":"2023-10-09T04:50:09.983086+0000","last_clean":"2023-10-09T04:50:09.983086+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:50:09.983086+0000","last_undegraded":"2023-10-09T04:50:09.983086+0000","last_fullsized":"2023-10-09T04:50:09.983086+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":297,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982896+0000","last_change":"2023-10-09T04:49:24.727794+0000","last_active":"2023-10-09T04:50:09.982896+0000","last_peered":"2023-10-09T04:50:09.982896+0000","last_clean":"2023-10-09T04:50:09.982896+0000","last_became_active":"2023-10-09T04:49:24.726934+0000","last_became_peered":"2023-10-09T04:49:24.726934+0000","last_unstale":"2023-10-09T04:50:09.982896+0000","last_undegraded":"2023-10-09T04:50:09.982896+0000","last_fullsized":"2023-10-09T04:50:09.982896+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:43.043505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":272,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670379+0000","last_change":"2023-10-09T04:49:36.252722+0000","last_active":"2023-10-09T04:50:11.670379+0000","last_peered":"2023-10-09T04:50:11.670379+0000","last_clean":"2023-10-09T04:50:11.670379+0000","last_became_active":"2023-10-09T04:49:36.251598+0000","last_became_peered":"2023-10-09T04:49:36.251598+0000","last_unstale":"2023-10-09T04:50:11.670379+0000","last_undegraded":"2023-10-09T04:50:11.670379+0000","last_fullsized":"2023-10-09T04:50:11.670379+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.458714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":315,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981701+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:50:09.981701+0000","last_peered":"2023-10-09T04:50:09.981701+0000","last_clean":"2023-10-09T04:50:09.981701+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:50:09.981701+0000","last_undegraded":"2023-10-09T04:50:09.981701+0000","last_fullsized":"2023-10-09T04:50:09.981701+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":306,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.698309+0000","last_change":"2023-10-09T04:49:24.728045+0000","last_active":"2023-10-09T04:50:11.698309+0000","last_peered":"2023-10-09T04:50:11.698309+0000","last_clean":"2023-10-09T04:50:11.698309+0000","last_became_active":"2023-10-09T04:49:24.727278+0000","last_became_peered":"2023-10-09T04:49:24.727278+0000","last_unstale":"2023-10-09T04:50:11.698309+0000","last_undegraded":"2023-10-09T04:50:11.698309+0000","last_fullsized":"2023-10-09T04:50:11.698309+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:17.700648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":398,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.701312+0000","last_change":"2023-10-09T04:49:24.751066+0000","last_active":"2023-10-09T04:50:11.701312+0000","last_peered":"2023-10-09T04:50:11.701312+0000","last_clean":"2023-10-09T04:50:11.701312+0000","last_became_active":"2023-10-09T04:49:24.731080+0000","last_became_peered":"2023-10-09T04:49:24.731080+0000","last_unstale":"2023-10-09T04:50:11.701312+0000","last_undegraded":"2023-10-09T04:50:11.701312+0000","last_fullsized":"2023-10-09T04:50:11.701312+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:47.656154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":251,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670646+0000","last_change":"2023-10-09T04:49:36.255264+0000","last_active":"2023-10-09T04:50:11.670646+0000","last_peered":"2023-10-09T04:50:11.670646+0000","last_clean":"2023-10-09T04:50:11.670646+0000","last_became_active":"2023-10-09T04:49:36.254631+0000","last_became_peered":"2023-10-09T04:49:36.254631+0000","last_unstale":"2023-10-09T04:50:11.670646+0000","last_undegraded":"2023-10-09T04:50:11.670646+0000","last_fullsized":"2023-10-09T04:50:11.670646+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:26.380768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":153,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780369+0000","last_change":"2023-10-09T04:49:24.739093+0000","last_active":"2023-10-09T04:50:09.780369+0000","last_peered":"2023-10-09T04:50:09.780369+0000","last_clean":"2023-10-09T04:50:09.780369+0000","last_became_active":"2023-10-09T04:49:24.723103+0000","last_became_peered":"2023-10-09T04:49:24.723103+0000","last_unstale":"2023-10-09T04:50:09.780369+0000","last_undegraded":"2023-10-09T04:50:09.780369+0000","last_fullsized":"2023-10-09T04:50:09.780369+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:41.664075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":325,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780344+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:50:09.780344+0000","last_peered":"2023-10-09T04:50:09.780344+0000","last_clean":"2023-10-09T04:50:09.780344+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:50:09.780344+0000","last_undegraded":"2023-10-09T04:50:09.780344+0000","last_fullsized":"2023-10-09T04:50:09.780344+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":382,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.698655+0000","last_change":"2023-10-09T04:49:24.748611+0000","last_active":"2023-10-09T04:50:11.698655+0000","last_peered":"2023-10-09T04:50:11.698655+0000","last_clean":"2023-10-09T04:50:11.698655+0000","last_became_active":"2023-10-09T04:49:24.730507+0000","last_became_peered":"2023-10-09T04:49:24.730507+0000","last_unstale":"2023-10-09T04:50:11.698655+0000","last_undegraded":"2023-10-09T04:50:11.698655+0000","last_fullsized":"2023-10-09T04:50:11.698655+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:22.219826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670966+0000","last_change":"2023-10-09T04:49:36.255342+0000","last_active":"2023-10-09T04:50:11.670966+0000","last_peered":"2023-10-09T04:50:11.670966+0000","last_clean":"2023-10-09T04:50:11.670966+0000","last_became_active":"2023-10-09T04:49:36.254372+0000","last_became_peered":"2023-10-09T04:49:36.254372+0000","last_unstale":"2023-10-09T04:50:11.670966+0000","last_undegraded":"2023-10-09T04:50:11.670966+0000","last_fullsized":"2023-10-09T04:50:11.670966+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:37.972615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":298,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.698449+0000","last_change":"2023-10-09T04:49:24.731694+0000","last_active":"2023-10-09T04:50:11.698449+0000","last_peered":"2023-10-09T04:50:11.698449+0000","last_clean":"2023-10-09T04:50:11.698449+0000","last_became_active":"2023-10-09T04:49:24.731338+0000","last_became_peered":"2023-10-09T04:49:24.731338+0000","last_unstale":"2023-10-09T04:50:11.698449+0000","last_undegraded":"2023-10-09T04:50:11.698449+0000","last_fullsized":"2023-10-09T04:50:11.698449+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:45.968733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981248+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:50:09.981248+0000","last_peered":"2023-10-09T04:50:09.981248+0000","last_clean":"2023-10-09T04:50:09.981248+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:50:09.981248+0000","last_undegraded":"2023-10-09T04:50:09.981248+0000","last_fullsized":"2023-10-09T04:50:09.981248+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":328,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.698225+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:11.698225+0000","last_peered":"2023-10-09T04:50:11.698225+0000","last_clean":"2023-10-09T04:50:11.698225+0000","last_became_active":"2023-10-09T04:49:24.727752+0000","last_became_peered":"2023-10-09T04:49:24.727752+0000","last_unstale":"2023-10-09T04:50:11.698225+0000","last_undegraded":"2023-10-09T04:50:11.698225+0000","last_fullsized":"2023-10-09T04:50:11.698225+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:37.448051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":186,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.702075+0000","last_change":"2023-10-09T04:49:24.734384+0000","last_active":"2023-10-09T04:50:11.702075+0000","last_peered":"2023-10-09T04:50:11.702075+0000","last_clean":"2023-10-09T04:50:11.702075+0000","last_became_active":"2023-10-09T04:49:24.733948+0000","last_became_peered":"2023-10-09T04:49:24.733948+0000","last_unstale":"2023-10-09T04:50:11.702075+0000","last_undegraded":"2023-10-09T04:50:11.702075+0000","last_fullsized":"2023-10-09T04:50:11.702075+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:55.081133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":156,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982161+0000","last_change":"2023-10-09T04:49:24.743137+0000","last_active":"2023-10-09T04:50:09.982161+0000","last_peered":"2023-10-09T04:50:09.982161+0000","last_clean":"2023-10-09T04:50:09.982161+0000","last_became_active":"2023-10-09T04:49:24.726930+0000","last_became_peered":"2023-10-09T04:49:24.726930+0000","last_unstale":"2023-10-09T04:50:09.982161+0000","last_undegraded":"2023-10-09T04:50:09.982161+0000","last_fullsized":"2023-10-09T04:50:09.982161+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:38.154496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":159,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.704134+0000","last_change":"2023-10-09T04:49:24.731691+0000","last_active":"2023-10-09T04:50:11.704134+0000","last_peered":"2023-10-09T04:50:11.704134+0000","last_clean":"2023-10-09T04:50:11.704134+0000","last_became_active":"2023-10-09T04:49:24.731374+0000","last_became_peered":"2023-10-09T04:49:24.731374+0000","last_unstale":"2023-10-09T04:50:11.704134+0000","last_undegraded":"2023-10-09T04:50:11.704134+0000","last_fullsized":"2023-10-09T04:50:11.704134+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:39.511857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"355'3867","reported_seq":3895,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.665437+0000","last_change":"2023-10-09T04:49:24.722035+0000","last_active":"2023-10-09T04:50:19.665437+0000","last_peered":"2023-10-09T04:50:19.665437+0000","last_clean":"2023-10-09T04:50:19.665437+0000","last_became_active":"2023-10-09T04:49:24.721093+0000","last_became_peered":"2023-10-09T04:49:24.721093+0000","last_unstale":"2023-10-09T04:50:19.665437+0000","last_undegraded":"2023-10-09T04:50:19.665437+0000","last_fullsized":"2023-10-09T04:50:19.665437+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3867,"log_dups_size":0,"ondisk_log_size":3867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:16.988764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253427712,"num_objects":3867,"num_object_clones":0,"num_object_copies":11601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3867,"num_write_kb":247488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":386,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.703960+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:50:11.703960+0000","last_peered":"2023-10-09T04:50:11.703960+0000","last_clean":"2023-10-09T04:50:11.703960+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:50:11.703960+0000","last_undegraded":"2023-10-09T04:50:11.703960+0000","last_fullsized":"2023-10-09T04:50:11.703960+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":307,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981899+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:50:09.981899+0000","last_peered":"2023-10-09T04:50:09.981899+0000","last_clean":"2023-10-09T04:50:09.981899+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:50:09.981899+0000","last_undegraded":"2023-10-09T04:50:09.981899+0000","last_fullsized":"2023-10-09T04:50:09.981899+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"356'3873","reported_seq":3911,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:20.238209+0000","last_change":"2023-10-09T04:49:24.729511+0000","last_active":"2023-10-09T04:50:20.238209+0000","last_peered":"2023-10-09T04:50:20.238209+0000","last_clean":"2023-10-09T04:50:20.238209+0000","last_became_active":"2023-10-09T04:49:24.728925+0000","last_became_peered":"2023-10-09T04:49:24.728925+0000","last_unstale":"2023-10-09T04:50:20.238209+0000","last_undegraded":"2023-10-09T04:50:20.238209+0000","last_fullsized":"2023-10-09T04:50:20.238209+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3873,"log_dups_size":0,"ondisk_log_size":3873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.606555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253820928,"num_objects":3873,"num_object_clones":0,"num_object_copies":11619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3873,"num_write_kb":247872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":345,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.701919+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:50:11.701919+0000","last_peered":"2023-10-09T04:50:11.701919+0000","last_clean":"2023-10-09T04:50:11.701919+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:50:11.701919+0000","last_undegraded":"2023-10-09T04:50:11.701919+0000","last_fullsized":"2023-10-09T04:50:11.701919+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":352,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780136+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:50:09.780136+0000","last_peered":"2023-10-09T04:50:09.780136+0000","last_clean":"2023-10-09T04:50:09.780136+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:50:09.780136+0000","last_undegraded":"2023-10-09T04:50:09.780136+0000","last_fullsized":"2023-10-09T04:50:09.780136+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"355'3836","reported_seq":3863,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.642896+0000","last_change":"2023-10-09T04:49:17.684070+0000","last_active":"2023-10-09T04:50:19.642896+0000","last_peered":"2023-10-09T04:50:19.642896+0000","last_clean":"2023-10-09T04:50:19.642896+0000","last_became_active":"2023-10-09T04:49:17.683772+0000","last_became_peered":"2023-10-09T04:49:17.683772+0000","last_unstale":"2023-10-09T04:50:19.642896+0000","last_undegraded":"2023-10-09T04:50:19.642896+0000","last_fullsized":"2023-10-09T04:50:19.642896+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3836,"log_dups_size":0,"ondisk_log_size":3836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:47.529254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251396096,"num_objects":3836,"num_object_clones":0,"num_object_copies":11508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3836,"num_write_kb":245504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":327,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.701129+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:50:11.701129+0000","last_peered":"2023-10-09T04:50:11.701129+0000","last_clean":"2023-10-09T04:50:11.701129+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:50:11.701129+0000","last_undegraded":"2023-10-09T04:50:11.701129+0000","last_fullsized":"2023-10-09T04:50:11.701129+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":224,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670281+0000","last_change":"2023-10-09T04:49:36.255047+0000","last_active":"2023-10-09T04:50:11.670281+0000","last_peered":"2023-10-09T04:50:11.670281+0000","last_clean":"2023-10-09T04:50:11.670281+0000","last_became_active":"2023-10-09T04:49:36.254324+0000","last_became_peered":"2023-10-09T04:49:36.254324+0000","last_unstale":"2023-10-09T04:50:11.670281+0000","last_undegraded":"2023-10-09T04:50:11.670281+0000","last_fullsized":"2023-10-09T04:50:11.670281+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:55.547119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"355'3867","reported_seq":3894,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.537524+0000","last_change":"2023-10-09T04:49:17.682749+0000","last_active":"2023-10-09T04:50:19.537524+0000","last_peered":"2023-10-09T04:50:19.537524+0000","last_clean":"2023-10-09T04:50:19.537524+0000","last_became_active":"2023-10-09T04:49:17.682418+0000","last_became_peered":"2023-10-09T04:49:17.682418+0000","last_unstale":"2023-10-09T04:50:19.537524+0000","last_undegraded":"2023-10-09T04:50:19.537524+0000","last_fullsized":"2023-10-09T04:50:19.537524+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3867,"log_dups_size":0,"ondisk_log_size":3867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:33.471194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253427712,"num_objects":3867,"num_object_clones":0,"num_object_copies":11601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3867,"num_write_kb":247488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":355,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982138+0000","last_change":"2023-10-09T04:49:24.728445+0000","last_active":"2023-10-09T04:50:09.982138+0000","last_peered":"2023-10-09T04:50:09.982138+0000","last_clean":"2023-10-09T04:50:09.982138+0000","last_became_active":"2023-10-09T04:49:24.728130+0000","last_became_peered":"2023-10-09T04:49:24.728130+0000","last_unstale":"2023-10-09T04:50:09.982138+0000","last_undegraded":"2023-10-09T04:50:09.982138+0000","last_fullsized":"2023-10-09T04:50:09.982138+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:04.777758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":71,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.672283+0000","last_change":"2023-10-09T04:49:36.253080+0000","last_active":"2023-10-09T04:50:11.672283+0000","last_peered":"2023-10-09T04:50:11.672283+0000","last_clean":"2023-10-09T04:50:11.672283+0000","last_became_active":"2023-10-09T04:49:36.252682+0000","last_became_peered":"2023-10-09T04:49:36.252682+0000","last_unstale":"2023-10-09T04:50:11.672283+0000","last_undegraded":"2023-10-09T04:50:11.672283+0000","last_fullsized":"2023-10-09T04:50:11.672283+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:31.712210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.5","version":"355'3771","reported_seq":3797,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.671971+0000","last_change":"2023-10-09T04:49:17.681809+0000","last_active":"2023-10-09T04:50:19.671971+0000","last_peered":"2023-10-09T04:50:19.671971+0000","last_clean":"2023-10-09T04:50:19.671971+0000","last_became_active":"2023-10-09T04:49:17.681477+0000","last_became_peered":"2023-10-09T04:49:17.681477+0000","last_unstale":"2023-10-09T04:50:19.671971+0000","last_undegraded":"2023-10-09T04:50:19.671971+0000","last_fullsized":"2023-10-09T04:50:19.671971+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3771,"log_dups_size":0,"ondisk_log_size":3771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:07.783547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247136256,"num_objects":3771,"num_object_clones":0,"num_object_copies":11313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3771,"num_write_kb":241344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":94,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780390+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:50:09.780390+0000","last_peered":"2023-10-09T04:50:09.780390+0000","last_clean":"2023-10-09T04:50:09.780390+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:50:09.780390+0000","last_undegraded":"2023-10-09T04:50:09.780390+0000","last_fullsized":"2023-10-09T04:50:09.780390+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780422+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:50:09.780422+0000","last_peered":"2023-10-09T04:50:09.780422+0000","last_clean":"2023-10-09T04:50:09.780422+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:50:09.780422+0000","last_undegraded":"2023-10-09T04:50:09.780422+0000","last_fullsized":"2023-10-09T04:50:09.780422+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"356'3919","reported_seq":3958,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:23.296916+0000","last_change":"2023-10-09T04:49:36.269590+0000","last_active":"2023-10-09T04:50:23.296916+0000","last_peered":"2023-10-09T04:50:23.296916+0000","last_clean":"2023-10-09T04:50:23.296916+0000","last_became_active":"2023-10-09T04:49:36.252739+0000","last_became_peered":"2023-10-09T04:49:36.252739+0000","last_unstale":"2023-10-09T04:50:23.296916+0000","last_undegraded":"2023-10-09T04:50:23.296916+0000","last_fullsized":"2023-10-09T04:50:23.296916+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3919,"log_dups_size":0,"ondisk_log_size":3919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:26.767295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256835584,"num_objects":3919,"num_object_clones":0,"num_object_copies":11757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3919,"num_write_kb":250816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":316,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780070+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:50:09.780070+0000","last_peered":"2023-10-09T04:50:09.780070+0000","last_clean":"2023-10-09T04:50:09.780070+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:50:09.780070+0000","last_undegraded":"2023-10-09T04:50:09.780070+0000","last_fullsized":"2023-10-09T04:50:09.780070+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":113,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780543+0000","last_change":"2023-10-09T04:49:24.712121+0000","last_active":"2023-10-09T04:50:09.780543+0000","last_peered":"2023-10-09T04:50:09.780543+0000","last_clean":"2023-10-09T04:50:09.780543+0000","last_became_active":"2023-10-09T04:49:24.711716+0000","last_became_peered":"2023-10-09T04:49:24.711716+0000","last_unstale":"2023-10-09T04:50:09.780543+0000","last_undegraded":"2023-10-09T04:50:09.780543+0000","last_fullsized":"2023-10-09T04:50:09.780543+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:37.141689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"356'3825","reported_seq":3852,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:20.237362+0000","last_change":"2023-10-09T04:49:17.686417+0000","last_active":"2023-10-09T04:50:20.237362+0000","last_peered":"2023-10-09T04:50:20.237362+0000","last_clean":"2023-10-09T04:50:20.237362+0000","last_became_active":"2023-10-09T04:49:17.686126+0000","last_became_peered":"2023-10-09T04:49:17.686126+0000","last_unstale":"2023-10-09T04:50:20.237362+0000","last_undegraded":"2023-10-09T04:50:20.237362+0000","last_fullsized":"2023-10-09T04:50:20.237362+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3825,"log_dups_size":0,"ondisk_log_size":3825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:58.636951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250675200,"num_objects":3825,"num_object_clones":0,"num_object_copies":11475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3825,"num_write_kb":244800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780052+0000","last_change":"2023-10-09T04:49:24.721842+0000","last_active":"2023-10-09T04:50:09.780052+0000","last_peered":"2023-10-09T04:50:09.780052+0000","last_clean":"2023-10-09T04:50:09.780052+0000","last_became_active":"2023-10-09T04:49:24.720666+0000","last_became_peered":"2023-10-09T04:49:24.720666+0000","last_unstale":"2023-10-09T04:50:09.780052+0000","last_undegraded":"2023-10-09T04:50:09.780052+0000","last_fullsized":"2023-10-09T04:50:09.780052+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:16.572332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":335,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780540+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:50:09.780540+0000","last_peered":"2023-10-09T04:50:09.780540+0000","last_clean":"2023-10-09T04:50:09.780540+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:50:09.780540+0000","last_undegraded":"2023-10-09T04:50:09.780540+0000","last_fullsized":"2023-10-09T04:50:09.780540+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"355'3892","reported_seq":3929,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.644671+0000","last_change":"2023-10-09T04:49:24.721346+0000","last_active":"2023-10-09T04:50:19.644671+0000","last_peered":"2023-10-09T04:50:19.644671+0000","last_clean":"2023-10-09T04:50:19.644671+0000","last_became_active":"2023-10-09T04:49:24.720955+0000","last_became_peered":"2023-10-09T04:49:24.720955+0000","last_unstale":"2023-10-09T04:50:19.644671+0000","last_undegraded":"2023-10-09T04:50:19.644671+0000","last_fullsized":"2023-10-09T04:50:19.644671+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3892,"log_dups_size":0,"ondisk_log_size":3892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:58:55.342602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255066112,"num_objects":3892,"num_object_clones":0,"num_object_copies":11676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3892,"num_write_kb":249088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670293+0000","last_change":"2023-10-09T04:49:36.253172+0000","last_active":"2023-10-09T04:50:11.670293+0000","last_peered":"2023-10-09T04:50:11.670293+0000","last_clean":"2023-10-09T04:50:11.670293+0000","last_became_active":"2023-10-09T04:49:36.252175+0000","last_became_peered":"2023-10-09T04:49:36.252175+0000","last_unstale":"2023-10-09T04:50:11.670293+0000","last_undegraded":"2023-10-09T04:50:11.670293+0000","last_fullsized":"2023-10-09T04:50:11.670293+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:38.269236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":261,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.671512+0000","last_change":"2023-10-09T04:49:36.251118+0000","last_active":"2023-10-09T04:50:11.671512+0000","last_peered":"2023-10-09T04:50:11.671512+0000","last_clean":"2023-10-09T04:50:11.671512+0000","last_became_active":"2023-10-09T04:49:36.250803+0000","last_became_peered":"2023-10-09T04:49:36.250803+0000","last_unstale":"2023-10-09T04:50:11.671512+0000","last_undegraded":"2023-10-09T04:50:11.671512+0000","last_fullsized":"2023-10-09T04:50:11.671512+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:38.765478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"355'3852","reported_seq":3889,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.526930+0000","last_change":"2023-10-09T04:49:24.727773+0000","last_active":"2023-10-09T04:50:19.526930+0000","last_peered":"2023-10-09T04:50:19.526930+0000","last_clean":"2023-10-09T04:50:19.526930+0000","last_became_active":"2023-10-09T04:49:24.727411+0000","last_became_peered":"2023-10-09T04:49:24.727411+0000","last_unstale":"2023-10-09T04:50:19.526930+0000","last_undegraded":"2023-10-09T04:50:19.526930+0000","last_fullsized":"2023-10-09T04:50:19.526930+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3852,"log_dups_size":0,"ondisk_log_size":3852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:21.626311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252444672,"num_objects":3852,"num_object_clones":0,"num_object_copies":11556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3852,"num_write_kb":246528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":250,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.671278+0000","last_change":"2023-10-09T04:49:36.250912+0000","last_active":"2023-10-09T04:50:11.671278+0000","last_peered":"2023-10-09T04:50:11.671278+0000","last_clean":"2023-10-09T04:50:11.671278+0000","last_became_active":"2023-10-09T04:49:36.250530+0000","last_became_peered":"2023-10-09T04:49:36.250530+0000","last_unstale":"2023-10-09T04:50:11.671278+0000","last_undegraded":"2023-10-09T04:50:11.671278+0000","last_fullsized":"2023-10-09T04:50:11.671278+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:26.975687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":369,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981170+0000","last_change":"2023-10-09T04:49:24.727377+0000","last_active":"2023-10-09T04:50:09.981170+0000","last_peered":"2023-10-09T04:50:09.981170+0000","last_clean":"2023-10-09T04:50:09.981170+0000","last_became_active":"2023-10-09T04:49:24.724195+0000","last_became_peered":"2023-10-09T04:49:24.724195+0000","last_unstale":"2023-10-09T04:50:09.981170+0000","last_undegraded":"2023-10-09T04:50:09.981170+0000","last_fullsized":"2023-10-09T04:50:09.981170+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:46.895156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":245,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670725+0000","last_change":"2023-10-09T04:49:36.254992+0000","last_active":"2023-10-09T04:50:11.670725+0000","last_peered":"2023-10-09T04:50:11.670725+0000","last_clean":"2023-10-09T04:50:11.670725+0000","last_became_active":"2023-10-09T04:49:36.254288+0000","last_became_peered":"2023-10-09T04:49:36.254288+0000","last_unstale":"2023-10-09T04:50:11.670725+0000","last_undegraded":"2023-10-09T04:50:11.670725+0000","last_fullsized":"2023-10-09T04:50:11.670725+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:13.355684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"355'3916","reported_seq":3943,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.642878+0000","last_change":"2023-10-09T04:49:17.684748+0000","last_active":"2023-10-09T04:50:19.642878+0000","last_peered":"2023-10-09T04:50:19.642878+0000","last_clean":"2023-10-09T04:50:19.642878+0000","last_became_active":"2023-10-09T04:49:17.684346+0000","last_became_peered":"2023-10-09T04:49:17.684346+0000","last_unstale":"2023-10-09T04:50:19.642878+0000","last_undegraded":"2023-10-09T04:50:19.642878+0000","last_fullsized":"2023-10-09T04:50:19.642878+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3916,"log_dups_size":0,"ondisk_log_size":3916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:39.897521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256638976,"num_objects":3916,"num_object_clones":0,"num_object_copies":11748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3916,"num_write_kb":250624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":339,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.984931+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:50:09.984931+0000","last_peered":"2023-10-09T04:50:09.984931+0000","last_clean":"2023-10-09T04:50:09.984931+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:50:09.984931+0000","last_undegraded":"2023-10-09T04:50:09.984931+0000","last_fullsized":"2023-10-09T04:50:09.984931+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780032+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:50:09.780032+0000","last_peered":"2023-10-09T04:50:09.780032+0000","last_clean":"2023-10-09T04:50:09.780032+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:50:09.780032+0000","last_undegraded":"2023-10-09T04:50:09.780032+0000","last_fullsized":"2023-10-09T04:50:09.780032+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":299,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670896+0000","last_change":"2023-10-09T04:49:36.254929+0000","last_active":"2023-10-09T04:50:11.670896+0000","last_peered":"2023-10-09T04:50:11.670896+0000","last_clean":"2023-10-09T04:50:11.670896+0000","last_became_active":"2023-10-09T04:49:36.252868+0000","last_became_peered":"2023-10-09T04:49:36.252868+0000","last_unstale":"2023-10-09T04:50:11.670896+0000","last_undegraded":"2023-10-09T04:50:11.670896+0000","last_fullsized":"2023-10-09T04:50:11.670896+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:26.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"356'3920","reported_seq":3947,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:20.202158+0000","last_change":"2023-10-09T04:49:17.686557+0000","last_active":"2023-10-09T04:50:20.202158+0000","last_peered":"2023-10-09T04:50:20.202158+0000","last_clean":"2023-10-09T04:50:20.202158+0000","last_became_active":"2023-10-09T04:49:17.686236+0000","last_became_peered":"2023-10-09T04:49:17.686236+0000","last_unstale":"2023-10-09T04:50:20.202158+0000","last_undegraded":"2023-10-09T04:50:20.202158+0000","last_fullsized":"2023-10-09T04:50:20.202158+0000","mapping_epoch":336,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3920,"log_dups_size":0,"ondisk_log_size":3920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:02.935353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256901120,"num_objects":3920,"num_object_clones":0,"num_object_copies":11760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3920,"num_write_kb":250880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":300,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670332+0000","last_change":"2023-10-09T04:49:36.250480+0000","last_active":"2023-10-09T04:50:11.670332+0000","last_peered":"2023-10-09T04:50:11.670332+0000","last_clean":"2023-10-09T04:50:11.670332+0000","last_became_active":"2023-10-09T04:49:36.250048+0000","last_became_peered":"2023-10-09T04:49:36.250048+0000","last_unstale":"2023-10-09T04:50:11.670332+0000","last_undegraded":"2023-10-09T04:50:11.670332+0000","last_fullsized":"2023-10-09T04:50:11.670332+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:21.602101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":299,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670463+0000","last_change":"2023-10-09T04:49:36.253118+0000","last_active":"2023-10-09T04:50:11.670463+0000","last_peered":"2023-10-09T04:50:11.670463+0000","last_clean":"2023-10-09T04:50:11.670463+0000","last_became_active":"2023-10-09T04:49:36.252036+0000","last_became_peered":"2023-10-09T04:49:36.252036+0000","last_unstale":"2023-10-09T04:50:11.670463+0000","last_undegraded":"2023-10-09T04:50:11.670463+0000","last_fullsized":"2023-10-09T04:50:11.670463+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.648245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":241,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.671435+0000","last_change":"2023-10-09T04:49:36.252116+0000","last_active":"2023-10-09T04:50:11.671435+0000","last_peered":"2023-10-09T04:50:11.671435+0000","last_clean":"2023-10-09T04:50:11.671435+0000","last_became_active":"2023-10-09T04:49:36.251630+0000","last_became_peered":"2023-10-09T04:49:36.251630+0000","last_unstale":"2023-10-09T04:50:11.671435+0000","last_undegraded":"2023-10-09T04:50:11.671435+0000","last_fullsized":"2023-10-09T04:50:11.671435+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:48.681072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"356'4069","reported_seq":4118,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:23.281947+0000","last_change":"2023-10-09T04:49:36.276163+0000","last_active":"2023-10-09T04:50:23.281947+0000","last_peered":"2023-10-09T04:50:23.281947+0000","last_clean":"2023-10-09T04:50:23.281947+0000","last_became_active":"2023-10-09T04:49:36.253314+0000","last_became_peered":"2023-10-09T04:49:36.253314+0000","last_unstale":"2023-10-09T04:50:23.281947+0000","last_undegraded":"2023-10-09T04:50:23.281947+0000","last_fullsized":"2023-10-09T04:50:23.281947+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4069,"log_dups_size":0,"ondisk_log_size":4069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:44.334582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266665984,"num_objects":4069,"num_object_clones":0,"num_object_copies":12207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4069,"num_write_kb":260416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":346,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.984029+0000","last_change":"2023-10-09T04:49:24.727573+0000","last_active":"2023-10-09T04:50:09.984029+0000","last_peered":"2023-10-09T04:50:09.984029+0000","last_clean":"2023-10-09T04:50:09.984029+0000","last_became_active":"2023-10-09T04:49:24.727151+0000","last_became_peered":"2023-10-09T04:49:24.727151+0000","last_unstale":"2023-10-09T04:50:09.984029+0000","last_undegraded":"2023-10-09T04:50:09.984029+0000","last_fullsized":"2023-10-09T04:50:09.984029+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:53.790164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":826,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.695449+0000","last_change":"2023-10-09T04:49:27.734937+0000","last_active":"2023-10-09T04:50:11.695449+0000","last_peered":"2023-10-09T04:50:11.695449+0000","last_clean":"2023-10-09T04:50:11.695449+0000","last_became_active":"2023-10-09T04:49:27.734587+0000","last_became_peered":"2023-10-09T04:49:27.734587+0000","last_unstale":"2023-10-09T04:50:11.695449+0000","last_undegraded":"2023-10-09T04:50:11.695449+0000","last_fullsized":"2023-10-09T04:50:11.695449+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:38.000046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":285,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.985119+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:50:09.985119+0000","last_peered":"2023-10-09T04:50:09.985119+0000","last_clean":"2023-10-09T04:50:09.985119+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:50:09.985119+0000","last_undegraded":"2023-10-09T04:50:09.985119+0000","last_fullsized":"2023-10-09T04:50:09.985119+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":298,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780511+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:50:09.780511+0000","last_peered":"2023-10-09T04:50:09.780511+0000","last_clean":"2023-10-09T04:50:09.780511+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:50:09.780511+0000","last_undegraded":"2023-10-09T04:50:09.780511+0000","last_fullsized":"2023-10-09T04:50:09.780511+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"356'3826","reported_seq":3864,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:20.198428+0000","last_change":"2023-10-09T04:49:24.733991+0000","last_active":"2023-10-09T04:50:20.198428+0000","last_peered":"2023-10-09T04:50:20.198428+0000","last_clean":"2023-10-09T04:50:20.198428+0000","last_became_active":"2023-10-09T04:49:24.733661+0000","last_became_peered":"2023-10-09T04:49:24.733661+0000","last_unstale":"2023-10-09T04:50:20.198428+0000","last_undegraded":"2023-10-09T04:50:20.198428+0000","last_fullsized":"2023-10-09T04:50:20.198428+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3826,"log_dups_size":0,"ondisk_log_size":3826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:22.030285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250740736,"num_objects":3826,"num_object_clones":0,"num_object_copies":11478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3826,"num_write_kb":244864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":366,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.695347+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:50:11.695347+0000","last_peered":"2023-10-09T04:50:11.695347+0000","last_clean":"2023-10-09T04:50:11.695347+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:50:11.695347+0000","last_undegraded":"2023-10-09T04:50:11.695347+0000","last_fullsized":"2023-10-09T04:50:11.695347+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780155+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:50:09.780155+0000","last_peered":"2023-10-09T04:50:09.780155+0000","last_clean":"2023-10-09T04:50:09.780155+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:50:09.780155+0000","last_undegraded":"2023-10-09T04:50:09.780155+0000","last_fullsized":"2023-10-09T04:50:09.780155+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":126,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.671046+0000","last_change":"2023-10-09T04:49:36.252316+0000","last_active":"2023-10-09T04:50:11.671046+0000","last_peered":"2023-10-09T04:50:11.671046+0000","last_clean":"2023-10-09T04:50:11.671046+0000","last_became_active":"2023-10-09T04:49:36.251771+0000","last_became_peered":"2023-10-09T04:49:36.251771+0000","last_unstale":"2023-10-09T04:50:11.671046+0000","last_undegraded":"2023-10-09T04:50:11.671046+0000","last_fullsized":"2023-10-09T04:50:11.671046+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:08.690791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"356'4062","reported_seq":4099,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-09T04:50:23.247460+0000","last_change":"2023-10-09T04:49:36.254761+0000","last_active":"2023-10-09T04:50:23.247460+0000","last_peered":"2023-10-09T04:50:23.247460+0000","last_clean":"2023-10-09T04:50:23.247460+0000","last_became_active":"2023-10-09T04:49:36.250489+0000","last_became_peered":"2023-10-09T04:49:36.250489+0000","last_unstale":"2023-10-09T04:50:23.247460+0000","last_undegraded":"2023-10-09T04:50:23.247460+0000","last_fullsized":"2023-10-09T04:50:23.247460+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4062,"log_dups_size":0,"ondisk_log_size":4062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:19.664434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266207232,"num_objects":4062,"num_object_clones":0,"num_object_copies":12186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4062,"num_write_kb":259968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":369,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.701214+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:50:11.701214+0000","last_peered":"2023-10-09T04:50:11.701214+0000","last_clean":"2023-10-09T04:50:11.701214+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:50:11.701214+0000","last_undegraded":"2023-10-09T04:50:11.701214+0000","last_fullsized":"2023-10-09T04:50:11.701214+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":252,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670817+0000","last_change":"2023-10-09T04:49:36.254730+0000","last_active":"2023-10-09T04:50:11.670817+0000","last_peered":"2023-10-09T04:50:11.670817+0000","last_clean":"2023-10-09T04:50:11.670817+0000","last_became_active":"2023-10-09T04:49:36.252242+0000","last_became_peered":"2023-10-09T04:49:36.252242+0000","last_unstale":"2023-10-09T04:50:11.670817+0000","last_undegraded":"2023-10-09T04:50:11.670817+0000","last_fullsized":"2023-10-09T04:50:11.670817+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.106203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982167+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:50:09.982167+0000","last_peered":"2023-10-09T04:50:09.982167+0000","last_clean":"2023-10-09T04:50:09.982167+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:50:09.982167+0000","last_undegraded":"2023-10-09T04:50:09.982167+0000","last_fullsized":"2023-10-09T04:50:09.982167+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"355'3820","reported_seq":3857,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.553163+0000","last_change":"2023-10-09T04:49:24.718431+0000","last_active":"2023-10-09T04:50:19.553163+0000","last_peered":"2023-10-09T04:50:19.553163+0000","last_clean":"2023-10-09T04:50:19.553163+0000","last_became_active":"2023-10-09T04:49:24.718108+0000","last_became_peered":"2023-10-09T04:49:24.718108+0000","last_unstale":"2023-10-09T04:50:19.553163+0000","last_undegraded":"2023-10-09T04:50:19.553163+0000","last_fullsized":"2023-10-09T04:50:19.553163+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3820,"log_dups_size":0,"ondisk_log_size":3820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.309575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250347520,"num_objects":3820,"num_object_clones":0,"num_object_copies":11460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3820,"num_write_kb":244480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":104,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.704221+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:50:11.704221+0000","last_peered":"2023-10-09T04:50:11.704221+0000","last_clean":"2023-10-09T04:50:11.704221+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:50:11.704221+0000","last_undegraded":"2023-10-09T04:50:11.704221+0000","last_fullsized":"2023-10-09T04:50:11.704221+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":326,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670756+0000","last_change":"2023-10-09T04:49:36.251899+0000","last_active":"2023-10-09T04:50:11.670756+0000","last_peered":"2023-10-09T04:50:11.670756+0000","last_clean":"2023-10-09T04:50:11.670756+0000","last_became_active":"2023-10-09T04:49:36.251283+0000","last_became_peered":"2023-10-09T04:49:36.251283+0000","last_unstale":"2023-10-09T04:50:11.670756+0000","last_undegraded":"2023-10-09T04:50:11.670756+0000","last_fullsized":"2023-10-09T04:50:11.670756+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:52.505049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.695563+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:50:11.695563+0000","last_peered":"2023-10-09T04:50:11.695563+0000","last_clean":"2023-10-09T04:50:11.695563+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:50:11.695563+0000","last_undegraded":"2023-10-09T04:50:11.695563+0000","last_fullsized":"2023-10-09T04:50:11.695563+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"355'3791","reported_seq":3819,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:19.503334+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:19.503334+0000","last_peered":"2023-10-09T04:50:19.503334+0000","last_clean":"2023-10-09T04:50:19.503334+0000","last_became_active":"2023-10-09T04:49:24.727688+0000","last_became_peered":"2023-10-09T04:49:24.727688+0000","last_unstale":"2023-10-09T04:50:19.503334+0000","last_undegraded":"2023-10-09T04:50:19.503334+0000","last_fullsized":"2023-10-09T04:50:19.503334+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3791,"log_dups_size":0,"ondisk_log_size":3791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:00.374022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248446976,"num_objects":3791,"num_object_clones":0,"num_object_copies":11373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3791,"num_write_kb":242624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":288,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.703861+0000","last_change":"2023-10-09T04:49:24.730477+0000","last_active":"2023-10-09T04:50:11.703861+0000","last_peered":"2023-10-09T04:50:11.703861+0000","last_clean":"2023-10-09T04:50:11.703861+0000","last_became_active":"2023-10-09T04:49:24.730115+0000","last_became_peered":"2023-10-09T04:49:24.730115+0000","last_unstale":"2023-10-09T04:50:11.703861+0000","last_undegraded":"2023-10-09T04:50:11.703861+0000","last_fullsized":"2023-10-09T04:50:11.703861+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:49.730424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":125,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670898+0000","last_change":"2023-10-09T04:49:36.250949+0000","last_active":"2023-10-09T04:50:11.670898+0000","last_peered":"2023-10-09T04:50:11.670898+0000","last_clean":"2023-10-09T04:50:11.670898+0000","last_became_active":"2023-10-09T04:49:36.250519+0000","last_became_peered":"2023-10-09T04:49:36.250519+0000","last_unstale":"2023-10-09T04:50:11.670898+0000","last_undegraded":"2023-10-09T04:50:11.670898+0000","last_fullsized":"2023-10-09T04:50:11.670898+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:51.821472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.779986+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:50:09.779986+0000","last_peered":"2023-10-09T04:50:09.779986+0000","last_clean":"2023-10-09T04:50:09.779986+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:50:09.779986+0000","last_undegraded":"2023-10-09T04:50:09.779986+0000","last_fullsized":"2023-10-09T04:50:09.779986+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":205,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780263+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:50:09.780263+0000","last_peered":"2023-10-09T04:50:09.780263+0000","last_clean":"2023-10-09T04:50:09.780263+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:50:09.780263+0000","last_undegraded":"2023-10-09T04:50:09.780263+0000","last_fullsized":"2023-10-09T04:50:09.780263+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":292,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670797+0000","last_change":"2023-10-09T04:49:36.254629+0000","last_active":"2023-10-09T04:50:11.670797+0000","last_peered":"2023-10-09T04:50:11.670797+0000","last_clean":"2023-10-09T04:50:11.670797+0000","last_became_active":"2023-10-09T04:49:36.251774+0000","last_became_peered":"2023-10-09T04:49:36.251774+0000","last_unstale":"2023-10-09T04:50:11.670797+0000","last_undegraded":"2023-10-09T04:50:11.670797+0000","last_fullsized":"2023-10-09T04:50:11.670797+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:24.682458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":345,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.982303+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:50:09.982303+0000","last_peered":"2023-10-09T04:50:09.982303+0000","last_clean":"2023-10-09T04:50:09.982303+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:50:09.982303+0000","last_undegraded":"2023-10-09T04:50:09.982303+0000","last_fullsized":"2023-10-09T04:50:09.982303+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780177+0000","last_change":"2023-10-09T04:49:24.721368+0000","last_active":"2023-10-09T04:50:09.780177+0000","last_peered":"2023-10-09T04:50:09.780177+0000","last_clean":"2023-10-09T04:50:09.780177+0000","last_became_active":"2023-10-09T04:49:24.721024+0000","last_became_peered":"2023-10-09T04:49:24.721024+0000","last_unstale":"2023-10-09T04:50:09.780177+0000","last_undegraded":"2023-10-09T04:50:09.780177+0000","last_fullsized":"2023-10-09T04:50:09.780177+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:57.736030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":314,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.981090+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:50:09.981090+0000","last_peered":"2023-10-09T04:50:09.981090+0000","last_clean":"2023-10-09T04:50:09.981090+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:50:09.981090+0000","last_undegraded":"2023-10-09T04:50:09.981090+0000","last_fullsized":"2023-10-09T04:50:09.981090+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":133,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.780114+0000","last_change":"2023-10-09T04:49:24.711760+0000","last_active":"2023-10-09T04:50:09.780114+0000","last_peered":"2023-10-09T04:50:09.780114+0000","last_clean":"2023-10-09T04:50:09.780114+0000","last_became_active":"2023-10-09T04:49:24.711383+0000","last_became_peered":"2023-10-09T04:49:24.711383+0000","last_unstale":"2023-10-09T04:50:09.780114+0000","last_undegraded":"2023-10-09T04:50:09.780114+0000","last_fullsized":"2023-10-09T04:50:09.780114+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:04.370272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":309,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-09T04:50:11.670620+0000","last_change":"2023-10-09T04:49:24.725553+0000","last_active":"2023-10-09T04:50:11.670620+0000","last_peered":"2023-10-09T04:50:11.670620+0000","last_clean":"2023-10-09T04:50:11.670620+0000","last_became_active":"2023-10-09T04:49:24.725179+0000","last_became_peered":"2023-10-09T04:49:24.725179+0000","last_unstale":"2023-10-09T04:50:11.670620+0000","last_undegraded":"2023-10-09T04:50:11.670620+0000","last_fullsized":"2023-10-09T04:50:11.670620+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.045784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.980974+0000","last_change":"2023-10-09T04:49:24.718316+0000","last_active":"2023-10-09T04:50:09.980974+0000","last_peered":"2023-10-09T04:50:09.980974+0000","last_clean":"2023-10-09T04:50:09.980974+0000","last_became_active":"2023-10-09T04:49:24.717948+0000","last_became_peered":"2023-10-09T04:49:24.717948+0000","last_unstale":"2023-10-09T04:50:09.980974+0000","last_undegraded":"2023-10-09T04:50:09.980974+0000","last_fullsized":"2023-10-09T04:50:09.980974+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:48.454201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":261,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.983175+0000","last_change":"2023-10-09T04:49:24.745757+0000","last_active":"2023-10-09T04:50:09.983175+0000","last_peered":"2023-10-09T04:50:09.983175+0000","last_clean":"2023-10-09T04:50:09.983175+0000","last_became_active":"2023-10-09T04:49:24.727010+0000","last_became_peered":"2023-10-09T04:49:24.727010+0000","last_unstale":"2023-10-09T04:50:09.983175+0000","last_undegraded":"2023-10-09T04:50:09.983175+0000","last_fullsized":"2023-10-09T04:50:09.983175+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:51.654447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":133,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-09T04:50:09.779919+0000","last_change":"2023-10-09T04:49:24.710860+0000","last_active":"2023-10-09T04:50:09.779919+0000","last_peered":"2023-10-09T04:50:09.779919+0000","last_clean":"2023-10-09T04:50:09.779919+0000","last_became_active":"2023-10-09T04:49:24.710496+0000","last_became_peered":"2023-10-09T04:49:24.710496+0000","last_unstale":"2023-10-09T04:50:09.779919+0000","last_undegraded":"2023-10-09T04:50:09.779919+0000","last_fullsized":"2023-10-09T04:50:09.779919+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:24.901415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4070178816,"num_objects":62106,"num_object_clones":0,"num_object_copies":186318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62106,"num_write_kb":3974784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12225478656,"data_stored":12225478656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62106,"ondisk_log_size":62106,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16666,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234521,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4481572,"kb_used_data":2976412,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":89263580,"statfs":{"total":95995035648,"available":91405905920,"internally_reserved":0,"allocated":3047845888,"data_stored":3047094923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.4039999999999999,"15min":2.0910000000000002},"min":{"1min":0.31,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":1.141,"5min":14.970000000000001,"15min":68.495999999999995},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.048,"15min":2.6200000000000001},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.249},"max":{"1min":2.1800000000000002,"5min":19.152000000000001,"15min":68.540000000000006},"last":0.54700000000000004}]},{"osd":1,"last update":"Mon Oct 9 04:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":4.5730000000000004,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":1.179,"5min":128.601,"15min":264.62},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":4.2990000000000004,"15min":4.2069999999999999},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":1.264,"5min":128.381,"15min":264.65899999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":6.6449999999999996,"15min":5.2489999999999997},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":7.2400000000000002,"5min":129.10400000000001,"15min":184.72},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.669,"5min":6.8280000000000003,"15min":5.548},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.28100000000000003},"max":{"1min":5.2210000000000001,"5min":129.15100000000001,"15min":184.697},"last":0.501}]},{"osd":3,"last update":"Mon Oct 9 04:49:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.9790000000000001,"15min":3.7010000000000001},"min":{"1min":0.41799999999999998,"5min":0.33900000000000002,"15min":0.29499999999999998},"max":{"1min":12.324,"5min":12.324,"15min":182.839},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.1319999999999999,"5min":2.4950000000000001,"15min":4.1040000000000001},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.7400000000000002,"5min":14.866,"15min":182.678},"last":0.56799999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.4449999999999998,"15min":1.772},"min":{"1min":0.34100000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.367000000000001,"5min":14.811999999999999,"15min":14.811999999999999},"last":12.367000000000001},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.669,"15min":1.377},"min":{"1min":0.313,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":4.5389999999999997,"5min":13.523,"15min":13.523},"last":4.5389999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.5189999999999999,"15min":1.3260000000000001},"min":{"1min":0.38,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.1020000000000003,"5min":12.202999999999999,"15min":13.319000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.548,"5min":2.1579999999999999,"15min":1.8340000000000001},"min":{"1min":0.432,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":10.544,"5min":12.081,"15min":18.408000000000001},"last":0.82699999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.2809999999999999,"15min":1.2889999999999999},"min":{"1min":0.39500000000000002,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":5.2350000000000003,"5min":10.047000000000001,"15min":11.388},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.363,"5min":1.9039999999999999,"15min":1.859},"min":{"1min":0.39700000000000002,"5min":0.34499999999999997,"15min":0.30099999999999999},"max":{"1min":7.3620000000000001,"5min":11.680999999999999,"15min":18.311},"last":0.52600000000000002}]}]},{"osd":6,"up_from":0,"seq":197568496155,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141245,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3413508,"kb_used_data":2550844,"kb_used_omap":4,"kb_used_meta":862651,"kb_avail":90331644,"statfs":{"total":95995035648,"available":92499603456,"internally_reserved":0,"allocated":2612064256,"data_stored":2610469853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":883355534},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.484,"15min":2.9529999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.14799999999999999},"max":{"1min":1.78,"5min":11.653,"15min":93.171999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.613,"15min":3.0219999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.193},"max":{"1min":2.1739999999999999,"5min":10.765000000000001,"15min":93.147000000000006},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":5.0199999999999996,"15min":4.8129999999999997},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":1.3899999999999999,"5min":151.41900000000001,"15min":263.60500000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":5.0780000000000003,"15min":5.1609999999999996},"min":{"1min":0.29599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.4299999999999999,"5min":146.749,"15min":263.65199999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:49:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.893,"15min":5.5119999999999996},"min":{"1min":0.28799999999999998,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":5.5880000000000001,"5min":11.68,"15min":379.80900000000003},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":2.343,"15min":6.085},"min":{"1min":0.34000000000000002,"5min":0.248,"15min":0.246},"max":{"1min":4.0510000000000002,"5min":18.285,"15min":379.95800000000003},"last":0.432}]},{"osd":4,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.371,"15min":2.657},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":4.1109999999999998,"5min":19.959,"15min":170.072},"last":3.8530000000000002},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.96,"15min":2.484},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":2.7040000000000002,"5min":11.598000000000001,"15min":169.94499999999999},"last":0.67500000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":2.0600000000000001,"15min":2.4319999999999999},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":3.2170000000000001,"5min":9.782,"15min":171.036},"last":1.3740000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":2.4849999999999999,"15min":2.762},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":1.407,"5min":20.021000000000001,"15min":170.88},"last":0.35699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.954,"15min":2.4159999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":2.786,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.81799999999999995,"5min":2.5739999999999998,"15min":2.7989999999999999},"min":{"1min":0.45800000000000002,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":2.802,"5min":20.164999999999999,"15min":132.17699999999999},"last":0.39700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":2.8460000000000001,"15min":3.4540000000000002},"min":{"1min":0.438,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":3.2280000000000002,"5min":20.055,"15min":174.75999999999999},"last":3.9380000000000002},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.254,"15min":3.1160000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.377},"max":{"1min":3.2109999999999999,"5min":16.238,"15min":174.779},"last":3.919}]}]},{"osd":0,"up_from":0,"seq":390842024302,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":0,"seq":949187772578,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":609885356303,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019429,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4916772,"kb_used_data":3413020,"kb_used_omap":0,"kb_used_meta":1503744,"kb_avail":88828380,"statfs":{"total":95995035648,"available":90960261120,"internally_reserved":0,"allocated":3494932480,"data_stored":3494181515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1539833856},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.7410000000000001,"15min":4.0309999999999997},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":1.0529999999999999,"5min":12.795,"15min":171.13499999999999},"last":2.3300000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.9350000000000001,"15min":4.0229999999999997},"min":{"1min":0.40999999999999998,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":3.4009999999999998,"5min":16.004000000000001,"15min":171.11799999999999},"last":1.0509999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:49:53 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":9.6300000000000008,"15min":5.351},"min":{"1min":0.45200000000000001,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":37.944000000000003,"5min":385.78199999999998,"15min":385.78199999999998},"last":1.492},{"interface":"front","average":{"1min":2.4700000000000002,"5min":9.2650000000000006,"15min":5.3330000000000002},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":37.408999999999999,"5min":386.09399999999999,"15min":386.09399999999999},"last":2.3809999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:50:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":11.032999999999999,"15min":6.5179999999999998},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.131,"5min":440.75599999999997,"15min":440.75599999999997},"last":2.3580000000000001},{"interface":"front","average":{"1min":1.0089999999999999,"5min":11.414,"15min":6.367},"min":{"1min":0.374,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":3.4399999999999999,"5min":440.61900000000003,"15min":440.61900000000003},"last":1.0740000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":4.2069999999999999,"15min":5.4000000000000004},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":7.2670000000000003,"5min":141.334,"15min":357.125},"last":1.099},{"interface":"front","average":{"1min":1.306,"5min":4.242,"15min":5.3310000000000004},"min":{"1min":0.44500000000000001,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":6.1319999999999997,"5min":141.18299999999999,"15min":359.02800000000002},"last":1.518}]},{"osd":5,"last update":"Mon Oct 9 04:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.994,"15min":1.8440000000000001},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":6.1040000000000001,"5min":24.234999999999999,"15min":107.476},"last":1.4570000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.6359999999999999,"15min":1.802},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":7.1769999999999996,"5min":18.785,"15min":107.414},"last":1.226}]},{"osd":6,"last update":"Mon Oct 9 04:49:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.9690000000000001,"15min":1.9019999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.29599999999999999},"max":{"1min":5.9349999999999996,"5min":22.920999999999999,"15min":65.75},"last":1.5429999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.5329999999999999,"15min":1.7430000000000001},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":1.2669999999999999,"5min":15.095000000000001,"15min":65.849999999999994},"last":2.2959999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:49:22 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":2.0369999999999999,"15min":1.5489999999999999},"min":{"1min":0.34599999999999997,"5min":0.33800000000000002,"15min":0.28000000000000003},"max":{"1min":7.298,"5min":16.109000000000002,"15min":16.109000000000002},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.6870000000000001,"5min":2.444,"15min":1.7529999999999999},"min":{"1min":0.36899999999999999,"5min":0.34300000000000003,"15min":0.32100000000000001},"max":{"1min":8.0570000000000004,"5min":22.858000000000001,"15min":22.858000000000001},"last":5.1210000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757793,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4121560,"kb_used_data":2942864,"kb_used_omap":0,"kb_used_meta":1178688,"kb_avail":89623592,"statfs":{"total":95995035648,"available":91774558208,"internally_reserved":0,"allocated":3013492736,"data_stored":3012753459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1206976512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.851,"15min":4.798},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23100000000000001},"max":{"1min":8.7309999999999999,"5min":9.6890000000000001,"15min":210.85499999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.6950000000000001,"15min":5.0359999999999996},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.5489999999999999,"5min":8.4749999999999996,"15min":210.23500000000001},"last":2.286}]},{"osd":1,"last update":"Mon Oct 9 04:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":8.3260000000000005,"15min":5.8259999999999996},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.252},"max":{"1min":1.0900000000000001,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.2110000000000001,"5min":8.5410000000000004,"15min":6.0119999999999996},"min":{"1min":0.33200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":8.6620000000000008,"5min":211.435,"15min":211.435},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":5.5179999999999998,"15min":4.9299999999999997},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":86.488,"15min":90.763999999999996},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.385,"5min":5.5199999999999996,"15min":5.0369999999999999},"min":{"1min":0.34100000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.7590000000000003,"5min":86.510999999999996,"15min":91.340000000000003},"last":0.34599999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:49:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":2.778,"15min":5.1289999999999996},"min":{"1min":0.32300000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":3.605,"5min":46.497,"15min":268.63299999999998},"last":2.3799999999999999},{"interface":"front","average":{"1min":1.2909999999999999,"5min":2.133,"15min":4.7380000000000004},"min":{"1min":0.40600000000000003,"5min":0.38400000000000001,"15min":0.29799999999999999},"max":{"1min":7.2469999999999999,"5min":46.125999999999998,"15min":268.92500000000001},"last":0.64300000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:49:26 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.3700000000000001,"15min":2.5299999999999998},"min":{"1min":0.314,"5min":0.307,"15min":0.23599999999999999},"max":{"1min":4.1799999999999997,"5min":10.268000000000001,"15min":361.09399999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.03,"5min":1.544,"15min":2.8380000000000001},"min":{"1min":0.308,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":4.4269999999999996,"5min":12.609999999999999,"15min":361.71300000000002},"last":3.669}]},{"osd":6,"last update":"Mon Oct 9 04:50:02 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.242,"15min":1.228},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.23100000000000001},"max":{"1min":3.5579999999999998,"5min":10.295999999999999,"15min":13.162000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.262,"15min":1.321},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.28000000000000003},"max":{"1min":3.6259999999999999,"5min":7.3040000000000003,"15min":13.231999999999999},"last":2.125}]},{"osd":7,"last update":"Mon Oct 9 04:49:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.5449999999999999,"15min":1.704},"min":{"1min":0.34399999999999997,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":3.5150000000000001,"5min":68.411000000000001,"15min":68.411000000000001},"last":2.0920000000000001},{"interface":"front","average":{"1min":1.2509999999999999,"5min":2.8679999999999999,"15min":1.7609999999999999},"min":{"1min":0.41199999999999998,"5min":0.38300000000000001,"15min":0.30599999999999999},"max":{"1min":4.157,"5min":68.435000000000002,"15min":68.435000000000002},"last":0.69199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2629304320,"data_stored":2629304320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3529048064,"data_stored":3529048064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3010723840,"data_stored":3010723840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3056402432,"data_stored":3056402432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:50:27.693 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T04:50:27.693 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:50:27.693 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:50:27.693 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 1.000000 2023-10-09T04:50:27.693 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T04:50:27.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:27.841 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:27.875 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:27.933 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:50:27.933 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:50:27.933 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:50:27.933 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:50:27.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:28.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:28.009 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 68679 68663 72.7284 58.75 0.0150408 0.0137453 2023-10-09T04:50:28.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:28.172 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:28.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:28.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:28.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:28.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:28.434 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:29.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:50:29.010221+0000 min lat: 0.0013453 max lat: 1.02767 avg lat: 0.0138355 2023-10-09T04:50:29.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:50:29.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 69389 69373 72.2558 44.375 0.0162358 0.0138355 2023-10-09T04:50:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[216903]: pgmap v1371: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T04:50:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3240760732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[223858]: pgmap v1371: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T04:50:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3240760732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:29.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:28 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:29.193 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:29.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:28 smithi177 ceph-mon[195797]: pgmap v1371: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 803 op/s 2023-10-09T04:50:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:50:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3240760732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:28 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:50:29.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:29.327 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:29.384 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:29.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:29.453 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:50:29.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:29.537 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:29.558 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:50:29.660 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:29.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:29.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:29.847 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:29.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:29.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:29.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:30.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 70261 70245 71.9647 54.5 0.0184881 0.0138906 2023-10-09T04:50:30.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:30.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:29 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:30.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:29 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:50:30.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:29 smithi029 ceph-mon[223858]: osdmap e357: 8 total, 8 up, 4 in 2023-10-09T04:50:30.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:29 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:30.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:29 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:50:30.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:29 smithi029 ceph-mon[216903]: osdmap e357: 8 total, 8 up, 4 in 2023-10-09T04:50:30.102 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:29 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:29 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:50:30.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:29 smithi177 ceph-mon[195797]: osdmap e357: 8 total, 8 up, 4 in 2023-10-09T04:50:30.397 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:31.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 70926 70910 71.4743 41.5625 0.0213049 0.0139856 2023-10-09T04:50:31.162 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:31.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[216903]: pgmap v1372: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 52 MiB/s wr, 829 op/s 2023-10-09T04:50:31.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[216903]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[216903]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[216903]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[216903]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[223858]: pgmap v1372: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 52 MiB/s wr, 829 op/s 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[223858]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[223858]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[223858]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:30 smithi029 ceph-mon[223858]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:31.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:31.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:30 smithi177 ceph-mon[195797]: pgmap v1372: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 52 MiB/s wr, 829 op/s 2023-10-09T04:50:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:30 smithi177 ceph-mon[195797]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:30 smithi177 ceph-mon[195797]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:30 smithi177 ceph-mon[195797]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:30 smithi177 ceph-mon[195797]: failed to encode map e357 with expected crc 2023-10-09T04:50:31.345 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:31.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:31.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:31.628 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:31.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:31.705 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:31.868 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:31.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:31.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:32.001 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:31 smithi029 ceph-mon[216903]: failed to encode map e357 with expected crc 2023-10-09T04:50:32.002 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:31 smithi029 ceph-mon[223858]: failed to encode map e357 with expected crc 2023-10-09T04:50:32.005 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:32.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 71806 71790 71.2127 55 0.0255162 0.0140375 2023-10-09T04:50:32.205 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:32.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:31 smithi177 ceph-mon[195797]: failed to encode map e357 with expected crc 2023-10-09T04:50:32.309 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:32.395 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:32.536 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:32.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:32.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:32.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:32.682 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:33.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 72473 72457 70.7513 41.6875 0.0102511 0.0141299 2023-10-09T04:50:33.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:32 smithi029 ceph-mon[216903]: pgmap v1374: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-09T04:50:33.070 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:50:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:50:32] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.43.0" 2023-10-09T04:50:33.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:32 smithi029 ceph-mon[223858]: pgmap v1374: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-09T04:50:33.081 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:50:33.106 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:50:33.138 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:50:33.169 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:50:33.197 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:50:33.238 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:50:33.251 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:33.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:33.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:33.268 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:50:33.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:32 smithi177 ceph-mon[195797]: pgmap v1374: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-09T04:50:33.294 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:50:33.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.298 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.299 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:50:33.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:33.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:50:33.456 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:33.727 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:33.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:33.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:33.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:33.887 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:34.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:34.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 73324 73308 70.481 53.1875 0.0203327 0.0141828 2023-10-09T04:50:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:34.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:34.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3322473859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:34.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:33 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:34.020 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:33 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:34.121 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3322473859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3322473859' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:34.212 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:34.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:34.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:34.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:34.436 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:34.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:34.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:34.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:34.764 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:34.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:34.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:34.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:35.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:35.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 74053 74037 70.1034 45.5625 0.00883836 0.0142586 2023-10-09T04:50:35.090 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:34 smithi029 ceph-mon[216903]: pgmap v1375: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 53 MiB/s wr, 843 op/s 2023-10-09T04:50:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:34 smithi029 ceph-mon[223858]: pgmap v1375: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 53 MiB/s wr, 843 op/s 2023-10-09T04:50:35.218 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:35.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:35.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:35.233 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:34 smithi177 ceph-mon[195797]: pgmap v1375: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 53 MiB/s wr, 843 op/s 2023-10-09T04:50:35.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:35.367 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:35.765 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:35.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:35.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:35.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:35.907 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:35.979 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:35.980 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:35.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:36.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 74986 74970 69.9274 58.3125 0.0166027 0.0142961 2023-10-09T04:50:36.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:36.117 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:36.203 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:50:36.203 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:50:36.278 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:36.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:36.382 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:36.463 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:36.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:36.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:36.685 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T04:50:36.688 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:36.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:36 smithi177 ceph-mon[195797]: pgmap v1376: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T04:50:36.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1451985157' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:36.769 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:36 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:36.778 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:36.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:36.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:36.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:36 smithi029 ceph-mon[216903]: pgmap v1376: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T04:50:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1451985157' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:36 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:36 smithi029 ceph-mon[223858]: pgmap v1376: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-09T04:50:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1451985157' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:36 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T04:50:36.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:37.010 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 75900 75884 69.7391 57.125 0.0202023 0.0143353 2023-10-09T04:50:37.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:37.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:37.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:37.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:37.392 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:37.507 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:37.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:37.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: failed to encode map e358 with expected crc 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: osdmap e358: 8 total, 8 up, 4 in 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: failed to encode map e358 with expected crc 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: failed to encode map e358 with expected crc 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: failed to encode map e358 with expected crc 2023-10-09T04:50:37.700 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:37 smithi177 ceph-mon[195797]: failed to encode map e358 with expected crc 2023-10-09T04:50:37.701 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:37.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:50:37.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:37.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:37.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:37.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:38.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 76778 76762 69.5236 54.875 0.0180191 0.0143788 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: osdmap e358: 8 total, 8 up, 4 in 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[216903]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: osdmap e358: 8 total, 8 up, 4 in 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:37 smithi029 ceph-mon[223858]: failed to encode map e358 with expected crc 2023-10-09T04:50:38.028 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:38.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:38.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:38.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:38.223 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.248 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:38.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:38.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:38.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:38.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:50:38.460 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:38.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:38.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:38.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:38.606 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:38.672 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:38.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:38.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:38.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:38.916 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:38.947 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:39.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 77424 77408 69.1072 40.375 0.0159928 0.0144652 2023-10-09T04:50:39.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:38 smithi177 ceph-mon[195797]: pgmap v1378: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T04:50:39.166 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:38 smithi029 ceph-mon[223858]: pgmap v1378: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T04:50:39.167 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:38 smithi029 ceph-mon[216903]: pgmap v1378: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 56 MiB/s wr, 903 op/s 2023-10-09T04:50:39.215 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:39.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:39.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:39.405 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:39.517 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:39.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:39.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:39.675 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:39.784 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:39.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:39.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:39.814 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:39.865 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:39.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:39.966 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:40.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 78349 78333 68.948 57.8125 0.0228825 0.0144995 2023-10-09T04:50:40.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:40.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:40.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:40.273 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:40.438 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:40.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:40.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:40.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:40.583 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:40.719 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:50:41.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 79302 79286 68.8176 59.5625 0.0214825 0.0145265 2023-10-09T04:50:41.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:40 smithi177 ceph-mon[195797]: pgmap v1379: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T04:50:41.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1844270075' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:40 smithi029 ceph-mon[216903]: pgmap v1379: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T04:50:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1844270075' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:40 smithi029 ceph-mon[223858]: pgmap v1379: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 810 op/s 2023-10-09T04:50:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1844270075' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:50:42.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 79994 79978 68.4673 43.25 0.0115259 0.014601 2023-10-09T04:50:42.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:42.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1844270075' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:42.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: osdmap e359: 8 total, 8 up, 4 in 2023-10-09T04:50:42.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:41 smithi177 ceph-mon[195797]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1844270075' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: osdmap e359: 8 total, 8 up, 4 in 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[216903]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1844270075' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:50:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: osdmap e359: 8 total, 8 up, 4 in 2023-10-09T04:50:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:41 smithi029 ceph-mon[223858]: failed to encode map e359 with expected crc 2023-10-09T04:50:42.621 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:42.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:42.726 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:42.803 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:42.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:42 smithi029 ceph-mon[216903]: pgmap v1381: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T04:50:42.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:42 smithi029 ceph-mon[223858]: pgmap v1381: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T04:50:42.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:42.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:43.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 80929 80913 68.3317 58.4375 0.0166256 0.01463 2023-10-09T04:50:43.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:42 smithi177 ceph-mon[195797]: pgmap v1381: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-09T04:50:43.037 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:43.119 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:50:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:50:42] "GET /metrics HTTP/1.1" 200 36526 "" "Prometheus/2.43.0" 2023-10-09T04:50:43.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:43.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:43.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:43.387 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:43.768 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:43.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:43.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:43.933 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:50:43.933 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:50:43.946 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:44.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 81808 81792 68.1531 54.9375 0.0252649 0.0146685 2023-10-09T04:50:44.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:44.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:44.167 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:50:44.191 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:50:44.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:44.218 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:50:44.237 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7] out_osds: [6, 3, 0, 2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:50:44.237 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:50:44.238 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T04:50:44.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:50:44.241 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:50:44.257 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:44.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:50:44.343 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:44.356 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:50:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:44.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:44.387 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:50:44.415 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:50:44.415 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.415 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.415 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.415 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.415 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.416 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.416 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.416 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:50:44.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:44.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:44.673 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:44.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:44.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:44.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:44.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[223858]: pgmap v1382: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 39 MiB/s wr, 630 op/s 2023-10-09T04:50:44.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:44.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:44.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3958827564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:44.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:44.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:44.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[216903]: pgmap v1382: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 39 MiB/s wr, 630 op/s 2023-10-09T04:50:44.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:44.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:44.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3958827564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:44.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:44.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:44.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:45.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 82667 82651 67.9627 53.6875 0.0104191 0.0147086 2023-10-09T04:50:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:44 smithi177 ceph-mon[195797]: pgmap v1382: 89 pgs: 89 active+clean; 4.8 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 39 MiB/s wr, 630 op/s 2023-10-09T04:50:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3958827564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:45.387 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:50:46.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 83340 83324 67.6263 42.0625 0.014288 0.0147819 2023-10-09T04:50:46.096 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:50:46.123 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:46.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:46.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:46.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:46.290 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:46.337 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:46.382 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:46.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:46.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:46.493 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:46.527 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:46.593 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:46.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:46.701 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:46.762 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:46.877 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:46.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:46.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[216903]: pgmap v1383: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T04:50:46.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:46.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:46.983 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:46.983 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[223858]: pgmap v1383: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T04:50:46.984 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:46.984 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:46.984 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:46.991 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:47.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 84245 84229 67.4844 56.5625 0.0170589 0.0148137 2023-10-09T04:50:47.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:46 smithi177 ceph-mon[195797]: pgmap v1383: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-09T04:50:47.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:50:47.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:50:47.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:50:47.069 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:47.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:47.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:47.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:47.337 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:47.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:47.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:47.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:47.504 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:47.571 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:47.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:47.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:47.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:47.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:47.907 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:48.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:48.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 85011 84995 67.2361 47.875 0.00855307 0.0148689 2023-10-09T04:50:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:48.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:48.061 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:48.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:48.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:50:48.163 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:48.859 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:48.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:48.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:48.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:48 smithi029 ceph-mon[216903]: pgmap v1384: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:50:48.997 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:48 smithi029 ceph-mon[223858]: pgmap v1384: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:50:49.000 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:49.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:50:49.011876+0000 min lat: 0.0013453 max lat: 1.02767 avg lat: 0.0149229 2023-10-09T04:50:49.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:50:49.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 85772 85756 66.9901 47.5625 0.027435 0.0149229 2023-10-09T04:50:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:48 smithi177 ceph-mon[195797]: pgmap v1384: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-09T04:50:49.060 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:49.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:49.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:49.248 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:49.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:49.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:49.483 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:49.555 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:49.651 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:49.676 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:49.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:49.748 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:50:49.751 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:50:49.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:49.830 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:49.992 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:49.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:50.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 86688 86672 66.8698 57.25 0.0169398 0.01495 2023-10-09T04:50:50.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:50.189 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:50.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:50.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:50.414 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:50.491 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:50.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:50.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:50.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:50.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:50.696 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:50:50.700 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:50.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:50 smithi177 ceph-mon[195797]: pgmap v1385: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T04:50:50.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:50 smithi177 ceph-mon[195797]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:50.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1943230853' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:50.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:50.828 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:50.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:50.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:50.969 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[216903]: pgmap v1385: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T04:50:50.969 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[216903]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:50.969 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1943230853' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:50.969 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:50.969 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[223858]: pgmap v1385: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-09T04:50:50.969 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[223858]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:50:50.970 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1943230853' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:50.970 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:50:50.972 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:51.011 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 87332 87316 66.5451 40.25 0.00505902 0.0149995 2023-10-09T04:50:51.783 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:51.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:51.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:51.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:51.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:51 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:51.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:51 smithi177 ceph-mon[195797]: failed to encode map e360 with expected crc 2023-10-09T04:50:51.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:51 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:50:51.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:51 smithi177 ceph-mon[195797]: osdmap e360: 8 total, 8 up, 4 in 2023-10-09T04:50:51.963 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:52.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 88268 88252 66.4482 58.5 0.0193918 0.0150437 2023-10-09T04:50:52.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:52.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:52.164 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[216903]: failed to encode map e360 with expected crc 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[216903]: osdmap e360: 8 total, 8 up, 4 in 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[223858]: failed to encode map e360 with expected crc 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:50:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:51 smithi029 ceph-mon[223858]: osdmap e360: 8 total, 8 up, 4 in 2023-10-09T04:50:52.200 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:52.565 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1385,"stamp":"2023-10-09T04:50:49.386627+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5440149408,"num_objects":83025,"num_object_clones":0,"num_object_copies":249075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83025,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":83301,"num_write_kb":5314509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":4870682,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73517,"ondisk_log_size":73517,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":21010856,"kb_used_data":15880200,"kb_used_omap":4,"kb_used_meta":5130619,"kb_avail":353969752,"statfs":{"total":383980142592,"available":362465026048,"internally_reserved":0,"allocated":16261324800,"data_stored":16257474219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":5253754766},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":562888704,"num_objects":8589,"num_object_clones":0,"num_object_copies":25767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8589,"num_write_kb":549696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002824"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":117,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.699802+0000","last_change":"2023-10-09T04:49:24.722690+0000","last_active":"2023-10-09T04:50:36.699802+0000","last_peered":"2023-10-09T04:50:36.699802+0000","last_clean":"2023-10-09T04:50:36.699802+0000","last_became_active":"2023-10-09T04:49:24.722128+0000","last_became_peered":"2023-10-09T04:49:24.722128+0000","last_unstale":"2023-10-09T04:50:36.699802+0000","last_undegraded":"2023-10-09T04:50:36.699802+0000","last_fullsized":"2023-10-09T04:50:36.699802+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:52.116095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":111,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.681162+0000","last_change":"2023-10-09T04:49:36.252176+0000","last_active":"2023-10-09T04:50:36.681162+0000","last_peered":"2023-10-09T04:50:36.681162+0000","last_clean":"2023-10-09T04:50:36.681162+0000","last_became_active":"2023-10-09T04:49:36.251166+0000","last_became_peered":"2023-10-09T04:49:36.251166+0000","last_unstale":"2023-10-09T04:50:36.681162+0000","last_undegraded":"2023-10-09T04:50:36.681162+0000","last_fullsized":"2023-10-09T04:50:36.681162+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:40.935795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":357,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:40.713587+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:50:40.713587+0000","last_peered":"2023-10-09T04:50:40.713587+0000","last_clean":"2023-10-09T04:50:40.713587+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:50:40.713587+0000","last_undegraded":"2023-10-09T04:50:40.713587+0000","last_fullsized":"2023-10-09T04:50:40.713587+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":292,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.687228+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:50:36.687228+0000","last_peered":"2023-10-09T04:50:36.687228+0000","last_clean":"2023-10-09T04:50:36.687228+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:50:36.687228+0000","last_undegraded":"2023-10-09T04:50:36.687228+0000","last_fullsized":"2023-10-09T04:50:36.687228+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":153,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.679636+0000","last_change":"2023-10-09T04:49:36.253446+0000","last_active":"2023-10-09T04:50:36.679636+0000","last_peered":"2023-10-09T04:50:36.679636+0000","last_clean":"2023-10-09T04:50:36.679636+0000","last_became_active":"2023-10-09T04:49:36.252717+0000","last_became_peered":"2023-10-09T04:49:36.252717+0000","last_unstale":"2023-10-09T04:50:36.679636+0000","last_undegraded":"2023-10-09T04:50:36.679636+0000","last_fullsized":"2023-10-09T04:50:36.679636+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:30.302532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":159,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.698459+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:50:36.698459+0000","last_peered":"2023-10-09T04:50:36.698459+0000","last_clean":"2023-10-09T04:50:36.698459+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:50:36.698459+0000","last_undegraded":"2023-10-09T04:50:36.698459+0000","last_fullsized":"2023-10-09T04:50:36.698459+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":301,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.698281+0000","last_change":"2023-10-09T04:49:24.727794+0000","last_active":"2023-10-09T04:50:36.698281+0000","last_peered":"2023-10-09T04:50:36.698281+0000","last_clean":"2023-10-09T04:50:36.698281+0000","last_became_active":"2023-10-09T04:49:24.726934+0000","last_became_peered":"2023-10-09T04:49:24.726934+0000","last_unstale":"2023-10-09T04:50:36.698281+0000","last_undegraded":"2023-10-09T04:50:36.698281+0000","last_fullsized":"2023-10-09T04:50:36.698281+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:43.043505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":275,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.679835+0000","last_change":"2023-10-09T04:49:36.252722+0000","last_active":"2023-10-09T04:50:36.679835+0000","last_peered":"2023-10-09T04:50:36.679835+0000","last_clean":"2023-10-09T04:50:36.679835+0000","last_became_active":"2023-10-09T04:49:36.251598+0000","last_became_peered":"2023-10-09T04:49:36.251598+0000","last_unstale":"2023-10-09T04:50:36.679835+0000","last_undegraded":"2023-10-09T04:50:36.679835+0000","last_fullsized":"2023-10-09T04:50:36.679835+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.458714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":319,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.696673+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:50:36.696673+0000","last_peered":"2023-10-09T04:50:36.696673+0000","last_clean":"2023-10-09T04:50:36.696673+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:50:36.696673+0000","last_undegraded":"2023-10-09T04:50:36.696673+0000","last_fullsized":"2023-10-09T04:50:36.696673+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":309,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686494+0000","last_change":"2023-10-09T04:49:24.728045+0000","last_active":"2023-10-09T04:50:36.686494+0000","last_peered":"2023-10-09T04:50:36.686494+0000","last_clean":"2023-10-09T04:50:36.686494+0000","last_became_active":"2023-10-09T04:49:24.727278+0000","last_became_peered":"2023-10-09T04:49:24.727278+0000","last_unstale":"2023-10-09T04:50:36.686494+0000","last_undegraded":"2023-10-09T04:50:36.686494+0000","last_fullsized":"2023-10-09T04:50:36.686494+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:17.700648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":401,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.687024+0000","last_change":"2023-10-09T04:49:24.751066+0000","last_active":"2023-10-09T04:50:36.687024+0000","last_peered":"2023-10-09T04:50:36.687024+0000","last_clean":"2023-10-09T04:50:36.687024+0000","last_became_active":"2023-10-09T04:49:24.731080+0000","last_became_peered":"2023-10-09T04:49:24.731080+0000","last_unstale":"2023-10-09T04:50:36.687024+0000","last_undegraded":"2023-10-09T04:50:36.687024+0000","last_fullsized":"2023-10-09T04:50:36.687024+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:47.656154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":254,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680003+0000","last_change":"2023-10-09T04:49:36.255264+0000","last_active":"2023-10-09T04:50:36.680003+0000","last_peered":"2023-10-09T04:50:36.680003+0000","last_clean":"2023-10-09T04:50:36.680003+0000","last_became_active":"2023-10-09T04:49:36.254631+0000","last_became_peered":"2023-10-09T04:49:36.254631+0000","last_unstale":"2023-10-09T04:50:36.680003+0000","last_undegraded":"2023-10-09T04:50:36.680003+0000","last_fullsized":"2023-10-09T04:50:36.680003+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:26.380768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":157,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.700287+0000","last_change":"2023-10-09T04:49:24.739093+0000","last_active":"2023-10-09T04:50:36.700287+0000","last_peered":"2023-10-09T04:50:36.700287+0000","last_clean":"2023-10-09T04:50:36.700287+0000","last_became_active":"2023-10-09T04:49:24.723103+0000","last_became_peered":"2023-10-09T04:49:24.723103+0000","last_unstale":"2023-10-09T04:50:36.700287+0000","last_undegraded":"2023-10-09T04:50:36.700287+0000","last_fullsized":"2023-10-09T04:50:36.700287+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:41.664075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":329,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.700293+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:50:36.700293+0000","last_peered":"2023-10-09T04:50:36.700293+0000","last_clean":"2023-10-09T04:50:36.700293+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:50:36.700293+0000","last_undegraded":"2023-10-09T04:50:36.700293+0000","last_fullsized":"2023-10-09T04:50:36.700293+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":385,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686439+0000","last_change":"2023-10-09T04:49:24.748611+0000","last_active":"2023-10-09T04:50:36.686439+0000","last_peered":"2023-10-09T04:50:36.686439+0000","last_clean":"2023-10-09T04:50:36.686439+0000","last_became_active":"2023-10-09T04:49:24.730507+0000","last_became_peered":"2023-10-09T04:49:24.730507+0000","last_unstale":"2023-10-09T04:50:36.686439+0000","last_undegraded":"2023-10-09T04:50:36.686439+0000","last_fullsized":"2023-10-09T04:50:36.686439+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:22.219826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":159,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.681096+0000","last_change":"2023-10-09T04:49:36.255342+0000","last_active":"2023-10-09T04:50:36.681096+0000","last_peered":"2023-10-09T04:50:36.681096+0000","last_clean":"2023-10-09T04:50:36.681096+0000","last_became_active":"2023-10-09T04:49:36.254372+0000","last_became_peered":"2023-10-09T04:49:36.254372+0000","last_unstale":"2023-10-09T04:50:36.681096+0000","last_undegraded":"2023-10-09T04:50:36.681096+0000","last_fullsized":"2023-10-09T04:50:36.681096+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:37.972615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":301,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686482+0000","last_change":"2023-10-09T04:49:24.731694+0000","last_active":"2023-10-09T04:50:36.686482+0000","last_peered":"2023-10-09T04:50:36.686482+0000","last_clean":"2023-10-09T04:50:36.686482+0000","last_became_active":"2023-10-09T04:49:24.731338+0000","last_became_peered":"2023-10-09T04:49:24.731338+0000","last_unstale":"2023-10-09T04:50:36.686482+0000","last_undegraded":"2023-10-09T04:50:36.686482+0000","last_fullsized":"2023-10-09T04:50:36.686482+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:45.968733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":338,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.696174+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:50:36.696174+0000","last_peered":"2023-10-09T04:50:36.696174+0000","last_clean":"2023-10-09T04:50:36.696174+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:50:36.696174+0000","last_undegraded":"2023-10-09T04:50:36.696174+0000","last_fullsized":"2023-10-09T04:50:36.696174+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":331,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686420+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:36.686420+0000","last_peered":"2023-10-09T04:50:36.686420+0000","last_clean":"2023-10-09T04:50:36.686420+0000","last_became_active":"2023-10-09T04:49:24.727752+0000","last_became_peered":"2023-10-09T04:49:24.727752+0000","last_unstale":"2023-10-09T04:50:36.686420+0000","last_undegraded":"2023-10-09T04:50:36.686420+0000","last_fullsized":"2023-10-09T04:50:36.686420+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:37.448051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":189,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.687308+0000","last_change":"2023-10-09T04:49:24.734384+0000","last_active":"2023-10-09T04:50:36.687308+0000","last_peered":"2023-10-09T04:50:36.687308+0000","last_clean":"2023-10-09T04:50:36.687308+0000","last_became_active":"2023-10-09T04:49:24.733948+0000","last_became_peered":"2023-10-09T04:49:24.733948+0000","last_unstale":"2023-10-09T04:50:36.687308+0000","last_undegraded":"2023-10-09T04:50:36.687308+0000","last_fullsized":"2023-10-09T04:50:36.687308+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:55.081133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":160,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.693791+0000","last_change":"2023-10-09T04:49:24.743137+0000","last_active":"2023-10-09T04:50:36.693791+0000","last_peered":"2023-10-09T04:50:36.693791+0000","last_clean":"2023-10-09T04:50:36.693791+0000","last_became_active":"2023-10-09T04:49:24.726930+0000","last_became_peered":"2023-10-09T04:49:24.726930+0000","last_unstale":"2023-10-09T04:50:36.693791+0000","last_undegraded":"2023-10-09T04:50:36.693791+0000","last_fullsized":"2023-10-09T04:50:36.693791+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:38.154496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":162,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.688053+0000","last_change":"2023-10-09T04:49:24.731691+0000","last_active":"2023-10-09T04:50:36.688053+0000","last_peered":"2023-10-09T04:50:36.688053+0000","last_clean":"2023-10-09T04:50:36.688053+0000","last_became_active":"2023-10-09T04:49:24.731374+0000","last_became_peered":"2023-10-09T04:49:24.731374+0000","last_unstale":"2023-10-09T04:50:36.688053+0000","last_undegraded":"2023-10-09T04:50:36.688053+0000","last_fullsized":"2023-10-09T04:50:36.688053+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:39.511857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"359'5160","reported_seq":5192,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.683615+0000","last_change":"2023-10-09T04:49:24.722035+0000","last_active":"2023-10-09T04:50:44.683615+0000","last_peered":"2023-10-09T04:50:44.683615+0000","last_clean":"2023-10-09T04:50:44.683615+0000","last_became_active":"2023-10-09T04:49:24.721093+0000","last_became_peered":"2023-10-09T04:49:24.721093+0000","last_unstale":"2023-10-09T04:50:44.683615+0000","last_undegraded":"2023-10-09T04:50:44.683615+0000","last_fullsized":"2023-10-09T04:50:44.683615+0000","mapping_epoch":340,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4660,"log_dups_size":0,"ondisk_log_size":4660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:16.988764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338165760,"num_objects":5160,"num_object_clones":0,"num_object_copies":15480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5160,"num_write_kb":330240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":389,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.687896+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:50:36.687896+0000","last_peered":"2023-10-09T04:50:36.687896+0000","last_clean":"2023-10-09T04:50:36.687896+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:50:36.687896+0000","last_undegraded":"2023-10-09T04:50:36.687896+0000","last_fullsized":"2023-10-09T04:50:36.687896+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":311,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.693579+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:50:36.693579+0000","last_peered":"2023-10-09T04:50:36.693579+0000","last_clean":"2023-10-09T04:50:36.693579+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:50:36.693579+0000","last_undegraded":"2023-10-09T04:50:36.693579+0000","last_fullsized":"2023-10-09T04:50:36.693579+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"359'5173","reported_seq":5214,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:45.112848+0000","last_change":"2023-10-09T04:49:24.729511+0000","last_active":"2023-10-09T04:50:45.112848+0000","last_peered":"2023-10-09T04:50:45.112848+0000","last_clean":"2023-10-09T04:50:45.112848+0000","last_became_active":"2023-10-09T04:49:24.728925+0000","last_became_peered":"2023-10-09T04:49:24.728925+0000","last_unstale":"2023-10-09T04:50:45.112848+0000","last_undegraded":"2023-10-09T04:50:45.112848+0000","last_fullsized":"2023-10-09T04:50:45.112848+0000","mapping_epoch":340,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4673,"log_dups_size":0,"ondisk_log_size":4673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.606555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339017728,"num_objects":5173,"num_object_clones":0,"num_object_copies":15519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5173,"num_write_kb":331072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":348,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.687151+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:50:36.687151+0000","last_peered":"2023-10-09T04:50:36.687151+0000","last_clean":"2023-10-09T04:50:36.687151+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:50:36.687151+0000","last_undegraded":"2023-10-09T04:50:36.687151+0000","last_fullsized":"2023-10-09T04:50:36.687151+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":356,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.702017+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:50:36.702017+0000","last_peered":"2023-10-09T04:50:36.702017+0000","last_clean":"2023-10-09T04:50:36.702017+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:50:36.702017+0000","last_undegraded":"2023-10-09T04:50:36.702017+0000","last_fullsized":"2023-10-09T04:50:36.702017+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"359'5089","reported_seq":5120,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.686009+0000","last_change":"2023-10-09T04:49:17.684070+0000","last_active":"2023-10-09T04:50:44.686009+0000","last_peered":"2023-10-09T04:50:44.686009+0000","last_clean":"2023-10-09T04:50:44.686009+0000","last_became_active":"2023-10-09T04:49:17.683772+0000","last_became_peered":"2023-10-09T04:49:17.683772+0000","last_unstale":"2023-10-09T04:50:44.686009+0000","last_undegraded":"2023-10-09T04:50:44.686009+0000","last_fullsized":"2023-10-09T04:50:44.686009+0000","mapping_epoch":336,"log_start":"345'400","ondisk_log_start":"345'400","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4689,"log_dups_size":0,"ondisk_log_size":4689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:47.529254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333512704,"num_objects":5089,"num_object_clones":0,"num_object_copies":15267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5089,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":330,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686848+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:50:36.686848+0000","last_peered":"2023-10-09T04:50:36.686848+0000","last_clean":"2023-10-09T04:50:36.686848+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:50:36.686848+0000","last_undegraded":"2023-10-09T04:50:36.686848+0000","last_fullsized":"2023-10-09T04:50:36.686848+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":227,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.679616+0000","last_change":"2023-10-09T04:49:36.255047+0000","last_active":"2023-10-09T04:50:36.679616+0000","last_peered":"2023-10-09T04:50:36.679616+0000","last_clean":"2023-10-09T04:50:36.679616+0000","last_became_active":"2023-10-09T04:49:36.254324+0000","last_became_peered":"2023-10-09T04:49:36.254324+0000","last_unstale":"2023-10-09T04:50:36.679616+0000","last_undegraded":"2023-10-09T04:50:36.679616+0000","last_fullsized":"2023-10-09T04:50:36.679616+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:55.547119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"359'5145","reported_seq":5175,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.571592+0000","last_change":"2023-10-09T04:49:17.682749+0000","last_active":"2023-10-09T04:50:44.571592+0000","last_peered":"2023-10-09T04:50:44.571592+0000","last_clean":"2023-10-09T04:50:44.571592+0000","last_became_active":"2023-10-09T04:49:17.682418+0000","last_became_peered":"2023-10-09T04:49:17.682418+0000","last_unstale":"2023-10-09T04:50:44.571592+0000","last_undegraded":"2023-10-09T04:50:44.571592+0000","last_fullsized":"2023-10-09T04:50:44.571592+0000","mapping_epoch":336,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4645,"log_dups_size":0,"ondisk_log_size":4645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:33.471194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337182720,"num_objects":5145,"num_object_clones":0,"num_object_copies":15435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5145,"num_write_kb":329280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":359,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.697887+0000","last_change":"2023-10-09T04:49:24.728445+0000","last_active":"2023-10-09T04:50:36.697887+0000","last_peered":"2023-10-09T04:50:36.697887+0000","last_clean":"2023-10-09T04:50:36.697887+0000","last_became_active":"2023-10-09T04:49:24.728130+0000","last_became_peered":"2023-10-09T04:49:24.728130+0000","last_unstale":"2023-10-09T04:50:36.697887+0000","last_undegraded":"2023-10-09T04:50:36.697887+0000","last_fullsized":"2023-10-09T04:50:36.697887+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:04.777758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":74,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680258+0000","last_change":"2023-10-09T04:49:36.253080+0000","last_active":"2023-10-09T04:50:36.680258+0000","last_peered":"2023-10-09T04:50:36.680258+0000","last_clean":"2023-10-09T04:50:36.680258+0000","last_became_active":"2023-10-09T04:49:36.252682+0000","last_became_peered":"2023-10-09T04:49:36.252682+0000","last_unstale":"2023-10-09T04:50:36.680258+0000","last_undegraded":"2023-10-09T04:50:36.680258+0000","last_fullsized":"2023-10-09T04:50:36.680258+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:31.712210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.5","version":"359'5161","reported_seq":5192,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.697809+0000","last_change":"2023-10-09T04:49:17.681809+0000","last_active":"2023-10-09T04:50:44.697809+0000","last_peered":"2023-10-09T04:50:44.697809+0000","last_clean":"2023-10-09T04:50:44.697809+0000","last_became_active":"2023-10-09T04:49:17.681477+0000","last_became_peered":"2023-10-09T04:49:17.681477+0000","last_unstale":"2023-10-09T04:50:44.697809+0000","last_undegraded":"2023-10-09T04:50:44.697809+0000","last_fullsized":"2023-10-09T04:50:44.697809+0000","mapping_epoch":336,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4661,"log_dups_size":0,"ondisk_log_size":4661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:07.783547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338231296,"num_objects":5161,"num_object_clones":0,"num_object_copies":15483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5161,"num_write_kb":330304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":98,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.700376+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:50:36.700376+0000","last_peered":"2023-10-09T04:50:36.700376+0000","last_clean":"2023-10-09T04:50:36.700376+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:50:36.700376+0000","last_undegraded":"2023-10-09T04:50:36.700376+0000","last_fullsized":"2023-10-09T04:50:36.700376+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":127,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.700361+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:50:36.700361+0000","last_peered":"2023-10-09T04:50:36.700361+0000","last_clean":"2023-10-09T04:50:36.700361+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:50:36.700361+0000","last_undegraded":"2023-10-09T04:50:36.700361+0000","last_fullsized":"2023-10-09T04:50:36.700361+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"359'5221","reported_seq":5263,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:48.284851+0000","last_change":"2023-10-09T04:49:36.269590+0000","last_active":"2023-10-09T04:50:48.284851+0000","last_peered":"2023-10-09T04:50:48.284851+0000","last_clean":"2023-10-09T04:50:48.284851+0000","last_became_active":"2023-10-09T04:49:36.252739+0000","last_became_peered":"2023-10-09T04:49:36.252739+0000","last_unstale":"2023-10-09T04:50:48.284851+0000","last_undegraded":"2023-10-09T04:50:48.284851+0000","last_fullsized":"2023-10-09T04:50:48.284851+0000","mapping_epoch":346,"log_start":"347'1000","ondisk_log_start":"347'1000","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4221,"log_dups_size":0,"ondisk_log_size":4221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:26.767295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342163456,"num_objects":5221,"num_object_clones":0,"num_object_copies":15663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5221,"num_write_kb":334144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":320,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.699887+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:50:36.699887+0000","last_peered":"2023-10-09T04:50:36.699887+0000","last_clean":"2023-10-09T04:50:36.699887+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:50:36.699887+0000","last_undegraded":"2023-10-09T04:50:36.699887+0000","last_fullsized":"2023-10-09T04:50:36.699887+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":117,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.701101+0000","last_change":"2023-10-09T04:49:24.712121+0000","last_active":"2023-10-09T04:50:36.701101+0000","last_peered":"2023-10-09T04:50:36.701101+0000","last_clean":"2023-10-09T04:50:36.701101+0000","last_became_active":"2023-10-09T04:49:24.711716+0000","last_became_peered":"2023-10-09T04:49:24.711716+0000","last_unstale":"2023-10-09T04:50:36.701101+0000","last_undegraded":"2023-10-09T04:50:36.701101+0000","last_fullsized":"2023-10-09T04:50:36.701101+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:37.141689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"359'5147","reported_seq":5178,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:45.132999+0000","last_change":"2023-10-09T04:49:17.686417+0000","last_active":"2023-10-09T04:50:45.132999+0000","last_peered":"2023-10-09T04:50:45.132999+0000","last_clean":"2023-10-09T04:50:45.132999+0000","last_became_active":"2023-10-09T04:49:17.686126+0000","last_became_peered":"2023-10-09T04:49:17.686126+0000","last_unstale":"2023-10-09T04:50:45.132999+0000","last_undegraded":"2023-10-09T04:50:45.132999+0000","last_fullsized":"2023-10-09T04:50:45.132999+0000","mapping_epoch":336,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4647,"log_dups_size":0,"ondisk_log_size":4647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:58.636951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.701906+0000","last_change":"2023-10-09T04:49:24.721842+0000","last_active":"2023-10-09T04:50:36.701906+0000","last_peered":"2023-10-09T04:50:36.701906+0000","last_clean":"2023-10-09T04:50:36.701906+0000","last_became_active":"2023-10-09T04:49:24.720666+0000","last_became_peered":"2023-10-09T04:49:24.720666+0000","last_unstale":"2023-10-09T04:50:36.701906+0000","last_undegraded":"2023-10-09T04:50:36.701906+0000","last_fullsized":"2023-10-09T04:50:36.701906+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:16.572332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":339,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.701604+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:50:36.701604+0000","last_peered":"2023-10-09T04:50:36.701604+0000","last_clean":"2023-10-09T04:50:36.701604+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:50:36.701604+0000","last_undegraded":"2023-10-09T04:50:36.701604+0000","last_fullsized":"2023-10-09T04:50:36.701604+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"359'5259","reported_seq":5300,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.602381+0000","last_change":"2023-10-09T04:49:24.721346+0000","last_active":"2023-10-09T04:50:44.602381+0000","last_peered":"2023-10-09T04:50:44.602381+0000","last_clean":"2023-10-09T04:50:44.602381+0000","last_became_active":"2023-10-09T04:49:24.720955+0000","last_became_peered":"2023-10-09T04:49:24.720955+0000","last_unstale":"2023-10-09T04:50:44.602381+0000","last_undegraded":"2023-10-09T04:50:44.602381+0000","last_fullsized":"2023-10-09T04:50:44.602381+0000","mapping_epoch":340,"log_start":"345'600","ondisk_log_start":"345'600","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4659,"log_dups_size":0,"ondisk_log_size":4659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:58:55.342602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344653824,"num_objects":5259,"num_object_clones":0,"num_object_copies":15777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5259,"num_write_kb":336576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":112,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.679765+0000","last_change":"2023-10-09T04:49:36.253172+0000","last_active":"2023-10-09T04:50:36.679765+0000","last_peered":"2023-10-09T04:50:36.679765+0000","last_clean":"2023-10-09T04:50:36.679765+0000","last_became_active":"2023-10-09T04:49:36.252175+0000","last_became_peered":"2023-10-09T04:49:36.252175+0000","last_unstale":"2023-10-09T04:50:36.679765+0000","last_undegraded":"2023-10-09T04:50:36.679765+0000","last_fullsized":"2023-10-09T04:50:36.679765+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:38.269236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":264,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680967+0000","last_change":"2023-10-09T04:49:36.251118+0000","last_active":"2023-10-09T04:50:36.680967+0000","last_peered":"2023-10-09T04:50:36.680967+0000","last_clean":"2023-10-09T04:50:36.680967+0000","last_became_active":"2023-10-09T04:49:36.250803+0000","last_became_peered":"2023-10-09T04:49:36.250803+0000","last_unstale":"2023-10-09T04:50:36.680967+0000","last_undegraded":"2023-10-09T04:50:36.680967+0000","last_fullsized":"2023-10-09T04:50:36.680967+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:38.765478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"359'5139","reported_seq":5180,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.576747+0000","last_change":"2023-10-09T04:49:24.727773+0000","last_active":"2023-10-09T04:50:44.576747+0000","last_peered":"2023-10-09T04:50:44.576747+0000","last_clean":"2023-10-09T04:50:44.576747+0000","last_became_active":"2023-10-09T04:49:24.727411+0000","last_became_peered":"2023-10-09T04:49:24.727411+0000","last_unstale":"2023-10-09T04:50:44.576747+0000","last_undegraded":"2023-10-09T04:50:44.576747+0000","last_fullsized":"2023-10-09T04:50:44.576747+0000","mapping_epoch":340,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4639,"log_dups_size":0,"ondisk_log_size":4639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:21.626311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336789504,"num_objects":5139,"num_object_clones":0,"num_object_copies":15417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5139,"num_write_kb":328896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":253,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680997+0000","last_change":"2023-10-09T04:49:36.250912+0000","last_active":"2023-10-09T04:50:36.680997+0000","last_peered":"2023-10-09T04:50:36.680997+0000","last_clean":"2023-10-09T04:50:36.680997+0000","last_became_active":"2023-10-09T04:49:36.250530+0000","last_became_peered":"2023-10-09T04:49:36.250530+0000","last_unstale":"2023-10-09T04:50:36.680997+0000","last_undegraded":"2023-10-09T04:50:36.680997+0000","last_fullsized":"2023-10-09T04:50:36.680997+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:26.975687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":373,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.696096+0000","last_change":"2023-10-09T04:49:24.727377+0000","last_active":"2023-10-09T04:50:36.696096+0000","last_peered":"2023-10-09T04:50:36.696096+0000","last_clean":"2023-10-09T04:50:36.696096+0000","last_became_active":"2023-10-09T04:49:24.724195+0000","last_became_peered":"2023-10-09T04:49:24.724195+0000","last_unstale":"2023-10-09T04:50:36.696096+0000","last_undegraded":"2023-10-09T04:50:36.696096+0000","last_fullsized":"2023-10-09T04:50:36.696096+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:46.895156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":248,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680105+0000","last_change":"2023-10-09T04:49:36.254992+0000","last_active":"2023-10-09T04:50:36.680105+0000","last_peered":"2023-10-09T04:50:36.680105+0000","last_clean":"2023-10-09T04:50:36.680105+0000","last_became_active":"2023-10-09T04:49:36.254288+0000","last_became_peered":"2023-10-09T04:49:36.254288+0000","last_unstale":"2023-10-09T04:50:36.680105+0000","last_undegraded":"2023-10-09T04:50:36.680105+0000","last_fullsized":"2023-10-09T04:50:36.680105+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:13.355684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"359'5173","reported_seq":5204,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.690707+0000","last_change":"2023-10-09T04:49:17.684748+0000","last_active":"2023-10-09T04:50:44.690707+0000","last_peered":"2023-10-09T04:50:44.690707+0000","last_clean":"2023-10-09T04:50:44.690707+0000","last_became_active":"2023-10-09T04:49:17.684346+0000","last_became_peered":"2023-10-09T04:49:17.684346+0000","last_unstale":"2023-10-09T04:50:44.690707+0000","last_undegraded":"2023-10-09T04:50:44.690707+0000","last_fullsized":"2023-10-09T04:50:44.690707+0000","mapping_epoch":336,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4673,"log_dups_size":0,"ondisk_log_size":4673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:39.897521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339017728,"num_objects":5173,"num_object_clones":0,"num_object_copies":15519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5173,"num_write_kb":331072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":343,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.696076+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:50:36.696076+0000","last_peered":"2023-10-09T04:50:36.696076+0000","last_clean":"2023-10-09T04:50:36.696076+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:50:36.696076+0000","last_undegraded":"2023-10-09T04:50:36.696076+0000","last_fullsized":"2023-10-09T04:50:36.696076+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.699853+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:50:36.699853+0000","last_peered":"2023-10-09T04:50:36.699853+0000","last_clean":"2023-10-09T04:50:36.699853+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:50:36.699853+0000","last_undegraded":"2023-10-09T04:50:36.699853+0000","last_fullsized":"2023-10-09T04:50:36.699853+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":302,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680918+0000","last_change":"2023-10-09T04:49:36.254929+0000","last_active":"2023-10-09T04:50:36.680918+0000","last_peered":"2023-10-09T04:50:36.680918+0000","last_clean":"2023-10-09T04:50:36.680918+0000","last_became_active":"2023-10-09T04:49:36.252868+0000","last_became_peered":"2023-10-09T04:49:36.252868+0000","last_unstale":"2023-10-09T04:50:36.680918+0000","last_undegraded":"2023-10-09T04:50:36.680918+0000","last_fullsized":"2023-10-09T04:50:36.680918+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:26.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"359'5280","reported_seq":5311,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:45.142467+0000","last_change":"2023-10-09T04:49:17.686557+0000","last_active":"2023-10-09T04:50:45.142467+0000","last_peered":"2023-10-09T04:50:45.142467+0000","last_clean":"2023-10-09T04:50:45.142467+0000","last_became_active":"2023-10-09T04:49:17.686236+0000","last_became_peered":"2023-10-09T04:49:17.686236+0000","last_unstale":"2023-10-09T04:50:45.142467+0000","last_undegraded":"2023-10-09T04:50:45.142467+0000","last_fullsized":"2023-10-09T04:50:45.142467+0000","mapping_epoch":336,"log_start":"345'600","ondisk_log_start":"345'600","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4680,"log_dups_size":0,"ondisk_log_size":4680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:02.935353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5280,"num_write_kb":337920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":303,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680571+0000","last_change":"2023-10-09T04:49:36.250480+0000","last_active":"2023-10-09T04:50:36.680571+0000","last_peered":"2023-10-09T04:50:36.680571+0000","last_clean":"2023-10-09T04:50:36.680571+0000","last_became_active":"2023-10-09T04:49:36.250048+0000","last_became_peered":"2023-10-09T04:49:36.250048+0000","last_unstale":"2023-10-09T04:50:36.680571+0000","last_undegraded":"2023-10-09T04:50:36.680571+0000","last_fullsized":"2023-10-09T04:50:36.680571+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:21.602101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":302,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.679913+0000","last_change":"2023-10-09T04:49:36.253118+0000","last_active":"2023-10-09T04:50:36.679913+0000","last_peered":"2023-10-09T04:50:36.679913+0000","last_clean":"2023-10-09T04:50:36.679913+0000","last_became_active":"2023-10-09T04:49:36.252036+0000","last_became_peered":"2023-10-09T04:49:36.252036+0000","last_unstale":"2023-10-09T04:50:36.679913+0000","last_undegraded":"2023-10-09T04:50:36.679913+0000","last_fullsized":"2023-10-09T04:50:36.679913+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.648245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":244,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680888+0000","last_change":"2023-10-09T04:49:36.252116+0000","last_active":"2023-10-09T04:50:36.680888+0000","last_peered":"2023-10-09T04:50:36.680888+0000","last_clean":"2023-10-09T04:50:36.680888+0000","last_became_active":"2023-10-09T04:49:36.251630+0000","last_became_peered":"2023-10-09T04:49:36.251630+0000","last_unstale":"2023-10-09T04:50:36.680888+0000","last_undegraded":"2023-10-09T04:50:36.680888+0000","last_fullsized":"2023-10-09T04:50:36.680888+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:48.681072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"359'5378","reported_seq":5431,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:48.253889+0000","last_change":"2023-10-09T04:49:36.276163+0000","last_active":"2023-10-09T04:50:48.253889+0000","last_peered":"2023-10-09T04:50:48.253889+0000","last_clean":"2023-10-09T04:50:48.253889+0000","last_became_active":"2023-10-09T04:49:36.253314+0000","last_became_peered":"2023-10-09T04:49:36.253314+0000","last_unstale":"2023-10-09T04:50:48.253889+0000","last_undegraded":"2023-10-09T04:50:48.253889+0000","last_fullsized":"2023-10-09T04:50:48.253889+0000","mapping_epoch":346,"log_start":"347'1200","ondisk_log_start":"347'1200","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4178,"log_dups_size":0,"ondisk_log_size":4178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:44.334582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452608,"num_objects":5378,"num_object_clones":0,"num_object_copies":16134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5378,"num_write_kb":344192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":350,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.697926+0000","last_change":"2023-10-09T04:49:24.727573+0000","last_active":"2023-10-09T04:50:36.697926+0000","last_peered":"2023-10-09T04:50:36.697926+0000","last_clean":"2023-10-09T04:50:36.697926+0000","last_became_active":"2023-10-09T04:49:24.727151+0000","last_became_peered":"2023-10-09T04:49:24.727151+0000","last_unstale":"2023-10-09T04:50:36.697926+0000","last_undegraded":"2023-10-09T04:50:36.697926+0000","last_fullsized":"2023-10-09T04:50:36.697926+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:53.790164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":829,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686223+0000","last_change":"2023-10-09T04:49:27.734937+0000","last_active":"2023-10-09T04:50:36.686223+0000","last_peered":"2023-10-09T04:50:36.686223+0000","last_clean":"2023-10-09T04:50:36.686223+0000","last_became_active":"2023-10-09T04:49:27.734587+0000","last_became_peered":"2023-10-09T04:49:27.734587+0000","last_unstale":"2023-10-09T04:50:36.686223+0000","last_undegraded":"2023-10-09T04:50:36.686223+0000","last_fullsized":"2023-10-09T04:50:36.686223+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:38.000046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":289,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.696241+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:50:36.696241+0000","last_peered":"2023-10-09T04:50:36.696241+0000","last_clean":"2023-10-09T04:50:36.696241+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:50:36.696241+0000","last_undegraded":"2023-10-09T04:50:36.696241+0000","last_fullsized":"2023-10-09T04:50:36.696241+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":302,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.701034+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:50:36.701034+0000","last_peered":"2023-10-09T04:50:36.701034+0000","last_clean":"2023-10-09T04:50:36.701034+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:50:36.701034+0000","last_undegraded":"2023-10-09T04:50:36.701034+0000","last_fullsized":"2023-10-09T04:50:36.701034+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"359'5195","reported_seq":5236,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:45.127622+0000","last_change":"2023-10-09T04:49:24.733991+0000","last_active":"2023-10-09T04:50:45.127622+0000","last_peered":"2023-10-09T04:50:45.127622+0000","last_clean":"2023-10-09T04:50:45.127622+0000","last_became_active":"2023-10-09T04:49:24.733661+0000","last_became_peered":"2023-10-09T04:49:24.733661+0000","last_unstale":"2023-10-09T04:50:45.127622+0000","last_undegraded":"2023-10-09T04:50:45.127622+0000","last_fullsized":"2023-10-09T04:50:45.127622+0000","mapping_epoch":340,"log_start":"345'500","ondisk_log_start":"345'500","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:22.030285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340459520,"num_objects":5195,"num_object_clones":0,"num_object_copies":15585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5195,"num_write_kb":332480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":369,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686159+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:50:36.686159+0000","last_peered":"2023-10-09T04:50:36.686159+0000","last_clean":"2023-10-09T04:50:36.686159+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:50:36.686159+0000","last_undegraded":"2023-10-09T04:50:36.686159+0000","last_fullsized":"2023-10-09T04:50:36.686159+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.701988+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:50:36.701988+0000","last_peered":"2023-10-09T04:50:36.701988+0000","last_clean":"2023-10-09T04:50:36.701988+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:50:36.701988+0000","last_undegraded":"2023-10-09T04:50:36.701988+0000","last_fullsized":"2023-10-09T04:50:36.701988+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":129,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.681161+0000","last_change":"2023-10-09T04:49:36.252316+0000","last_active":"2023-10-09T04:50:36.681161+0000","last_peered":"2023-10-09T04:50:36.681161+0000","last_clean":"2023-10-09T04:50:36.681161+0000","last_became_active":"2023-10-09T04:49:36.251771+0000","last_became_peered":"2023-10-09T04:49:36.251771+0000","last_unstale":"2023-10-09T04:50:36.681161+0000","last_undegraded":"2023-10-09T04:50:36.681161+0000","last_fullsized":"2023-10-09T04:50:36.681161+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:08.690791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"359'5310","reported_seq":5350,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:48.281745+0000","last_change":"2023-10-09T04:49:36.254761+0000","last_active":"2023-10-09T04:50:48.281745+0000","last_peered":"2023-10-09T04:50:48.281745+0000","last_clean":"2023-10-09T04:50:48.281745+0000","last_became_active":"2023-10-09T04:49:36.250489+0000","last_became_peered":"2023-10-09T04:49:36.250489+0000","last_unstale":"2023-10-09T04:50:48.281745+0000","last_undegraded":"2023-10-09T04:50:48.281745+0000","last_fullsized":"2023-10-09T04:50:48.281745+0000","mapping_epoch":346,"log_start":"347'1100","ondisk_log_start":"347'1100","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4210,"log_dups_size":0,"ondisk_log_size":4210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:19.664434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347996160,"num_objects":5310,"num_object_clones":0,"num_object_copies":15930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5310,"num_write_kb":339840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":372,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686935+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:50:36.686935+0000","last_peered":"2023-10-09T04:50:36.686935+0000","last_clean":"2023-10-09T04:50:36.686935+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:50:36.686935+0000","last_undegraded":"2023-10-09T04:50:36.686935+0000","last_fullsized":"2023-10-09T04:50:36.686935+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680839+0000","last_change":"2023-10-09T04:49:36.254730+0000","last_active":"2023-10-09T04:50:36.680839+0000","last_peered":"2023-10-09T04:50:36.680839+0000","last_clean":"2023-10-09T04:50:36.680839+0000","last_became_active":"2023-10-09T04:49:36.252242+0000","last_became_peered":"2023-10-09T04:49:36.252242+0000","last_unstale":"2023-10-09T04:50:36.680839+0000","last_undegraded":"2023-10-09T04:50:36.680839+0000","last_fullsized":"2023-10-09T04:50:36.680839+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.106203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.697793+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:50:36.697793+0000","last_peered":"2023-10-09T04:50:36.697793+0000","last_clean":"2023-10-09T04:50:36.697793+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:50:36.697793+0000","last_undegraded":"2023-10-09T04:50:36.697793+0000","last_fullsized":"2023-10-09T04:50:36.697793+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"359'5067","reported_seq":5108,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.521161+0000","last_change":"2023-10-09T04:49:24.718431+0000","last_active":"2023-10-09T04:50:44.521161+0000","last_peered":"2023-10-09T04:50:44.521161+0000","last_clean":"2023-10-09T04:50:44.521161+0000","last_became_active":"2023-10-09T04:49:24.718108+0000","last_became_peered":"2023-10-09T04:49:24.718108+0000","last_unstale":"2023-10-09T04:50:44.521161+0000","last_undegraded":"2023-10-09T04:50:44.521161+0000","last_fullsized":"2023-10-09T04:50:44.521161+0000","mapping_epoch":340,"log_start":"345'400","ondisk_log_start":"345'400","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4667,"log_dups_size":0,"ondisk_log_size":4667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.309575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332070912,"num_objects":5067,"num_object_clones":0,"num_object_copies":15201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5067,"num_write_kb":324288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":107,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.688136+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:50:36.688136+0000","last_peered":"2023-10-09T04:50:36.688136+0000","last_clean":"2023-10-09T04:50:36.688136+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:50:36.688136+0000","last_undegraded":"2023-10-09T04:50:36.688136+0000","last_fullsized":"2023-10-09T04:50:36.688136+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":329,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680837+0000","last_change":"2023-10-09T04:49:36.251899+0000","last_active":"2023-10-09T04:50:36.680837+0000","last_peered":"2023-10-09T04:50:36.680837+0000","last_clean":"2023-10-09T04:50:36.680837+0000","last_became_active":"2023-10-09T04:49:36.251283+0000","last_became_peered":"2023-10-09T04:49:36.251283+0000","last_unstale":"2023-10-09T04:50:36.680837+0000","last_undegraded":"2023-10-09T04:50:36.680837+0000","last_fullsized":"2023-10-09T04:50:36.680837+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:52.505049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":205,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.686294+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:50:36.686294+0000","last_peered":"2023-10-09T04:50:36.686294+0000","last_clean":"2023-10-09T04:50:36.686294+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:50:36.686294+0000","last_undegraded":"2023-10-09T04:50:36.686294+0000","last_fullsized":"2023-10-09T04:50:36.686294+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"359'5104","reported_seq":5136,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-09T04:50:44.524799+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:44.524799+0000","last_peered":"2023-10-09T04:50:44.524799+0000","last_clean":"2023-10-09T04:50:44.524799+0000","last_became_active":"2023-10-09T04:49:24.727688+0000","last_became_peered":"2023-10-09T04:49:24.727688+0000","last_unstale":"2023-10-09T04:50:44.524799+0000","last_undegraded":"2023-10-09T04:50:44.524799+0000","last_fullsized":"2023-10-09T04:50:44.524799+0000","mapping_epoch":340,"log_start":"345'400","ondisk_log_start":"345'400","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4704,"log_dups_size":0,"ondisk_log_size":4704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:00.374022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334495744,"num_objects":5104,"num_object_clones":0,"num_object_copies":15312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5104,"num_write_kb":326656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":291,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.687813+0000","last_change":"2023-10-09T04:49:24.730477+0000","last_active":"2023-10-09T04:50:36.687813+0000","last_peered":"2023-10-09T04:50:36.687813+0000","last_clean":"2023-10-09T04:50:36.687813+0000","last_became_active":"2023-10-09T04:49:24.730115+0000","last_became_peered":"2023-10-09T04:49:24.730115+0000","last_unstale":"2023-10-09T04:50:36.687813+0000","last_undegraded":"2023-10-09T04:50:36.687813+0000","last_fullsized":"2023-10-09T04:50:36.687813+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:49.730424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":128,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.681001+0000","last_change":"2023-10-09T04:49:36.250949+0000","last_active":"2023-10-09T04:50:36.681001+0000","last_peered":"2023-10-09T04:50:36.681001+0000","last_clean":"2023-10-09T04:50:36.681001+0000","last_became_active":"2023-10-09T04:49:36.250519+0000","last_became_peered":"2023-10-09T04:49:36.250519+0000","last_unstale":"2023-10-09T04:50:36.681001+0000","last_undegraded":"2023-10-09T04:50:36.681001+0000","last_fullsized":"2023-10-09T04:50:36.681001+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:51.821472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":127,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.701889+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:50:36.701889+0000","last_peered":"2023-10-09T04:50:36.701889+0000","last_clean":"2023-10-09T04:50:36.701889+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:50:36.701889+0000","last_undegraded":"2023-10-09T04:50:36.701889+0000","last_fullsized":"2023-10-09T04:50:36.701889+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":209,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.700164+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:50:36.700164+0000","last_peered":"2023-10-09T04:50:36.700164+0000","last_clean":"2023-10-09T04:50:36.700164+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:50:36.700164+0000","last_undegraded":"2023-10-09T04:50:36.700164+0000","last_fullsized":"2023-10-09T04:50:36.700164+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":295,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680746+0000","last_change":"2023-10-09T04:49:36.254629+0000","last_active":"2023-10-09T04:50:36.680746+0000","last_peered":"2023-10-09T04:50:36.680746+0000","last_clean":"2023-10-09T04:50:36.680746+0000","last_became_active":"2023-10-09T04:49:36.251774+0000","last_became_peered":"2023-10-09T04:49:36.251774+0000","last_unstale":"2023-10-09T04:50:36.680746+0000","last_undegraded":"2023-10-09T04:50:36.680746+0000","last_fullsized":"2023-10-09T04:50:36.680746+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:24.682458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":349,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.698022+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:50:36.698022+0000","last_peered":"2023-10-09T04:50:36.698022+0000","last_clean":"2023-10-09T04:50:36.698022+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:50:36.698022+0000","last_undegraded":"2023-10-09T04:50:36.698022+0000","last_fullsized":"2023-10-09T04:50:36.698022+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":127,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.700029+0000","last_change":"2023-10-09T04:49:24.721368+0000","last_active":"2023-10-09T04:50:36.700029+0000","last_peered":"2023-10-09T04:50:36.700029+0000","last_clean":"2023-10-09T04:50:36.700029+0000","last_became_active":"2023-10-09T04:49:24.721024+0000","last_became_peered":"2023-10-09T04:49:24.721024+0000","last_unstale":"2023-10-09T04:50:36.700029+0000","last_undegraded":"2023-10-09T04:50:36.700029+0000","last_fullsized":"2023-10-09T04:50:36.700029+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:57.736030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":318,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.696011+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:50:36.696011+0000","last_peered":"2023-10-09T04:50:36.696011+0000","last_clean":"2023-10-09T04:50:36.696011+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:50:36.696011+0000","last_undegraded":"2023-10-09T04:50:36.696011+0000","last_fullsized":"2023-10-09T04:50:36.696011+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":137,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.699973+0000","last_change":"2023-10-09T04:49:24.711760+0000","last_active":"2023-10-09T04:50:36.699973+0000","last_peered":"2023-10-09T04:50:36.699973+0000","last_clean":"2023-10-09T04:50:36.699973+0000","last_became_active":"2023-10-09T04:49:24.711383+0000","last_became_peered":"2023-10-09T04:49:24.711383+0000","last_unstale":"2023-10-09T04:50:36.699973+0000","last_undegraded":"2023-10-09T04:50:36.699973+0000","last_fullsized":"2023-10-09T04:50:36.699973+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:04.370272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":312,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.680686+0000","last_change":"2023-10-09T04:49:24.725553+0000","last_active":"2023-10-09T04:50:36.680686+0000","last_peered":"2023-10-09T04:50:36.680686+0000","last_clean":"2023-10-09T04:50:36.680686+0000","last_became_active":"2023-10-09T04:49:24.725179+0000","last_became_peered":"2023-10-09T04:49:24.725179+0000","last_unstale":"2023-10-09T04:50:36.680686+0000","last_undegraded":"2023-10-09T04:50:36.680686+0000","last_fullsized":"2023-10-09T04:50:36.680686+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.045784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":278,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.695732+0000","last_change":"2023-10-09T04:49:24.718316+0000","last_active":"2023-10-09T04:50:36.695732+0000","last_peered":"2023-10-09T04:50:36.695732+0000","last_clean":"2023-10-09T04:50:36.695732+0000","last_became_active":"2023-10-09T04:49:24.717948+0000","last_became_peered":"2023-10-09T04:49:24.717948+0000","last_unstale":"2023-10-09T04:50:36.695732+0000","last_undegraded":"2023-10-09T04:50:36.695732+0000","last_fullsized":"2023-10-09T04:50:36.695732+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:48.454201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":265,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.698550+0000","last_change":"2023-10-09T04:49:24.745757+0000","last_active":"2023-10-09T04:50:36.698550+0000","last_peered":"2023-10-09T04:50:36.698550+0000","last_clean":"2023-10-09T04:50:36.698550+0000","last_became_active":"2023-10-09T04:49:24.727010+0000","last_became_peered":"2023-10-09T04:49:24.727010+0000","last_unstale":"2023-10-09T04:50:36.698550+0000","last_undegraded":"2023-10-09T04:50:36.698550+0000","last_fullsized":"2023-10-09T04:50:36.698550+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:51.654447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-09T04:50:36.699589+0000","last_change":"2023-10-09T04:49:24.710860+0000","last_active":"2023-10-09T04:50:36.699589+0000","last_peered":"2023-10-09T04:50:36.699589+0000","last_clean":"2023-10-09T04:50:36.699589+0000","last_became_active":"2023-10-09T04:49:24.710496+0000","last_became_peered":"2023-10-09T04:49:24.710496+0000","last_unstale":"2023-10-09T04:50:36.699589+0000","last_undegraded":"2023-10-09T04:50:36.699589+0000","last_fullsized":"2023-10-09T04:50:36.699589+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:24.901415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5439553536,"num_objects":83001,"num_object_clones":0,"num_object_copies":249003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":83001,"num_write_kb":5312064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16328556544,"data_stored":16328556544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73301,"ondisk_log_size":73301,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16666,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234526,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5502788,"kb_used_data":3989436,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":88242364,"statfs":{"total":95995035648,"available":90360180736,"internally_reserved":0,"allocated":4085182464,"data_stored":4084430563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":1.4039999999999999,"15min":2.0910000000000002},"min":{"1min":0.31,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":1.141,"5min":14.970000000000001,"15min":68.495999999999995},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.71199999999999997,"5min":2.048,"15min":2.6200000000000001},"min":{"1min":0.30599999999999999,"5min":0.249,"15min":0.249},"max":{"1min":2.1800000000000002,"5min":19.152000000000001,"15min":68.540000000000006},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 04:49:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":4.5730000000000004,"15min":4.1440000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":1.179,"5min":128.601,"15min":264.62},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":4.2990000000000004,"15min":4.2069999999999999},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.23799999999999999},"max":{"1min":1.264,"5min":128.381,"15min":264.65899999999999},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":6.6449999999999996,"15min":5.2489999999999997},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":7.2400000000000002,"5min":129.10400000000001,"15min":184.72},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.669,"5min":6.8280000000000003,"15min":5.548},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.28100000000000003},"max":{"1min":5.2210000000000001,"5min":129.15100000000001,"15min":184.697},"last":0.47699999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:50:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.0110000000000001,"15min":3.5819999999999999},"min":{"1min":0.41899999999999998,"5min":0.34899999999999998,"15min":0.29499999999999998},"max":{"1min":7.1959999999999997,"5min":12.324,"15min":182.839},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.3599999999999999,"15min":4},"min":{"1min":0.38900000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.0999999999999996,"5min":14.866,"15min":182.678},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.4449999999999998,"15min":1.772},"min":{"1min":0.34100000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.367000000000001,"5min":14.811999999999999,"15min":14.811999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.669,"15min":1.377},"min":{"1min":0.313,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":4.5389999999999997,"5min":13.523,"15min":13.523},"last":8.2949999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.5189999999999999,"15min":1.3260000000000001},"min":{"1min":0.38,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.1020000000000003,"5min":12.202999999999999,"15min":13.319000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.548,"5min":2.1579999999999999,"15min":1.8340000000000001},"min":{"1min":0.432,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":10.544,"5min":12.081,"15min":18.408000000000001},"last":7.9909999999999997}]},{"osd":6,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.2809999999999999,"15min":1.2889999999999999},"min":{"1min":0.39500000000000002,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":5.2350000000000003,"5min":10.047000000000001,"15min":11.388},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.363,"5min":1.9039999999999999,"15min":1.859},"min":{"1min":0.39700000000000002,"5min":0.34499999999999997,"15min":0.30099999999999999},"max":{"1min":7.3620000000000001,"5min":11.680999999999999,"15min":18.311},"last":0.44900000000000001}]}]},{"osd":6,"up_from":0,"seq":197568496160,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141250,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4319976,"kb_used_data":3385120,"kb_used_omap":4,"kb_used_meta":934843,"kb_avail":89425176,"statfs":{"total":95995035648,"available":91571380224,"internally_reserved":0,"allocated":3466362880,"data_stored":3464756314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":957280142},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.484,"15min":2.9529999999999998},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.14799999999999999},"max":{"1min":1.78,"5min":11.653,"15min":93.171999999999997},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.613,"15min":3.0219999999999998},"min":{"1min":0.246,"5min":0.245,"15min":0.193},"max":{"1min":2.1739999999999999,"5min":10.765000000000001,"15min":93.147000000000006},"last":0.25700000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":5.0199999999999996,"15min":4.8129999999999997},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":1.3899999999999999,"5min":151.41900000000001,"15min":263.60500000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":5.0780000000000003,"15min":5.1609999999999996},"min":{"1min":0.29599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.4299999999999999,"5min":146.749,"15min":263.65199999999999},"last":0.32800000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.224,"15min":5.3040000000000003},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":2.6669999999999998,"5min":9.0709999999999997,"15min":379.80900000000003},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.738,"15min":5.8440000000000003},"min":{"1min":0.35899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.2370000000000001,"5min":12.615,"15min":379.95800000000003},"last":0.377}]},{"osd":4,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.371,"15min":2.657},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":4.1109999999999998,"5min":19.959,"15min":170.072},"last":0.31},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.96,"15min":2.484},"min":{"1min":0.38900000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":2.7040000000000002,"5min":11.598000000000001,"15min":169.94499999999999},"last":1.0740000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":2.0600000000000001,"15min":2.4319999999999999},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.28299999999999997},"max":{"1min":3.2170000000000001,"5min":9.782,"15min":171.036},"last":1.2849999999999999},{"interface":"front","average":{"1min":0.71999999999999997,"5min":2.4849999999999999,"15min":2.762},"min":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":1.407,"5min":20.021000000000001,"15min":170.88},"last":1.0149999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":1.954,"15min":2.4159999999999999},"min":{"1min":0.39900000000000002,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":2.786,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.81799999999999995,"5min":2.5739999999999998,"15min":2.7989999999999999},"min":{"1min":0.45800000000000002,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":2.802,"5min":20.164999999999999,"15min":132.17699999999999},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:50:01 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":2.8460000000000001,"15min":3.4540000000000002},"min":{"1min":0.438,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":3.2280000000000002,"5min":20.055,"15min":174.75999999999999},"last":1.353},{"interface":"front","average":{"1min":0.91200000000000003,"5min":2.254,"15min":3.1160000000000001},"min":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.377},"max":{"1min":3.2109999999999999,"5min":16.238,"15min":174.779},"last":1.367}]}]},{"osd":0,"up_from":0,"seq":390842024307,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":0,"seq":949187772583,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":609885356308,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019434,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6095300,"kb_used_data":4595644,"kb_used_omap":0,"kb_used_meta":1499648,"kb_avail":87649852,"statfs":{"total":95995035648,"available":89753448448,"internally_reserved":0,"allocated":4705939456,"data_stored":4705187555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1535639552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.7410000000000001,"15min":4.0309999999999997},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.23100000000000001},"max":{"1min":1.0529999999999999,"5min":12.795,"15min":171.13499999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.9350000000000001,"15min":4.0229999999999997},"min":{"1min":0.40999999999999998,"5min":0.26000000000000001,"15min":0.23200000000000001},"max":{"1min":3.4009999999999998,"5min":16.004000000000001,"15min":171.11799999999999},"last":0.38}]},{"osd":1,"last update":"Mon Oct 9 04:49:53 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":9.6300000000000008,"15min":5.351},"min":{"1min":0.45200000000000001,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":37.944000000000003,"5min":385.78199999999998,"15min":385.78199999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.4700000000000002,"5min":9.2650000000000006,"15min":5.3330000000000002},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":37.408999999999999,"5min":386.09399999999999,"15min":386.09399999999999},"last":0.311}]},{"osd":2,"last update":"Mon Oct 9 04:50:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":11.032999999999999,"15min":6.5179999999999998},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.131,"5min":440.75599999999997,"15min":440.75599999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.0089999999999999,"5min":11.414,"15min":6.367},"min":{"1min":0.374,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":3.4399999999999999,"5min":440.61900000000003,"15min":440.61900000000003},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":3.746,"15min":5.1360000000000001},"min":{"1min":0.32600000000000001,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":4.4429999999999996,"5min":141.334,"15min":357.125},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.2999999999999998,"5min":4.1529999999999996,"15min":5.1429999999999998},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":7.1760000000000002,"5min":141.18299999999999,"15min":359.02800000000002},"last":0.74399999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":10.875999999999999,"5min":3.9660000000000002,"15min":2.1349999999999998},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":169.44200000000001,"5min":169.44200000000001,"15min":169.44200000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":10.66,"5min":3.5630000000000002,"15min":2.073},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.24099999999999999},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:49:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.9690000000000001,"15min":1.9019999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.29599999999999999},"max":{"1min":5.9349999999999996,"5min":22.920999999999999,"15min":65.75},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.5329999999999999,"15min":1.7430000000000001},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":1.2669999999999999,"5min":15.095000000000001,"15min":65.849999999999994},"last":0.80200000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:50:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":2.093,"15min":1.5960000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28000000000000003},"max":{"1min":4.8250000000000002,"5min":16.109000000000002,"15min":16.109000000000002},"last":4.1660000000000004},{"interface":"front","average":{"1min":3.1720000000000002,"5min":2.8109999999999999,"15min":1.903},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.2060000000000004,"5min":22.858000000000001,"15min":22.858000000000001},"last":0.65500000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757798,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5092792,"kb_used_data":3910000,"kb_used_omap":0,"kb_used_meta":1182784,"kb_avail":88652360,"statfs":{"total":95995035648,"available":90780016640,"internally_reserved":0,"allocated":4003840000,"data_stored":4003099787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1211170816},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:49:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.851,"15min":4.798},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23100000000000001},"max":{"1min":8.7309999999999999,"5min":9.6890000000000001,"15min":210.85499999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.6950000000000001,"15min":5.0359999999999996},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.5489999999999999,"5min":8.4749999999999996,"15min":210.23500000000001},"last":4.9390000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:49:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":8.3260000000000005,"15min":5.8259999999999996},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.252},"max":{"1min":1.0900000000000001,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.2110000000000001,"5min":8.5410000000000004,"15min":6.0119999999999996},"min":{"1min":0.33200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":8.6620000000000008,"5min":211.435,"15min":211.435},"last":0.57899999999999996}]},{"osd":2,"last update":"Mon Oct 9 04:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":5.5179999999999998,"15min":4.9299999999999997},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":86.488,"15min":90.763999999999996},"last":8.8940000000000001},{"interface":"front","average":{"1min":1.385,"5min":5.5199999999999996,"15min":5.0369999999999999},"min":{"1min":0.34100000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.7590000000000003,"5min":86.510999999999996,"15min":91.340000000000003},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.9790000000000001,"15min":4.9880000000000004},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":6.2140000000000004,"5min":13.711,"15min":268.63299999999998},"last":4.9660000000000002},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.484,"15min":4.6040000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.29799999999999999},"max":{"1min":2.2189999999999999,"5min":10.227,"15min":268.92500000000001},"last":0.65000000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":6.6760000000000002,"5min":2.552,"15min":2.8690000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23599999999999999},"max":{"1min":116.054,"5min":116.054,"15min":361.09399999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":8.5060000000000002,"5min":3.0499999999999998,"15min":3.2930000000000001},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.253},"max":{"1min":118.47,"5min":118.47,"15min":361.71300000000002},"last":1.784}]},{"osd":6,"last update":"Mon Oct 9 04:50:02 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.242,"15min":1.228},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.23100000000000001},"max":{"1min":3.5579999999999998,"5min":10.295999999999999,"15min":13.162000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.262,"15min":1.321},"min":{"1min":0.34499999999999997,"5min":0.314,"15min":0.28000000000000003},"max":{"1min":3.6259999999999999,"5min":7.3040000000000003,"15min":13.231999999999999},"last":4.9020000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":11.656000000000001,"5min":4.7309999999999999,"15min":2.3740000000000001},"min":{"1min":0.47399999999999998,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":110.521,"5min":110.521,"15min":110.521},"last":4.8470000000000004},{"interface":"front","average":{"1min":11.186999999999999,"5min":4.9630000000000001,"15min":2.3809999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":110.746,"5min":110.746,"15min":110.746},"last":0.67700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3480289280,"data_stored":3480289280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4724031488,"data_stored":4724031488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4043046912,"data_stored":4043046912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4081188864,"data_stored":4081188864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:50:52.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-10-09T04:50:52.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:52 smithi029 ceph-mon[216903]: pgmap v1387: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T04:50:52.789 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:52 smithi029 ceph-mon[223858]: pgmap v1387: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T04:50:52.872 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:52.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:52.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:50:53.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 89180 89164 66.3356 57 0.0133344 0.0150699 2023-10-09T04:50:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:52 smithi177 ceph-mon[195797]: pgmap v1387: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 52 MiB/s wr, 833 op/s 2023-10-09T04:50:53.039 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:53.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:50:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:53.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:50:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:50:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:50:52] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-10-09T04:50:53.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:53.214 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:53.328 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:53.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:53.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:53.473 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:53.539 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:53.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:53.579 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:50:53.579 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:50:53.579 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:50:53.579 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:50:53.644 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:53.682 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:53.938 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:53.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:54.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 89930 89914 66.1066 46.875 0.0267984 0.0151224 2023-10-09T04:50:54.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:54.075 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:54.391 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:54.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:54.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:54.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:54.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:50:54.537 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:54.658 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:54.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:54.679 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:54.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:50:54.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:54.834 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:54 smithi029 ceph-mon[216903]: pgmap v1388: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-09T04:50:54.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:54 smithi029 ceph-mon[223858]: pgmap v1388: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-09T04:50:54.958 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:50:54.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:54.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:54.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:54.987 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:54 smithi177 ceph-mon[195797]: pgmap v1388: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-09T04:50:54.988 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:50:55.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 90859 90843 66.013 58.0625 0.0333456 0.0151442 2023-10-09T04:50:55.015 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:50:55.041 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:50:55.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:55.091 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:50:55.131 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:55.147 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:50:55.163 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:50:55.204 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.209 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:50:55.346 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:55.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:55.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:55.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:55.540 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:55.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:55.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:55.735 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:55.766 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:56.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 91644 91628 65.8182 49.0625 0.014111 0.0151889 2023-10-09T04:50:56.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/262740604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/262740604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/262740604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:50:56.227 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:56.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:56.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:56.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:50:56.363 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:56.756 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:50:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:56.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:50:56.859 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:50:56.895 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:56.927 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:50:56.942 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:57.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 92512 92496 65.6866 54.25 0.0354982 0.015219 2023-10-09T04:50:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:56 smithi177 ceph-mon[195797]: pgmap v1389: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T04:50:57.056 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:50:57.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:57.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:50:57.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:56 smithi029 ceph-mon[216903]: pgmap v1389: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T04:50:57.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:56 smithi029 ceph-mon[223858]: pgmap v1389: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T04:50:57.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:57.208 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:57.616 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:57.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:57.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:50:57.756 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:58.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 93320 93304 65.5159 50.5 0.00362553 0.0152492 2023-10-09T04:50:58.465 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:50:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:58.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:50:58.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:58.601 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:58.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:50:58.966 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:50:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:58.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:50:58.991 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:58 smithi029 ceph-mon[216903]: pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T04:50:58.991 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:58.991 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:58 smithi029 ceph-mon[223858]: pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T04:50:58.991 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:59.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0077 2023-10-09T04:50:59.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 94257 2023-10-09T04:50:59.012 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 65.4507 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 27.6414 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 172.5 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 35.875 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1047 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 442.262 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 2760 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 574 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.015275 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0256686 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 1.02767 2023-10-09T04:50:59.013 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.0013453 2023-10-09T04:50:59.017 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:50:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:58 smithi177 ceph-mon[195797]: pgmap v1390: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-09T04:50:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:50:59.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:50:59.132 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:59.175 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:50:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:59.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:50:59.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:50:59.343 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:59.469 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:50:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:59.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:50:59.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:50:59.617 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:50:59.745 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:50:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/847374514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:50:59.758 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pgp_num to 1 2023-10-09T04:50:59.786 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:50:59.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:50:59.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:50:59.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:50:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:50:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/847374514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:50:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:50:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/847374514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-09T04:50:59.928 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:00.089 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:00.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:00.196 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:00.279 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:00.327 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:51:00.328 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:51:00.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:00.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:00.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:00.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:00.587 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:00.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:00.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:00.793 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:00.826 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: pgmap v1391: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T04:51:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/847374514' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:51:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: osdmap e361: 8 total, 8 up, 4 in 2023-10-09T04:51:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:00 smithi177 ceph-mon[195797]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: pgmap v1391: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/847374514' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: osdmap e361: 8 total, 8 up, 4 in 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[216903]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: pgmap v1391: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 52 MiB/s wr, 826 op/s 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/847374514' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: osdmap e361: 8 total, 8 up, 4 in 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: failed to encode map e361 with expected crc 2023-10-09T04:51:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:00 smithi029 ceph-mon[223858]: failed to encode map e361 with expected crc 2023-10-09T04:51:02.054 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:02.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:51:02.288 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:02.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:02.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:02.464 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:02.547 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:02.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:02.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:02.733 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:02.779 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:51:02.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:02.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:02.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:02 smithi177 ceph-mon[195797]: pgmap v1393: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 102 B/s rd, 49 MiB/s wr, 843 op/s 2023-10-09T04:51:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/635478193' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:03.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:03.043 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:02 smithi029 ceph-mon[216903]: pgmap v1393: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 102 B/s rd, 49 MiB/s wr, 843 op/s 2023-10-09T04:51:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/635478193' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:03.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:02 smithi029 ceph-mon[223858]: pgmap v1393: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 102 B/s rd, 49 MiB/s wr, 843 op/s 2023-10-09T04:51:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/635478193' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:03.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:03.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:51:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:51:02] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-10-09T04:51:03.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:03.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:03.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:03.320 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:03.419 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:03.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:03.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:03.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:03.599 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:03.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:03.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:03.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:03.829 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: osdmap e362: 8 total, 8 up, 4 in 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: failed to encode map e362 with expected crc 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: failed to encode map e362 with expected crc 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[216903]: failed to encode map e362 with expected crc 2023-10-09T04:51:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:51:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: osdmap e362: 8 total, 8 up, 4 in 2023-10-09T04:51:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: failed to encode map e362 with expected crc 2023-10-09T04:51:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: failed to encode map e362 with expected crc 2023-10-09T04:51:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:03 smithi029 ceph-mon[223858]: failed to encode map e362 with expected crc 2023-10-09T04:51:04.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:51:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: osdmap e362: 8 total, 8 up, 4 in 2023-10-09T04:51:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: failed to encode map e362 with expected crc 2023-10-09T04:51:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: failed to encode map e362 with expected crc 2023-10-09T04:51:04.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:03 smithi177 ceph-mon[195797]: failed to encode map e362 with expected crc 2023-10-09T04:51:04.325 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:04.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:04.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:04.430 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:04.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:04.496 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:04.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:04.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:04.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:04.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:04.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:04.774 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:04 smithi177 ceph-mon[195797]: pgmap v1395: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 29 MiB/s wr, 530 op/s 2023-10-09T04:51:04.774 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:04 smithi177 ceph-mon[195797]: failed to encode map e362 with expected crc 2023-10-09T04:51:05.022 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:51:05.046 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:51:05.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:04 smithi029 ceph-mon[216903]: pgmap v1395: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 29 MiB/s wr, 530 op/s 2023-10-09T04:51:05.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:04 smithi029 ceph-mon[216903]: failed to encode map e362 with expected crc 2023-10-09T04:51:05.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:04 smithi029 ceph-mon[223858]: pgmap v1395: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 29 MiB/s wr, 530 op/s 2023-10-09T04:51:05.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:04 smithi029 ceph-mon[223858]: failed to encode map e362 with expected crc 2023-10-09T04:51:05.060 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:05.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:05.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:05.075 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:51:05.103 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:51:05.141 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:51:05.167 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:51:05.168 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:05.193 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:51:05.221 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:51:05.222 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.222 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.223 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.223 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.223 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.223 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.223 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.223 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:51:05.245 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:05.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:05.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:05.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:05.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:05.568 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:05.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:05.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:05.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:05.807 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1933651461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1933651461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:06.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1933651461' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:06.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:51:06.565 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:06.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:06.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:06.698 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:06.740 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:06.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:06.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:06.873 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:06.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:06.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:06 smithi177 ceph-mon[195797]: pgmap v1396: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 30 MiB/s wr, 1.08k op/s 2023-10-09T04:51:06.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:06.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:07.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:07.081 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7] out_osds: [6, 3, 0, 2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:51:07.081 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:51:07.081 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2023-10-09T04:51:07.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:51:07.109 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:06 smithi029 ceph-mon[216903]: pgmap v1396: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 30 MiB/s wr, 1.08k op/s 2023-10-09T04:51:07.109 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:06 smithi029 ceph-mon[223858]: pgmap v1396: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 30 MiB/s wr, 1.08k op/s 2023-10-09T04:51:07.152 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:07.285 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:07.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:07.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:07.483 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:07.538 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:07.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:07.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:07.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:07.774 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:07.900 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:29917 12.8 e288/284 12:1f6a57ad:::benchmark_data_smithi029_442458_object29916:head v 288'1844, mlcod=288'1841)", 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:43:57.034493+0000", 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 430.86436056600002, 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.219557849, 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.901 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034493+0000", 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034493+0000", 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034492+0000", 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034513+0000", 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0849e-05 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034523+0000", 2023-10-09T04:51:07.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9250000000000007e-06 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034529+0000", 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4409999999999998e-06 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034569+0000", 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9972999999999998e-05 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034589+0000", 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0043000000000001e-05 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.254003+0000", 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21941329000000001 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.254051+0000", 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.799e-05 2023-10-09T04:51:07.905 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:29918 12.8 e288/284 12:1656cc53:::benchmark_data_smithi029_442458_object29917:head v 288'1845, mlcod=288'1841)", 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:43:57.034542+0000", 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 430.86431125299998, 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21953012199999999, 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034542+0000", 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034542+0000", 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034542+0000", 2023-10-09T04:51:07.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034561+0000", 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9683000000000001e-05 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034568+0000", 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3679999999999998e-06 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.034573+0000", 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7659999999999998e-06 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.035694+0000", 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011215769999999999 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.035713+0000", 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8570999999999999e-05 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.254055+0000", 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.218342652 2023-10-09T04:51:07.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:43:57.254072+0000", 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6809000000000001e-05 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122048 12.8 e308/307 12:12f0bcb9:::benchmark_data_smithi029_442458_object26798:head v 308'7621, mlcod=0'0)", 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:25.304138+0000", 2023-10-09T04:51:07.911 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 282.59471552299999, 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.32250182100000002, 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.304138+0000", 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.304138+0000", 2023-10-09T04:51:07.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.304137+0000", 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.304147+0000", 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9320000000000008e-06 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.304169+0000", 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2580999999999999e-05 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.304180+0000", 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0674e-05 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.333489+0000", 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029309465999999999 2023-10-09T04:51:07.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.333520+0000", 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0843999999999998e-05 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626593+0000", 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29307282200000001 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626640+0000", 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6646e-05 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122058 12.8 e308/307 12:106bae43:::benchmark_data_smithi029_442458_object26808:head v 308'7622, mlcod=0'0)", 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:25.314898+0000", 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 282.58395549099998, 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.31176883100000002, 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.916 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314898+0000", 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314898+0000", 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314896+0000", 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314909+0000", 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3337e-05 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314932+0000", 2023-10-09T04:51:07.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2382999999999999e-05 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314943+0000", 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1558e-05 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.438450+0000", 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.123506719 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.438492+0000", 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1638999999999999e-05 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626645+0000", 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.188153652 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626667+0000", 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1373e-05 2023-10-09T04:51:07.920 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122070 12.8 e308/307 12:1cf21352:::benchmark_data_smithi029_442458_object26820:head v 308'7623, mlcod=0'0)", 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:25.314952+0000", 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 282.58390111099999, 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.311735284, 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314952+0000", 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314952+0000", 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314952+0000", 2023-10-09T04:51:07.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314957+0000", 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7199999999999997e-06 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314962+0000", 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1839999999999998e-06 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.314966+0000", 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.014e-06 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.533769+0000", 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.218803053 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.533798+0000", 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9198000000000001e-05 2023-10-09T04:51:07.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626669+0000", 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.092870675999999999 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626687+0000", 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8683000000000001e-05 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.925 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122077 12.8 e308/307 12:17464ea6:::benchmark_data_smithi029_442458_object26827:head v 308'7624, mlcod=0'0)", 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:25.318788+0000", 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 282.58006495000001, 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30798629799999999, 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318788+0000", 2023-10-09T04:51:07.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318788+0000", 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318787+0000", 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318800+0000", 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3008e-05 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318822+0000", 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1724000000000001e-05 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.318832+0000", 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0519e-05 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.623091+0000", 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30425862100000001 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.623120+0000", 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9611000000000002e-05 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626738+0000", 2023-10-09T04:51:07.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003617234 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.626775+0000", 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7030999999999998e-05 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122197 12.6 e309/308 12:63479ba0:::benchmark_data_smithi029_442458_object26947:head v 309'7555, mlcod=309'7553)", 2023-10-09T04:51:07.930 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:26.749356+0000", 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.149496967, 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33801140499999999, 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749356+0000", 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749356+0000", 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749355+0000", 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749366+0000", 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1e-05 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749389+0000", 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2483000000000001e-05 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749398+0000", 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5070000000000006e-06 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749474+0000", 2023-10-09T04:51:07.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5611999999999994e-05 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.749505+0000", 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0808e-05 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.087332+0000", 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33782718699999997 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.087368+0000", 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6041000000000003e-05 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122199 12.6 e309/308 12:67cf0c95:::benchmark_data_smithi029_442458_object26949:head v 309'7556, mlcod=309'7553)", 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:26.755244+0000", 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.14360968400001, 2023-10-09T04:51:07.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33411501199999999, 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.755244+0000", 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.755244+0000", 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.755242+0000", 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.755254+0000", 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2065000000000001e-05 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.755277+0000", 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2832000000000001e-05 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.755287+0000", 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3940000000000002e-06 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.837332+0000", 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.082045119 2023-10-09T04:51:07.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.837353+0000", 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1557e-05 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.089320+0000", 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25196613200000001 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.089359+0000", 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9119000000000003e-05 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122209 12.e e309/308 12:77a81543:::benchmark_data_smithi029_442458_object26959:head v 309'7552, mlcod=309'7550)", 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:26.780492+0000", 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.11836118399998, 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.31059092300000002, 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.940 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.780492+0000", 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.780492+0000", 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.780491+0000", 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.780505+0000", 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3874e-05 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.780528+0000", 2023-10-09T04:51:07.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3180000000000002e-05 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.780538+0000", 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5899999999999997e-06 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.837321+0000", 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.056783004999999998 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.837347+0000", 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5936999999999999e-05 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.091036+0000", 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25368961800000001 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.091083+0000", 2023-10-09T04:51:07.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6894999999999997e-05 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122213 12.e e309/308 12:76b8fcd2:::benchmark_data_smithi029_442458_object26963:head v 309'7553, mlcod=309'7550)", 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:26.792429+0000", 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.106423812, 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.29867986600000002, 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.792429+0000", 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.792429+0000", 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.792428+0000", 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.792441+0000", 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2374e-05 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.792453+0000", 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2458e-05 2023-10-09T04:51:07.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.792460+0000", 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9380000000000003e-06 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.964195+0000", 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17173485599999999 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.964214+0000", 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8794999999999999e-05 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.091087+0000", 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.126873241 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.091109+0000", 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2184e-05 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.949 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122225 12.3 e309/308 12:c93d3a84:::benchmark_data_smithi029_442458_object26975:head v 309'7663, mlcod=288'1688)", 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:26.838745+0000", 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.06010805800003, 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25119665400000002, 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838745+0000", 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838745+0000", 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838744+0000", 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:07.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838756+0000", 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1987999999999999e-05 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838769+0000", 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2506e-05 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838777+0000", 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3420000000000008e-06 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838863+0000", 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5935999999999998e-05 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.838891+0000", 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7583999999999999e-05 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.089906+0000", 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25101567800000002 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.089942+0000", 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5660000000000001e-05 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.954 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:122226 12.3 e309/308 12:c17dd3ff:::benchmark_data_smithi029_442458_object26976:head v 309'7664, mlcod=288'1688)", 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:26.868281+0000", 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 281.03057278900002, 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.22168258399999999, 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.868281+0000", 2023-10-09T04:51:07.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.868281+0000", 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.868276+0000", 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999952 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.868293+0000", 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6806000000000001e-05 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.868317+0000", 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4536999999999999e-05 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.868327+0000", 2023-10-09T04:51:07.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5810000000000003e-06 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.964264+0000", 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.095937731999999998 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.964293+0000", 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8235999999999999e-05 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.089946+0000", 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12565301200000001 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:27.089963+0000", 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7484000000000001e-05 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:138978 12.8 e317/307 12:1765ab08:::benchmark_data_smithi029_442458_object43728:head v 317'8713, mlcod=317'8709)", 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.864771+0000", 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 237.034082074, 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022326006999999998, 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864771+0000", 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864771+0000", 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864770+0000", 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864779+0000", 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5960000000000004e-06 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864799+0000", 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0324999999999999e-05 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864808+0000", 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1339999999999993e-06 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.866430+0000", 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016219019999999999 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.866460+0000", 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9974000000000001e-05 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.887047+0000", 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.02058714 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.887097+0000", 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9845000000000001e-05 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:149161 12.8 e320/307 12:1a4a23ee:::benchmark_data_smithi029_442458_object53911:head v 320'9364, mlcod=320'9361)", 2023-10-09T04:51:07.964 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:33.441480+0000", 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 214.457373684, 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023034631, 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441480+0000", 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441480+0000", 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441479+0000", 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:07.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441488+0000", 2023-10-09T04:51:07.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9889999999999998e-06 2023-10-09T04:51:07.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441501+0000", 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3811e-05 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441509+0000", 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9889999999999992e-06 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.448849+0000", 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073399670000000002 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.448876+0000", 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6962e-05 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.464468+0000", 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015592168 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.464514+0000", 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5738999999999997e-05 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:168534 12.8 e324/307 12:15729ae9:::benchmark_data_smithi029_442458_object73284:head v 324'10567, mlcod=324'10565)", 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:12.796708+0000", 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 175.10214507000001, 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026130677000000001, 2023-10-09T04:51:07.970 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796708+0000", 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796708+0000", 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:07.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796707+0000", 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796718+0000", 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1420000000000001e-05 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:07.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796742+0000", 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3431999999999999e-05 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796751+0000", 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4760000000000004e-06 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.798093+0000", 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013422600000000001 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.798126+0000", 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2511000000000001e-05 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.822781+0000", 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024655274000000001 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:07.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.822839+0000", 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7722000000000001e-05 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:07.975 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:51:07.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:07.976 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:07.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:08.049 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:08.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:08.119 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:08.230 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:08.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:08.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:08.418 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:08.545 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:08.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:08.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:08.732 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:08.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:08.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:08 smithi029 ceph-mon[216903]: pgmap v1397: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 23 MiB/s wr, 978 op/s 2023-10-09T04:51:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:08 smithi029 ceph-mon[223858]: pgmap v1397: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 23 MiB/s wr, 978 op/s 2023-10-09T04:51:08.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:09.013 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:08 smithi177 ceph-mon[195797]: pgmap v1397: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 127 B/s rd, 23 MiB/s wr, 978 op/s 2023-10-09T04:51:09.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:51:10.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:51:10.533 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:10.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:10.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:10.667 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:10 smithi029 ceph-mon[216903]: pgmap v1398: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 5.9 MiB/s wr, 658 op/s 2023-10-09T04:51:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:10 smithi029 ceph-mon[223858]: pgmap v1398: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 5.9 MiB/s wr, 658 op/s 2023-10-09T04:51:11.184 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:11.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:11.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:11.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:10 smithi177 ceph-mon[195797]: pgmap v1398: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 5.9 MiB/s wr, 658 op/s 2023-10-09T04:51:11.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:11.363 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:11.463 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:11.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:11.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:11.627 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:11.644 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:11.656 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:11.729 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:11.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:11.847 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:11.894 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:12.034 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:12.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:12.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:12.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:12.179 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:12.296 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:51:12.308 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:51:12.824 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:51:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:51:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:51:12] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-10-09T04:51:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[216903]: pgmap v1399: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 582 KiB/s wr, 876 op/s 2023-10-09T04:51:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1030460542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[216903]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:51:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[223858]: pgmap v1399: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 582 KiB/s wr, 876 op/s 2023-10-09T04:51:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1030460542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:12 smithi029 ceph-mon[223858]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:51:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:12 smithi177 ceph-mon[195797]: pgmap v1399: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 582 KiB/s wr, 876 op/s 2023-10-09T04:51:13.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1030460542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:51:13.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:12 smithi177 ceph-mon[195797]: from='client.36711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:51:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:51:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: osdmap e363: 8 total, 8 up, 4 in 2023-10-09T04:51:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: osdmap e363: 8 total, 8 up, 4 in 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: osdmap e363: 8 total, 8 up, 4 in 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: failed to encode map e363 with expected crc 2023-10-09T04:51:14.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:14.312 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:14.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:14.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:14.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:14.476 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:14.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:14.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:14.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:14.710 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:14.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:14.817 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1399,"stamp":"2023-10-09T04:51:11.391912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5573449656,"num_objects":85060,"num_object_clones":0,"num_object_copies":255180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85060,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":103780,"num_write_kb":6034894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":4870682,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73396,"ondisk_log_size":73396,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":21836064,"kb_used_data":16402624,"kb_used_omap":4,"kb_used_meta":5432955,"kb_avail":353144544,"statfs":{"total":383980142592,"available":361620013056,"internally_reserved":0,"allocated":16796286976,"data_stored":16792410272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":5563346830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,2,4,1,2,2,5,0,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-562757632,"num_objects":-8587,"num_object_clones":0,"num_object_copies":-25761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8769,"num_write_kb":5824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002587"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":120,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761359+0000","last_change":"2023-10-09T04:49:24.722690+0000","last_active":"2023-10-09T04:50:59.761359+0000","last_peered":"2023-10-09T04:50:59.761359+0000","last_clean":"2023-10-09T04:50:59.761359+0000","last_became_active":"2023-10-09T04:49:24.722128+0000","last_became_peered":"2023-10-09T04:49:24.722128+0000","last_unstale":"2023-10-09T04:50:59.761359+0000","last_undegraded":"2023-10-09T04:50:59.761359+0000","last_fullsized":"2023-10-09T04:50:59.761359+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:52.116095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":114,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753826+0000","last_change":"2023-10-09T04:49:36.252176+0000","last_active":"2023-10-09T04:50:59.753826+0000","last_peered":"2023-10-09T04:50:59.753826+0000","last_clean":"2023-10-09T04:50:59.753826+0000","last_became_active":"2023-10-09T04:49:36.251166+0000","last_became_peered":"2023-10-09T04:49:36.251166+0000","last_unstale":"2023-10-09T04:50:59.753826+0000","last_undegraded":"2023-10-09T04:50:59.753826+0000","last_fullsized":"2023-10-09T04:50:59.753826+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:40.935795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":360,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.752672+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:50:59.752672+0000","last_peered":"2023-10-09T04:50:59.752672+0000","last_clean":"2023-10-09T04:50:59.752672+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:50:59.752672+0000","last_undegraded":"2023-10-09T04:50:59.752672+0000","last_fullsized":"2023-10-09T04:50:59.752672+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":295,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.764144+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:50:59.764144+0000","last_peered":"2023-10-09T04:50:59.764144+0000","last_clean":"2023-10-09T04:50:59.764144+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:50:59.764144+0000","last_undegraded":"2023-10-09T04:50:59.764144+0000","last_fullsized":"2023-10-09T04:50:59.764144+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":156,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.750920+0000","last_change":"2023-10-09T04:49:36.253446+0000","last_active":"2023-10-09T04:50:59.750920+0000","last_peered":"2023-10-09T04:50:59.750920+0000","last_clean":"2023-10-09T04:50:59.750920+0000","last_became_active":"2023-10-09T04:49:36.252717+0000","last_became_peered":"2023-10-09T04:49:36.252717+0000","last_unstale":"2023-10-09T04:50:59.750920+0000","last_undegraded":"2023-10-09T04:50:59.750920+0000","last_fullsized":"2023-10-09T04:50:59.750920+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:30.302532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.769627+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:50:59.769627+0000","last_peered":"2023-10-09T04:50:59.769627+0000","last_clean":"2023-10-09T04:50:59.769627+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:50:59.769627+0000","last_undegraded":"2023-10-09T04:50:59.769627+0000","last_fullsized":"2023-10-09T04:50:59.769627+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":304,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.769460+0000","last_change":"2023-10-09T04:49:24.727794+0000","last_active":"2023-10-09T04:50:59.769460+0000","last_peered":"2023-10-09T04:50:59.769460+0000","last_clean":"2023-10-09T04:50:59.769460+0000","last_became_active":"2023-10-09T04:49:24.726934+0000","last_became_peered":"2023-10-09T04:49:24.726934+0000","last_unstale":"2023-10-09T04:50:59.769460+0000","last_undegraded":"2023-10-09T04:50:59.769460+0000","last_fullsized":"2023-10-09T04:50:59.769460+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:43.043505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":278,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.751055+0000","last_change":"2023-10-09T04:49:36.252722+0000","last_active":"2023-10-09T04:50:59.751055+0000","last_peered":"2023-10-09T04:50:59.751055+0000","last_clean":"2023-10-09T04:50:59.751055+0000","last_became_active":"2023-10-09T04:49:36.251598+0000","last_became_peered":"2023-10-09T04:49:36.251598+0000","last_unstale":"2023-10-09T04:50:59.751055+0000","last_undegraded":"2023-10-09T04:50:59.751055+0000","last_fullsized":"2023-10-09T04:50:59.751055+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.458714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":322,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.769901+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:50:59.769901+0000","last_peered":"2023-10-09T04:50:59.769901+0000","last_clean":"2023-10-09T04:50:59.769901+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:50:59.769901+0000","last_undegraded":"2023-10-09T04:50:59.769901+0000","last_fullsized":"2023-10-09T04:50:59.769901+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":312,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762306+0000","last_change":"2023-10-09T04:49:24.728045+0000","last_active":"2023-10-09T04:50:59.762306+0000","last_peered":"2023-10-09T04:50:59.762306+0000","last_clean":"2023-10-09T04:50:59.762306+0000","last_became_active":"2023-10-09T04:49:24.727278+0000","last_became_peered":"2023-10-09T04:49:24.727278+0000","last_unstale":"2023-10-09T04:50:59.762306+0000","last_undegraded":"2023-10-09T04:50:59.762306+0000","last_fullsized":"2023-10-09T04:50:59.762306+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:17.700648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":404,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762281+0000","last_change":"2023-10-09T04:49:24.751066+0000","last_active":"2023-10-09T04:50:59.762281+0000","last_peered":"2023-10-09T04:50:59.762281+0000","last_clean":"2023-10-09T04:50:59.762281+0000","last_became_active":"2023-10-09T04:49:24.731080+0000","last_became_peered":"2023-10-09T04:49:24.731080+0000","last_unstale":"2023-10-09T04:50:59.762281+0000","last_undegraded":"2023-10-09T04:50:59.762281+0000","last_fullsized":"2023-10-09T04:50:59.762281+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:47.656154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":257,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753198+0000","last_change":"2023-10-09T04:49:36.255264+0000","last_active":"2023-10-09T04:50:59.753198+0000","last_peered":"2023-10-09T04:50:59.753198+0000","last_clean":"2023-10-09T04:50:59.753198+0000","last_became_active":"2023-10-09T04:49:36.254631+0000","last_became_peered":"2023-10-09T04:49:36.254631+0000","last_unstale":"2023-10-09T04:50:59.753198+0000","last_undegraded":"2023-10-09T04:50:59.753198+0000","last_fullsized":"2023-10-09T04:50:59.753198+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:26.380768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":160,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761658+0000","last_change":"2023-10-09T04:49:24.739093+0000","last_active":"2023-10-09T04:50:59.761658+0000","last_peered":"2023-10-09T04:50:59.761658+0000","last_clean":"2023-10-09T04:50:59.761658+0000","last_became_active":"2023-10-09T04:49:24.723103+0000","last_became_peered":"2023-10-09T04:49:24.723103+0000","last_unstale":"2023-10-09T04:50:59.761658+0000","last_undegraded":"2023-10-09T04:50:59.761658+0000","last_fullsized":"2023-10-09T04:50:59.761658+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:41.664075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":332,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762132+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:50:59.762132+0000","last_peered":"2023-10-09T04:50:59.762132+0000","last_clean":"2023-10-09T04:50:59.762132+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:50:59.762132+0000","last_undegraded":"2023-10-09T04:50:59.762132+0000","last_fullsized":"2023-10-09T04:50:59.762132+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":388,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762133+0000","last_change":"2023-10-09T04:49:24.748611+0000","last_active":"2023-10-09T04:50:59.762133+0000","last_peered":"2023-10-09T04:50:59.762133+0000","last_clean":"2023-10-09T04:50:59.762133+0000","last_became_active":"2023-10-09T04:49:24.730507+0000","last_became_peered":"2023-10-09T04:49:24.730507+0000","last_unstale":"2023-10-09T04:50:59.762133+0000","last_undegraded":"2023-10-09T04:50:59.762133+0000","last_fullsized":"2023-10-09T04:50:59.762133+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:22.219826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":162,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753702+0000","last_change":"2023-10-09T04:49:36.255342+0000","last_active":"2023-10-09T04:50:59.753702+0000","last_peered":"2023-10-09T04:50:59.753702+0000","last_clean":"2023-10-09T04:50:59.753702+0000","last_became_active":"2023-10-09T04:49:36.254372+0000","last_became_peered":"2023-10-09T04:49:36.254372+0000","last_unstale":"2023-10-09T04:50:59.753702+0000","last_undegraded":"2023-10-09T04:50:59.753702+0000","last_fullsized":"2023-10-09T04:50:59.753702+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:37.972615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":304,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762068+0000","last_change":"2023-10-09T04:49:24.731694+0000","last_active":"2023-10-09T04:50:59.762068+0000","last_peered":"2023-10-09T04:50:59.762068+0000","last_clean":"2023-10-09T04:50:59.762068+0000","last_became_active":"2023-10-09T04:49:24.731338+0000","last_became_peered":"2023-10-09T04:49:24.731338+0000","last_unstale":"2023-10-09T04:50:59.762068+0000","last_undegraded":"2023-10-09T04:50:59.762068+0000","last_fullsized":"2023-10-09T04:50:59.762068+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:45.968733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":341,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768908+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:50:59.768908+0000","last_peered":"2023-10-09T04:50:59.768908+0000","last_clean":"2023-10-09T04:50:59.768908+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:50:59.768908+0000","last_undegraded":"2023-10-09T04:50:59.768908+0000","last_fullsized":"2023-10-09T04:50:59.768908+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":334,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762229+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:50:59.762229+0000","last_peered":"2023-10-09T04:50:59.762229+0000","last_clean":"2023-10-09T04:50:59.762229+0000","last_became_active":"2023-10-09T04:49:24.727752+0000","last_became_peered":"2023-10-09T04:49:24.727752+0000","last_unstale":"2023-10-09T04:50:59.762229+0000","last_undegraded":"2023-10-09T04:50:59.762229+0000","last_fullsized":"2023-10-09T04:50:59.762229+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:37.448051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":192,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.764212+0000","last_change":"2023-10-09T04:49:24.734384+0000","last_active":"2023-10-09T04:50:59.764212+0000","last_peered":"2023-10-09T04:50:59.764212+0000","last_clean":"2023-10-09T04:50:59.764212+0000","last_became_active":"2023-10-09T04:49:24.733948+0000","last_became_peered":"2023-10-09T04:49:24.733948+0000","last_unstale":"2023-10-09T04:50:59.764212+0000","last_undegraded":"2023-10-09T04:50:59.764212+0000","last_fullsized":"2023-10-09T04:50:59.764212+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:55.081133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":163,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.767896+0000","last_change":"2023-10-09T04:49:24.743137+0000","last_active":"2023-10-09T04:50:59.767896+0000","last_peered":"2023-10-09T04:50:59.767896+0000","last_clean":"2023-10-09T04:50:59.767896+0000","last_became_active":"2023-10-09T04:49:24.726930+0000","last_became_peered":"2023-10-09T04:49:24.726930+0000","last_unstale":"2023-10-09T04:50:59.767896+0000","last_undegraded":"2023-10-09T04:50:59.767896+0000","last_fullsized":"2023-10-09T04:50:59.767896+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:38.154496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":165,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.765502+0000","last_change":"2023-10-09T04:49:24.731691+0000","last_active":"2023-10-09T04:50:59.765502+0000","last_peered":"2023-10-09T04:50:59.765502+0000","last_clean":"2023-10-09T04:50:59.765502+0000","last_became_active":"2023-10-09T04:49:24.731374+0000","last_became_peered":"2023-10-09T04:49:24.731374+0000","last_unstale":"2023-10-09T04:50:59.765502+0000","last_undegraded":"2023-10-09T04:50:59.765502+0000","last_fullsized":"2023-10-09T04:50:59.765502+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:39.511857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"362'6429","reported_seq":6464,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.550488+0000","last_change":"2023-10-09T04:49:24.722035+0000","last_active":"2023-10-09T04:51:09.550488+0000","last_peered":"2023-10-09T04:51:09.550488+0000","last_clean":"2023-10-09T04:51:09.550488+0000","last_became_active":"2023-10-09T04:49:24.721093+0000","last_became_peered":"2023-10-09T04:49:24.721093+0000","last_unstale":"2023-10-09T04:51:09.550488+0000","last_undegraded":"2023-10-09T04:51:09.550488+0000","last_fullsized":"2023-10-09T04:51:09.550488+0000","mapping_epoch":340,"log_start":"348'1800","ondisk_log_start":"348'1800","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4629,"log_dups_size":0,"ondisk_log_size":4629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:02:16.988764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349765632,"num_objects":5337,"num_object_clones":0,"num_object_copies":16011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6429,"num_write_kb":376512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":392,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.765332+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:50:59.765332+0000","last_peered":"2023-10-09T04:50:59.765332+0000","last_clean":"2023-10-09T04:50:59.765332+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:50:59.765332+0000","last_undegraded":"2023-10-09T04:50:59.765332+0000","last_fullsized":"2023-10-09T04:50:59.765332+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":314,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.767657+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:50:59.767657+0000","last_peered":"2023-10-09T04:50:59.767657+0000","last_clean":"2023-10-09T04:50:59.767657+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:50:59.767657+0000","last_undegraded":"2023-10-09T04:50:59.767657+0000","last_fullsized":"2023-10-09T04:50:59.767657+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"362'6521","reported_seq":6565,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:10.280480+0000","last_change":"2023-10-09T04:49:24.729511+0000","last_active":"2023-10-09T04:51:10.280480+0000","last_peered":"2023-10-09T04:51:10.280480+0000","last_clean":"2023-10-09T04:51:10.280480+0000","last_became_active":"2023-10-09T04:49:24.728925+0000","last_became_peered":"2023-10-09T04:49:24.728925+0000","last_unstale":"2023-10-09T04:51:10.280480+0000","last_undegraded":"2023-10-09T04:51:10.280480+0000","last_fullsized":"2023-10-09T04:51:10.280480+0000","mapping_epoch":340,"log_start":"348'1900","ondisk_log_start":"348'1900","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4621,"log_dups_size":0,"ondisk_log_size":4621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:48.606555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343867392,"num_objects":5247,"num_object_clones":0,"num_object_copies":15741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6521,"num_write_kb":376576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":351,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.764075+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:50:59.764075+0000","last_peered":"2023-10-09T04:50:59.764075+0000","last_clean":"2023-10-09T04:50:59.764075+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:50:59.764075+0000","last_undegraded":"2023-10-09T04:50:59.764075+0000","last_fullsized":"2023-10-09T04:50:59.764075+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":359,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.763531+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:50:59.763531+0000","last_peered":"2023-10-09T04:50:59.763531+0000","last_clean":"2023-10-09T04:50:59.763531+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:50:59.763531+0000","last_undegraded":"2023-10-09T04:50:59.763531+0000","last_fullsized":"2023-10-09T04:50:59.763531+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"362'6393","reported_seq":6427,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.578774+0000","last_change":"2023-10-09T04:49:17.684070+0000","last_active":"2023-10-09T04:51:09.578774+0000","last_peered":"2023-10-09T04:51:09.578774+0000","last_clean":"2023-10-09T04:51:09.578774+0000","last_became_active":"2023-10-09T04:49:17.683772+0000","last_became_peered":"2023-10-09T04:49:17.683772+0000","last_unstale":"2023-10-09T04:51:09.578774+0000","last_undegraded":"2023-10-09T04:51:09.578774+0000","last_fullsized":"2023-10-09T04:51:09.578774+0000","mapping_epoch":336,"log_start":"348'1700","ondisk_log_start":"348'1700","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:47.529254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343736320,"num_objects":5245,"num_object_clones":0,"num_object_copies":15735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6393,"num_write_kb":372416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":333,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762107+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:50:59.762107+0000","last_peered":"2023-10-09T04:50:59.762107+0000","last_clean":"2023-10-09T04:50:59.762107+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:50:59.762107+0000","last_undegraded":"2023-10-09T04:50:59.762107+0000","last_fullsized":"2023-10-09T04:50:59.762107+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":230,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.752765+0000","last_change":"2023-10-09T04:49:36.255047+0000","last_active":"2023-10-09T04:50:59.752765+0000","last_peered":"2023-10-09T04:50:59.752765+0000","last_clean":"2023-10-09T04:50:59.752765+0000","last_became_active":"2023-10-09T04:49:36.254324+0000","last_became_peered":"2023-10-09T04:49:36.254324+0000","last_unstale":"2023-10-09T04:50:59.752765+0000","last_undegraded":"2023-10-09T04:50:59.752765+0000","last_fullsized":"2023-10-09T04:50:59.752765+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:55.547119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"362'6533","reported_seq":6567,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.565932+0000","last_change":"2023-10-09T04:49:17.682749+0000","last_active":"2023-10-09T04:51:09.565932+0000","last_peered":"2023-10-09T04:51:09.565932+0000","last_clean":"2023-10-09T04:51:09.565932+0000","last_became_active":"2023-10-09T04:49:17.682418+0000","last_became_peered":"2023-10-09T04:49:17.682418+0000","last_unstale":"2023-10-09T04:51:09.565932+0000","last_undegraded":"2023-10-09T04:51:09.565932+0000","last_fullsized":"2023-10-09T04:51:09.565932+0000","mapping_epoch":336,"log_start":"348'1900","ondisk_log_start":"348'1900","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4633,"log_dups_size":0,"ondisk_log_size":4633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:33.471194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346226688,"num_objects":5283,"num_object_clones":0,"num_object_copies":15849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6533,"num_write_kb":378112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":362,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768901+0000","last_change":"2023-10-09T04:49:24.728445+0000","last_active":"2023-10-09T04:50:59.768901+0000","last_peered":"2023-10-09T04:50:59.768901+0000","last_clean":"2023-10-09T04:50:59.768901+0000","last_became_active":"2023-10-09T04:49:24.728130+0000","last_became_peered":"2023-10-09T04:49:24.728130+0000","last_unstale":"2023-10-09T04:50:59.768901+0000","last_undegraded":"2023-10-09T04:50:59.768901+0000","last_fullsized":"2023-10-09T04:50:59.768901+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:04.777758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":77,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753197+0000","last_change":"2023-10-09T04:49:36.253080+0000","last_active":"2023-10-09T04:50:59.753197+0000","last_peered":"2023-10-09T04:50:59.753197+0000","last_clean":"2023-10-09T04:50:59.753197+0000","last_became_active":"2023-10-09T04:49:36.252682+0000","last_became_peered":"2023-10-09T04:49:36.252682+0000","last_unstale":"2023-10-09T04:50:59.753197+0000","last_undegraded":"2023-10-09T04:50:59.753197+0000","last_fullsized":"2023-10-09T04:50:59.753197+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:31.712210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.5","version":"362'6477","reported_seq":6511,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.739459+0000","last_change":"2023-10-09T04:49:17.681809+0000","last_active":"2023-10-09T04:51:09.739459+0000","last_peered":"2023-10-09T04:51:09.739459+0000","last_clean":"2023-10-09T04:51:09.739459+0000","last_became_active":"2023-10-09T04:49:17.681477+0000","last_became_peered":"2023-10-09T04:49:17.681477+0000","last_unstale":"2023-10-09T04:51:09.739459+0000","last_undegraded":"2023-10-09T04:51:09.739459+0000","last_fullsized":"2023-10-09T04:51:09.739459+0000","mapping_epoch":336,"log_start":"348'1800","ondisk_log_start":"348'1800","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4677,"log_dups_size":0,"ondisk_log_size":4677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:07.783547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351731712,"num_objects":5367,"num_object_clones":0,"num_object_copies":16101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6477,"num_write_kb":379008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":101,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762221+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:50:59.762221+0000","last_peered":"2023-10-09T04:50:59.762221+0000","last_clean":"2023-10-09T04:50:59.762221+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:50:59.762221+0000","last_undegraded":"2023-10-09T04:50:59.762221+0000","last_fullsized":"2023-10-09T04:50:59.762221+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":130,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761739+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:50:59.761739+0000","last_peered":"2023-10-09T04:50:59.761739+0000","last_clean":"2023-10-09T04:50:59.761739+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:50:59.761739+0000","last_undegraded":"2023-10-09T04:50:59.761739+0000","last_fullsized":"2023-10-09T04:50:59.761739+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"362'6319","reported_seq":6364,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:08.469565+0000","last_change":"2023-10-09T04:49:36.269590+0000","last_active":"2023-10-09T04:51:08.469565+0000","last_peered":"2023-10-09T04:51:08.469565+0000","last_clean":"2023-10-09T04:51:08.469565+0000","last_became_active":"2023-10-09T04:49:36.252739+0000","last_became_peered":"2023-10-09T04:49:36.252739+0000","last_unstale":"2023-10-09T04:51:08.469565+0000","last_undegraded":"2023-10-09T04:51:08.469565+0000","last_fullsized":"2023-10-09T04:51:08.469565+0000","mapping_epoch":346,"log_start":"349'2100","ondisk_log_start":"349'2100","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4219,"log_dups_size":0,"ondisk_log_size":4219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:08:26.767295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346750976,"num_objects":5291,"num_object_clones":0,"num_object_copies":15873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6319,"num_write_kb":371520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":323,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761450+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:50:59.761450+0000","last_peered":"2023-10-09T04:50:59.761450+0000","last_clean":"2023-10-09T04:50:59.761450+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:50:59.761450+0000","last_undegraded":"2023-10-09T04:50:59.761450+0000","last_fullsized":"2023-10-09T04:50:59.761450+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761508+0000","last_change":"2023-10-09T04:49:24.712121+0000","last_active":"2023-10-09T04:50:59.761508+0000","last_peered":"2023-10-09T04:50:59.761508+0000","last_clean":"2023-10-09T04:50:59.761508+0000","last_became_active":"2023-10-09T04:49:24.711716+0000","last_became_peered":"2023-10-09T04:49:24.711716+0000","last_unstale":"2023-10-09T04:50:59.761508+0000","last_undegraded":"2023-10-09T04:50:59.761508+0000","last_fullsized":"2023-10-09T04:50:59.761508+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:37.141689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"362'6473","reported_seq":6507,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:10.277191+0000","last_change":"2023-10-09T04:49:17.686417+0000","last_active":"2023-10-09T04:51:10.277191+0000","last_peered":"2023-10-09T04:51:10.277191+0000","last_clean":"2023-10-09T04:51:10.277191+0000","last_became_active":"2023-10-09T04:49:17.686126+0000","last_became_peered":"2023-10-09T04:49:17.686126+0000","last_unstale":"2023-10-09T04:51:10.277191+0000","last_undegraded":"2023-10-09T04:51:10.277191+0000","last_fullsized":"2023-10-09T04:51:10.277191+0000","mapping_epoch":336,"log_start":"348'1800","ondisk_log_start":"348'1800","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4673,"log_dups_size":0,"ondisk_log_size":4673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:06:58.636951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047040,"num_objects":5265,"num_object_clones":0,"num_object_copies":15795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6473,"num_write_kb":375616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.763433+0000","last_change":"2023-10-09T04:49:24.721842+0000","last_active":"2023-10-09T04:50:59.763433+0000","last_peered":"2023-10-09T04:50:59.763433+0000","last_clean":"2023-10-09T04:50:59.763433+0000","last_became_active":"2023-10-09T04:49:24.720666+0000","last_became_peered":"2023-10-09T04:49:24.720666+0000","last_unstale":"2023-10-09T04:50:59.763433+0000","last_undegraded":"2023-10-09T04:50:59.763433+0000","last_fullsized":"2023-10-09T04:50:59.763433+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:16.572332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":342,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761804+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:50:59.761804+0000","last_peered":"2023-10-09T04:50:59.761804+0000","last_clean":"2023-10-09T04:50:59.761804+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:50:59.761804+0000","last_undegraded":"2023-10-09T04:50:59.761804+0000","last_fullsized":"2023-10-09T04:50:59.761804+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"362'6602","reported_seq":6646,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.639062+0000","last_change":"2023-10-09T04:49:24.721346+0000","last_active":"2023-10-09T04:51:09.639062+0000","last_peered":"2023-10-09T04:51:09.639062+0000","last_clean":"2023-10-09T04:51:09.639062+0000","last_became_active":"2023-10-09T04:49:24.720955+0000","last_became_peered":"2023-10-09T04:49:24.720955+0000","last_unstale":"2023-10-09T04:51:09.639062+0000","last_undegraded":"2023-10-09T04:51:09.639062+0000","last_fullsized":"2023-10-09T04:51:09.639062+0000","mapping_epoch":340,"log_start":"348'1900","ondisk_log_start":"348'1900","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4702,"log_dups_size":0,"ondisk_log_size":4702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:58:55.342602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360710144,"num_objects":5504,"num_object_clones":0,"num_object_copies":16512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6602,"num_write_kb":387392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":115,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.750995+0000","last_change":"2023-10-09T04:49:36.253172+0000","last_active":"2023-10-09T04:50:59.750995+0000","last_peered":"2023-10-09T04:50:59.750995+0000","last_clean":"2023-10-09T04:50:59.750995+0000","last_became_active":"2023-10-09T04:49:36.252175+0000","last_became_peered":"2023-10-09T04:49:36.252175+0000","last_unstale":"2023-10-09T04:50:59.750995+0000","last_undegraded":"2023-10-09T04:50:59.750995+0000","last_fullsized":"2023-10-09T04:50:59.750995+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:38.269236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":267,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.754190+0000","last_change":"2023-10-09T04:49:36.251118+0000","last_active":"2023-10-09T04:50:59.754190+0000","last_peered":"2023-10-09T04:50:59.754190+0000","last_clean":"2023-10-09T04:50:59.754190+0000","last_became_active":"2023-10-09T04:49:36.250803+0000","last_became_peered":"2023-10-09T04:49:36.250803+0000","last_unstale":"2023-10-09T04:50:59.754190+0000","last_undegraded":"2023-10-09T04:50:59.754190+0000","last_fullsized":"2023-10-09T04:50:59.754190+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:38.765478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"362'6450","reported_seq":6492,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.553419+0000","last_change":"2023-10-09T04:49:24.727773+0000","last_active":"2023-10-09T04:51:09.553419+0000","last_peered":"2023-10-09T04:51:09.553419+0000","last_clean":"2023-10-09T04:51:09.553419+0000","last_became_active":"2023-10-09T04:49:24.727411+0000","last_became_peered":"2023-10-09T04:49:24.727411+0000","last_unstale":"2023-10-09T04:51:09.553419+0000","last_undegraded":"2023-10-09T04:51:09.553419+0000","last_fullsized":"2023-10-09T04:51:09.553419+0000","mapping_epoch":340,"log_start":"348'1800","ondisk_log_start":"348'1800","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4650,"log_dups_size":0,"ondisk_log_size":4650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:21.626311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346423296,"num_objects":5286,"num_object_clones":0,"num_object_copies":15858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6450,"num_write_kb":375552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":256,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753624+0000","last_change":"2023-10-09T04:49:36.250912+0000","last_active":"2023-10-09T04:50:59.753624+0000","last_peered":"2023-10-09T04:50:59.753624+0000","last_clean":"2023-10-09T04:50:59.753624+0000","last_became_active":"2023-10-09T04:49:36.250530+0000","last_became_peered":"2023-10-09T04:49:36.250530+0000","last_unstale":"2023-10-09T04:50:59.753624+0000","last_undegraded":"2023-10-09T04:50:59.753624+0000","last_fullsized":"2023-10-09T04:50:59.753624+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:26.975687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":376,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768833+0000","last_change":"2023-10-09T04:49:24.727377+0000","last_active":"2023-10-09T04:50:59.768833+0000","last_peered":"2023-10-09T04:50:59.768833+0000","last_clean":"2023-10-09T04:50:59.768833+0000","last_became_active":"2023-10-09T04:49:24.724195+0000","last_became_peered":"2023-10-09T04:49:24.724195+0000","last_unstale":"2023-10-09T04:50:59.768833+0000","last_undegraded":"2023-10-09T04:50:59.768833+0000","last_fullsized":"2023-10-09T04:50:59.768833+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:46.895156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":251,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753273+0000","last_change":"2023-10-09T04:49:36.254992+0000","last_active":"2023-10-09T04:50:59.753273+0000","last_peered":"2023-10-09T04:50:59.753273+0000","last_clean":"2023-10-09T04:50:59.753273+0000","last_became_active":"2023-10-09T04:49:36.254288+0000","last_became_peered":"2023-10-09T04:49:36.254288+0000","last_unstale":"2023-10-09T04:50:59.753273+0000","last_undegraded":"2023-10-09T04:50:59.753273+0000","last_fullsized":"2023-10-09T04:50:59.753273+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:13.355684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"362'6554","reported_seq":6589,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.735735+0000","last_change":"2023-10-09T04:49:17.684748+0000","last_active":"2023-10-09T04:51:09.735735+0000","last_peered":"2023-10-09T04:51:09.735735+0000","last_clean":"2023-10-09T04:51:09.735735+0000","last_became_active":"2023-10-09T04:49:17.684346+0000","last_became_peered":"2023-10-09T04:49:17.684346+0000","last_unstale":"2023-10-09T04:51:09.735735+0000","last_undegraded":"2023-10-09T04:51:09.735735+0000","last_fullsized":"2023-10-09T04:51:09.735735+0000","mapping_epoch":336,"log_start":"348'1900","ondisk_log_start":"348'1900","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4654,"log_dups_size":0,"ondisk_log_size":4654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:39.897521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346357784,"num_objects":5286,"num_object_clones":0,"num_object_copies":15858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5286,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6554,"num_write_kb":378817,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":346,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768301+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:50:59.768301+0000","last_peered":"2023-10-09T04:50:59.768301+0000","last_clean":"2023-10-09T04:50:59.768301+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:50:59.768301+0000","last_undegraded":"2023-10-09T04:50:59.768301+0000","last_fullsized":"2023-10-09T04:50:59.768301+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.763391+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:50:59.763391+0000","last_peered":"2023-10-09T04:50:59.763391+0000","last_clean":"2023-10-09T04:50:59.763391+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:50:59.763391+0000","last_undegraded":"2023-10-09T04:50:59.763391+0000","last_fullsized":"2023-10-09T04:50:59.763391+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":305,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753541+0000","last_change":"2023-10-09T04:49:36.254929+0000","last_active":"2023-10-09T04:50:59.753541+0000","last_peered":"2023-10-09T04:50:59.753541+0000","last_clean":"2023-10-09T04:50:59.753541+0000","last_became_active":"2023-10-09T04:49:36.252868+0000","last_became_peered":"2023-10-09T04:49:36.252868+0000","last_unstale":"2023-10-09T04:50:59.753541+0000","last_undegraded":"2023-10-09T04:50:59.753541+0000","last_fullsized":"2023-10-09T04:50:59.753541+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:26.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"362'6627","reported_seq":6661,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:10.217730+0000","last_change":"2023-10-09T04:49:17.686557+0000","last_active":"2023-10-09T04:51:10.217730+0000","last_peered":"2023-10-09T04:51:10.217730+0000","last_clean":"2023-10-09T04:51:10.217730+0000","last_became_active":"2023-10-09T04:49:17.686236+0000","last_became_peered":"2023-10-09T04:49:17.686236+0000","last_unstale":"2023-10-09T04:51:10.217730+0000","last_undegraded":"2023-10-09T04:51:10.217730+0000","last_fullsized":"2023-10-09T04:51:10.217730+0000","mapping_epoch":336,"log_start":"348'2000","ondisk_log_start":"348'2000","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4627,"log_dups_size":0,"ondisk_log_size":4627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:02.935353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349241344,"num_objects":5329,"num_object_clones":0,"num_object_copies":15987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6627,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":306,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.752750+0000","last_change":"2023-10-09T04:49:36.250480+0000","last_active":"2023-10-09T04:50:59.752750+0000","last_peered":"2023-10-09T04:50:59.752750+0000","last_clean":"2023-10-09T04:50:59.752750+0000","last_became_active":"2023-10-09T04:49:36.250048+0000","last_became_peered":"2023-10-09T04:49:36.250048+0000","last_unstale":"2023-10-09T04:50:59.752750+0000","last_undegraded":"2023-10-09T04:50:59.752750+0000","last_fullsized":"2023-10-09T04:50:59.752750+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:21.602101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":305,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.751117+0000","last_change":"2023-10-09T04:49:36.253118+0000","last_active":"2023-10-09T04:50:59.751117+0000","last_peered":"2023-10-09T04:50:59.751117+0000","last_clean":"2023-10-09T04:50:59.751117+0000","last_became_active":"2023-10-09T04:49:36.252036+0000","last_became_peered":"2023-10-09T04:49:36.252036+0000","last_unstale":"2023-10-09T04:50:59.751117+0000","last_undegraded":"2023-10-09T04:50:59.751117+0000","last_fullsized":"2023-10-09T04:50:59.751117+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.648245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":247,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.754185+0000","last_change":"2023-10-09T04:49:36.252116+0000","last_active":"2023-10-09T04:50:59.754185+0000","last_peered":"2023-10-09T04:50:59.754185+0000","last_clean":"2023-10-09T04:50:59.754185+0000","last_became_active":"2023-10-09T04:49:36.251630+0000","last_became_peered":"2023-10-09T04:49:36.251630+0000","last_unstale":"2023-10-09T04:50:59.754185+0000","last_undegraded":"2023-10-09T04:50:59.754185+0000","last_fullsized":"2023-10-09T04:50:59.754185+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:48.681072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"362'6440","reported_seq":6496,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:08.428263+0000","last_change":"2023-10-09T04:49:36.276163+0000","last_active":"2023-10-09T04:51:08.428263+0000","last_peered":"2023-10-09T04:51:08.428263+0000","last_clean":"2023-10-09T04:51:08.428263+0000","last_became_active":"2023-10-09T04:49:36.253314+0000","last_became_peered":"2023-10-09T04:49:36.253314+0000","last_unstale":"2023-10-09T04:51:08.428263+0000","last_undegraded":"2023-10-09T04:51:08.428263+0000","last_fullsized":"2023-10-09T04:51:08.428263+0000","mapping_epoch":346,"log_start":"349'2200","ondisk_log_start":"349'2200","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4240,"log_dups_size":0,"ondisk_log_size":4240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:44.334582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352976896,"num_objects":5386,"num_object_clones":0,"num_object_copies":16158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6440,"num_write_kb":378432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":353,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768529+0000","last_change":"2023-10-09T04:49:24.727573+0000","last_active":"2023-10-09T04:50:59.768529+0000","last_peered":"2023-10-09T04:50:59.768529+0000","last_clean":"2023-10-09T04:50:59.768529+0000","last_became_active":"2023-10-09T04:49:24.727151+0000","last_became_peered":"2023-10-09T04:49:24.727151+0000","last_unstale":"2023-10-09T04:50:59.768529+0000","last_undegraded":"2023-10-09T04:50:59.768529+0000","last_fullsized":"2023-10-09T04:50:59.768529+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:53.790164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":832,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762189+0000","last_change":"2023-10-09T04:49:27.734937+0000","last_active":"2023-10-09T04:50:59.762189+0000","last_peered":"2023-10-09T04:50:59.762189+0000","last_clean":"2023-10-09T04:50:59.762189+0000","last_became_active":"2023-10-09T04:49:27.734587+0000","last_became_peered":"2023-10-09T04:49:27.734587+0000","last_unstale":"2023-10-09T04:50:59.762189+0000","last_undegraded":"2023-10-09T04:50:59.762189+0000","last_fullsized":"2023-10-09T04:50:59.762189+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:38.000046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":292,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768447+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:50:59.768447+0000","last_peered":"2023-10-09T04:50:59.768447+0000","last_clean":"2023-10-09T04:50:59.768447+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:50:59.768447+0000","last_undegraded":"2023-10-09T04:50:59.768447+0000","last_fullsized":"2023-10-09T04:50:59.768447+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":305,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761496+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:50:59.761496+0000","last_peered":"2023-10-09T04:50:59.761496+0000","last_clean":"2023-10-09T04:50:59.761496+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:50:59.761496+0000","last_undegraded":"2023-10-09T04:50:59.761496+0000","last_fullsized":"2023-10-09T04:50:59.761496+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"362'6535","reported_seq":6579,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:10.273019+0000","last_change":"2023-10-09T04:49:24.733991+0000","last_active":"2023-10-09T04:51:10.273019+0000","last_peered":"2023-10-09T04:51:10.273019+0000","last_clean":"2023-10-09T04:51:10.273019+0000","last_became_active":"2023-10-09T04:49:24.733661+0000","last_became_peered":"2023-10-09T04:49:24.733661+0000","last_unstale":"2023-10-09T04:51:10.273019+0000","last_undegraded":"2023-10-09T04:51:10.273019+0000","last_fullsized":"2023-10-09T04:51:10.273019+0000","mapping_epoch":340,"log_start":"348'1900","ondisk_log_start":"348'1900","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4635,"log_dups_size":0,"ondisk_log_size":4635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:22.030285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348717056,"num_objects":5321,"num_object_clones":0,"num_object_copies":15963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6535,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":372,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762077+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:50:59.762077+0000","last_peered":"2023-10-09T04:50:59.762077+0000","last_clean":"2023-10-09T04:50:59.762077+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:50:59.762077+0000","last_undegraded":"2023-10-09T04:50:59.762077+0000","last_fullsized":"2023-10-09T04:50:59.762077+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":130,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.763513+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:50:59.763513+0000","last_peered":"2023-10-09T04:50:59.763513+0000","last_clean":"2023-10-09T04:50:59.763513+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:50:59.763513+0000","last_undegraded":"2023-10-09T04:50:59.763513+0000","last_fullsized":"2023-10-09T04:50:59.763513+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.754073+0000","last_change":"2023-10-09T04:49:36.252316+0000","last_active":"2023-10-09T04:50:59.754073+0000","last_peered":"2023-10-09T04:50:59.754073+0000","last_clean":"2023-10-09T04:50:59.754073+0000","last_became_active":"2023-10-09T04:49:36.251771+0000","last_became_peered":"2023-10-09T04:49:36.251771+0000","last_unstale":"2023-10-09T04:50:59.754073+0000","last_undegraded":"2023-10-09T04:50:59.754073+0000","last_fullsized":"2023-10-09T04:50:59.754073+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:08.690791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"362'6384","reported_seq":6427,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:08.444284+0000","last_change":"2023-10-09T04:49:36.254761+0000","last_active":"2023-10-09T04:51:08.444284+0000","last_peered":"2023-10-09T04:51:08.444284+0000","last_clean":"2023-10-09T04:51:08.444284+0000","last_became_active":"2023-10-09T04:49:36.250489+0000","last_became_peered":"2023-10-09T04:49:36.250489+0000","last_unstale":"2023-10-09T04:51:08.444284+0000","last_undegraded":"2023-10-09T04:51:08.444284+0000","last_fullsized":"2023-10-09T04:51:08.444284+0000","mapping_epoch":346,"log_start":"349'2200","ondisk_log_start":"349'2200","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4184,"log_dups_size":0,"ondisk_log_size":4184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:19.664434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349175808,"num_objects":5328,"num_object_clones":0,"num_object_copies":15984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6384,"num_write_kb":374784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":375,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762191+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:50:59.762191+0000","last_peered":"2023-10-09T04:50:59.762191+0000","last_clean":"2023-10-09T04:50:59.762191+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:50:59.762191+0000","last_undegraded":"2023-10-09T04:50:59.762191+0000","last_fullsized":"2023-10-09T04:50:59.762191+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":258,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753460+0000","last_change":"2023-10-09T04:49:36.254730+0000","last_active":"2023-10-09T04:50:59.753460+0000","last_peered":"2023-10-09T04:50:59.753460+0000","last_clean":"2023-10-09T04:50:59.753460+0000","last_became_active":"2023-10-09T04:49:36.252242+0000","last_became_peered":"2023-10-09T04:49:36.252242+0000","last_unstale":"2023-10-09T04:50:59.753460+0000","last_undegraded":"2023-10-09T04:50:59.753460+0000","last_fullsized":"2023-10-09T04:50:59.753460+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.106203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768444+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:50:59.768444+0000","last_peered":"2023-10-09T04:50:59.768444+0000","last_clean":"2023-10-09T04:50:59.768444+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:50:59.768444+0000","last_undegraded":"2023-10-09T04:50:59.768444+0000","last_fullsized":"2023-10-09T04:50:59.768444+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"362'6311","reported_seq":6355,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.565869+0000","last_change":"2023-10-09T04:49:24.718431+0000","last_active":"2023-10-09T04:51:09.565869+0000","last_peered":"2023-10-09T04:51:09.565869+0000","last_clean":"2023-10-09T04:51:09.565869+0000","last_became_active":"2023-10-09T04:49:24.718108+0000","last_became_peered":"2023-10-09T04:49:24.718108+0000","last_unstale":"2023-10-09T04:51:09.565869+0000","last_undegraded":"2023-10-09T04:51:09.565869+0000","last_fullsized":"2023-10-09T04:51:09.565869+0000","mapping_epoch":340,"log_start":"348'1600","ondisk_log_start":"348'1600","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4711,"log_dups_size":0,"ondisk_log_size":4711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:51:59.309575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344260608,"num_objects":5253,"num_object_clones":0,"num_object_copies":15759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6311,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.765590+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:50:59.765590+0000","last_peered":"2023-10-09T04:50:59.765590+0000","last_clean":"2023-10-09T04:50:59.765590+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:50:59.765590+0000","last_undegraded":"2023-10-09T04:50:59.765590+0000","last_fullsized":"2023-10-09T04:50:59.765590+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":332,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753884+0000","last_change":"2023-10-09T04:49:36.251899+0000","last_active":"2023-10-09T04:50:59.753884+0000","last_peered":"2023-10-09T04:50:59.753884+0000","last_clean":"2023-10-09T04:50:59.753884+0000","last_became_active":"2023-10-09T04:49:36.251283+0000","last_became_peered":"2023-10-09T04:49:36.251283+0000","last_unstale":"2023-10-09T04:50:59.753884+0000","last_undegraded":"2023-10-09T04:50:59.753884+0000","last_fullsized":"2023-10-09T04:50:59.753884+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:52.505049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":208,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.762321+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:50:59.762321+0000","last_peered":"2023-10-09T04:50:59.762321+0000","last_clean":"2023-10-09T04:50:59.762321+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:50:59.762321+0000","last_undegraded":"2023-10-09T04:50:59.762321+0000","last_fullsized":"2023-10-09T04:50:59.762321+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"362'6432","reported_seq":6467,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-10-09T04:51:09.512709+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:51:09.512709+0000","last_peered":"2023-10-09T04:51:09.512709+0000","last_clean":"2023-10-09T04:51:09.512709+0000","last_became_active":"2023-10-09T04:49:24.727688+0000","last_became_peered":"2023-10-09T04:49:24.727688+0000","last_unstale":"2023-10-09T04:51:09.512709+0000","last_undegraded":"2023-10-09T04:51:09.512709+0000","last_fullsized":"2023-10-09T04:51:09.512709+0000","mapping_epoch":340,"log_start":"348'1800","ondisk_log_start":"348'1800","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4632,"log_dups_size":0,"ondisk_log_size":4632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:00.374022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347865088,"num_objects":5308,"num_object_clones":0,"num_object_copies":15924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6432,"num_write_kb":375680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":294,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.765243+0000","last_change":"2023-10-09T04:49:24.730477+0000","last_active":"2023-10-09T04:50:59.765243+0000","last_peered":"2023-10-09T04:50:59.765243+0000","last_clean":"2023-10-09T04:50:59.765243+0000","last_became_active":"2023-10-09T04:49:24.730115+0000","last_became_peered":"2023-10-09T04:49:24.730115+0000","last_unstale":"2023-10-09T04:50:59.765243+0000","last_undegraded":"2023-10-09T04:50:59.765243+0000","last_fullsized":"2023-10-09T04:50:59.765243+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:49.730424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":131,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753983+0000","last_change":"2023-10-09T04:49:36.250949+0000","last_active":"2023-10-09T04:50:59.753983+0000","last_peered":"2023-10-09T04:50:59.753983+0000","last_clean":"2023-10-09T04:50:59.753983+0000","last_became_active":"2023-10-09T04:49:36.250519+0000","last_became_peered":"2023-10-09T04:49:36.250519+0000","last_unstale":"2023-10-09T04:50:59.753983+0000","last_undegraded":"2023-10-09T04:50:59.753983+0000","last_fullsized":"2023-10-09T04:50:59.753983+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:51.821472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":130,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.763397+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:50:59.763397+0000","last_peered":"2023-10-09T04:50:59.763397+0000","last_clean":"2023-10-09T04:50:59.763397+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:50:59.763397+0000","last_undegraded":"2023-10-09T04:50:59.763397+0000","last_fullsized":"2023-10-09T04:50:59.763397+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":212,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761996+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:50:59.761996+0000","last_peered":"2023-10-09T04:50:59.761996+0000","last_clean":"2023-10-09T04:50:59.761996+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:50:59.761996+0000","last_undegraded":"2023-10-09T04:50:59.761996+0000","last_fullsized":"2023-10-09T04:50:59.761996+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":298,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753369+0000","last_change":"2023-10-09T04:49:36.254629+0000","last_active":"2023-10-09T04:50:59.753369+0000","last_peered":"2023-10-09T04:50:59.753369+0000","last_clean":"2023-10-09T04:50:59.753369+0000","last_became_active":"2023-10-09T04:49:36.251774+0000","last_became_peered":"2023-10-09T04:49:36.251774+0000","last_unstale":"2023-10-09T04:50:59.753369+0000","last_undegraded":"2023-10-09T04:50:59.753369+0000","last_fullsized":"2023-10-09T04:50:59.753369+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:24.682458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":352,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.769042+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:50:59.769042+0000","last_peered":"2023-10-09T04:50:59.769042+0000","last_clean":"2023-10-09T04:50:59.769042+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:50:59.769042+0000","last_undegraded":"2023-10-09T04:50:59.769042+0000","last_fullsized":"2023-10-09T04:50:59.769042+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":130,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761859+0000","last_change":"2023-10-09T04:49:24.721368+0000","last_active":"2023-10-09T04:50:59.761859+0000","last_peered":"2023-10-09T04:50:59.761859+0000","last_clean":"2023-10-09T04:50:59.761859+0000","last_became_active":"2023-10-09T04:49:24.721024+0000","last_became_peered":"2023-10-09T04:49:24.721024+0000","last_unstale":"2023-10-09T04:50:59.761859+0000","last_undegraded":"2023-10-09T04:50:59.761859+0000","last_fullsized":"2023-10-09T04:50:59.761859+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:57.736030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":321,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.768750+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:50:59.768750+0000","last_peered":"2023-10-09T04:50:59.768750+0000","last_clean":"2023-10-09T04:50:59.768750+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:50:59.768750+0000","last_undegraded":"2023-10-09T04:50:59.768750+0000","last_fullsized":"2023-10-09T04:50:59.768750+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":140,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761540+0000","last_change":"2023-10-09T04:49:24.711760+0000","last_active":"2023-10-09T04:50:59.761540+0000","last_peered":"2023-10-09T04:50:59.761540+0000","last_clean":"2023-10-09T04:50:59.761540+0000","last_became_active":"2023-10-09T04:49:24.711383+0000","last_became_peered":"2023-10-09T04:49:24.711383+0000","last_unstale":"2023-10-09T04:50:59.761540+0000","last_undegraded":"2023-10-09T04:50:59.761540+0000","last_fullsized":"2023-10-09T04:50:59.761540+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:04.370272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":315,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.753832+0000","last_change":"2023-10-09T04:49:24.725553+0000","last_active":"2023-10-09T04:50:59.753832+0000","last_peered":"2023-10-09T04:50:59.753832+0000","last_clean":"2023-10-09T04:50:59.753832+0000","last_became_active":"2023-10-09T04:49:24.725179+0000","last_became_peered":"2023-10-09T04:49:24.725179+0000","last_unstale":"2023-10-09T04:50:59.753832+0000","last_undegraded":"2023-10-09T04:50:59.753832+0000","last_fullsized":"2023-10-09T04:50:59.753832+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.045784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":281,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.767941+0000","last_change":"2023-10-09T04:49:24.718316+0000","last_active":"2023-10-09T04:50:59.767941+0000","last_peered":"2023-10-09T04:50:59.767941+0000","last_clean":"2023-10-09T04:50:59.767941+0000","last_became_active":"2023-10-09T04:49:24.717948+0000","last_became_peered":"2023-10-09T04:49:24.717948+0000","last_unstale":"2023-10-09T04:50:59.767941+0000","last_undegraded":"2023-10-09T04:50:59.767941+0000","last_fullsized":"2023-10-09T04:50:59.767941+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:48.454201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":268,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.769714+0000","last_change":"2023-10-09T04:49:24.745757+0000","last_active":"2023-10-09T04:50:59.769714+0000","last_peered":"2023-10-09T04:50:59.769714+0000","last_clean":"2023-10-09T04:50:59.769714+0000","last_became_active":"2023-10-09T04:49:24.727010+0000","last_became_peered":"2023-10-09T04:49:24.727010+0000","last_unstale":"2023-10-09T04:50:59.769714+0000","last_undegraded":"2023-10-09T04:50:59.769714+0000","last_fullsized":"2023-10-09T04:50:59.769714+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:51.654447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":140,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-10-09T04:50:59.761074+0000","last_change":"2023-10-09T04:49:24.710860+0000","last_active":"2023-10-09T04:50:59.761074+0000","last_peered":"2023-10-09T04:50:59.761074+0000","last_clean":"2023-10-09T04:50:59.761074+0000","last_became_active":"2023-10-09T04:49:24.710496+0000","last_became_peered":"2023-10-09T04:49:24.710496+0000","last_unstale":"2023-10-09T04:50:59.761074+0000","last_undegraded":"2023-10-09T04:50:59.761074+0000","last_fullsized":"2023-10-09T04:50:59.761074+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:24.901415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5572853784,"num_objects":85036,"num_object_clones":0,"num_object_copies":255108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85036,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103480,"num_write_kb":6032449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16720015360,"data_stored":16720003144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73180,"ondisk_log_size":73180,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16666,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234531,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5567584,"kb_used_data":4062360,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":88177568,"statfs":{"total":95995035648,"available":90293829632,"internally_reserved":0,"allocated":4159856640,"data_stored":4159099965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,2,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.3540000000000001,"15min":1.746},"min":{"1min":0.36099999999999999,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":5.181,"5min":14.970000000000001,"15min":44.122999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.1429999999999998,"15min":2.302},"min":{"1min":0.377,"5min":0.27700000000000002,"15min":0.249},"max":{"1min":7.96,"5min":19.152000000000001,"15min":43.313000000000002},"last":3.855}]},{"osd":1,"last update":"Mon Oct 9 04:50:50 2023","interfaces":[{"interface":"back","average":{"1min":12.518000000000001,"5min":5.5099999999999998,"15min":4.79},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":122.56,"5min":128.601,"15min":264.62},"last":0.57299999999999995},{"interface":"front","average":{"1min":12.481,"5min":5.4219999999999997,"15min":4.8360000000000003},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":122.666,"5min":128.381,"15min":264.65899999999999},"last":1.4319999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":6.6449999999999996,"15min":5.2489999999999997},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":7.2400000000000002,"5min":129.10400000000001,"15min":184.72},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.669,"5min":6.8280000000000003,"15min":5.548},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.28100000000000003},"max":{"1min":5.2210000000000001,"5min":129.15100000000001,"15min":184.697},"last":3.8170000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:50:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.0110000000000001,"15min":3.5819999999999999},"min":{"1min":0.41899999999999998,"5min":0.34899999999999998,"15min":0.29499999999999998},"max":{"1min":7.1959999999999997,"5min":12.324,"15min":182.839},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.3599999999999999,"15min":4},"min":{"1min":0.38900000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.0999999999999996,"5min":14.866,"15min":182.678},"last":3.9729999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.4449999999999998,"15min":1.772},"min":{"1min":0.34100000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":12.367000000000001,"5min":14.811999999999999,"15min":14.811999999999999},"last":10.49},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.669,"15min":1.377},"min":{"1min":0.313,"5min":0.25700000000000001,"15min":0.21199999999999999},"max":{"1min":4.5389999999999997,"5min":13.523,"15min":13.523},"last":0.72399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.5189999999999999,"15min":1.3260000000000001},"min":{"1min":0.38,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.1020000000000003,"5min":12.202999999999999,"15min":13.319000000000001},"last":0.90100000000000002},{"interface":"front","average":{"1min":2.548,"5min":2.1579999999999999,"15min":1.8340000000000001},"min":{"1min":0.432,"5min":0.29399999999999998,"15min":0.26000000000000001},"max":{"1min":10.544,"5min":12.081,"15min":18.408000000000001},"last":3.944}]},{"osd":6,"last update":"Mon Oct 9 04:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.2809999999999999,"15min":1.2889999999999999},"min":{"1min":0.39500000000000002,"5min":0.32200000000000001,"15min":0.29899999999999999},"max":{"1min":5.2350000000000003,"5min":10.047000000000001,"15min":11.388},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.363,"5min":1.9039999999999999,"15min":1.859},"min":{"1min":0.39700000000000002,"5min":0.34499999999999997,"15min":0.30099999999999999},"max":{"1min":7.3620000000000001,"5min":11.680999999999999,"15min":18.311},"last":3.891}]}]},{"osd":6,"up_from":0,"seq":197568496165,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141254,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4376144,"kb_used_data":3433096,"kb_used_omap":4,"kb_used_meta":943035,"kb_avail":89369008,"statfs":{"total":95995035648,"available":91513864192,"internally_reserved":0,"allocated":3515490304,"data_stored":3513867865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":965668750},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.1279999999999999,"15min":2.6139999999999999},"min":{"1min":0.23200000000000001,"5min":0.20799999999999999,"15min":0.14799999999999999},"max":{"1min":2.8660000000000001,"5min":9.0229999999999997,"15min":93.171999999999997},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.254,"15min":2.6619999999999999},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.193},"max":{"1min":3.524,"5min":8.8190000000000008,"15min":93.147000000000006},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":5.0199999999999996,"15min":4.8129999999999997},"min":{"1min":0.26100000000000001,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":1.3899999999999999,"5min":151.41900000000001,"15min":263.60500000000002},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":5.0780000000000003,"15min":5.1609999999999996},"min":{"1min":0.29599999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.4299999999999999,"5min":146.749,"15min":263.65199999999999},"last":0.61799999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.224,"15min":5.3040000000000003},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":2.6669999999999998,"5min":9.0709999999999997,"15min":379.80900000000003},"last":0.879},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.738,"15min":5.8440000000000003},"min":{"1min":0.35899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.2370000000000001,"5min":12.615,"15min":379.95800000000003},"last":0.64400000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":5.1600000000000001,"5min":2.7679999999999998,"15min":2.879},"min":{"1min":0.31,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":65.036000000000001,"5min":65.036000000000001,"15min":170.072},"last":2.6099999999999999},{"interface":"front","average":{"1min":5.6689999999999996,"5min":2.7719999999999998,"15min":2.7730000000000001},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":62.585000000000001,"5min":62.585000000000001,"15min":169.94499999999999},"last":0.72099999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":2.0619999999999998,"15min":2.4790000000000001},"min":{"1min":0.47199999999999998,"5min":0.34200000000000003,"15min":0.28299999999999997},"max":{"1min":4.4100000000000001,"5min":9.782,"15min":171.036},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.274,"5min":2.0619999999999998,"15min":2.7320000000000002},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":3.8340000000000001,"5min":13.728,"15min":170.88},"last":2.661}]},{"osd":6,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.883,"15min":2.3809999999999998},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":3.145,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.294,"5min":2.1600000000000001,"15min":2.7280000000000002},"min":{"1min":0.39700000000000002,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":3.0449999999999999,"5min":13.686999999999999,"15min":132.17699999999999},"last":2.7360000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":5.2480000000000002,"5min":3.2679999999999998,"15min":3.6560000000000001},"min":{"1min":0.54400000000000004,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":59.972000000000001,"5min":59.972000000000001,"15min":174.75999999999999},"last":2.6930000000000001},{"interface":"front","average":{"1min":5.3799999999999999,"5min":3.0049999999999999,"15min":3.3820000000000001},"min":{"1min":0.45000000000000001,"5min":0.39800000000000002,"15min":0.377},"max":{"1min":60.011000000000003,"5min":60.011000000000003,"15min":174.779},"last":0.747}]}]},{"osd":0,"up_from":0,"seq":390842024312,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":0,"seq":949187772587,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":609885356312,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019439,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6281120,"kb_used_data":4785368,"kb_used_omap":0,"kb_used_meta":1495552,"kb_avail":87464032,"statfs":{"total":95995035648,"available":89563168768,"internally_reserved":0,"allocated":4900216832,"data_stored":4899460157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1531445248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,5],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:59 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.6160000000000001,"15min":3.6349999999999998},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":18.585999999999999,"5min":18.585999999999999,"15min":171.13499999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.331,"5min":1.9339999999999999,"15min":3.6949999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":19.469000000000001,"5min":19.469000000000001,"15min":171.11799999999999},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:50:58 2023","interfaces":[{"interface":"back","average":{"1min":7.2590000000000003,"5min":10.019,"15min":5.5469999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":92.030000000000001,"5min":385.78199999999998,"15min":385.78199999999998},"last":41.412999999999997},{"interface":"front","average":{"1min":5.1390000000000002,"5min":9.3689999999999998,"15min":5.3700000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":91.619,"5min":386.09399999999999,"15min":386.09399999999999},"last":41.262}]},{"osd":2,"last update":"Mon Oct 9 04:50:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":11.032999999999999,"15min":6.5179999999999998},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.131,"5min":440.75599999999997,"15min":440.75599999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.0089999999999999,"5min":11.414,"15min":6.367},"min":{"1min":0.374,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":3.4399999999999999,"5min":440.61900000000003,"15min":440.61900000000003},"last":0.55900000000000005}]},{"osd":3,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":3.746,"15min":5.1360000000000001},"min":{"1min":0.32600000000000001,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":4.4429999999999996,"5min":141.334,"15min":357.125},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.2999999999999998,"5min":4.1529999999999996,"15min":5.1429999999999998},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":7.1760000000000002,"5min":141.18299999999999,"15min":359.02800000000002},"last":0.56499999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":10.875999999999999,"5min":3.9660000000000002,"15min":2.1349999999999998},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":169.44200000000001,"5min":169.44200000000001,"15min":169.44200000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":10.66,"5min":3.5630000000000002,"15min":2.073},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.24099999999999999},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.376}]},{"osd":6,"last update":"Mon Oct 9 04:50:59 2023","interfaces":[{"interface":"back","average":{"1min":4.7569999999999997,"5min":2.5030000000000001,"15min":1.7410000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":18.738,"5min":18.738,"15min":22.920999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.645,"5min":1.629,"15min":1.415},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":18.789000000000001,"5min":18.789000000000001,"15min":18.789000000000001},"last":0.35499999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:50:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":2.093,"15min":1.5960000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28000000000000003},"max":{"1min":4.8250000000000002,"5min":16.109000000000002,"15min":16.109000000000002},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.1720000000000002,"5min":2.8109999999999999,"15min":1.903},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.2060000000000004,"5min":22.858000000000001,"15min":22.858000000000001},"last":0.40100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757803,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5611216,"kb_used_data":4121800,"kb_used_omap":0,"kb_used_meta":1489216,"kb_avail":88133936,"statfs":{"total":95995035648,"available":90249150464,"internally_reserved":0,"allocated":4220723200,"data_stored":4219982285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1524957184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.782,"15min":4.5250000000000004},"min":{"1min":0.251,"5min":0.251,"15min":0.23100000000000001},"max":{"1min":5.0300000000000002,"5min":9.6890000000000001,"15min":210.85499999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.6060000000000001,"15min":4.7409999999999997},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.9390000000000001,"5min":8.4749999999999996,"15min":210.23500000000001},"last":1.49}]},{"osd":1,"last update":"Mon Oct 9 04:50:57 2023","interfaces":[{"interface":"back","average":{"1min":3.7999999999999998,"5min":7.7080000000000002,"15min":5.851},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.252},"max":{"1min":55.234000000000002,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":4.3109999999999999,"5min":8.0549999999999997,"15min":5.9880000000000004},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.255},"max":{"1min":54.994999999999997,"5min":211.435,"15min":211.435},"last":0.56299999999999994}]},{"osd":2,"last update":"Mon Oct 9 04:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":5.5179999999999998,"15min":4.9299999999999997},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":86.488,"15min":90.763999999999996},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.385,"5min":5.5199999999999996,"15min":5.0369999999999999},"min":{"1min":0.34100000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.7590000000000003,"5min":86.510999999999996,"15min":91.340000000000003},"last":0.61899999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.9790000000000001,"15min":4.9880000000000004},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":6.2140000000000004,"5min":13.711,"15min":268.63299999999998},"last":1.5149999999999999},{"interface":"front","average":{"1min":0.74099999999999999,"5min":1.484,"15min":4.6040000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.29799999999999999},"max":{"1min":2.2189999999999999,"5min":10.227,"15min":268.92500000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":6.6760000000000002,"5min":2.552,"15min":2.8690000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.23599999999999999},"max":{"1min":116.054,"5min":116.054,"15min":361.09399999999999},"last":0.745},{"interface":"front","average":{"1min":8.5060000000000002,"5min":3.0499999999999998,"15min":3.2930000000000001},"min":{"1min":0.307,"5min":0.30299999999999999,"15min":0.253},"max":{"1min":118.47,"5min":118.47,"15min":361.71300000000002},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.373,"15min":1.1499999999999999},"min":{"1min":0.33400000000000002,"5min":0.32800000000000001,"15min":0.23100000000000001},"max":{"1min":5.7149999999999999,"5min":10.295999999999999,"15min":13.162000000000001},"last":2.1349999999999998},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.371,"15min":1.25},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28000000000000003},"max":{"1min":4.9020000000000001,"5min":7.3040000000000003,"15min":13.231999999999999},"last":1.5580000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":11.656000000000001,"5min":4.7309999999999999,"15min":2.3740000000000001},"min":{"1min":0.47399999999999998,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":110.521,"5min":110.521,"15min":110.521},"last":3.0009999999999999},{"interface":"front","average":{"1min":11.186999999999999,"5min":4.9630000000000001,"15min":2.3809999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":110.746,"5min":110.746,"15min":110.746},"last":1.3460000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3507687424,"data_stored":3507683352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4872867840,"data_stored":4872863768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4183883776,"data_stored":4183883776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4155576320,"data_stored":4155572248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:51:14.823 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:51:14.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pg_num 26 2023-10-09T04:51:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:14.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:14.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:14 smithi029 ceph-mon[216903]: pgmap v1401: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 582 KiB/s wr, 876 op/s 2023-10-09T04:51:14.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:14 smithi029 ceph-mon[223858]: pgmap v1401: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 582 KiB/s wr, 876 op/s 2023-10-09T04:51:14.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:14.960 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:15.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:14 smithi177 ceph-mon[195797]: pgmap v1401: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 582 KiB/s wr, 876 op/s 2023-10-09T04:51:15.770 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:15.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:15.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:15.875 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:15.939 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:16.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:16.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:16.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:16.245 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:16.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:16.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:16.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:16.461 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:16.514 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:51:16.537 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:51:16.538 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:16.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:51:16.591 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:51:16.621 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:51:16.647 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:51:16.653 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:16.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:16.668 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:16.677 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:51:16.718 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:51:16.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.719 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:51:16.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:16.785 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:51:16.785 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:51:16.804 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:16.921 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:16.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:17.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:17.068 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:16 smithi029 ceph-mon[216903]: pgmap v1402: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T04:51:17.068 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1958295164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:17.069 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:16 smithi029 ceph-mon[223858]: pgmap v1402: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T04:51:17.069 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1958295164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:17.073 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:16 smithi177 ceph-mon[195797]: pgmap v1402: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T04:51:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1958295164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:18.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:51:18.269 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:18.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:18.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:18.434 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:18.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:18.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:18.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:18.619 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:18.889 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:18 smithi029 ceph-mon[223858]: pgmap v1403: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T04:51:18.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:18 smithi029 ceph-mon[216903]: pgmap v1403: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T04:51:19.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:18 smithi177 ceph-mon[195797]: pgmap v1403: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 874 op/s 2023-10-09T04:51:19.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:51:19.378 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:19.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:19.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:19.559 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:19.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:19.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:19.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:19.740 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:20.138 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:20.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:20.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:20.242 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:20.279 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:20.348 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:20.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:20.354 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:20.454 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:20.487 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:20.518 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:20.611 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:20.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:20.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:20.745 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:20.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:51:20.964 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:20.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:21.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:20 smithi029 ceph-mon[216903]: pgmap v1404: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:51:21.058 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:20 smithi029 ceph-mon[223858]: pgmap v1404: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:51:21.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:21.114 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:21.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:20 smithi177 ceph-mon[195797]: pgmap v1404: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 867 op/s 2023-10-09T04:51:21.296 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:21.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:21.405 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:21.499 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:21.629 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:21.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:21.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:21.778 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:21.871 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 13 pg_num to 26 2023-10-09T04:51:22.134 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:22.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/448487398' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:51:22.136 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/448487398' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:51:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:22.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:22.237 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:22.270 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:22.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/448487398' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:51:22.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:51:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:51:22] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-10-09T04:51:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: pgmap v1405: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T04:51:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/448487398' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:51:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: osdmap e364: 8 total, 8 up, 4 in 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[223858]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: pgmap v1405: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/448487398' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: osdmap e364: 8 total, 8 up, 4 in 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: failed to encode map e364 with expected crc 2023-10-09T04:51:22.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:22 smithi029 ceph-mon[216903]: failed to encode map e364 with expected crc 2023-10-09T04:51:23.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: pgmap v1405: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/448487398' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: osdmap e364: 8 total, 8 up, 4 in 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: failed to encode map e364 with expected crc 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: failed to encode map e364 with expected crc 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: failed to encode map e364 with expected crc 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: failed to encode map e364 with expected crc 2023-10-09T04:51:23.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:22 smithi177 ceph-mon[195797]: failed to encode map e364 with expected crc 2023-10-09T04:51:23.429 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:23.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:23.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:23.564 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:24.140 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:24.142 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:51:24.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:51:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:24.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:24.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:24.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:51:24.609 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:24.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:24.718 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:24.734 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:24.748 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:24.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:24.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:24.829 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:51:24.830 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:51:24.886 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:51:24.900 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:24.938 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: pgmap v1407: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: osdmap e365: 8 total, 8 up, 4 in 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3031918165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: pgmap v1407: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T04:51:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: osdmap e365: 8 total, 8 up, 4 in 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3031918165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: pgmap v1407: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 881 op/s 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: osdmap e365: 8 total, 8 up, 4 in 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: failed to encode map e365 with expected crc 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3031918165' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:25.396 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:25.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:25.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:25.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:25.785 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:26.168 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:26.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:26.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: osdmap e366: 8 total, 8 up, 4 in 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[216903]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: osdmap e366: 8 total, 8 up, 4 in 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:25 smithi029 ceph-mon[223858]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:26.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:26.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: osdmap e366: 8 total, 8 up, 4 in 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.282 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:25 smithi177 ceph-mon[195797]: failed to encode map e366 with expected crc 2023-10-09T04:51:26.305 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:27.009 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:27.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:27.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:26 smithi177 ceph-mon[195797]: pgmap v1410: 99 pgs: 7 peering, 8 unknown, 84 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:27.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:27.141 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:26 smithi029 ceph-mon[216903]: pgmap v1410: 99 pgs: 7 peering, 8 unknown, 84 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:27.142 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:26 smithi029 ceph-mon[223858]: pgmap v1410: 99 pgs: 7 peering, 8 unknown, 84 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:27.144 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:27.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:51:27.877 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:27.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:27.980 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:28.050 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:28.161 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:28.186 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:28.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:28.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:51:28.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:28.332 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:28.557 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:28.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:28.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:28.562 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:51:28.589 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:51:28.616 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:51:28.648 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:51:28.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:28.686 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:51:28.715 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:51:28.732 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:28.740 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:51:28.767 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:51:28.768 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.769 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:51:28.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:28.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:28.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:28.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:29.006 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:29.010 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:28 smithi177 ceph-mon[195797]: pgmap v1411: 99 pgs: 7 peering, 8 unknown, 84 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1760139942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:29.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:28 smithi029 ceph-mon[216903]: pgmap v1411: 99 pgs: 7 peering, 8 unknown, 84 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1760139942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:28 smithi029 ceph-mon[223858]: pgmap v1411: 99 pgs: 7 peering, 8 unknown, 84 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1760139942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:29.189 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7] out_osds: [6, 3, 0, 2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:51:29.189 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:51:29.189 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_8 2023-10-09T04:51:29.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:51:29.272 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:29.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:29.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:29.447 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:29.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:29.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:29.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:29.701 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:30.874 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:30.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:30.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:31.036 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:31.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:31.134 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:30 smithi029 ceph-mon[216903]: pgmap v1412: 99 pgs: 7 peering, 7 unknown, 85 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:31.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:30 smithi029 ceph-mon[223858]: pgmap v1412: 99 pgs: 7 peering, 7 unknown, 85 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:31.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:31.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:30 smithi177 ceph-mon[195797]: pgmap v1412: 99 pgs: 7 peering, 7 unknown, 85 active+clean; 3.4 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:31.318 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:31.409 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:31.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:31.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:31.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:31.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:31.601 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:31.728 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:31.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:31.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:31.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:31.903 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:31 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:51:31.910 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:32.016 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:32.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:32.128 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:32.167 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:32.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:31 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:51:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:31 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-09T04:51:32.234 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:32.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:32.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:32.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:32.372 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:32.599 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:32.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:32.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:32.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:32.734 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:32.885 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:32.886 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:51:32.886 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:51:32.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122040 12.6 12:61901a80:::benchmark_data_smithi029_442458_object26790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.129024+0000", 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 308.75536106499999, 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3996381749999998, 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122040 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129024+0000", 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129024+0000", 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129023+0000", 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129031+0000", 2023-10-09T04:51:32.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0590000000000003e-06 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129048+0000", 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7487e-05 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129059+0000", 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0854e-05 2023-10-09T04:51:32.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129213+0000", 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154024 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129339+0000", 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000126391 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129452+0000", 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112485 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.133565+0000", 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041133230000000003 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.158583+0000", 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025017377 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.158595+0000", 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2877000000000001e-05 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132748+0000", 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97415217899999995 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132763+0000", 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5218999999999999e-05 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528607+0000", 2023-10-09T04:51:32.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3958441239999999 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528663+0000", 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5662999999999999e-05 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.895 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122080 12.6 12:68c22d6d:::benchmark_data_smithi029_442458_object26830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.317667+0000", 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 308.56671843999999, 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.227455092, 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122080 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317667+0000", 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317667+0000", 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317666+0000", 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317672+0000", 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3090000000000001e-06 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317685+0000", 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3329999999999999e-05 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317692+0000", 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.477e-06 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317740+0000", 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8185e-05 2023-10-09T04:51:32.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317748+0000", 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9640000000000003e-06 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132836+0000", 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81508806 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132846+0000", 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4609999999999997e-06 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528868+0000", 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3960220569999999 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528955+0000", 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7470000000000001e-05 2023-10-09T04:51:32.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.529238+0000", 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000283262 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.538506+0000", 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092673530000000007 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.541036+0000", 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025307760000000002 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.542769+0000", 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001732305 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.544942+0000", 2023-10-09T04:51:32.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021731960000000001 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.545004+0000", 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2064999999999996e-05 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.545122+0000", 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011807900000000001 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.904 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122083 12.6 12:6ccb75f6:::benchmark_data_smithi029_442458_object26833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:25.488467+0000", 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 307.39591815400001, 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0625703580000001, 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:51:32.905 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122083 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488467+0000", 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488467+0000", 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488466+0000", 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488474+0000", 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0849999999999997e-06 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488492+0000", 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8292999999999999e-05 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488500+0000", 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0490000000000004e-06 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488574+0000", 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.415e-05 2023-10-09T04:51:32.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.488590+0000", 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6192999999999999e-05 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.536220+0000", 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.047629511 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.536375+0000", 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015522800000000001 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.536539+0000", 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016353900000000001 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.541043+0000", 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045043419999999997 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.542731+0000", 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001687947 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.548561+0000", 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005829852 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.550748+0000", 2023-10-09T04:51:32.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002187268 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.550909+0000", 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000161189 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.551038+0000", 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012851099999999999 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.912 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:131919 12.6 12:6e32fc05:::benchmark_data_smithi029_442458_object36669:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:53.831862+0000", 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 279.05252318300001, 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.84946584400000003, 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:51:32.913 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 131919 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831862+0000", 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831862+0000", 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831861+0000", 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831869+0000", 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.96e-06 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831891+0000", 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2560000000000001e-05 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831902+0000", 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0395e-05 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831957+0000", 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5736999999999998e-05 2023-10-09T04:51:32.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:53.831968+0000", 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1056999999999999e-05 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.671734+0000", 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83976587199999997 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.671857+0000", 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000123261 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.671959+0000", 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010180599999999999 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.675692+0000", 2023-10-09T04:51:32.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037328940000000001 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.681184+0000", 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005491569 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.681240+0000", 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6532e-05 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.681293+0000", 2023-10-09T04:51:32.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2717999999999999e-05 2023-10-09T04:51:32.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:54.681328+0000", 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5176999999999999e-05 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13424 13.4 13:2beeaa67:::benchmark_data_smithi029_495570_object13423:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.922 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.255879+0000", 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.628506594, 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0084037800000001, 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13424 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255879+0000", 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255879+0000", 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255877+0000", 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255906+0000", 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9221e-05 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255919+0000", 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3185e-05 2023-10-09T04:51:32.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255928+0000", 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6440000000000006e-06 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255982+0000", 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4150000000000002e-05 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255994+0000", 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2024999999999999e-05 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260900+0000", 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0049060270000001 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260990+0000", 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9687999999999999e-05 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264118+0000", 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031284989999999999 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264283+0000", 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016422600000000001 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13427 13.c 13:33f20e12:::benchmark_data_smithi029_495570_object13426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.255942+0000", 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.628443029, 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.003376204, 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13427 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255942+0000", 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255942+0000", 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255942+0000", 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255963+0000", 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.126e-05 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255970+0000", 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9750000000000001e-06 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255976+0000", 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4160000000000003e-06 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256710+0000", 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00073457000000000004 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256723+0000", 2023-10-09T04:51:32.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2918999999999999e-05 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255490+0000", 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.998766406 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255605+0000", 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115757 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259184+0000", 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035784609999999998 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259319+0000", 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013487500000000001 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13433 13.a 13:5844af57:::benchmark_data_smithi029_495570_object13432:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.255980+0000", 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.628405011, 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.000670315, 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13433 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.935 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255980+0000", 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255980+0000", 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.255980+0000", 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256005+0000", 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4567999999999999e-05 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256018+0000", 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3546e-05 2023-10-09T04:51:32.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256024+0000", 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2950000000000001e-06 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256066+0000", 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2131e-05 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256074+0000", 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0399999999999993e-06 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.251542+0000", 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99546844800000001 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.252068+0000", 2023-10-09T04:51:32.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052575400000000002 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.252299+0000", 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023068900000000001 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.254847+0000", 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025483789999999999 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256187+0000", 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013396090000000001 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256594+0000", 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00040763399999999998 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256628+0000", 2023-10-09T04:51:32.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3293000000000001e-05 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256651+0000", 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3278e-05 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.942 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13435 13.c 13:39b045a7:::benchmark_data_smithi029_495570_object13434:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.256041+0000", 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.62834495600001, 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.00353314, 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13435 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256041+0000", 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256041+0000", 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256040+0000", 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:32.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256057+0000", 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6588000000000001e-05 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256064+0000", 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9840000000000004e-06 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256069+0000", 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2480000000000004e-06 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256795+0000", 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00072606799999999996 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256803+0000", 2023-10-09T04:51:32.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5429999999999994e-06 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255649+0000", 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99884576800000002 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255793+0000", 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000143511 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255957+0000", 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016401600000000001 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.258567+0000", 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026104729999999999 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259347+0000", 2023-10-09T04:51:32.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00077929300000000002 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259405+0000", 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8316999999999997e-05 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259462+0000", 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7445000000000003e-05 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259574+0000", 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011134299999999999 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13441 13.4 13:2ff450f1:::benchmark_data_smithi029_495570_object13440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.256073+0000", 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.62831219500001, 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.01235561, 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.950 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13441 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256073+0000", 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256073+0000", 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256073+0000", 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256093+0000", 2023-10-09T04:51:32.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9933999999999999e-05 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256099+0000", 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9309999999999996e-06 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256106+0000", 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1110000000000002e-06 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256751+0000", 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064513999999999999 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.256760+0000", 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5150000000000006e-06 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261019+0000", 2023-10-09T04:51:32.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.004259545 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261100+0000", 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0939999999999994e-05 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261203+0000", 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102457 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.263875+0000", 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026721539999999999 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266425+0000", 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025499680000000001 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268255+0000", 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018302870000000001 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268315+0000", 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0398e-05 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268429+0000", 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113581 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13453 13.c 13:3be0679b:::benchmark_data_smithi029_495570_object13452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.262803+0000", 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.621582068, 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99708689299999997, 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.958 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13453 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262803+0000", 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262803+0000", 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262802+0000", 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262840+0000", 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7812999999999999e-05 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262856+0000", 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6099999999999998e-05 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262869+0000", 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2999999999999999e-05 2023-10-09T04:51:32.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262943+0000", 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4324000000000002e-05 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.262965+0000", 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2164e-05 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256923+0000", 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99395784099999995 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.257068+0000", 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014486699999999999 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.257237+0000", 2023-10-09T04:51:32.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000169128 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259314+0000", 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020767630000000001 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259666+0000", 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000351753 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259719+0000", 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3443000000000003e-05 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259782+0000", 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2602999999999997e-05 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259890+0000", 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000108609 2023-10-09T04:51:32.965 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13459 13.4 13:24022a87:::benchmark_data_smithi029_495570_object13458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.264479+0000", 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.619906844, 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.004122658, 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.966 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13459 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264479+0000", 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264479+0000", 2023-10-09T04:51:32.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264477+0000", 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264506+0000", 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9215000000000001e-05 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264526+0000", 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0006000000000001e-05 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264536+0000", 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0613e-05 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264606+0000", 2023-10-09T04:51:32.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9756000000000001e-05 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.264618+0000", 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2081e-05 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261767+0000", 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99714880500000003 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261859+0000", 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1973999999999997e-05 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.262000+0000", 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014071799999999999 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264067+0000", 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020675469999999999 2023-10-09T04:51:32.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.267244+0000", 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031763429999999999 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268476+0000", 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012327900000000001 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268510+0000", 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3756e-05 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268601+0000", 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1068000000000003e-05 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13461 13.4 13:23ea307f:::benchmark_data_smithi029_495570_object13460:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.973 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.267437+0000", 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.616948901, 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0013615600000001, 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13461 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267437+0000", 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267437+0000", 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267435+0000", 2023-10-09T04:51:32.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267487+0000", 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1940000000000001e-05 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267503+0000", 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6456e-05 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267511+0000", 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5240000000000003e-06 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267614+0000", 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010281999999999999 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.267633+0000", 2023-10-09T04:51:32.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8760999999999999e-05 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.262470+0000", 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99483698399999998 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.262619+0000", 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014931699999999999 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.262810+0000", 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000191294 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264243+0000", 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014325589999999999 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.267298+0000", 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030551939999999998 2023-10-09T04:51:32.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268666+0000", 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001368085 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268699+0000", 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3358000000000001e-05 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268798+0000", 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8747999999999997e-05 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13467 13.4 13:212181d8:::benchmark_data_smithi029_495570_object13466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.268210+0000", 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.616175477, 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.000848975, 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.981 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13467 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268210+0000", 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268210+0000", 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268209+0000", 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268230+0000", 2023-10-09T04:51:32.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1229999999999998e-05 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268242+0000", 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1891e-05 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268248+0000", 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3550000000000001e-06 2023-10-09T04:51:32.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268300+0000", 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2002000000000001e-05 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.268316+0000", 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5831000000000001e-05 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264343+0000", 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99602629799999998 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264453+0000", 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0001102 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264567+0000", 2023-10-09T04:51:32.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011464999999999999 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266436+0000", 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018682919999999999 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268874+0000", 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024387089999999998 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268918+0000", 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4045999999999998e-05 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268951+0000", 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2737000000000002e-05 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269059+0000", 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000107804 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13477 13.4 13:276fea61:::benchmark_data_smithi029_495570_object13476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.274510+0000", 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.60987524399999, 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99528455500000002, 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.989 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13477 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274510+0000", 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274510+0000", 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274509+0000", 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274541+0000", 2023-10-09T04:51:32.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2101999999999999e-05 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274557+0000", 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6546999999999999e-05 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274566+0000", 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0529999999999996e-06 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274636+0000", 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9690999999999994e-05 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.274652+0000", 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5637e-05 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.265039+0000", 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99038737700000001 2023-10-09T04:51:32.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.265181+0000", 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000142127 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.265359+0000", 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000178114 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:32.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.267191+0000", 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018315899999999999 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269127+0000", 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001936349 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269703+0000", 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057547999999999996 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269753+0000", 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0757999999999998e-05 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269795+0000", 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1349e-05 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13480 13.c 13:368109b6:::benchmark_data_smithi029_495570_object13479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.275027+0000", 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.60935866600001, 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98518214999999998, 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13480 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.997 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275027+0000", 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275027+0000", 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275026+0000", 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275057+0000", 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0871000000000003e-05 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275072+0000", 2023-10-09T04:51:32.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4989e-05 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275079+0000", 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2189999999999998e-06 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275118+0000", 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9317999999999999e-05 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.275132+0000", 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3275000000000001e-05 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.257879+0000", 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98274788400000002 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.257983+0000", 2023-10-09T04:51:33.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103275 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.258089+0000", 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000106628 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259592+0000", 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015024070000000001 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259941+0000", 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034906899999999997 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.259998+0000", 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7272000000000003e-05 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260101+0000", 2023-10-09T04:51:33.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000102938 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260209+0000", 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000107897 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13484 13.4 13:26448c6a:::benchmark_data_smithi029_495570_object13483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:33.004 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.277745+0000", 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.606640766, 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99215241099999996, 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13484 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:33.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277745+0000", 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277745+0000", 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277743+0000", 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:33.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277781+0000", 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8297000000000003e-05 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277795+0000", 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3733e-05 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:33.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277802+0000", 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2189999999999998e-06 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277871+0000", 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8891000000000002e-05 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.277884+0000", 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2731999999999999e-05 2023-10-09T04:51:33.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.265814+0000", 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98793027200000005 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.265905+0000", 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0317999999999998e-05 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:33.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266013+0000", 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010849 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.267308+0000", 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012949260000000001 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269186+0000", 2023-10-09T04:51:33.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001877994 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269831+0000", 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064464000000000004 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269867+0000", 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5944e-05 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.269897+0000", 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0530000000000001e-05 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13500 13.c 13:3c7a1795:::benchmark_data_smithi029_495570_object13499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.286381+0000", 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.59800399300001, 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97764515799999996, 2023-10-09T04:51:33.012 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13500 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286381+0000", 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286381+0000", 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286380+0000", 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286403+0000", 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2751e-05 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286413+0000", 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0525e-05 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286419+0000", 2023-10-09T04:51:33.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8649999999999998e-06 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286475+0000", 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5973000000000001e-05 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.286487+0000", 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1744e-05 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260257+0000", 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97376989199999997 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260357+0000", 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010027 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260465+0000", 2023-10-09T04:51:33.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010850500000000001 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.263361+0000", 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002895159 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.263423+0000", 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2680000000000003e-05 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.263908+0000", 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00048461100000000003 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.263947+0000", 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8896999999999997e-05 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264027+0000", 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9938999999999995e-05 2023-10-09T04:51:33.019 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13502 13.a 13:5727847c:::benchmark_data_smithi029_495570_object13501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.287894+0000", 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.596491144, 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96900139500000004, 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:33.020 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13502 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.287894+0000", 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:33.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.287894+0000", 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.287893+0000", 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.287923+0000", 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9813999999999999e-05 2023-10-09T04:51:33.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.287940+0000", 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6738000000000001e-05 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.287948+0000", 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8890000000000008e-06 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.288036+0000", 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7658000000000007e-05 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.288081+0000", 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5247999999999998e-05 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255222+0000", 2023-10-09T04:51:33.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96714019799999995 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255397+0000", 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017514200000000001 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.255534+0000", 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137143 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256229+0000", 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00069514900000000005 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256790+0000", 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00056052399999999996 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256832+0000", 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2756999999999997e-05 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256874+0000", 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1570999999999999e-05 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.256896+0000", 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1877000000000001e-05 2023-10-09T04:51:33.027 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13511 13.4 13:2c176134:::benchmark_data_smithi029_495570_object13510:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.296979+0000", 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.58740680699999, 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97701428800000001, 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:33.028 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13511 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.296979+0000", 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.296979+0000", 2023-10-09T04:51:33.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.296977+0000", 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.297001+0000", 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4498000000000002e-05 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.297026+0000", 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5072e-05 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.297036+0000", 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5200000000000003e-06 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.297091+0000", 2023-10-09T04:51:33.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4851000000000001e-05 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.297099+0000", 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4959999999999996e-06 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266463+0000", 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96936363400000003 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266597+0000", 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013417600000000001 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266762+0000", 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016544699999999999 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.268626+0000", 2023-10-09T04:51:33.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018641179999999999 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.272554+0000", 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039271890000000002 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.273914+0000", 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013606950000000001 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.273964+0000", 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0148000000000002e-05 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.273993+0000", 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8504999999999999e-05 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.035 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:13515 13.c 13:327b8052:::benchmark_data_smithi029_495570_object13514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:49:35.303386+0000", 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 117.58099975099999, 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96314922400000003, 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 13515 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303386+0000", 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303386+0000", 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303384+0000", 2023-10-09T04:51:33.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303413+0000", 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9652e-05 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303434+0000", 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0676999999999999e-05 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303445+0000", 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1257000000000001e-05 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303497+0000", 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1588999999999997e-05 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:35.303505+0000", 2023-10-09T04:51:33.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4589999999999998e-06 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.260919+0000", 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95741322500000003 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261081+0000", 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016243900000000001 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.261264+0000", 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000182924 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264054+0000", 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027897769999999998 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.264089+0000", 2023-10-09T04:51:33.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5633000000000003e-05 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266426+0000", 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002336079 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266458+0000", 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2787000000000003e-05 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:51:33.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:49:36.266535+0000", 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6643000000000005e-05 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:51:33.043 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:51:33.043 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:51:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:51:32] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-10-09T04:51:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:33.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:33.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:51:33.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: pgmap v1413: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:51:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: osdmap e367: 8 total, 8 up, 4 in 2023-10-09T04:51:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[216903]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: pgmap v1413: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: osdmap e367: 8 total, 8 up, 4 in 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:32 smithi029 ceph-mon[223858]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.174 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: pgmap v1413: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-09T04:51:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: osdmap e367: 8 total, 8 up, 4 in 2023-10-09T04:51:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:32 smithi177 ceph-mon[195797]: failed to encode map e367 with expected crc 2023-10-09T04:51:33.776 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:33.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:33.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:33.880 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:33.909 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:33.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:33.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:33.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:34.064 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:34.092 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[216903]: osdmap e368: 8 total, 8 up, 4 in 2023-10-09T04:51:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[216903]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[216903]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[216903]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:51:34.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[223858]: osdmap e368: 8 total, 8 up, 4 in 2023-10-09T04:51:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[223858]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[223858]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[223858]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:33 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:51:34.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:34.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:33 smithi177 ceph-mon[195797]: osdmap e368: 8 total, 8 up, 4 in 2023-10-09T04:51:34.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:33 smithi177 ceph-mon[195797]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:33 smithi177 ceph-mon[195797]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:33 smithi177 ceph-mon[195797]: failed to encode map e368 with expected crc 2023-10-09T04:51:34.206 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:33 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-09T04:51:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:34.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:34.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:34.346 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:34.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:34.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:34.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:34.546 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:34.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:51:34.815 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:34.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:34.903 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:34.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:34.969 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:35.043 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:35.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:35.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:35.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:35.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: pgmap v1416: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 161 MiB/s wr, 3.81k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:35.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: failed to encode map e368 with expected crc 2023-10-09T04:51:35.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: 13.10 starting backfill to osd.5 from (0'0,0'0] MAX to 366'7499 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: osdmap e369: 8 total, 8 up, 4 in 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: pgmap v1416: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 161 MiB/s wr, 3.81k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: failed to encode map e368 with expected crc 2023-10-09T04:51:35.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: 13.10 starting backfill to osd.5 from (0'0,0'0] MAX to 366'7499 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: osdmap e369: 8 total, 8 up, 4 in 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:35.187 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:35.245 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:35.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: pgmap v1416: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 161 MiB/s wr, 3.81k op/s; 0 B/s, 0 objects/s recovering 2023-10-09T04:51:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: failed to encode map e368 with expected crc 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: 13.10 starting backfill to osd.5 from (0'0,0'0] MAX to 366'7499 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: osdmap e369: 8 total, 8 up, 4 in 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: failed to encode map e369 with expected crc 2023-10-09T04:51:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:35.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:35.384 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:35.447 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:35.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:35.551 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:51:35.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:35.553 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:51:35.563 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:35.579 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:35.625 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:35.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:35.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:35.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:35.762 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:35.942 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:51:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[216903]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[216903]: osdmap e370: 8 total, 8 up, 4 in 2023-10-09T04:51:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[216903]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[216903]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2884236198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[223858]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[223858]: osdmap e370: 8 total, 8 up, 4 in 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[223858]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[223858]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2884236198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:36.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:35 smithi177 ceph-mon[195797]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:35 smithi177 ceph-mon[195797]: osdmap e370: 8 total, 8 up, 4 in 2023-10-09T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:35 smithi177 ceph-mon[195797]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:35 smithi177 ceph-mon[195797]: failed to encode map e370 with expected crc 2023-10-09T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2884236198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:36.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:51:36.370 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:36.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:36.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:36.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:36.506 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: pgmap v1419: 99 pgs: 1 remapped+peering, 98 active+clean; 3.9 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: from='client.46637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: 13.11 starting backfill to osd.5 from (0'0,0'0] MAX to 367'7480 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: osdmap e371: 8 total, 8 up, 4 in 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:36 smithi177 ceph-mon[195797]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: pgmap v1419: 99 pgs: 1 remapped+peering, 98 active+clean; 3.9 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: from='client.46637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: 13.11 starting backfill to osd.5 from (0'0,0'0] MAX to 367'7480 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: osdmap e371: 8 total, 8 up, 4 in 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[216903]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: pgmap v1419: 99 pgs: 1 remapped+peering, 98 active+clean; 3.9 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 802 op/s 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: from='client.46637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: 13.11 starting backfill to osd.5 from (0'0,0'0] MAX to 367'7480 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: osdmap e371: 8 total, 8 up, 4 in 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: failed to encode map e371 with expected crc 2023-10-09T04:51:37.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:36 smithi029 ceph-mon[223858]: failed to encode map e371 with expected crc 2023-10-09T04:51:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:37 smithi177 ceph-mon[195797]: 13.3 scrub starts 2023-10-09T04:51:38.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:37 smithi029 ceph-mon[216903]: 13.3 scrub starts 2023-10-09T04:51:38.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:37 smithi029 ceph-mon[223858]: 13.3 scrub starts 2023-10-09T04:51:39.170 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:39.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:39.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:39.275 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:39.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:38 smithi177 ceph-mon[195797]: 13.7 scrub starts 2023-10-09T04:51:39.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:38 smithi177 ceph-mon[195797]: pgmap v1421: 99 pgs: 1 remapped+peering, 98 active+clean; 3.9 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:51:39.283 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:38 smithi177 ceph-mon[195797]: 13.0 scrub starts 2023-10-09T04:51:39.355 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:38 smithi029 ceph-mon[216903]: 13.7 scrub starts 2023-10-09T04:51:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:38 smithi029 ceph-mon[216903]: pgmap v1421: 99 pgs: 1 remapped+peering, 98 active+clean; 3.9 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:51:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:38 smithi029 ceph-mon[216903]: 13.0 scrub starts 2023-10-09T04:51:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:38 smithi029 ceph-mon[223858]: 13.7 scrub starts 2023-10-09T04:51:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:38 smithi029 ceph-mon[223858]: pgmap v1421: 99 pgs: 1 remapped+peering, 98 active+clean; 3.9 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 715 op/s 2023-10-09T04:51:39.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:38 smithi029 ceph-mon[223858]: 13.0 scrub starts 2023-10-09T04:51:39.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:39.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:39.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:39.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:39.635 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:39.751 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1419,"stamp":"2023-10-09T04:51:35.397676+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4194375608,"num_objects":64017,"num_object_clones":0,"num_object_copies":192051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64017,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":124823,"num_write_kb":6034894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":4870682,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122462,"ondisk_log_size":122462,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":17767320,"kb_used_data":12298232,"kb_used_omap":4,"kb_used_meta":5469051,"kb_avail":357213288,"statfs":{"total":383980142592,"available":365786406912,"internally_reserved":0,"allocated":12593389568,"data_stored":12589484796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":5600309134},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,6,2,4,2,0,0,0,1,10,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-210370560,"num_objects":-3210,"num_object_clones":0,"num_object_copies":-9630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3210,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.001044"},"pg_stats":[{"pgid":"13.14","version":"367'7399","reported_seq":7460,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.303821+0000","last_change":"2023-10-09T04:51:24.960331+0000","last_active":"2023-10-09T04:51:32.303821+0000","last_peered":"2023-10-09T04:51:32.303821+0000","last_clean":"2023-10-09T04:51:32.303821+0000","last_became_active":"2023-10-09T04:51:24.950988+0000","last_became_peered":"2023-10-09T04:51:24.950988+0000","last_unstale":"2023-10-09T04:51:32.303821+0000","last_undegraded":"2023-10-09T04:51:32.303821+0000","last_fullsized":"2023-10-09T04:51:32.303821+0000","mapping_epoch":365,"log_start":"353'2900","ondisk_log_start":"353'2900","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4499,"log_dups_size":0,"ondisk_log_size":4499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129761280,"num_objects":1980,"num_object_clones":0,"num_object_copies":5940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3825,"num_write_kb":185760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":127,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922414+0000","last_change":"2023-10-09T04:49:24.722690+0000","last_active":"2023-10-09T04:51:32.922414+0000","last_peered":"2023-10-09T04:51:32.922414+0000","last_clean":"2023-10-09T04:51:32.922414+0000","last_became_active":"2023-10-09T04:49:24.722128+0000","last_became_peered":"2023-10-09T04:49:24.722128+0000","last_unstale":"2023-10-09T04:51:32.922414+0000","last_undegraded":"2023-10-09T04:51:32.922414+0000","last_fullsized":"2023-10-09T04:51:32.922414+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:52.116095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":120,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911291+0000","last_change":"2023-10-09T04:49:36.252176+0000","last_active":"2023-10-09T04:51:31.911291+0000","last_peered":"2023-10-09T04:51:31.911291+0000","last_clean":"2023-10-09T04:51:31.911291+0000","last_became_active":"2023-10-09T04:49:36.251166+0000","last_became_peered":"2023-10-09T04:49:36.251166+0000","last_unstale":"2023-10-09T04:51:31.911291+0000","last_undegraded":"2023-10-09T04:51:31.911291+0000","last_fullsized":"2023-10-09T04:51:31.911291+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:23:40.935795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.17","version":"369'7487","reported_seq":7539,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.515704+0000","last_change":"2023-10-09T04:51:24.987100+0000","last_active":"2023-10-09T04:51:34.515704+0000","last_peered":"2023-10-09T04:51:34.515704+0000","last_clean":"2023-10-09T04:51:34.515704+0000","last_became_active":"2023-10-09T04:51:24.976848+0000","last_became_peered":"2023-10-09T04:51:24.976848+0000","last_unstale":"2023-10-09T04:51:34.515704+0000","last_undegraded":"2023-10-09T04:51:34.515704+0000","last_fullsized":"2023-10-09T04:51:34.515704+0000","mapping_epoch":365,"log_start":"350'2500","ondisk_log_start":"350'2500","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4987,"log_dups_size":0,"ondisk_log_size":4987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130318336,"num_objects":1989,"num_object_clones":0,"num_object_copies":5967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3881,"num_write_kb":187808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":366,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911473+0000","last_change":"2023-10-09T04:45:45.641946+0000","last_active":"2023-10-09T04:51:31.911473+0000","last_peered":"2023-10-09T04:51:31.911473+0000","last_clean":"2023-10-09T04:51:31.911473+0000","last_became_active":"2023-10-09T04:45:45.641426+0000","last_became_peered":"2023-10-09T04:45:45.641426+0000","last_unstale":"2023-10-09T04:51:31.911473+0000","last_undegraded":"2023-10-09T04:51:31.911473+0000","last_fullsized":"2023-10-09T04:51:31.911473+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:18.000468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":303,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.926682+0000","last_change":"2023-10-09T04:46:25.193789+0000","last_active":"2023-10-09T04:51:33.926682+0000","last_peered":"2023-10-09T04:51:33.926682+0000","last_clean":"2023-10-09T04:51:33.926682+0000","last_became_active":"2023-10-09T04:46:25.193461+0000","last_became_peered":"2023-10-09T04:46:25.193461+0000","last_unstale":"2023-10-09T04:51:33.926682+0000","last_undegraded":"2023-10-09T04:51:33.926682+0000","last_fullsized":"2023-10-09T04:51:33.926682+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:01:27.658570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.16","version":"369'7662","reported_seq":7723,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.316169+0000","last_change":"2023-10-09T04:51:24.987178+0000","last_active":"2023-10-09T04:51:34.316169+0000","last_peered":"2023-10-09T04:51:34.316169+0000","last_clean":"2023-10-09T04:51:34.316169+0000","last_became_active":"2023-10-09T04:51:24.972218+0000","last_became_peered":"2023-10-09T04:51:24.972218+0000","last_unstale":"2023-10-09T04:51:34.316169+0000","last_undegraded":"2023-10-09T04:51:34.316169+0000","last_fullsized":"2023-10-09T04:51:34.316169+0000","mapping_epoch":365,"log_start":"351'2700","ondisk_log_start":"351'2700","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4962,"log_dups_size":0,"ondisk_log_size":4962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136773632,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3966,"num_write_kb":193696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":162,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.912143+0000","last_change":"2023-10-09T04:49:36.253446+0000","last_active":"2023-10-09T04:51:31.912143+0000","last_peered":"2023-10-09T04:51:31.912143+0000","last_clean":"2023-10-09T04:51:31.912143+0000","last_became_active":"2023-10-09T04:49:36.252717+0000","last_became_peered":"2023-10-09T04:49:36.252717+0000","last_unstale":"2023-10-09T04:51:31.912143+0000","last_undegraded":"2023-10-09T04:51:31.912143+0000","last_fullsized":"2023-10-09T04:51:31.912143+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:30.302532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":169,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924603+0000","last_change":"2023-10-09T04:46:25.177190+0000","last_active":"2023-10-09T04:51:32.924603+0000","last_peered":"2023-10-09T04:51:32.924603+0000","last_clean":"2023-10-09T04:51:32.924603+0000","last_became_active":"2023-10-09T04:46:25.176231+0000","last_became_peered":"2023-10-09T04:46:25.176231+0000","last_unstale":"2023-10-09T04:51:32.924603+0000","last_undegraded":"2023-10-09T04:51:32.924603+0000","last_fullsized":"2023-10-09T04:51:32.924603+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:22.296242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.19","version":"369'7392","reported_seq":7453,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.507792+0000","last_change":"2023-10-09T04:51:24.960248+0000","last_active":"2023-10-09T04:51:34.507792+0000","last_peered":"2023-10-09T04:51:34.507792+0000","last_clean":"2023-10-09T04:51:34.507792+0000","last_became_active":"2023-10-09T04:51:24.951613+0000","last_became_peered":"2023-10-09T04:51:24.951613+0000","last_unstale":"2023-10-09T04:51:34.507792+0000","last_undegraded":"2023-10-09T04:51:34.507792+0000","last_fullsized":"2023-10-09T04:51:34.507792+0000","mapping_epoch":365,"log_start":"350'2400","ondisk_log_start":"350'2400","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4992,"log_dups_size":0,"ondisk_log_size":4992,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127500288,"num_objects":1946,"num_object_clones":0,"num_object_copies":5838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3837,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":311,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924546+0000","last_change":"2023-10-09T04:49:24.727794+0000","last_active":"2023-10-09T04:51:32.924546+0000","last_peered":"2023-10-09T04:51:32.924546+0000","last_clean":"2023-10-09T04:51:32.924546+0000","last_became_active":"2023-10-09T04:49:24.726934+0000","last_became_peered":"2023-10-09T04:49:24.726934+0000","last_unstale":"2023-10-09T04:51:32.924546+0000","last_undegraded":"2023-10-09T04:51:32.924546+0000","last_fullsized":"2023-10-09T04:51:32.924546+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:43.043505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":284,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.912247+0000","last_change":"2023-10-09T04:49:36.252722+0000","last_active":"2023-10-09T04:51:31.912247+0000","last_peered":"2023-10-09T04:51:31.912247+0000","last_clean":"2023-10-09T04:51:31.912247+0000","last_became_active":"2023-10-09T04:49:36.251598+0000","last_became_peered":"2023-10-09T04:49:36.251598+0000","last_unstale":"2023-10-09T04:51:31.912247+0000","last_undegraded":"2023-10-09T04:51:31.912247+0000","last_fullsized":"2023-10-09T04:51:31.912247+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:51.458714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.18","version":"369'7488","reported_seq":7540,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.252676+0000","last_change":"2023-10-09T04:51:24.954080+0000","last_active":"2023-10-09T04:51:34.252676+0000","last_peered":"2023-10-09T04:51:34.252676+0000","last_clean":"2023-10-09T04:51:34.252676+0000","last_became_active":"2023-10-09T04:51:24.944899+0000","last_became_peered":"2023-10-09T04:51:24.944899+0000","last_unstale":"2023-10-09T04:51:34.252676+0000","last_undegraded":"2023-10-09T04:51:34.252676+0000","last_fullsized":"2023-10-09T04:51:34.252676+0000","mapping_epoch":365,"log_start":"351'2600","ondisk_log_start":"351'2600","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4888,"log_dups_size":0,"ondisk_log_size":4888,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130809856,"num_objects":1996,"num_object_clones":0,"num_object_copies":5988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3874,"num_write_kb":187840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":329,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924391+0000","last_change":"2023-10-09T04:24:07.926463+0000","last_active":"2023-10-09T04:51:32.924391+0000","last_peered":"2023-10-09T04:51:32.924391+0000","last_clean":"2023-10-09T04:51:32.924391+0000","last_became_active":"2023-10-09T04:24:07.926119+0000","last_became_peered":"2023-10-09T04:24:07.926119+0000","last_unstale":"2023-10-09T04:51:32.924391+0000","last_undegraded":"2023-10-09T04:51:32.924391+0000","last_fullsized":"2023-10-09T04:51:32.924391+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:12.778606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":320,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.929871+0000","last_change":"2023-10-09T04:49:24.728045+0000","last_active":"2023-10-09T04:51:33.929871+0000","last_peered":"2023-10-09T04:51:33.929871+0000","last_clean":"2023-10-09T04:51:33.929871+0000","last_became_active":"2023-10-09T04:49:24.727278+0000","last_became_peered":"2023-10-09T04:49:24.727278+0000","last_unstale":"2023-10-09T04:51:33.929871+0000","last_undegraded":"2023-10-09T04:51:33.929871+0000","last_fullsized":"2023-10-09T04:51:33.929871+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:17.700648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":412,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.929008+0000","last_change":"2023-10-09T04:49:24.751066+0000","last_active":"2023-10-09T04:51:33.929008+0000","last_peered":"2023-10-09T04:51:33.929008+0000","last_clean":"2023-10-09T04:51:33.929008+0000","last_became_active":"2023-10-09T04:49:24.731080+0000","last_became_peered":"2023-10-09T04:49:24.731080+0000","last_unstale":"2023-10-09T04:51:33.929008+0000","last_undegraded":"2023-10-09T04:51:33.929008+0000","last_fullsized":"2023-10-09T04:51:33.929008+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:58:47.656154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":263,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.910852+0000","last_change":"2023-10-09T04:49:36.255264+0000","last_active":"2023-10-09T04:51:31.910852+0000","last_peered":"2023-10-09T04:51:31.910852+0000","last_clean":"2023-10-09T04:51:31.910852+0000","last_became_active":"2023-10-09T04:49:36.254631+0000","last_became_peered":"2023-10-09T04:49:36.254631+0000","last_unstale":"2023-10-09T04:51:31.910852+0000","last_undegraded":"2023-10-09T04:51:31.910852+0000","last_fullsized":"2023-10-09T04:51:31.910852+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:26.380768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":167,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922749+0000","last_change":"2023-10-09T04:49:24.739093+0000","last_active":"2023-10-09T04:51:32.922749+0000","last_peered":"2023-10-09T04:51:32.922749+0000","last_clean":"2023-10-09T04:51:32.922749+0000","last_became_active":"2023-10-09T04:49:24.723103+0000","last_became_peered":"2023-10-09T04:49:24.723103+0000","last_unstale":"2023-10-09T04:51:32.922749+0000","last_undegraded":"2023-10-09T04:51:32.922749+0000","last_fullsized":"2023-10-09T04:51:32.922749+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:41.664075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924773+0000","last_change":"2023-10-09T04:44:38.777835+0000","last_active":"2023-10-09T04:51:32.924773+0000","last_peered":"2023-10-09T04:51:32.924773+0000","last_clean":"2023-10-09T04:51:32.924773+0000","last_became_active":"2023-10-09T04:44:38.777500+0000","last_became_peered":"2023-10-09T04:44:38.777500+0000","last_unstale":"2023-10-09T04:51:32.924773+0000","last_undegraded":"2023-10-09T04:51:32.924773+0000","last_fullsized":"2023-10-09T04:51:32.924773+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:16.017295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":396,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.928825+0000","last_change":"2023-10-09T04:49:24.748611+0000","last_active":"2023-10-09T04:51:33.928825+0000","last_peered":"2023-10-09T04:51:33.928825+0000","last_clean":"2023-10-09T04:51:33.928825+0000","last_became_active":"2023-10-09T04:49:24.730507+0000","last_became_peered":"2023-10-09T04:49:24.730507+0000","last_unstale":"2023-10-09T04:51:33.928825+0000","last_undegraded":"2023-10-09T04:51:33.928825+0000","last_fullsized":"2023-10-09T04:51:33.928825+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:21:22.219826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":168,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911720+0000","last_change":"2023-10-09T04:49:36.255342+0000","last_active":"2023-10-09T04:51:31.911720+0000","last_peered":"2023-10-09T04:51:31.911720+0000","last_clean":"2023-10-09T04:51:31.911720+0000","last_became_active":"2023-10-09T04:49:36.254372+0000","last_became_peered":"2023-10-09T04:49:36.254372+0000","last_unstale":"2023-10-09T04:51:31.911720+0000","last_undegraded":"2023-10-09T04:51:31.911720+0000","last_fullsized":"2023-10-09T04:51:31.911720+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:37.972615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":312,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927198+0000","last_change":"2023-10-09T04:49:24.731694+0000","last_active":"2023-10-09T04:51:33.927198+0000","last_peered":"2023-10-09T04:51:33.927198+0000","last_clean":"2023-10-09T04:51:33.927198+0000","last_became_active":"2023-10-09T04:49:24.731338+0000","last_became_peered":"2023-10-09T04:49:24.731338+0000","last_unstale":"2023-10-09T04:51:33.927198+0000","last_undegraded":"2023-10-09T04:51:33.927198+0000","last_fullsized":"2023-10-09T04:51:33.927198+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:28:45.968733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":348,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922541+0000","last_change":"2023-10-09T04:46:25.171177+0000","last_active":"2023-10-09T04:51:32.922541+0000","last_peered":"2023-10-09T04:51:32.922541+0000","last_clean":"2023-10-09T04:51:32.922541+0000","last_became_active":"2023-10-09T04:46:25.170857+0000","last_became_peered":"2023-10-09T04:46:25.170857+0000","last_unstale":"2023-10-09T04:51:32.922541+0000","last_undegraded":"2023-10-09T04:51:32.922541+0000","last_fullsized":"2023-10-09T04:51:32.922541+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:13:25.997048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":342,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.929836+0000","last_change":"2023-10-09T04:49:24.728084+0000","last_active":"2023-10-09T04:51:33.929836+0000","last_peered":"2023-10-09T04:51:33.929836+0000","last_clean":"2023-10-09T04:51:33.929836+0000","last_became_active":"2023-10-09T04:49:24.727752+0000","last_became_peered":"2023-10-09T04:49:24.727752+0000","last_unstale":"2023-10-09T04:51:33.929836+0000","last_undegraded":"2023-10-09T04:51:33.929836+0000","last_fullsized":"2023-10-09T04:51:33.929836+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:12:37.448051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":200,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.926766+0000","last_change":"2023-10-09T04:49:24.734384+0000","last_active":"2023-10-09T04:51:33.926766+0000","last_peered":"2023-10-09T04:51:33.926766+0000","last_clean":"2023-10-09T04:51:33.926766+0000","last_became_active":"2023-10-09T04:49:24.733948+0000","last_became_peered":"2023-10-09T04:49:24.733948+0000","last_unstale":"2023-10-09T04:51:33.926766+0000","last_undegraded":"2023-10-09T04:51:33.926766+0000","last_fullsized":"2023-10-09T04:51:33.926766+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:25:55.081133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":170,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923410+0000","last_change":"2023-10-09T04:49:24.743137+0000","last_active":"2023-10-09T04:51:32.923410+0000","last_peered":"2023-10-09T04:51:32.923410+0000","last_clean":"2023-10-09T04:51:32.923410+0000","last_became_active":"2023-10-09T04:49:24.726930+0000","last_became_peered":"2023-10-09T04:49:24.726930+0000","last_unstale":"2023-10-09T04:51:32.923410+0000","last_undegraded":"2023-10-09T04:51:32.923410+0000","last_fullsized":"2023-10-09T04:51:32.923410+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:29:38.154496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":173,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927529+0000","last_change":"2023-10-09T04:49:24.731691+0000","last_active":"2023-10-09T04:51:33.927529+0000","last_peered":"2023-10-09T04:51:33.927529+0000","last_clean":"2023-10-09T04:51:33.927529+0000","last_became_active":"2023-10-09T04:49:24.731374+0000","last_became_peered":"2023-10-09T04:49:24.731374+0000","last_unstale":"2023-10-09T04:51:33.927529+0000","last_undegraded":"2023-10-09T04:51:33.927529+0000","last_fullsized":"2023-10-09T04:51:33.927529+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:49:39.511857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.1","version":"369'7482","reported_seq":7535,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.391991+0000","last_change":"2023-10-09T04:51:24.924955+0000","last_active":"2023-10-09T04:51:34.391991+0000","last_peered":"2023-10-09T04:51:34.391991+0000","last_clean":"2023-10-09T04:51:34.391991+0000","last_became_active":"2023-10-09T04:51:24.903201+0000","last_became_peered":"2023-10-09T04:51:24.903201+0000","last_unstale":"2023-10-09T04:51:34.391991+0000","last_undegraded":"2023-10-09T04:51:34.391991+0000","last_fullsized":"2023-10-09T04:51:34.391991+0000","mapping_epoch":340,"log_start":"351'2600","ondisk_log_start":"351'2600","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132546560,"num_objects":2022,"num_object_clones":0,"num_object_copies":6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3860,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":400,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927341+0000","last_change":"2023-10-09T04:46:25.239452+0000","last_active":"2023-10-09T04:51:33.927341+0000","last_peered":"2023-10-09T04:51:33.927341+0000","last_clean":"2023-10-09T04:51:33.927341+0000","last_became_active":"2023-10-09T04:46:25.170563+0000","last_became_peered":"2023-10-09T04:46:25.170563+0000","last_unstale":"2023-10-09T04:51:33.927341+0000","last_undegraded":"2023-10-09T04:51:33.927341+0000","last_fullsized":"2023-10-09T04:51:33.927341+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:48:08.388072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":321,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923049+0000","last_change":"2023-10-09T04:44:38.779565+0000","last_active":"2023-10-09T04:51:32.923049+0000","last_peered":"2023-10-09T04:51:32.923049+0000","last_clean":"2023-10-09T04:51:32.923049+0000","last_became_active":"2023-10-09T04:44:38.779316+0000","last_became_peered":"2023-10-09T04:44:38.779316+0000","last_unstale":"2023-10-09T04:51:32.923049+0000","last_undegraded":"2023-10-09T04:51:32.923049+0000","last_fullsized":"2023-10-09T04:51:32.923049+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:01:11.213103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"369'7509","reported_seq":7574,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.567910+0000","last_change":"2023-10-09T04:51:24.959920+0000","last_active":"2023-10-09T04:51:34.567910+0000","last_peered":"2023-10-09T04:51:34.567910+0000","last_clean":"2023-10-09T04:51:34.567910+0000","last_became_active":"2023-10-09T04:51:24.920783+0000","last_became_peered":"2023-10-09T04:51:24.920783+0000","last_unstale":"2023-10-09T04:51:34.567910+0000","last_undegraded":"2023-10-09T04:51:34.567910+0000","last_fullsized":"2023-10-09T04:51:34.567910+0000","mapping_epoch":340,"log_start":"351'2600","ondisk_log_start":"351'2600","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4909,"log_dups_size":0,"ondisk_log_size":4909,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131465216,"num_objects":2006,"num_object_clones":0,"num_object_copies":6018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3878,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":359,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.926595+0000","last_change":"2023-10-09T04:41:02.743927+0000","last_active":"2023-10-09T04:51:33.926595+0000","last_peered":"2023-10-09T04:51:33.926595+0000","last_clean":"2023-10-09T04:51:33.926595+0000","last_became_active":"2023-10-09T04:41:02.629658+0000","last_became_peered":"2023-10-09T04:41:02.629658+0000","last_unstale":"2023-10-09T04:51:33.926595+0000","last_undegraded":"2023-10-09T04:51:33.926595+0000","last_fullsized":"2023-10-09T04:51:33.926595+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:35:29.238572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":366,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924180+0000","last_change":"2023-10-09T04:46:25.178960+0000","last_active":"2023-10-09T04:51:32.924180+0000","last_peered":"2023-10-09T04:51:32.924180+0000","last_clean":"2023-10-09T04:51:32.924180+0000","last_became_active":"2023-10-09T04:46:25.178064+0000","last_became_peered":"2023-10-09T04:46:25.178064+0000","last_unstale":"2023-10-09T04:51:32.924180+0000","last_undegraded":"2023-10-09T04:51:32.924180+0000","last_fullsized":"2023-10-09T04:51:32.924180+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:10.430898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"369'7467","reported_seq":7518,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.343449+0000","last_change":"2023-10-09T04:51:24.984247+0000","last_active":"2023-10-09T04:51:34.343449+0000","last_peered":"2023-10-09T04:51:34.343449+0000","last_clean":"2023-10-09T04:51:34.343449+0000","last_became_active":"2023-10-09T04:51:24.948784+0000","last_became_peered":"2023-10-09T04:51:24.948784+0000","last_unstale":"2023-10-09T04:51:34.343449+0000","last_undegraded":"2023-10-09T04:51:34.343449+0000","last_fullsized":"2023-10-09T04:51:34.343449+0000","mapping_epoch":336,"log_start":"350'2500","ondisk_log_start":"350'2500","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127369216,"num_objects":1943,"num_object_clones":0,"num_object_copies":5829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3875,"num_write_kb":186208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":341,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.928841+0000","last_change":"2023-10-09T04:45:45.606198+0000","last_active":"2023-10-09T04:51:33.928841+0000","last_peered":"2023-10-09T04:51:33.928841+0000","last_clean":"2023-10-09T04:51:33.928841+0000","last_became_active":"2023-10-09T04:45:45.602519+0000","last_became_peered":"2023-10-09T04:45:45.602519+0000","last_unstale":"2023-10-09T04:51:33.928841+0000","last_undegraded":"2023-10-09T04:51:33.928841+0000","last_fullsized":"2023-10-09T04:51:33.928841+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:33:44.341047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.910426+0000","last_change":"2023-10-09T04:49:36.255047+0000","last_active":"2023-10-09T04:51:31.910426+0000","last_peered":"2023-10-09T04:51:31.910426+0000","last_clean":"2023-10-09T04:51:31.910426+0000","last_became_active":"2023-10-09T04:49:36.254324+0000","last_became_peered":"2023-10-09T04:49:36.254324+0000","last_unstale":"2023-10-09T04:51:31.910426+0000","last_undegraded":"2023-10-09T04:51:31.910426+0000","last_fullsized":"2023-10-09T04:51:31.910426+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:22:55.547119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"369'7573","reported_seq":7623,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.436644+0000","last_change":"2023-10-09T04:51:24.927568+0000","last_active":"2023-10-09T04:51:34.436644+0000","last_peered":"2023-10-09T04:51:34.436644+0000","last_clean":"2023-10-09T04:51:34.436644+0000","last_became_active":"2023-10-09T04:51:24.920109+0000","last_became_peered":"2023-10-09T04:51:24.920109+0000","last_unstale":"2023-10-09T04:51:34.436644+0000","last_undegraded":"2023-10-09T04:51:34.436644+0000","last_fullsized":"2023-10-09T04:51:34.436644+0000","mapping_epoch":336,"log_start":"351'2600","ondisk_log_start":"351'2600","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4973,"log_dups_size":0,"ondisk_log_size":4973,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130252800,"num_objects":1987,"num_object_clones":0,"num_object_copies":5961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3920,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":369,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924171+0000","last_change":"2023-10-09T04:49:24.728445+0000","last_active":"2023-10-09T04:51:32.924171+0000","last_peered":"2023-10-09T04:51:32.924171+0000","last_clean":"2023-10-09T04:51:32.924171+0000","last_became_active":"2023-10-09T04:49:24.728130+0000","last_became_peered":"2023-10-09T04:49:24.728130+0000","last_unstale":"2023-10-09T04:51:32.924171+0000","last_undegraded":"2023-10-09T04:51:32.924171+0000","last_fullsized":"2023-10-09T04:51:32.924171+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:04.777758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911024+0000","last_change":"2023-10-09T04:49:36.253080+0000","last_active":"2023-10-09T04:51:31.911024+0000","last_peered":"2023-10-09T04:51:31.911024+0000","last_clean":"2023-10-09T04:51:31.911024+0000","last_became_active":"2023-10-09T04:49:36.252682+0000","last_became_peered":"2023-10-09T04:49:36.252682+0000","last_unstale":"2023-10-09T04:51:31.911024+0000","last_undegraded":"2023-10-09T04:51:31.911024+0000","last_fullsized":"2023-10-09T04:51:31.911024+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:31.712210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.5","version":"369'7532","reported_seq":7582,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.403602+0000","last_change":"2023-10-09T04:51:24.956311+0000","last_active":"2023-10-09T04:51:34.403602+0000","last_peered":"2023-10-09T04:51:34.403602+0000","last_clean":"2023-10-09T04:51:34.403602+0000","last_became_active":"2023-10-09T04:51:24.942155+0000","last_became_peered":"2023-10-09T04:51:24.942155+0000","last_unstale":"2023-10-09T04:51:34.403602+0000","last_undegraded":"2023-10-09T04:51:34.403602+0000","last_fullsized":"2023-10-09T04:51:34.403602+0000","mapping_epoch":336,"log_start":"351'2600","ondisk_log_start":"351'2600","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4932,"log_dups_size":0,"ondisk_log_size":4932,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132251648,"num_objects":2018,"num_object_clones":0,"num_object_copies":6054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3904,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":108,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924853+0000","last_change":"2023-10-09T04:46:25.263244+0000","last_active":"2023-10-09T04:51:32.924853+0000","last_peered":"2023-10-09T04:51:32.924853+0000","last_clean":"2023-10-09T04:51:32.924853+0000","last_became_active":"2023-10-09T04:46:25.174616+0000","last_became_peered":"2023-10-09T04:46:25.174616+0000","last_unstale":"2023-10-09T04:51:32.924853+0000","last_undegraded":"2023-10-09T04:51:32.924853+0000","last_fullsized":"2023-10-09T04:51:32.924853+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:35:27.463644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":137,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922893+0000","last_change":"2023-10-09T04:45:45.604139+0000","last_active":"2023-10-09T04:51:32.922893+0000","last_peered":"2023-10-09T04:51:32.922893+0000","last_clean":"2023-10-09T04:51:32.922893+0000","last_became_active":"2023-10-09T04:45:45.603831+0000","last_became_peered":"2023-10-09T04:45:45.603831+0000","last_unstale":"2023-10-09T04:51:32.922893+0000","last_undegraded":"2023-10-09T04:51:32.922893+0000","last_fullsized":"2023-10-09T04:51:32.922893+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:16:26.472088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"367'7399","reported_seq":7460,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.308217+0000","last_change":"2023-10-09T04:51:24.907219+0000","last_active":"2023-10-09T04:51:32.308217+0000","last_peered":"2023-10-09T04:51:32.308217+0000","last_clean":"2023-10-09T04:51:32.308217+0000","last_became_active":"2023-10-09T04:51:24.897886+0000","last_became_peered":"2023-10-09T04:51:24.897886+0000","last_unstale":"2023-10-09T04:51:32.308217+0000","last_undegraded":"2023-10-09T04:51:32.308217+0000","last_fullsized":"2023-10-09T04:51:32.308217+0000","mapping_epoch":346,"log_start":"353'2900","ondisk_log_start":"353'2900","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4499,"log_dups_size":0,"ondisk_log_size":4499,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129761280,"num_objects":1980,"num_object_clones":0,"num_object_copies":5940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3825,"num_write_kb":185760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":330,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922495+0000","last_change":"2023-10-09T04:45:45.617184+0000","last_active":"2023-10-09T04:51:32.922495+0000","last_peered":"2023-10-09T04:51:32.922495+0000","last_clean":"2023-10-09T04:51:32.922495+0000","last_became_active":"2023-10-09T04:45:45.616914+0000","last_became_peered":"2023-10-09T04:45:45.616914+0000","last_unstale":"2023-10-09T04:51:32.922495+0000","last_undegraded":"2023-10-09T04:51:32.922495+0000","last_fullsized":"2023-10-09T04:51:32.922495+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:34:58.883882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":127,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923488+0000","last_change":"2023-10-09T04:49:24.712121+0000","last_active":"2023-10-09T04:51:32.923488+0000","last_peered":"2023-10-09T04:51:32.923488+0000","last_clean":"2023-10-09T04:51:32.923488+0000","last_became_active":"2023-10-09T04:49:24.711716+0000","last_became_peered":"2023-10-09T04:49:24.711716+0000","last_unstale":"2023-10-09T04:51:32.923488+0000","last_undegraded":"2023-10-09T04:51:32.923488+0000","last_fullsized":"2023-10-09T04:51:32.923488+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:37.141689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"369'7465","reported_seq":7518,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.360785+0000","last_change":"2023-10-09T04:51:24.932068+0000","last_active":"2023-10-09T04:51:34.360785+0000","last_peered":"2023-10-09T04:51:34.360785+0000","last_clean":"2023-10-09T04:51:34.360785+0000","last_became_active":"2023-10-09T04:51:24.919994+0000","last_became_peered":"2023-10-09T04:51:24.919994+0000","last_unstale":"2023-10-09T04:51:34.360785+0000","last_undegraded":"2023-10-09T04:51:34.360785+0000","last_fullsized":"2023-10-09T04:51:34.360785+0000","mapping_epoch":336,"log_start":"350'2500","ondisk_log_start":"350'2500","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4965,"log_dups_size":0,"ondisk_log_size":4965,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131760128,"num_objects":2010,"num_object_clones":0,"num_object_copies":6030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3858,"num_write_kb":187808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":127,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924101+0000","last_change":"2023-10-09T04:49:24.721842+0000","last_active":"2023-10-09T04:51:32.924101+0000","last_peered":"2023-10-09T04:51:32.924101+0000","last_clean":"2023-10-09T04:51:32.924101+0000","last_became_active":"2023-10-09T04:49:24.720666+0000","last_became_peered":"2023-10-09T04:49:24.720666+0000","last_unstale":"2023-10-09T04:51:32.924101+0000","last_undegraded":"2023-10-09T04:51:32.924101+0000","last_fullsized":"2023-10-09T04:51:32.924101+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:58:16.572332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":349,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923016+0000","last_change":"2023-10-09T04:45:45.600708+0000","last_active":"2023-10-09T04:51:32.923016+0000","last_peered":"2023-10-09T04:51:32.923016+0000","last_clean":"2023-10-09T04:51:32.923016+0000","last_became_active":"2023-10-09T04:45:45.600387+0000","last_became_peered":"2023-10-09T04:45:45.600387+0000","last_unstale":"2023-10-09T04:51:32.923016+0000","last_undegraded":"2023-10-09T04:51:32.923016+0000","last_fullsized":"2023-10-09T04:51:32.923016+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:57:44.255035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"369'7683","reported_seq":7745,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.405975+0000","last_change":"2023-10-09T04:51:24.917742+0000","last_active":"2023-10-09T04:51:34.405975+0000","last_peered":"2023-10-09T04:51:34.405975+0000","last_clean":"2023-10-09T04:51:34.405975+0000","last_became_active":"2023-10-09T04:51:24.910637+0000","last_became_peered":"2023-10-09T04:51:24.910637+0000","last_unstale":"2023-10-09T04:51:34.405975+0000","last_undegraded":"2023-10-09T04:51:34.405975+0000","last_fullsized":"2023-10-09T04:51:34.405975+0000","mapping_epoch":340,"log_start":"351'2700","ondisk_log_start":"351'2700","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4983,"log_dups_size":0,"ondisk_log_size":4983,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3987,"num_write_kb":193696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":121,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.912246+0000","last_change":"2023-10-09T04:49:36.253172+0000","last_active":"2023-10-09T04:51:31.912246+0000","last_peered":"2023-10-09T04:51:31.912246+0000","last_clean":"2023-10-09T04:51:31.912246+0000","last_became_active":"2023-10-09T04:49:36.252175+0000","last_became_peered":"2023-10-09T04:49:36.252175+0000","last_unstale":"2023-10-09T04:51:31.912246+0000","last_undegraded":"2023-10-09T04:51:31.912246+0000","last_fullsized":"2023-10-09T04:51:31.912246+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:38.269236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":273,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911686+0000","last_change":"2023-10-09T04:49:36.251118+0000","last_active":"2023-10-09T04:51:31.911686+0000","last_peered":"2023-10-09T04:51:31.911686+0000","last_clean":"2023-10-09T04:51:31.911686+0000","last_became_active":"2023-10-09T04:49:36.250803+0000","last_became_peered":"2023-10-09T04:49:36.250803+0000","last_unstale":"2023-10-09T04:51:31.911686+0000","last_undegraded":"2023-10-09T04:51:31.911686+0000","last_fullsized":"2023-10-09T04:51:31.911686+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:38.765478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"369'7762","reported_seq":7813,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.503704+0000","last_change":"2023-10-09T04:49:24.727773+0000","last_active":"2023-10-09T04:51:34.503704+0000","last_peered":"2023-10-09T04:51:34.503704+0000","last_clean":"2023-10-09T04:51:34.503704+0000","last_became_active":"2023-10-09T04:49:24.727411+0000","last_became_peered":"2023-10-09T04:49:24.727411+0000","last_unstale":"2023-10-09T04:51:34.503704+0000","last_undegraded":"2023-10-09T04:51:34.503704+0000","last_fullsized":"2023-10-09T04:51:34.503704+0000","mapping_epoch":340,"log_start":"355'3694","ondisk_log_start":"355'3694","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4068,"log_dups_size":0,"ondisk_log_size":4068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:29:21.626311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260440064,"num_objects":3974,"num_object_clones":0,"num_object_copies":11922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7762,"num_write_kb":375552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":262,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911649+0000","last_change":"2023-10-09T04:49:36.250912+0000","last_active":"2023-10-09T04:51:31.911649+0000","last_peered":"2023-10-09T04:51:31.911649+0000","last_clean":"2023-10-09T04:51:31.911649+0000","last_became_active":"2023-10-09T04:49:36.250530+0000","last_became_peered":"2023-10-09T04:49:36.250530+0000","last_unstale":"2023-10-09T04:51:31.911649+0000","last_undegraded":"2023-10-09T04:51:31.911649+0000","last_fullsized":"2023-10-09T04:51:31.911649+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:23:26.975687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":383,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922469+0000","last_change":"2023-10-09T04:49:24.727377+0000","last_active":"2023-10-09T04:51:32.922469+0000","last_peered":"2023-10-09T04:51:32.922469+0000","last_clean":"2023-10-09T04:51:32.922469+0000","last_became_active":"2023-10-09T04:49:24.724195+0000","last_became_peered":"2023-10-09T04:49:24.724195+0000","last_unstale":"2023-10-09T04:51:32.922469+0000","last_undegraded":"2023-10-09T04:51:32.922469+0000","last_fullsized":"2023-10-09T04:51:32.922469+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:32:46.895156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":257,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.910926+0000","last_change":"2023-10-09T04:49:36.254992+0000","last_active":"2023-10-09T04:51:31.910926+0000","last_peered":"2023-10-09T04:51:31.910926+0000","last_clean":"2023-10-09T04:51:31.910926+0000","last_became_active":"2023-10-09T04:49:36.254288+0000","last_became_peered":"2023-10-09T04:49:36.254288+0000","last_unstale":"2023-10-09T04:51:31.910926+0000","last_undegraded":"2023-10-09T04:51:31.910926+0000","last_fullsized":"2023-10-09T04:51:31.910926+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:13.355684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"369'7893","reported_seq":7935,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.543868+0000","last_change":"2023-10-09T04:49:17.684748+0000","last_active":"2023-10-09T04:51:34.543868+0000","last_peered":"2023-10-09T04:51:34.543868+0000","last_clean":"2023-10-09T04:51:34.543868+0000","last_became_active":"2023-10-09T04:49:17.684346+0000","last_became_peered":"2023-10-09T04:49:17.684346+0000","last_unstale":"2023-10-09T04:51:34.543868+0000","last_undegraded":"2023-10-09T04:51:34.543868+0000","last_fullsized":"2023-10-09T04:51:34.543868+0000","mapping_epoch":336,"log_start":"355'3773","ondisk_log_start":"355'3773","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4120,"log_dups_size":0,"ondisk_log_size":4120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:47:39.897521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605080,"num_objects":3947,"num_object_clones":0,"num_object_copies":11841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3947,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7893,"num_write_kb":378817,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":353,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.921779+0000","last_change":"2023-10-09T04:46:25.171158+0000","last_active":"2023-10-09T04:51:32.921779+0000","last_peered":"2023-10-09T04:51:32.921779+0000","last_clean":"2023-10-09T04:51:32.921779+0000","last_became_active":"2023-10-09T04:46:25.170888+0000","last_became_peered":"2023-10-09T04:46:25.170888+0000","last_unstale":"2023-10-09T04:51:32.921779+0000","last_undegraded":"2023-10-09T04:51:32.921779+0000","last_fullsized":"2023-10-09T04:51:32.921779+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:39:48.638953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":137,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922009+0000","last_change":"2023-10-09T04:46:25.172975+0000","last_active":"2023-10-09T04:51:32.922009+0000","last_peered":"2023-10-09T04:51:32.922009+0000","last_clean":"2023-10-09T04:51:32.922009+0000","last_became_active":"2023-10-09T04:46:25.172192+0000","last_became_peered":"2023-10-09T04:46:25.172192+0000","last_unstale":"2023-10-09T04:51:32.922009+0000","last_undegraded":"2023-10-09T04:51:32.922009+0000","last_fullsized":"2023-10-09T04:51:32.922009+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:46.256423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":311,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911604+0000","last_change":"2023-10-09T04:49:36.254929+0000","last_active":"2023-10-09T04:51:31.911604+0000","last_peered":"2023-10-09T04:51:31.911604+0000","last_clean":"2023-10-09T04:51:31.911604+0000","last_became_active":"2023-10-09T04:49:36.252868+0000","last_became_peered":"2023-10-09T04:49:36.252868+0000","last_unstale":"2023-10-09T04:51:31.911604+0000","last_undegraded":"2023-10-09T04:51:31.911604+0000","last_fullsized":"2023-10-09T04:51:31.911604+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:53:26.997588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"369'7950","reported_seq":7992,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.493790+0000","last_change":"2023-10-09T04:49:17.686557+0000","last_active":"2023-10-09T04:51:34.493790+0000","last_peered":"2023-10-09T04:51:34.493790+0000","last_clean":"2023-10-09T04:51:34.493790+0000","last_became_active":"2023-10-09T04:49:17.686236+0000","last_became_peered":"2023-10-09T04:49:17.686236+0000","last_unstale":"2023-10-09T04:51:34.493790+0000","last_undegraded":"2023-10-09T04:51:34.493790+0000","last_fullsized":"2023-10-09T04:51:34.493790+0000","mapping_epoch":336,"log_start":"355'3762","ondisk_log_start":"355'3762","created":336,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4188,"log_dups_size":0,"ondisk_log_size":4188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:46:02.935353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262537216,"num_objects":4006,"num_object_clones":0,"num_object_copies":12018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7950,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":312,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911741+0000","last_change":"2023-10-09T04:49:36.250480+0000","last_active":"2023-10-09T04:51:31.911741+0000","last_peered":"2023-10-09T04:51:31.911741+0000","last_clean":"2023-10-09T04:51:31.911741+0000","last_became_active":"2023-10-09T04:49:36.250048+0000","last_became_peered":"2023-10-09T04:49:36.250048+0000","last_unstale":"2023-10-09T04:51:31.911741+0000","last_undegraded":"2023-10-09T04:51:31.911741+0000","last_fullsized":"2023-10-09T04:51:31.911741+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:21.602101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":311,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.912331+0000","last_change":"2023-10-09T04:49:36.253118+0000","last_active":"2023-10-09T04:51:31.912331+0000","last_peered":"2023-10-09T04:51:31.912331+0000","last_clean":"2023-10-09T04:51:31.912331+0000","last_became_active":"2023-10-09T04:49:36.252036+0000","last_became_peered":"2023-10-09T04:49:36.252036+0000","last_unstale":"2023-10-09T04:51:31.912331+0000","last_undegraded":"2023-10-09T04:51:31.912331+0000","last_fullsized":"2023-10-09T04:51:31.912331+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:50.648245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":253,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911586+0000","last_change":"2023-10-09T04:49:36.252116+0000","last_active":"2023-10-09T04:51:31.911586+0000","last_peered":"2023-10-09T04:51:31.911586+0000","last_clean":"2023-10-09T04:51:31.911586+0000","last_became_active":"2023-10-09T04:49:36.251630+0000","last_became_peered":"2023-10-09T04:49:36.251630+0000","last_unstale":"2023-10-09T04:51:31.911586+0000","last_undegraded":"2023-10-09T04:51:31.911586+0000","last_fullsized":"2023-10-09T04:51:31.911586+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:28:48.681072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"367'7787","reported_seq":7849,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.318435+0000","last_change":"2023-10-09T04:49:36.276163+0000","last_active":"2023-10-09T04:51:32.318435+0000","last_peered":"2023-10-09T04:51:32.318435+0000","last_clean":"2023-10-09T04:51:32.318435+0000","last_became_active":"2023-10-09T04:49:36.253314+0000","last_became_peered":"2023-10-09T04:49:36.253314+0000","last_unstale":"2023-10-09T04:51:32.318435+0000","last_undegraded":"2023-10-09T04:51:32.318435+0000","last_fullsized":"2023-10-09T04:51:32.318435+0000","mapping_epoch":346,"log_start":"355'3868","ondisk_log_start":"355'3868","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3919,"log_dups_size":0,"ondisk_log_size":3919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:21:44.334582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264699904,"num_objects":4039,"num_object_clones":0,"num_object_copies":12117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7787,"num_write_kb":378432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":360,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923222+0000","last_change":"2023-10-09T04:49:24.727573+0000","last_active":"2023-10-09T04:51:32.923222+0000","last_peered":"2023-10-09T04:51:32.923222+0000","last_clean":"2023-10-09T04:51:32.923222+0000","last_became_active":"2023-10-09T04:49:24.727151+0000","last_became_peered":"2023-10-09T04:49:24.727151+0000","last_unstale":"2023-10-09T04:51:32.923222+0000","last_undegraded":"2023-10-09T04:51:32.923222+0000","last_fullsized":"2023-10-09T04:51:32.923222+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:53.790164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":840,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927849+0000","last_change":"2023-10-09T04:49:27.734937+0000","last_active":"2023-10-09T04:51:33.927849+0000","last_peered":"2023-10-09T04:51:33.927849+0000","last_clean":"2023-10-09T04:51:33.927849+0000","last_became_active":"2023-10-09T04:49:27.734587+0000","last_became_peered":"2023-10-09T04:49:27.734587+0000","last_unstale":"2023-10-09T04:51:33.927849+0000","last_undegraded":"2023-10-09T04:51:33.927849+0000","last_fullsized":"2023-10-09T04:51:33.927849+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:38.000046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":299,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922056+0000","last_change":"2023-10-09T04:45:45.626127+0000","last_active":"2023-10-09T04:51:32.922056+0000","last_peered":"2023-10-09T04:51:32.922056+0000","last_clean":"2023-10-09T04:51:32.922056+0000","last_became_active":"2023-10-09T04:45:45.625780+0000","last_became_peered":"2023-10-09T04:45:45.625780+0000","last_unstale":"2023-10-09T04:51:32.922056+0000","last_undegraded":"2023-10-09T04:51:32.922056+0000","last_fullsized":"2023-10-09T04:51:32.922056+0000","mapping_epoch":302,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:18.939556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":312,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923409+0000","last_change":"2023-10-09T04:46:25.172058+0000","last_active":"2023-10-09T04:51:32.923409+0000","last_peered":"2023-10-09T04:51:32.923409+0000","last_clean":"2023-10-09T04:51:32.923409+0000","last_became_active":"2023-10-09T04:46:25.171749+0000","last_became_peered":"2023-10-09T04:46:25.171749+0000","last_unstale":"2023-10-09T04:51:32.923409+0000","last_undegraded":"2023-10-09T04:51:32.923409+0000","last_fullsized":"2023-10-09T04:51:32.923409+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:08:07.864030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"369'7854","reported_seq":7906,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.533021+0000","last_change":"2023-10-09T04:49:24.733991+0000","last_active":"2023-10-09T04:51:34.533021+0000","last_peered":"2023-10-09T04:51:34.533021+0000","last_clean":"2023-10-09T04:51:34.533021+0000","last_became_active":"2023-10-09T04:49:24.733661+0000","last_became_peered":"2023-10-09T04:49:24.733661+0000","last_unstale":"2023-10-09T04:51:34.533021+0000","last_undegraded":"2023-10-09T04:51:34.533021+0000","last_fullsized":"2023-10-09T04:51:34.533021+0000","mapping_epoch":340,"log_start":"355'3641","ondisk_log_start":"355'3641","created":336,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4213,"log_dups_size":0,"ondisk_log_size":4213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:22.030285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262275072,"num_objects":4002,"num_object_clones":0,"num_object_copies":12006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7854,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":380,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927788+0000","last_change":"2023-10-09T04:46:25.171269+0000","last_active":"2023-10-09T04:51:33.927788+0000","last_peered":"2023-10-09T04:51:33.927788+0000","last_clean":"2023-10-09T04:51:33.927788+0000","last_became_active":"2023-10-09T04:46:25.170213+0000","last_became_peered":"2023-10-09T04:46:25.170213+0000","last_unstale":"2023-10-09T04:51:33.927788+0000","last_undegraded":"2023-10-09T04:51:33.927788+0000","last_fullsized":"2023-10-09T04:51:33.927788+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:55:39.464606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":137,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924144+0000","last_change":"2023-10-09T04:46:25.179133+0000","last_active":"2023-10-09T04:51:32.924144+0000","last_peered":"2023-10-09T04:51:32.924144+0000","last_clean":"2023-10-09T04:51:32.924144+0000","last_became_active":"2023-10-09T04:46:25.178290+0000","last_became_peered":"2023-10-09T04:46:25.178290+0000","last_unstale":"2023-10-09T04:51:32.924144+0000","last_undegraded":"2023-10-09T04:51:32.924144+0000","last_fullsized":"2023-10-09T04:51:32.924144+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:02.166116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":138,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911367+0000","last_change":"2023-10-09T04:49:36.252316+0000","last_active":"2023-10-09T04:51:31.911367+0000","last_peered":"2023-10-09T04:51:31.911367+0000","last_clean":"2023-10-09T04:51:31.911367+0000","last_became_active":"2023-10-09T04:49:36.251771+0000","last_became_peered":"2023-10-09T04:49:36.251771+0000","last_unstale":"2023-10-09T04:51:31.911367+0000","last_undegraded":"2023-10-09T04:51:31.911367+0000","last_fullsized":"2023-10-09T04:51:31.911367+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:43:08.690791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"367'7759","reported_seq":7808,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.340550+0000","last_change":"2023-10-09T04:49:36.254761+0000","last_active":"2023-10-09T04:51:32.340550+0000","last_peered":"2023-10-09T04:51:32.340550+0000","last_clean":"2023-10-09T04:51:32.340550+0000","last_became_active":"2023-10-09T04:49:36.250489+0000","last_became_peered":"2023-10-09T04:49:36.250489+0000","last_unstale":"2023-10-09T04:51:32.340550+0000","last_undegraded":"2023-10-09T04:51:32.340550+0000","last_fullsized":"2023-10-09T04:51:32.340550+0000","mapping_epoch":346,"log_start":"355'3880","ondisk_log_start":"355'3880","created":336,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":3879,"log_dups_size":0,"ondisk_log_size":3879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:24:19.664434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259063808,"num_objects":3953,"num_object_clones":0,"num_object_copies":11859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7759,"num_write_kb":374784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":383,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.928917+0000","last_change":"2023-10-09T04:46:25.168614+0000","last_active":"2023-10-09T04:51:33.928917+0000","last_peered":"2023-10-09T04:51:33.928917+0000","last_clean":"2023-10-09T04:51:33.928917+0000","last_became_active":"2023-10-09T04:46:25.168299+0000","last_became_peered":"2023-10-09T04:46:25.168299+0000","last_unstale":"2023-10-09T04:51:33.928917+0000","last_undegraded":"2023-10-09T04:51:33.928917+0000","last_fullsized":"2023-10-09T04:51:33.928917+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:21.843942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":264,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911553+0000","last_change":"2023-10-09T04:49:36.254730+0000","last_active":"2023-10-09T04:51:31.911553+0000","last_peered":"2023-10-09T04:51:31.911553+0000","last_clean":"2023-10-09T04:51:31.911553+0000","last_became_active":"2023-10-09T04:49:36.252242+0000","last_became_peered":"2023-10-09T04:49:36.252242+0000","last_unstale":"2023-10-09T04:51:31.911553+0000","last_undegraded":"2023-10-09T04:51:31.911553+0000","last_fullsized":"2023-10-09T04:51:31.911553+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:51.106203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":223,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923083+0000","last_change":"2023-10-09T04:46:25.175422+0000","last_active":"2023-10-09T04:51:32.923083+0000","last_peered":"2023-10-09T04:51:32.923083+0000","last_clean":"2023-10-09T04:51:32.923083+0000","last_became_active":"2023-10-09T04:46:25.175142+0000","last_became_peered":"2023-10-09T04:46:25.175142+0000","last_unstale":"2023-10-09T04:51:32.923083+0000","last_undegraded":"2023-10-09T04:51:32.923083+0000","last_fullsized":"2023-10-09T04:51:32.923083+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:31:35.761821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"369'7380","reported_seq":7442,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.448776+0000","last_change":"2023-10-09T04:51:24.915008+0000","last_active":"2023-10-09T04:51:34.448776+0000","last_peered":"2023-10-09T04:51:34.448776+0000","last_clean":"2023-10-09T04:51:34.448776+0000","last_became_active":"2023-10-09T04:51:24.908425+0000","last_became_peered":"2023-10-09T04:51:24.908425+0000","last_unstale":"2023-10-09T04:51:34.448776+0000","last_undegraded":"2023-10-09T04:51:34.448776+0000","last_fullsized":"2023-10-09T04:51:34.448776+0000","mapping_epoch":340,"log_start":"350'2400","ondisk_log_start":"350'2400","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4980,"log_dups_size":0,"ondisk_log_size":4980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128286720,"num_objects":1957,"num_object_clones":0,"num_object_copies":5871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3824,"num_write_kb":185024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":118,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927624+0000","last_change":"2023-10-09T04:46:25.213013+0000","last_active":"2023-10-09T04:51:33.927624+0000","last_peered":"2023-10-09T04:51:33.927624+0000","last_clean":"2023-10-09T04:51:33.927624+0000","last_became_active":"2023-10-09T04:46:25.167221+0000","last_became_peered":"2023-10-09T04:46:25.167221+0000","last_unstale":"2023-10-09T04:51:33.927624+0000","last_undegraded":"2023-10-09T04:51:33.927624+0000","last_fullsized":"2023-10-09T04:51:33.927624+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:32.939088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":338,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911030+0000","last_change":"2023-10-09T04:49:36.251899+0000","last_active":"2023-10-09T04:51:31.911030+0000","last_peered":"2023-10-09T04:51:31.911030+0000","last_clean":"2023-10-09T04:51:31.911030+0000","last_became_active":"2023-10-09T04:49:36.251283+0000","last_became_peered":"2023-10-09T04:49:36.251283+0000","last_unstale":"2023-10-09T04:51:31.911030+0000","last_undegraded":"2023-10-09T04:51:31.911030+0000","last_fullsized":"2023-10-09T04:51:31.911030+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:47:52.505049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927968+0000","last_change":"2023-10-09T04:46:25.170212+0000","last_active":"2023-10-09T04:51:33.927968+0000","last_peered":"2023-10-09T04:51:33.927968+0000","last_clean":"2023-10-09T04:51:33.927968+0000","last_became_active":"2023-10-09T04:46:25.169817+0000","last_became_peered":"2023-10-09T04:46:25.169817+0000","last_unstale":"2023-10-09T04:51:33.927968+0000","last_undegraded":"2023-10-09T04:51:33.927968+0000","last_fullsized":"2023-10-09T04:51:33.927968+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:42.257693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.8","version":"369'7501","reported_seq":7554,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.438872+0000","last_change":"2023-10-09T04:51:24.910232+0000","last_active":"2023-10-09T04:51:34.438872+0000","last_peered":"2023-10-09T04:51:34.438872+0000","last_clean":"2023-10-09T04:51:34.438872+0000","last_became_active":"2023-10-09T04:51:24.902820+0000","last_became_peered":"2023-10-09T04:51:24.902820+0000","last_unstale":"2023-10-09T04:51:34.438872+0000","last_undegraded":"2023-10-09T04:51:34.438872+0000","last_fullsized":"2023-10-09T04:51:34.438872+0000","mapping_epoch":340,"log_start":"351'2600","ondisk_log_start":"351'2600","created":336,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4901,"log_dups_size":0,"ondisk_log_size":4901,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129957888,"num_objects":1983,"num_object_clones":0,"num_object_copies":5949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3887,"num_write_kb":187840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":302,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:33.927248+0000","last_change":"2023-10-09T04:49:24.730477+0000","last_active":"2023-10-09T04:51:33.927248+0000","last_peered":"2023-10-09T04:51:33.927248+0000","last_clean":"2023-10-09T04:51:33.927248+0000","last_became_active":"2023-10-09T04:49:24.730115+0000","last_became_peered":"2023-10-09T04:49:24.730115+0000","last_unstale":"2023-10-09T04:51:33.927248+0000","last_undegraded":"2023-10-09T04:51:33.927248+0000","last_fullsized":"2023-10-09T04:51:33.927248+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:55:49.730424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":137,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911192+0000","last_change":"2023-10-09T04:49:36.250949+0000","last_active":"2023-10-09T04:51:31.911192+0000","last_peered":"2023-10-09T04:51:31.911192+0000","last_clean":"2023-10-09T04:51:31.911192+0000","last_became_active":"2023-10-09T04:49:36.250519+0000","last_became_peered":"2023-10-09T04:49:36.250519+0000","last_unstale":"2023-10-09T04:51:31.911192+0000","last_undegraded":"2023-10-09T04:51:31.911192+0000","last_fullsized":"2023-10-09T04:51:31.911192+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:51.821472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":137,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924016+0000","last_change":"2023-10-09T04:46:25.179307+0000","last_active":"2023-10-09T04:51:32.924016+0000","last_peered":"2023-10-09T04:51:32.924016+0000","last_clean":"2023-10-09T04:51:32.924016+0000","last_became_active":"2023-10-09T04:46:25.178568+0000","last_became_peered":"2023-10-09T04:46:25.178568+0000","last_unstale":"2023-10-09T04:51:32.924016+0000","last_undegraded":"2023-10-09T04:51:32.924016+0000","last_fullsized":"2023-10-09T04:51:32.924016+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:40.189934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.15","version":"369'7527","reported_seq":7581,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.416206+0000","last_change":"2023-10-09T04:51:24.983634+0000","last_active":"2023-10-09T04:51:34.416206+0000","last_peered":"2023-10-09T04:51:34.416206+0000","last_clean":"2023-10-09T04:51:34.416206+0000","last_became_active":"2023-10-09T04:51:24.961040+0000","last_became_peered":"2023-10-09T04:51:24.961040+0000","last_unstale":"2023-10-09T04:51:34.416206+0000","last_undegraded":"2023-10-09T04:51:34.416206+0000","last_fullsized":"2023-10-09T04:51:34.416206+0000","mapping_epoch":365,"log_start":"351'2600","ondisk_log_start":"351'2600","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3899,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":219,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924618+0000","last_change":"2023-10-09T04:46:25.172825+0000","last_active":"2023-10-09T04:51:32.924618+0000","last_peered":"2023-10-09T04:51:32.924618+0000","last_clean":"2023-10-09T04:51:32.924618+0000","last_became_active":"2023-10-09T04:46:25.171823+0000","last_became_peered":"2023-10-09T04:46:25.171823+0000","last_unstale":"2023-10-09T04:51:32.924618+0000","last_undegraded":"2023-10-09T04:51:32.924618+0000","last_fullsized":"2023-10-09T04:51:32.924618+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.467407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":304,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.911463+0000","last_change":"2023-10-09T04:49:36.254629+0000","last_active":"2023-10-09T04:51:31.911463+0000","last_peered":"2023-10-09T04:51:31.911463+0000","last_clean":"2023-10-09T04:51:31.911463+0000","last_became_active":"2023-10-09T04:49:36.251774+0000","last_became_peered":"2023-10-09T04:49:36.251774+0000","last_unstale":"2023-10-09T04:51:31.911463+0000","last_undegraded":"2023-10-09T04:51:31.911463+0000","last_fullsized":"2023-10-09T04:51:31.911463+0000","mapping_epoch":346,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":347,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:24.682458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.12","version":"369'7560","reported_seq":7609,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.495754+0000","last_change":"2023-10-09T04:51:24.968400+0000","last_active":"2023-10-09T04:51:34.495754+0000","last_peered":"2023-10-09T04:51:34.495754+0000","last_clean":"2023-10-09T04:51:34.495754+0000","last_became_active":"2023-10-09T04:51:24.956942+0000","last_became_peered":"2023-10-09T04:51:24.956942+0000","last_unstale":"2023-10-09T04:51:34.495754+0000","last_undegraded":"2023-10-09T04:51:34.495754+0000","last_fullsized":"2023-10-09T04:51:34.495754+0000","mapping_epoch":365,"log_start":"351'2600","ondisk_log_start":"351'2600","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4960,"log_dups_size":0,"ondisk_log_size":4960,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131104768,"num_objects":2001,"num_object_clones":0,"num_object_copies":6003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3908,"num_write_kb":189056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":359,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924313+0000","last_change":"2023-10-09T04:41:02.741535+0000","last_active":"2023-10-09T04:51:32.924313+0000","last_peered":"2023-10-09T04:51:32.924313+0000","last_clean":"2023-10-09T04:51:32.924313+0000","last_became_active":"2023-10-09T04:41:02.625484+0000","last_became_peered":"2023-10-09T04:41:02.625484+0000","last_unstale":"2023-10-09T04:51:32.924313+0000","last_undegraded":"2023-10-09T04:51:32.924313+0000","last_fullsized":"2023-10-09T04:51:32.924313+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:58:46.265542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":137,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924478+0000","last_change":"2023-10-09T04:49:24.721368+0000","last_active":"2023-10-09T04:51:32.924478+0000","last_peered":"2023-10-09T04:51:32.924478+0000","last_clean":"2023-10-09T04:51:32.924478+0000","last_became_active":"2023-10-09T04:49:24.721024+0000","last_became_peered":"2023-10-09T04:49:24.721024+0000","last_unstale":"2023-10-09T04:51:32.924478+0000","last_undegraded":"2023-10-09T04:51:32.924478+0000","last_fullsized":"2023-10-09T04:51:32.924478+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:33:57.736030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.13","version":"369'7462","reported_seq":7516,"reported_epoch":369,"state":"active+clean","last_fresh":"2023-10-09T04:51:34.299651+0000","last_change":"2023-10-09T04:51:25.010939+0000","last_active":"2023-10-09T04:51:34.299651+0000","last_peered":"2023-10-09T04:51:34.299651+0000","last_clean":"2023-10-09T04:51:34.299651+0000","last_became_active":"2023-10-09T04:51:24.989669+0000","last_became_peered":"2023-10-09T04:51:24.989669+0000","last_unstale":"2023-10-09T04:51:34.299651+0000","last_undegraded":"2023-10-09T04:51:34.299651+0000","last_fullsized":"2023-10-09T04:51:34.299651+0000","mapping_epoch":365,"log_start":"350'2500","ondisk_log_start":"350'2500","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4962,"log_dups_size":0,"ondisk_log_size":4962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127696896,"num_objects":1949,"num_object_clones":0,"num_object_copies":5847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3871,"num_write_kb":186208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":328,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922387+0000","last_change":"2023-10-09T04:44:38.814266+0000","last_active":"2023-10-09T04:51:32.922387+0000","last_peered":"2023-10-09T04:51:32.922387+0000","last_clean":"2023-10-09T04:51:32.922387+0000","last_became_active":"2023-10-09T04:44:38.783436+0000","last_became_peered":"2023-10-09T04:44:38.783436+0000","last_unstale":"2023-10-09T04:51:32.922387+0000","last_undegraded":"2023-10-09T04:51:32.922387+0000","last_fullsized":"2023-10-09T04:51:32.922387+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:22:46.708060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":147,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922579+0000","last_change":"2023-10-09T04:49:24.711760+0000","last_active":"2023-10-09T04:51:32.922579+0000","last_peered":"2023-10-09T04:51:32.922579+0000","last_clean":"2023-10-09T04:51:32.922579+0000","last_became_active":"2023-10-09T04:49:24.711383+0000","last_became_peered":"2023-10-09T04:49:24.711383+0000","last_unstale":"2023-10-09T04:51:32.922579+0000","last_undegraded":"2023-10-09T04:51:32.922579+0000","last_fullsized":"2023-10-09T04:51:32.922579+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:47:04.370272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.10","version":"366'7499","reported_seq":7561,"reported_epoch":368,"state":"remapped+peering","last_fresh":"2023-10-09T04:51:32.915167+0000","last_change":"2023-10-09T04:51:32.915167+0000","last_active":"2023-10-09T04:51:31.911755+0000","last_peered":"2023-10-09T04:51:31.889039+0000","last_clean":"2023-10-09T04:51:31.889039+0000","last_became_active":"2023-10-09T04:51:24.964592+0000","last_became_peered":"2023-10-09T04:51:24.964592+0000","last_unstale":"2023-10-09T04:51:32.915167+0000","last_undegraded":"2023-10-09T04:51:32.915167+0000","last_fullsized":"2023-10-09T04:51:32.915167+0000","mapping_epoch":368,"log_start":"351'2600","ondisk_log_start":"351'2600","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132120576,"num_objects":2016,"num_object_clones":0,"num_object_copies":6048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3868,"num_write_kb":188288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":322,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-10-09T04:51:31.910880+0000","last_change":"2023-10-09T04:49:24.725553+0000","last_active":"2023-10-09T04:51:31.910880+0000","last_peered":"2023-10-09T04:51:31.910880+0000","last_clean":"2023-10-09T04:51:31.910880+0000","last_became_active":"2023-10-09T04:49:24.725179+0000","last_became_peered":"2023-10-09T04:49:24.725179+0000","last_unstale":"2023-10-09T04:51:31.910880+0000","last_undegraded":"2023-10-09T04:51:31.910880+0000","last_fullsized":"2023-10-09T04:51:31.910880+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:32.045784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.923707+0000","last_change":"2023-10-09T04:49:24.718316+0000","last_active":"2023-10-09T04:51:32.923707+0000","last_peered":"2023-10-09T04:51:32.923707+0000","last_clean":"2023-10-09T04:51:32.923707+0000","last_became_active":"2023-10-09T04:49:24.717948+0000","last_became_peered":"2023-10-09T04:49:24.717948+0000","last_unstale":"2023-10-09T04:51:32.923707+0000","last_undegraded":"2023-10-09T04:51:32.923707+0000","last_fullsized":"2023-10-09T04:51:32.923707+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:56:48.454201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.11","version":"366'7392","reported_seq":7441,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-10-09T04:51:29.743502+0000","last_change":"2023-10-09T04:51:24.973604+0000","last_active":"2023-10-09T04:51:29.743502+0000","last_peered":"2023-10-09T04:51:29.743502+0000","last_clean":"2023-10-09T04:51:29.743502+0000","last_became_active":"2023-10-09T04:51:24.964954+0000","last_became_peered":"2023-10-09T04:51:24.964954+0000","last_unstale":"2023-10-09T04:51:29.743502+0000","last_undegraded":"2023-10-09T04:51:29.743502+0000","last_fullsized":"2023-10-09T04:51:29.743502+0000","mapping_epoch":365,"log_start":"351'2600","ondisk_log_start":"351'2600","created":365,"last_epoch_clean":366,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:49:16.656784+0000","last_clean_scrub_stamp":"2023-10-09T04:49:16.656784+0000","objects_scrubbed":0,"log_size":4792,"log_dups_size":0,"ondisk_log_size":4792,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138444800,"num_objects":2113,"num_object_clones":0,"num_object_copies":6339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3771,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":275,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.924650+0000","last_change":"2023-10-09T04:49:24.745757+0000","last_active":"2023-10-09T04:51:32.924650+0000","last_peered":"2023-10-09T04:51:32.924650+0000","last_clean":"2023-10-09T04:51:32.924650+0000","last_became_active":"2023-10-09T04:49:24.727010+0000","last_became_peered":"2023-10-09T04:49:24.727010+0000","last_unstale":"2023-10-09T04:51:32.924650+0000","last_undegraded":"2023-10-09T04:51:32.924650+0000","last_fullsized":"2023-10-09T04:51:32.924650+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:35:51.654447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":147,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-10-09T04:51:32.922397+0000","last_change":"2023-10-09T04:49:24.710860+0000","last_active":"2023-10-09T04:51:32.922397+0000","last_peered":"2023-10-09T04:51:32.922397+0000","last_clean":"2023-10-09T04:51:32.922397+0000","last_became_active":"2023-10-09T04:49:24.710496+0000","last_became_peered":"2023-10-09T04:49:24.710496+0000","last_unstale":"2023-10-09T04:51:32.922397+0000","last_undegraded":"2023-10-09T04:51:32.922397+0000","last_fullsized":"2023-10-09T04:51:32.922397+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:01:24.901415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":26,"stat_sum":{"num_bytes":4193779736,"num_objects":63993,"num_object_clones":0,"num_object_copies":191979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63993,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124523,"num_write_kb":6032449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12578664448,"data_stored":12578652232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122246,"ondisk_log_size":122246,"up":78,"acting":78,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16666,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":22750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234536,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4590880,"kb_used_data":3073432,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":89154272,"statfs":{"total":95995035648,"available":91293974528,"internally_reserved":0,"allocated":3147194368,"data_stored":3146435961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":1.3540000000000001,"15min":1.746},"min":{"1min":0.36099999999999999,"5min":0.224,"15min":0.21099999999999999},"max":{"1min":5.181,"5min":14.970000000000001,"15min":44.122999999999998},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.1429999999999998,"15min":2.302},"min":{"1min":0.377,"5min":0.27700000000000002,"15min":0.249},"max":{"1min":7.96,"5min":19.152000000000001,"15min":43.313000000000002},"last":0.35199999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:50:50 2023","interfaces":[{"interface":"back","average":{"1min":12.518000000000001,"5min":5.5099999999999998,"15min":4.79},"min":{"1min":0.27400000000000002,"5min":0.246,"15min":0.246},"max":{"1min":122.56,"5min":128.601,"15min":264.62},"last":0.47699999999999998},{"interface":"front","average":{"1min":12.481,"5min":5.4219999999999997,"15min":4.8360000000000003},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":122.666,"5min":128.381,"15min":264.65899999999999},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":6.2679999999999998,"15min":5.2489999999999997},"min":{"1min":0.372,"5min":0.29499999999999998,"15min":0.27100000000000002},"max":{"1min":9.0090000000000003,"5min":129.10400000000001,"15min":184.72},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.0979999999999999,"5min":6.5119999999999996,"15min":5.5339999999999998},"min":{"1min":0.39800000000000002,"5min":0.34399999999999997,"15min":0.28100000000000003},"max":{"1min":7.9939999999999998,"5min":129.15100000000001,"15min":184.697},"last":0.53600000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:51:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":2.0350000000000001,"15min":3.5619999999999998},"min":{"1min":0.40899999999999997,"5min":0.34899999999999998,"15min":0.29499999999999998},"max":{"1min":6.0339999999999998,"5min":12.324,"15min":182.839},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.823,"5min":2.4319999999999999,"15min":4.0529999999999999},"min":{"1min":0.40999999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":21.295999999999999,"5min":21.295999999999999,"15min":182.678},"last":0.45700000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:51:21 2023","interfaces":[{"interface":"back","average":{"1min":3.0859999999999999,"5min":2.8959999999999999,"15min":1.829},"min":{"1min":0.33900000000000002,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":10.49,"5min":14.811999999999999,"15min":14.811999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.7000000000000002,"5min":2.0950000000000002,"15min":1.4119999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.21199999999999999},"max":{"1min":9.3729999999999993,"5min":13.523,"15min":13.523},"last":0.28199999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:51:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.7170000000000001,"15min":1.335},"min":{"1min":0.316,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.7060000000000004,"5min":12.202999999999999,"15min":13.319000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":4.8479999999999999,"5min":2.8940000000000001,"15min":1.994},"min":{"1min":0.42799999999999999,"5min":0.33400000000000002,"15min":0.26000000000000001},"max":{"1min":13.673,"5min":13.673,"15min":18.408000000000001},"last":0.311}]},{"osd":6,"last update":"Mon Oct 9 04:51:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.5680000000000001,"15min":1.2969999999999999},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32200000000000001},"max":{"1min":6.4009999999999998,"5min":10.047000000000001,"15min":11.388},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.982,"5min":2.117,"15min":1.784},"min":{"1min":0.44900000000000001,"5min":0.34499999999999997,"15min":0.30099999999999999},"max":{"1min":8.5069999999999997,"5min":11.680999999999999,"15min":18.311},"last":0.51000000000000001}]}]},{"osd":6,"up_from":0,"seq":197568496170,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":118,"seq":506806141259,"num_pgs":78,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3535576,"kb_used_data":2564624,"kb_used_omap":4,"kb_used_meta":970939,"kb_avail":90209576,"statfs":{"total":95995035648,"available":92374605824,"internally_reserved":0,"allocated":2626174976,"data_stored":2624529329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":994242446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,10,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.1279999999999999,"15min":2.6139999999999999},"min":{"1min":0.23200000000000001,"5min":0.20799999999999999,"15min":0.14799999999999999},"max":{"1min":2.8660000000000001,"5min":9.0229999999999997,"15min":93.171999999999997},"last":0.442},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.254,"15min":2.6619999999999999},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.193},"max":{"1min":3.524,"5min":8.8190000000000008,"15min":93.147000000000006},"last":0.47399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:51:12 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":4.6230000000000002,"15min":4.6840000000000002},"min":{"1min":0.26700000000000002,"5min":0.223,"15min":0.20899999999999999},"max":{"1min":2.786,"5min":151.41900000000001,"15min":263.60500000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.753,"5min":4.5970000000000004,"15min":5.0140000000000002},"min":{"1min":0.26900000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.194,"5min":146.749,"15min":263.65199999999999},"last":0.57899999999999996}]},{"osd":3,"last update":"Mon Oct 9 04:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.224,"15min":5.3040000000000003},"min":{"1min":0.27500000000000002,"5min":0.26500000000000001,"15min":0.23000000000000001},"max":{"1min":2.6669999999999998,"5min":9.0709999999999997,"15min":379.80900000000003},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":1.738,"15min":5.8440000000000003},"min":{"1min":0.35899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":1.2370000000000001,"5min":12.615,"15min":379.95800000000003},"last":0.52100000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":5.1600000000000001,"5min":2.7679999999999998,"15min":2.879},"min":{"1min":0.31,"5min":0.28199999999999997,"15min":0.27100000000000002},"max":{"1min":65.036000000000001,"5min":65.036000000000001,"15min":170.072},"last":0.626},{"interface":"front","average":{"1min":5.6689999999999996,"5min":2.7719999999999998,"15min":2.7730000000000001},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":62.585000000000001,"5min":62.585000000000001,"15min":169.94499999999999},"last":0.56799999999999995}]},{"osd":5,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":2.0619999999999998,"15min":2.4790000000000001},"min":{"1min":0.47199999999999998,"5min":0.34200000000000003,"15min":0.28299999999999997},"max":{"1min":4.4100000000000001,"5min":9.782,"15min":171.036},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.274,"5min":2.0619999999999998,"15min":2.7320000000000002},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.30499999999999999},"max":{"1min":3.8340000000000001,"5min":13.728,"15min":170.88},"last":0.59699999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.883,"15min":2.3809999999999998},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.32000000000000001},"max":{"1min":3.145,"5min":9.7840000000000007,"15min":132.36099999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.294,"5min":2.1600000000000001,"15min":2.7280000000000002},"min":{"1min":0.39700000000000002,"5min":0.32400000000000001,"15min":0.317},"max":{"1min":3.0449999999999999,"5min":13.686999999999999,"15min":132.17699999999999},"last":0.64800000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:51:03 2023","interfaces":[{"interface":"back","average":{"1min":5.2480000000000002,"5min":3.2679999999999998,"15min":3.6560000000000001},"min":{"1min":0.54400000000000004,"5min":0.35899999999999999,"15min":0.34100000000000003},"max":{"1min":59.972000000000001,"5min":59.972000000000001,"15min":174.75999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":5.3799999999999999,"5min":3.0049999999999999,"15min":3.3820000000000001},"min":{"1min":0.45000000000000001,"5min":0.39800000000000002,"15min":0.377},"max":{"1min":60.011000000000003,"5min":60.011000000000003,"15min":174.779},"last":0.70299999999999996}]}]},{"osd":0,"up_from":0,"seq":390842024316,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":0,"seq":949187772592,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":609885356317,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019444,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5077080,"kb_used_data":3581520,"kb_used_omap":0,"kb_used_meta":1495552,"kb_avail":88668072,"statfs":{"total":95995035648,"available":90796105728,"internally_reserved":0,"allocated":3667476480,"data_stored":3666718241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1531445248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,2,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:59 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":1.6160000000000001,"15min":3.6349999999999998},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.23200000000000001},"max":{"1min":18.585999999999999,"5min":18.585999999999999,"15min":171.13499999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.331,"5min":1.9339999999999999,"15min":3.6949999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":19.469000000000001,"5min":19.469000000000001,"15min":171.11799999999999},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:50:58 2023","interfaces":[{"interface":"back","average":{"1min":7.2590000000000003,"5min":10.019,"15min":5.5469999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":92.030000000000001,"5min":385.78199999999998,"15min":385.78199999999998},"last":1.069},{"interface":"front","average":{"1min":5.1390000000000002,"5min":9.3689999999999998,"15min":5.3700000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":91.619,"5min":386.09399999999999,"15min":386.09399999999999},"last":0.47399999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:51:19 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":9.7699999999999996,"15min":6.375},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":18.617000000000001,"5min":440.75599999999997,"15min":440.75599999999997},"last":0.502},{"interface":"front","average":{"1min":3.4089999999999998,"5min":10.635,"15min":6.3929999999999998},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.30299999999999999},"max":{"1min":19.329999999999998,"5min":440.61900000000003,"15min":440.61900000000003},"last":0.54700000000000004}]},{"osd":3,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":3.746,"15min":5.1360000000000001},"min":{"1min":0.32600000000000001,"5min":0.315,"15min":0.27900000000000003},"max":{"1min":4.4429999999999996,"5min":141.334,"15min":357.125},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.2999999999999998,"5min":4.1529999999999996,"15min":5.1429999999999998},"min":{"1min":0.33100000000000002,"5min":0.32000000000000001,"15min":0.312},"max":{"1min":7.1760000000000002,"5min":141.18299999999999,"15min":359.02800000000002},"last":1.099}]},{"osd":5,"last update":"Mon Oct 9 04:50:31 2023","interfaces":[{"interface":"back","average":{"1min":10.875999999999999,"5min":3.9660000000000002,"15min":2.1349999999999998},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":169.44200000000001,"5min":169.44200000000001,"15min":169.44200000000001},"last":1.27},{"interface":"front","average":{"1min":10.66,"5min":3.5630000000000002,"15min":2.073},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.24099999999999999},"max":{"1min":172.36600000000001,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:50:59 2023","interfaces":[{"interface":"back","average":{"1min":4.7569999999999997,"5min":2.5030000000000001,"15min":1.7410000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":18.738,"5min":18.738,"15min":22.920999999999999},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.645,"5min":1.629,"15min":1.415},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":18.789000000000001,"5min":18.789000000000001,"15min":18.789000000000001},"last":0.52700000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:51:29 2023","interfaces":[{"interface":"back","average":{"1min":3.8399999999999999,"5min":2.6160000000000001,"15min":1.6870000000000001},"min":{"1min":0.377,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":19.111000000000001,"5min":19.111000000000001,"15min":19.111000000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":5.407,"5min":3.403,"15min":2.101},"min":{"1min":0.32800000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":19.611000000000001,"5min":19.611000000000001,"15min":22.858000000000001},"last":1.1419999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757808,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4563784,"kb_used_data":3078656,"kb_used_omap":0,"kb_used_meta":1485120,"kb_avail":89181368,"statfs":{"total":95995035648,"available":91321720832,"internally_reserved":0,"allocated":3152543744,"data_stored":3151801265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1520762880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.782,"15min":4.5250000000000004},"min":{"1min":0.251,"5min":0.251,"15min":0.23100000000000001},"max":{"1min":5.0300000000000002,"5min":9.6890000000000001,"15min":210.85499999999999},"last":0.496},{"interface":"front","average":{"1min":1.2569999999999999,"5min":1.6060000000000001,"15min":4.7409999999999997},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.9390000000000001,"5min":8.4749999999999996,"15min":210.23500000000001},"last":0.29899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:50:57 2023","interfaces":[{"interface":"back","average":{"1min":3.7999999999999998,"5min":7.7080000000000002,"15min":5.851},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.252},"max":{"1min":55.234000000000002,"5min":211.85400000000001,"15min":211.85400000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":4.3109999999999999,"5min":8.0549999999999997,"15min":5.9880000000000004},"min":{"1min":0.28899999999999998,"5min":0.255,"15min":0.255},"max":{"1min":54.994999999999997,"5min":211.435,"15min":211.435},"last":0.57199999999999995}]},{"osd":2,"last update":"Mon Oct 9 04:51:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":4.452,"15min":4.556},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.26800000000000002},"max":{"1min":8.8940000000000001,"5min":86.488,"15min":90.763999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.5349999999999999,"5min":4.3259999999999996,"15min":4.6420000000000003},"min":{"1min":0.34599999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2030000000000003,"5min":86.510999999999996,"15min":91.340000000000003},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:51:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.6799999999999999,"15min":4.7350000000000003},"min":{"1min":0.32200000000000001,"5min":0.31900000000000001,"15min":0.30599999999999999},"max":{"1min":4.9660000000000002,"5min":7.585,"15min":268.63299999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.5589999999999999,"15min":4.3680000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29799999999999999},"max":{"1min":5.6769999999999996,"5min":10.227,"15min":268.92500000000001},"last":0.54200000000000004}]},{"osd":4,"last update":"Mon Oct 9 04:51:32 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":2.75,"15min":1.389},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.23599999999999999},"max":{"1min":10.33,"5min":116.054,"15min":116.054},"last":0.47499999999999998},{"interface":"front","average":{"1min":3.008,"5min":3.4910000000000001,"15min":1.8919999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.253},"max":{"1min":11.888999999999999,"5min":118.47,"15min":118.47},"last":0.373}]},{"osd":6,"last update":"Mon Oct 9 04:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.373,"15min":1.1499999999999999},"min":{"1min":0.33400000000000002,"5min":0.32800000000000001,"15min":0.23100000000000001},"max":{"1min":5.7149999999999999,"5min":10.295999999999999,"15min":13.162000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.371,"15min":1.25},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28000000000000003},"max":{"1min":4.9020000000000001,"5min":7.3040000000000003,"15min":13.231999999999999},"last":0.52100000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:51:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3079999999999998,"5min":4.9909999999999997,"15min":2.4249999999999998},"min":{"1min":0.40000000000000002,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":6.6239999999999997,"5min":110.521,"15min":110.521},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.1739999999999999,"5min":5.3899999999999997,"15min":2.4329999999999998},"min":{"1min":0.38100000000000001,"5min":0.33300000000000002,"15min":0.30599999999999999},"max":{"1min":15.461,"5min":110.746,"15min":110.746},"last":0.56499999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2621116416,"data_stored":2621112344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3658813440,"data_stored":3658809368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3155623936,"data_stored":3155623936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3143110656,"data_stored":3143106584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:51:39.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pgp_num 26 2023-10-09T04:51:39.759 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:39.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:39.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:39.909 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:39.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:39 smithi177 ceph-mon[195797]: 13.5 scrub starts 2023-10-09T04:51:39.969 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:51:40.049 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:39 smithi029 ceph-mon[223858]: 13.5 scrub starts 2023-10-09T04:51:40.049 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:51:40.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:39 smithi029 ceph-mon[216903]: 13.5 scrub starts 2023-10-09T04:51:40.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-09T04:51:40.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:40.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:40.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:40.243 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:40.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:40.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:40.419 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:40.455 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:40.608 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:40.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:40.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:40.698 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:51:40.713 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:40.717 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:51:40.742 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:51:40.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:40.767 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:51:40.796 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:51:40.802 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:40.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:40.820 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:51:40.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:51:40.863 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:51:40.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.865 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.867 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.868 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.868 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.868 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.868 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:51:40.905 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:40.934 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:40.985 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:40.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:41.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:41.116 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:41.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:41.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: 13.1 scrub starts 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: pgmap v1422: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 2386/189267 objects misplaced (1.261%); 15 MiB/s, 246 objects/s recovering 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: osdmap e372: 8 total, 8 up, 4 in 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3428799270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[223858]: osdmap e373: 8 total, 8 up, 4 in 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: 13.1 scrub starts 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: pgmap v1422: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 2386/189267 objects misplaced (1.261%); 15 MiB/s, 246 objects/s recovering 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: osdmap e372: 8 total, 8 up, 4 in 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3428799270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:41.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:40 smithi029 ceph-mon[216903]: osdmap e373: 8 total, 8 up, 4 in 2023-10-09T04:51:41.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: 13.1 scrub starts 2023-10-09T04:51:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: pgmap v1422: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.8 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 2386/189267 objects misplaced (1.261%); 15 MiB/s, 246 objects/s recovering 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: osdmap e372: 8 total, 8 up, 4 in 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: failed to encode map e372 with expected crc 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3428799270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:40 smithi177 ceph-mon[195797]: osdmap e373: 8 total, 8 up, 4 in 2023-10-09T04:51:41.292 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:41.338 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:41.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:41.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:41.473 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:41.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:51:42.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.830 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.830 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.831 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.831 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.831 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: 13.17 scrub starts 2023-10-09T04:51:42.831 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:51:42.831 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:51:42.831 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:41 smithi177 ceph-mon[195797]: osdmap e374: 8 total, 8 up, 4 in 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: 13.17 scrub starts 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:51:42.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[223858]: osdmap e374: 8 total, 8 up, 4 in 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: failed to encode map e373 with expected crc 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: 13.17 scrub starts 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-09T04:51:42.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:41 smithi029 ceph-mon[216903]: osdmap e374: 8 total, 8 up, 4 in 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:51:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:51:42] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.43.0" 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: pgmap v1425: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.6 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 707 op/s; 2386/179307 objects misplaced (1.331%); 15 MiB/s, 246 objects/s recovering 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: 13.12 starting backfill to osd.1 from (0'0,0'0] MAX to 371'7697 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[223858]: osdmap e375: 8 total, 8 up, 4 in 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: pgmap v1425: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.6 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 707 op/s; 2386/179307 objects misplaced (1.331%); 15 MiB/s, 246 objects/s recovering 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: 13.12 starting backfill to osd.1 from (0'0,0'0] MAX to 371'7697 2023-10-09T04:51:43.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:42 smithi029 ceph-mon[216903]: osdmap e375: 8 total, 8 up, 4 in 2023-10-09T04:51:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: pgmap v1425: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.6 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 707 op/s; 2386/179307 objects misplaced (1.331%); 15 MiB/s, 246 objects/s recovering 2023-10-09T04:51:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: failed to encode map e374 with expected crc 2023-10-09T04:51:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: 13.12 starting backfill to osd.1 from (0'0,0'0] MAX to 371'7697 2023-10-09T04:51:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:42 smithi177 ceph-mon[195797]: osdmap e375: 8 total, 8 up, 4 in 2023-10-09T04:51:43.428 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:43.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:43.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:43.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:43.563 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:43.611 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:43.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:43.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:43.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:43.795 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:43.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:43.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:43.940 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:44.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:44.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:44.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: 13.7 scrub starts 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:51:44.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[216903]: osdmap e376: 8 total, 8 up, 4 in 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: 13.7 scrub starts 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:51:44.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:43 smithi029 ceph-mon[223858]: osdmap e376: 8 total, 8 up, 4 in 2023-10-09T04:51:44.141 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: failed to encode map e375 with expected crc 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: 13.7 scrub starts 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-09T04:51:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:43 smithi177 ceph-mon[195797]: osdmap e376: 8 total, 8 up, 4 in 2023-10-09T04:51:44.307 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:44.345 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:44.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:44.449 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:44.479 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:44.487 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:44.546 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:44.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:44.649 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:44.680 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:44.742 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:44.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:44.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:44.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:44.873 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:44.915 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:44.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:44.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:45.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:45.047 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:45.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:45.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:45.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: pgmap v1428: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.6 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 829 op/s; 2386/179307 objects misplaced (1.331%) 2023-10-09T04:51:45.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: 13.13 starting backfill to osd.1 from (0'0,0'0] MAX to 372'7591 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[216903]: osdmap e377: 8 total, 8 up, 4 in 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: pgmap v1428: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.6 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 829 op/s; 2386/179307 objects misplaced (1.331%) 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: 13.13 starting backfill to osd.1 from (0'0,0'0] MAX to 372'7591 2023-10-09T04:51:45.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:44 smithi029 ceph-mon[223858]: osdmap e377: 8 total, 8 up, 4 in 2023-10-09T04:51:45.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:45.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: pgmap v1428: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.6 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 829 op/s; 2386/179307 objects misplaced (1.331%) 2023-10-09T04:51:45.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: failed to encode map e376 with expected crc 2023-10-09T04:51:45.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: 13.13 starting backfill to osd.1 from (0'0,0'0] MAX to 372'7591 2023-10-09T04:51:45.225 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:44 smithi177 ceph-mon[195797]: osdmap e377: 8 total, 8 up, 4 in 2023-10-09T04:51:45.227 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:45.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:45.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:45.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:45.410 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:46.100 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:51:46.101 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:51:46.120 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:51:46.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[216903]: osdmap e378: 8 total, 8 up, 4 in 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:51:46.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:45 smithi029 ceph-mon[223858]: osdmap e378: 8 total, 8 up, 4 in 2023-10-09T04:51:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:46.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:46.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:46.256 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: failed to encode map e377 with expected crc 2023-10-09T04:51:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-09T04:51:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-09T04:51:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:45 smithi177 ceph-mon[195797]: osdmap e378: 8 total, 8 up, 4 in 2023-10-09T04:51:46.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:51:47.263 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:47.265 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: pgmap v1431: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+remapped, 96 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 432 op/s; 2135/174120 objects misplaced (1.226%); 60 MiB/s, 961 objects/s recovering 2023-10-09T04:51:47.265 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.265 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: 13.14 starting backfill to osd.7 from (0'0,0'0] MAX to 374'7559 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[216903]: osdmap e379: 8 total, 8 up, 4 in 2023-10-09T04:51:47.266 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: pgmap v1431: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+remapped, 96 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 432 op/s; 2135/174120 objects misplaced (1.226%); 60 MiB/s, 961 objects/s recovering 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: 13.14 starting backfill to osd.7 from (0'0,0'0] MAX to 374'7559 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:47.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:46 smithi029 ceph-mon[223858]: osdmap e379: 8 total, 8 up, 4 in 2023-10-09T04:51:47.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:47.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: pgmap v1431: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+remapped, 96 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 432 op/s; 2135/174120 objects misplaced (1.226%); 60 MiB/s, 961 objects/s recovering 2023-10-09T04:51:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: 13.14 starting backfill to osd.7 from (0'0,0'0] MAX to 374'7559 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: failed to encode map e378 with expected crc 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:51:47.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:46 smithi177 ceph-mon[195797]: osdmap e379: 8 total, 8 up, 4 in 2023-10-09T04:51:47.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:51:47.399 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T04:51:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:48 smithi177 ceph-mon[195797]: osdmap e380: 8 total, 8 up, 4 in 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[223858]: osdmap e380: 8 total, 8 up, 4 in 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: failed to encode map e379 with expected crc 2023-10-09T04:51:48.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-09T04:51:48.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-09T04:51:48.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:48 smithi029 ceph-mon[216903]: osdmap e380: 8 total, 8 up, 4 in 2023-10-09T04:51:49.093 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:51:49.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:49.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:51:49.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:49.229 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: 13.6 scrub starts 2023-10-09T04:51:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: 13.2 scrub starts 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: pgmap v1434: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+remapped, 96 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 432 op/s; 2135/174120 objects misplaced (1.226%); 60 MiB/s, 961 objects/s recovering 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: 13.15 starting backfill to osd.7 from (0'0,0'0] MAX to 377'7706 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:49 smithi177 ceph-mon[195797]: osdmap e381: 8 total, 8 up, 4 in 2023-10-09T04:51:49.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:49.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:49.381 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:49.410 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: 13.6 scrub starts 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: 13.2 scrub starts 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: pgmap v1434: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+remapped, 96 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 432 op/s; 2135/174120 objects misplaced (1.226%); 60 MiB/s, 961 objects/s recovering 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: 13.15 starting backfill to osd.7 from (0'0,0'0] MAX to 377'7706 2023-10-09T04:51:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[216903]: osdmap e381: 8 total, 8 up, 4 in 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: 13.6 scrub starts 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: 13.2 scrub starts 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: pgmap v1434: 99 pgs: 2 active+remapped+backfilling, 1 active+clean+remapped, 96 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 432 op/s; 2135/174120 objects misplaced (1.226%); 60 MiB/s, 961 objects/s recovering 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: 13.15 starting backfill to osd.7 from (0'0,0'0] MAX to 377'7706 2023-10-09T04:51:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: failed to encode map e380 with expected crc 2023-10-09T04:51:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:49 smithi029 ceph-mon[223858]: osdmap e381: 8 total, 8 up, 4 in 2023-10-09T04:51:49.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:49.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:49.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:49.591 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:49.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:49.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:49.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:49.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:49.830 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[216903]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[216903]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[216903]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[216903]: 13.4 scrub starts 2023-10-09T04:51:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[216903]: osdmap e382: 8 total, 8 up, 4 in 2023-10-09T04:51:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[223858]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[223858]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[223858]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[223858]: 13.4 scrub starts 2023-10-09T04:51:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:50 smithi029 ceph-mon[223858]: osdmap e382: 8 total, 8 up, 4 in 2023-10-09T04:51:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:50 smithi177 ceph-mon[195797]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:50 smithi177 ceph-mon[195797]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:50 smithi177 ceph-mon[195797]: failed to encode map e381 with expected crc 2023-10-09T04:51:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:50 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:50 smithi177 ceph-mon[195797]: 13.4 scrub starts 2023-10-09T04:51:50.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:50 smithi177 ceph-mon[195797]: osdmap e382: 8 total, 8 up, 4 in 2023-10-09T04:51:50.503 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:50.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:50.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:50.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:51:50.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:50.636 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:50.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:50.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:50.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:50.860 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:50.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:50.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:50.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:51.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: pgmap v1437: 99 pgs: 1 unknown, 3 active+remapped+backfilling, 95 active+clean; 3.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 2581/167622 objects misplaced (1.540%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:51:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: 13.16 starting backfill to osd.1 from (0'0,0'0] MAX to 379'7854 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: osdmap e383: 8 total, 8 up, 4 in 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: pgmap v1437: 99 pgs: 1 unknown, 3 active+remapped+backfilling, 95 active+clean; 3.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 2581/167622 objects misplaced (1.540%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: 13.16 starting backfill to osd.1 from (0'0,0'0] MAX to 379'7854 2023-10-09T04:51:51.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: osdmap e383: 8 total, 8 up, 4 in 2023-10-09T04:51:51.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:51.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:51.177 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: pgmap v1437: 99 pgs: 1 unknown, 3 active+remapped+backfilling, 95 active+clean; 3.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 2581/167622 objects misplaced (1.540%); 21 MiB/s, 337 objects/s recovering 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: failed to encode map e382 with expected crc 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: 13.16 starting backfill to osd.1 from (0'0,0'0] MAX to 379'7854 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: osdmap e383: 8 total, 8 up, 4 in 2023-10-09T04:51:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:51.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:51.288 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:51.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:51.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:51.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:51.478 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:51.573 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:51.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:51.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:51.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:51:51.768 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:51.882 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:51:51.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:51.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:51:51.962 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:52.000 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: 13.10 scrub starts 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2366280507' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:51:52.021 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2075341620' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:52.029 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:51:52.029 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 13 pgp_num to 26 2023-10-09T04:51:52.092 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: 13.10 scrub starts 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2366280507' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2075341620' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: failed to encode map e383 with expected crc 2023-10-09T04:51:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: 13.10 scrub starts 2023-10-09T04:51:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2366280507' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:51:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2075341620' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:51:52.220 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:51:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:52.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:51:52.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:52.417 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:52.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:52.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:52.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:52.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:52.662 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:52.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:51:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:51:52] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.43.0" 2023-10-09T04:51:53.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: pgmap v1440: 99 pgs: 1 active+remapped+backfilling, 2 peering, 1 active+remapped, 95 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 232 op/s; 3261/171333 objects misplaced (1.903%); 67 MiB/s, 1.06k objects/s recovering 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2366280507' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2075341620' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: osdmap e384: 8 total, 8 up, 4 in 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:53 smithi177 ceph-mon[195797]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: pgmap v1440: 99 pgs: 1 active+remapped+backfilling, 2 peering, 1 active+remapped, 95 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 232 op/s; 3261/171333 objects misplaced (1.903%); 67 MiB/s, 1.06k objects/s recovering 2023-10-09T04:51:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2366280507' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2075341620' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: osdmap e384: 8 total, 8 up, 4 in 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[216903]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: pgmap v1440: 99 pgs: 1 active+remapped+backfilling, 2 peering, 1 active+remapped, 95 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 232 op/s; 3261/171333 objects misplaced (1.903%); 67 MiB/s, 1.06k objects/s recovering 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2366280507' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2075341620' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: osdmap e384: 8 total, 8 up, 4 in 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: failed to encode map e384 with expected crc 2023-10-09T04:51:53.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:53 smithi029 ceph-mon[223858]: failed to encode map e384 with expected crc 2023-10-09T04:51:54.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:54 smithi029 ceph-mon[223858]: 13.8 scrub starts 2023-10-09T04:51:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:54 smithi029 ceph-mon[216903]: 13.8 scrub starts 2023-10-09T04:51:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:54 smithi177 ceph-mon[195797]: 13.8 scrub starts 2023-10-09T04:51:55.283 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:55.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:55.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:55 smithi029 ceph-mon[216903]: 13.11 scrub starts 2023-10-09T04:51:55.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:55 smithi029 ceph-mon[216903]: pgmap v1442: 99 pgs: 1 active+remapped+backfilling, 2 peering, 1 active+remapped, 95 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 40 MiB/s wr, 1.02k op/s; 3261/171333 objects misplaced (1.903%); 39 MiB/s, 618 objects/s recovering 2023-10-09T04:51:55.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:55 smithi029 ceph-mon[223858]: 13.11 scrub starts 2023-10-09T04:51:55.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:55 smithi029 ceph-mon[223858]: pgmap v1442: 99 pgs: 1 active+remapped+backfilling, 2 peering, 1 active+remapped, 95 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 40 MiB/s wr, 1.02k op/s; 3261/171333 objects misplaced (1.903%); 39 MiB/s, 618 objects/s recovering 2023-10-09T04:51:55.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:55.427 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:55.492 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:55.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:55 smithi177 ceph-mon[195797]: 13.11 scrub starts 2023-10-09T04:51:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:55 smithi177 ceph-mon[195797]: pgmap v1442: 99 pgs: 1 active+remapped+backfilling, 2 peering, 1 active+remapped, 95 active+clean; 3.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 40 MiB/s wr, 1.02k op/s; 3261/171333 objects misplaced (1.903%); 39 MiB/s, 618 objects/s recovering 2023-10-09T04:51:55.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:55.665 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:55.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:55.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:55.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:55.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:55.892 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:56 smithi029 ceph-mon[216903]: 13.4 scrub starts 2023-10-09T04:51:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:56 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T04:51:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:56 smithi029 ceph-mon[223858]: 13.4 scrub starts 2023-10-09T04:51:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:56 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T04:51:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:56 smithi177 ceph-mon[195797]: 13.4 scrub starts 2023-10-09T04:51:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:56 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-09T04:51:56.546 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:56.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:56.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:56.688 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:56.872 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:51:56.895 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:51:56.895 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:56.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:56.923 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:51:56.947 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:51:56.971 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:51:56.991 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:51:57.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:51:57.018 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:51:57.041 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:57.051 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.054 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: 13.10 scrub starts 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: pgmap v1443: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.2 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 32 MiB/s wr, 1.57k op/s; 2033/157407 objects misplaced (1.292%); 41 MiB/s, 653 objects/s recovering 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: osdmap e385: 8 total, 8 up, 4 in 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1202336862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: 13.10 scrub starts 2023-10-09T04:51:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: pgmap v1443: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.2 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 32 MiB/s wr, 1.57k op/s; 2033/157407 objects misplaced (1.292%); 41 MiB/s, 653 objects/s recovering 2023-10-09T04:51:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T04:51:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: osdmap e385: 8 total, 8 up, 4 in 2023-10-09T04:51:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1202336862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:57.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:51:57.285 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:51:57.286 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:51:57.286 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:51:57.286 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:51:57.286 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:51:57.463 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:51:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:57.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:51:57.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: 13.10 scrub starts 2023-10-09T04:51:57.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: pgmap v1443: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.2 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 32 MiB/s wr, 1.57k op/s; 2033/157407 objects misplaced (1.292%); 41 MiB/s, 653 objects/s recovering 2023-10-09T04:51:57.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-09T04:51:57.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: osdmap e385: 8 total, 8 up, 4 in 2023-10-09T04:51:57.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: failed to encode map e385 with expected crc 2023-10-09T04:51:57.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1202336862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:51:57.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:57.614 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:57.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:57.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:57.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:57.773 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:51:57.813 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:57.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:51:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:57.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:51:57.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:58.020 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:58 smithi177 ceph-mon[195797]: failed to encode map e385 with expected crc 2023-10-09T04:51:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:58 smithi177 ceph-mon[195797]: osdmap e386: 8 total, 8 up, 4 in 2023-10-09T04:51:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:58 smithi177 ceph-mon[195797]: failed to encode map e386 with expected crc 2023-10-09T04:51:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:58 smithi177 ceph-mon[195797]: failed to encode map e386 with expected crc 2023-10-09T04:51:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[216903]: failed to encode map e385 with expected crc 2023-10-09T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[216903]: osdmap e386: 8 total, 8 up, 4 in 2023-10-09T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[216903]: failed to encode map e386 with expected crc 2023-10-09T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[216903]: failed to encode map e386 with expected crc 2023-10-09T04:51:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T04:51:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[223858]: failed to encode map e385 with expected crc 2023-10-09T04:51:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[223858]: osdmap e386: 8 total, 8 up, 4 in 2023-10-09T04:51:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[223858]: failed to encode map e386 with expected crc 2023-10-09T04:51:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[223858]: failed to encode map e386 with expected crc 2023-10-09T04:51:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-09T04:51:59.303 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:51:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:59.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: pgmap v1446: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.2 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 773 op/s; 2033/157407 objects misplaced (1.292%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: osdmap e387: 8 total, 8 up, 4 in 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:59.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: pgmap v1446: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.2 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 773 op/s; 2033/157407 objects misplaced (1.292%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: osdmap e387: 8 total, 8 up, 4 in 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:59.326 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:51:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:59.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:51:59.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:51:59.431 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: pgmap v1446: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.2 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 773 op/s; 2033/157407 objects misplaced (1.292%); 10 MiB/s, 167 objects/s recovering 2023-10-09T04:51:59.431 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.431 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-09T04:51:59.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: osdmap e387: 8 total, 8 up, 4 in 2023-10-09T04:51:59.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: failed to encode map e387 with expected crc 2023-10-09T04:51:59.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:51:59.432 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:51:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:51:59.434 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:59.472 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:51:59.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:51:59.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:59.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:51:59.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:51:59.618 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:51:59.766 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:51:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:51:59.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:51:59.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:51:59.902 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:00.059 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:00.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:00.165 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[216903]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[216903]: osdmap e388: 8 total, 8 up, 4 in 2023-10-09T04:52:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[216903]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[216903]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T04:52:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[223858]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[223858]: osdmap e388: 8 total, 8 up, 4 in 2023-10-09T04:52:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[223858]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[223858]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:00 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T04:52:00.199 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:00.309 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7] out_osds: [6, 3, 0, 2] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:52:00.309 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:52:00.309 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T04:52:00.309 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T04:52:00.372 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:00.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:00.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:00.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:00 smithi177 ceph-mon[195797]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:00 smithi177 ceph-mon[195797]: osdmap e388: 8 total, 8 up, 4 in 2023-10-09T04:52:00.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:00 smithi177 ceph-mon[195797]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:00 smithi177 ceph-mon[195797]: failed to encode map e388 with expected crc 2023-10-09T04:52:00.507 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:00 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-09T04:52:00.509 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:00.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:00.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:00.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:00.692 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:01.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: 13.9 deep-scrub starts 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: pgmap v1449: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.1 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 235 op/s; 2033/154575 objects misplaced (1.315%) 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: osdmap e389: 8 total, 8 up, 4 in 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: 13.18 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7874 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[216903]: osdmap e390: 8 total, 8 up, 4 in 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: 13.9 deep-scrub starts 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: pgmap v1449: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.1 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 235 op/s; 2033/154575 objects misplaced (1.315%) 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: osdmap e389: 8 total, 8 up, 4 in 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: 13.18 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7874 2023-10-09T04:52:01.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:01 smithi029 ceph-mon[223858]: osdmap e390: 8 total, 8 up, 4 in 2023-10-09T04:52:01.501 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:01.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: 13.9 deep-scrub starts 2023-10-09T04:52:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: pgmap v1449: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 3.1 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 235 op/s; 2033/154575 objects misplaced (1.315%) 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: osdmap e389: 8 total, 8 up, 4 in 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: failed to encode map e389 with expected crc 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: 13.18 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7874 2023-10-09T04:52:01.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:01 smithi177 ceph-mon[195797]: osdmap e390: 8 total, 8 up, 4 in 2023-10-09T04:52:01.597 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:01.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:01.635 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:01.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:01.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:01.787 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:01.873 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:01.921 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:01.954 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:01.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:02.035 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:02.059 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:02.113 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:02.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: 13.17 scrub starts 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[216903]: osdmap e391: 8 total, 8 up, 4 in 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: 13.17 scrub starts 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.123 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:02 smithi029 ceph-mon[223858]: osdmap e391: 8 total, 8 up, 4 in 2023-10-09T04:52:02.242 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:02.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:02.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:02.401 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: 13.17 scrub starts 2023-10-09T04:52:02.401 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: failed to encode map e390 with expected crc 2023-10-09T04:52:02.402 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:02 smithi177 ceph-mon[195797]: osdmap e391: 8 total, 8 up, 4 in 2023-10-09T04:52:02.410 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:02.535 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:02.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:02.645 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:02.688 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:02.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:52:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:52:02] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T04:52:03.066 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:03.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:03.118 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:52:03.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:03.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: 13.11 scrub starts 2023-10-09T04:52:03.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: pgmap v1452: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 822 op/s; 1872/147540 objects misplaced (1.269%); 26 MiB/s, 419 objects/s recovering 2023-10-09T04:52:03.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: 13.19 starting backfill to osd.7 from (0'0,0'0] MAX to 387'7827 2023-10-09T04:52:03.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: 13.2 scrub starts 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1159845230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: 13.11 scrub starts 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: pgmap v1452: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 822 op/s; 1872/147540 objects misplaced (1.269%); 26 MiB/s, 419 objects/s recovering 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: 13.19 starting backfill to osd.7 from (0'0,0'0] MAX to 387'7827 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: 13.2 scrub starts 2023-10-09T04:52:03.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1159845230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:03.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:03.226 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:03.390 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:03.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:03.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: 13.11 scrub starts 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: pgmap v1452: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 822 op/s; 1872/147540 objects misplaced (1.269%); 26 MiB/s, 419 objects/s recovering 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: 13.19 starting backfill to osd.7 from (0'0,0'0] MAX to 387'7827 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: failed to encode map e391 with expected crc 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: 13.2 scrub starts 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1159845230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:03.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:03.529 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[223858]: osdmap e392: 8 total, 8 up, 4 in 2023-10-09T04:52:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[223858]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[223858]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[223858]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[216903]: osdmap e392: 8 total, 8 up, 4 in 2023-10-09T04:52:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[216903]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[216903]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:04 smithi029 ceph-mon[216903]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:04 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:04.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:04 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:04.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:04 smithi177 ceph-mon[195797]: osdmap e392: 8 total, 8 up, 4 in 2023-10-09T04:52:04.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:04 smithi177 ceph-mon[195797]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:04 smithi177 ceph-mon[195797]: failed to encode map e392 with expected crc 2023-10-09T04:52:04.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:04 smithi177 ceph-mon[195797]: failed to encode map e392 with expected crc 2023-10-09T04:52:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:05 smithi029 ceph-mon[216903]: pgmap v1455: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 586 op/s; 1872/147540 objects misplaced (1.269%); 26 MiB/s, 419 objects/s recovering 2023-10-09T04:52:05.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:05 smithi029 ceph-mon[223858]: pgmap v1455: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 586 op/s; 1872/147540 objects misplaced (1.269%); 26 MiB/s, 419 objects/s recovering 2023-10-09T04:52:05.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:05 smithi177 ceph-mon[195797]: pgmap v1455: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 586 op/s; 1872/147540 objects misplaced (1.269%); 26 MiB/s, 419 objects/s recovering 2023-10-09T04:52:06.614 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:06.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:06.718 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:06.751 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:06 smithi029 ceph-mon[216903]: pgmap v1456: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 820 op/s; 1399/136632 objects misplaced (1.024%); 22 MiB/s, 344 objects/s recovering 2023-10-09T04:52:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:06 smithi029 ceph-mon[223858]: pgmap v1456: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 820 op/s; 1399/136632 objects misplaced (1.024%); 22 MiB/s, 344 objects/s recovering 2023-10-09T04:52:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:07.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:06 smithi177 ceph-mon[195797]: pgmap v1456: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 820 op/s; 1399/136632 objects misplaced (1.024%); 22 MiB/s, 344 objects/s recovering 2023-10-09T04:52:07.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:07.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:07.312 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:07.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:07.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:07.476 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:07.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:07.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:07.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:07.680 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:08.270 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:08.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:08.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: osdmap e393: 8 total, 8 up, 4 in 2023-10-09T04:52:08.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:08.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: osdmap e393: 8 total, 8 up, 4 in 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:08.405 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:08.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:08 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:08.408 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: osdmap e393: 8 total, 8 up, 4 in 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: failed to encode map e393 with expected crc 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:52:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:08 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:08.859 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:08.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:192262 11.0 11:04d9d911:::benchmark_data_smithi029_392508_object91706:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.880 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:24.181778+0000", 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 584.69419577799999, 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.41528092300000002, 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 192262 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.881 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181778+0000", 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181778+0000", 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181778+0000", 2023-10-09T04:52:08.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181781+0000", 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.163e-06 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181788+0000", 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9639999999999997e-06 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181793+0000", 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.87e-06 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.192791+0000", 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010997816000000001 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.192937+0000", 2023-10-09T04:52:08.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014638100000000001 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.193074+0000", 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013740200000000001 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.203246+0000", 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010171072999999999 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.203260+0000", 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4521e-05 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.596960+0000", 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39369970599999998 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.597017+0000", 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7453e-05 2023-10-09T04:52:08.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.597059+0000", 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1990000000000003e-05 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:192265 11.0 11:0b803803:::benchmark_data_smithi029_392508_object91709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:24.181797+0000", 2023-10-09T04:52:08.887 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 584.69417698799998, 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.41540570199999999, 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 192265 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181797+0000", 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181797+0000", 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181797+0000", 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181799+0000", 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5890000000000001e-06 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181806+0000", 2023-10-09T04:52:08.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2879999999999998e-06 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.181811+0000", 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2619999999999998e-06 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.197518+0000", 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015707123999999999 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.197884+0000", 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036642599999999999 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.198631+0000", 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00074711799999999996 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.210327+0000", 2023-10-09T04:52:08.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011695110999999999 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.218322+0000", 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079950509999999995 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.597135+0000", 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37881310000000001 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.597171+0000", 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6072000000000003e-05 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:24.597203+0000", 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1928000000000003e-05 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.894 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195074 11.0 11:0c6f84fd:::benchmark_data_smithi029_392508_object94518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.037189+0000", 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.83878469599995, 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.44497853900000001, 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195074 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.037189+0000", 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.037189+0000", 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.037189+0000", 2023-10-09T04:52:08.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.037199+0000", 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0416e-05 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.037206+0000", 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4239999999999998e-06 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.037212+0000", 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8640000000000001e-06 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.085923+0000", 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.048711844999999997 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.086057+0000", 2023-10-09T04:52:08.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013376300000000001 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.086174+0000", 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011714699999999999 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.097677+0000", 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011503049 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477736+0000", 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38005816799999997 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482078+0000", 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043423460000000004 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482128+0000", 2023-10-09T04:52:08.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9895999999999998e-05 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482168+0000", 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0101000000000001e-05 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195075 11.a 11:598b09b1:::benchmark_data_smithi029_392508_object94519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.040052+0000", 2023-10-09T04:52:08.901 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.83592157199996, 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.43786720499999998, 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195075 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.040052+0000", 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.040052+0000", 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.040052+0000", 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.040056+0000", 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.639e-06 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082123+0000", 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042067018999999997 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082136+0000", 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2615e-05 2023-10-09T04:52:08.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.086053+0000", 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039168880000000003 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.086157+0000", 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010451299999999999 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.086260+0000", 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103053 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090537+0000", 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042762590000000001 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090543+0000", 2023-10-09T04:52:08.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3239999999999999e-06 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477828+0000", 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38728565100000001 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477878+0000", 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9153999999999997e-05 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477920+0000", 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2033000000000002e-05 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195078 11.0 11:0049093c:::benchmark_data_smithi029_392508_object94522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.082178+0000", 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.79379605199995, 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40631534400000002, 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.908 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195078 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082178+0000", 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082178+0000", 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082178+0000", 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082180+0000", 2023-10-09T04:52:08.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5560000000000002e-06 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082186+0000", 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5010000000000004e-06 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082191+0000", 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9699999999999998e-06 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089853+0000", 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076623810000000002 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089940+0000", 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7159999999999999e-05 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090046+0000", 2023-10-09T04:52:08.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010603999999999999 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477726+0000", 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38768016700000002 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482200+0000", 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004473505 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488412+0000", 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006212258 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488456+0000", 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3849e-05 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488493+0000", 2023-10-09T04:52:08.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7319e-05 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195089 11.0 11:060a0fd9:::benchmark_data_smithi029_392508_object94533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.082240+0000", 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.79373385600002, 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.406322085, 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.915 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195089 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082240+0000", 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082240+0000", 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082240+0000", 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082242+0000", 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5280000000000002e-06 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082247+0000", 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0340000000000004e-06 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.082252+0000", 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6539999999999998e-06 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.093310+0000", 2023-10-09T04:52:08.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011058213000000001 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.093424+0000", 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113836 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.093534+0000", 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109448 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482047+0000", 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38851347600000002 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482227+0000", 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000179977 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488500+0000", 2023-10-09T04:52:08.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062731250000000001 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488532+0000", 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2301000000000001e-05 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488562+0000", 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9793000000000001e-05 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.921 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195090 11.9 11:914730d6:::benchmark_data_smithi029_392508_object94534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.089058+0000", 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.78691576799997, 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39641441500000002, 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.922 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195090 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089058+0000", 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089058+0000", 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089057+0000", 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089066+0000", 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1200000000000008e-06 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089076+0000", 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0743e-05 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089083+0000", 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9480000000000002e-06 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.095050+0000", 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059666119999999996 2023-10-09T04:52:08.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.095140+0000", 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9948e-05 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.095248+0000", 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010821 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477733+0000", 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38248516500000002 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.485360+0000", 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076266260000000001 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.485394+0000", 2023-10-09T04:52:08.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4508999999999998e-05 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.485436+0000", 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1238000000000001e-05 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.485473+0000", 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7033000000000003e-05 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.928 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195091 11.8 11:15d7c752:::benchmark_data_smithi029_392508_object94535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.089089+0000", 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.78688542400005, 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40059936299999999, 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.929 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195091 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089089+0000", 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089089+0000", 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089088+0000", 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089091+0000", 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2819999999999999e-06 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089096+0000", 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1619999999999999e-06 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.089100+0000", 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5290000000000002e-06 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.097699+0000", 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085991639999999994 2023-10-09T04:52:08.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.097805+0000", 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000105967 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.097917+0000", 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011205399999999999 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488598+0000", 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39068101399999999 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488605+0000", 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8789999999999997e-06 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.489609+0000", 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001003943 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.489652+0000", 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3241999999999997e-05 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.489688+0000", 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5510999999999999e-05 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195092 11.8 11:10c2f99e:::benchmark_data_smithi029_392508_object94536:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.090827+0000", 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.78514680199999, 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40741083099999997, 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195092 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.936 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090827+0000", 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090827+0000", 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090826+0000", 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090831+0000", 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5029999999999999e-06 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090842+0000", 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0739e-05 2023-10-09T04:52:08.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.090849+0000", 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2579999999999998e-06 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.484063+0000", 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39321411699999997 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.484186+0000", 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012250799999999999 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.484306+0000", 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000120112 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.489723+0000", 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054170700000000004 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.498082+0000", 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083590999999999995 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.498167+0000", 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5506000000000004e-05 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.498204+0000", 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6411000000000001e-05 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.498238+0000", 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4246999999999999e-05 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195093 11.0 11:0ae87ac3:::benchmark_data_smithi029_392508_object94537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.942 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.092067+0000", 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.78390678599999, 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40113712200000001, 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195093 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.092067+0000", 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.092067+0000", 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.092066+0000", 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:52:08.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.092071+0000", 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7190000000000001e-06 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.092082+0000", 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0545e-05 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.092089+0000", 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4399999999999999e-06 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482250+0000", 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39016112200000003 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482373+0000", 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000122688 2023-10-09T04:52:08.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.482493+0000", 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000119794 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.488610+0000", 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061173169999999997 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.493092+0000", 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044823409999999999 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.493127+0000", 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4779e-05 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.493166+0000", 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8588999999999999e-05 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.493204+0000", 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8572999999999998e-05 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.45869.0:195096 11.9 11:9f8c6b5f:::benchmark_data_smithi029_392508_object94540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:42:29.099425+0000", 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 579.77654939299998, 2023-10-09T04:52:08.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.39709063300000003, 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.45869", 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2177249681", 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 195096 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.099425+0000", 2023-10-09T04:52:08.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.099425+0000", 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.099423+0000", 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.099429+0000", 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2890000000000003e-06 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477675+0000", 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37824587100000001 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477704+0000", 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8789000000000002e-05 2023-10-09T04:52:08.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.477979+0000", 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00027517999999999998 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.478137+0000", 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015777400000000001 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.478853+0000", 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00071575700000000003 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.487064+0000", 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082112569999999996 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.489316+0000", 2023-10-09T04:52:08.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022515220000000002 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.496423+0000", 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071071370000000004 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.496474+0000", 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1251000000000001e-05 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:42:29.496515+0000", 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1300000000000001e-05 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:80159 12.4 12:2c44e365:::benchmark_data_smithi029_442458_object80158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:44:49.016795+0000", 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 439.85917861799999, 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.41110931899999997, 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.956 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80159 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.016795+0000", 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.016795+0000", 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.016795+0000", 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.016813+0000", 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8590999999999999e-05 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.016828+0000", 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.468e-05 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.016837+0000", 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8230000000000001e-06 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.395338+0000", 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37850092099999999 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.395545+0000", 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020729799999999999 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.395755+0000", 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020998600000000001 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.409843+0000", 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014088108 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.409855+0000", 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1705000000000001e-05 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.424195+0000", 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014339747 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.424256+0000", 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1431000000000002e-05 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:44:49.427905+0000", 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036487949999999998 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:88474 12.1 12:8e16fb5d:::benchmark_data_smithi029_442458_object88473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:03.448906+0000", 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 425.42706780899999, 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.42881274400000002, 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.963 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88474 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.448906+0000", 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.448906+0000", 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.448905+0000", 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.448927+0000", 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2410000000000001e-05 2023-10-09T04:52:08.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.448940+0000", 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2762000000000001e-05 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.448946+0000", 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9329999999999998e-06 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.456337+0000", 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073909010000000001 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.456502+0000", 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016513300000000001 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.456677+0000", 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017523699999999999 2023-10-09T04:52:08.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.459983+0000", 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033055279999999999 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.460761+0000", 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00077847799999999998 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.874981+0000", 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.414220329 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.875078+0000", 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7007000000000001e-05 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.877719+0000", 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026406279999999999 2023-10-09T04:52:08.969 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:88492 12.1 12:8f9e5b4b:::benchmark_data_smithi029_442458_object88491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e297)", 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:03.484782+0000", 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 425.39119189500002, 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.43009987500000002, 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.970 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 88492 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484782+0000", 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484782+0000", 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484780+0000", 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484810+0000", 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9702000000000001e-05 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484824+0000", 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4357e-05 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484833+0000", 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7509999999999997e-06 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.484879+0000", 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6146000000000002e-05 2023-10-09T04:52:08.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.485019+0000", 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139942 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.485149+0000", 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013014600000000001 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.487756+0000", 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026070289999999999 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.488174+0000", 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041752800000000002 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.912490+0000", 2023-10-09T04:52:08.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.42431628199999999 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.912540+0000", 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9648999999999998e-05 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:03.914882+0000", 2023-10-09T04:52:08.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023420559999999999 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111552 12.b 12:de0741f0:::benchmark_data_smithi029_442458_object16302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.816332+0000", 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 384.05964236400001, 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76530434700000005, 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.977 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111552 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.816332+0000", 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.816332+0000", 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.816330+0000", 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.816338+0000", 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0690000000000002e-06 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.816353+0000", 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4598e-05 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.816363+0000", 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7750000000000004e-06 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.817960+0000", 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015973020000000001 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.818150+0000", 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018948200000000001 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.818320+0000", 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017073699999999999 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.827562+0000", 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092418919999999998 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.827572+0000", 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7440000000000002e-06 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882134+0000", 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054562174999999997 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882147+0000", 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2446000000000001e-05 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581622+0000", 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.699475823 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581636+0000", 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3645000000000001e-05 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111562 12.b 12:dfc0490f:::benchmark_data_smithi029_442458_object16312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.826469+0000", 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 384.04950501000002, 2023-10-09T04:52:08.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75545808999999997, 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111562 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.826469+0000", 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.826469+0000", 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.826467+0000", 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.826474+0000", 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8850000000000001e-06 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.826491+0000", 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.677e-05 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.826499+0000", 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.875e-06 2023-10-09T04:52:08.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.827601+0000", 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001102282 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.827806+0000", 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000205274 2023-10-09T04:52:08.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.827979+0000", 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017312299999999999 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839357+0000", 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011377490000000001 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.843925+0000", 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045684569999999997 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882888+0000", 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038962661000000003 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:52:08.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882895+0000", 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7619999999999998e-06 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581914+0000", 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.69901929399999996 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581927+0000", 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3037000000000001e-05 2023-10-09T04:52:08.991 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111568 12.b 12:db8ebbfb:::benchmark_data_smithi029_442458_object16318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e301)", 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.839456+0000", 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 384.036518102, 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.74251056500000001, 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.992 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111568 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839456+0000", 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839456+0000", 2023-10-09T04:52:08.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839454+0000", 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839461+0000", 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7270000000000001e-06 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839471+0000", 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0484e-05 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839479+0000", 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5360000000000004e-06 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839655+0000", 2023-10-09T04:52:08.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000176009 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.839843+0000", 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018772999999999999 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.840043+0000", 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000200414 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.843998+0000", 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039550899999999996 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882929+0000", 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038930640000000002 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882934+0000", 2023-10-09T04:52:08.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5520000000000004e-06 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581954+0000", 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.69901997900000001 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581966+0000", 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2111e-05 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:08.998 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111574 12.b 12.b65b037b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.875756+0000", 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 384.00021791299997, 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.70624134500000002, 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:08.999 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111574 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875756+0000", 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875756+0000", 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875754+0000", 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875760+0000", 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7790000000000001e-06 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875771+0000", 2023-10-09T04:52:09.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0883000000000001e-05 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875781+0000", 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6579999999999997e-06 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882951+0000", 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071706970000000002 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.882956+0000", 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6410000000000001e-06 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581988+0000", 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.69903150999999997 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.581997+0000", 2023-10-09T04:52:09.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8309999999999996e-06 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:111575 12.4 12.6a7137f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e302)", 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:45:44.875792+0000", 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 384.000182516, 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.70196411599999997, 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:52:09.004 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 111575 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875792+0000", 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875792+0000", 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875791+0000", 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875794+0000", 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1810000000000001e-06 2023-10-09T04:52:09.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875801+0000", 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0310000000000001e-06 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.875807+0000", 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6699999999999999e-06 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:09.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.881283+0000", 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054756919999999999 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:44.881297+0000", 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3906999999999999e-05 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.577734+0000", 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.69643721800000002 2023-10-09T04:52:09.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:45:45.577756+0000", 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1878999999999998e-05 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:129941 12.4 e311/303 12:2805a2fd:::benchmark_data_smithi029_442458_object34691:head v 311'8121, mlcod=289'3151)", 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:47.762857+0000", 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 321.11311704399998, 2023-10-09T04:52:09.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.325737732, 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:47.762857+0000", 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:47.762857+0000", 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:52:09.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:47.762856+0000", 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:47.762865+0000", 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2580000000000001e-06 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:47.762878+0000", 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2376e-05 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:47.762886+0000", 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0940000000000008e-06 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:48.083309+0000", 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.32042334700000003 2023-10-09T04:52:09.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:48.083345+0000", 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6205000000000003e-05 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:48.088562+0000", 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052168580000000004 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:52:09.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:48.088595+0000", 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2589999999999998e-05 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:52:09.014 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:52:09.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:09.088 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[223858]: pgmap v1458: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 605 op/s; 1399/136632 objects misplaced (1.024%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[223858]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[223858]: osdmap e394: 8 total, 8 up, 4 in 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[223858]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.135 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[223858]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:09.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:09.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:09.344 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[216903]: pgmap v1458: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 605 op/s; 1399/136632 objects misplaced (1.024%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:52:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[216903]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[216903]: osdmap e394: 8 total, 8 up, 4 in 2023-10-09T04:52:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[216903]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:09 smithi029 ceph-mon[216903]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:09 smithi177 ceph-mon[195797]: pgmap v1458: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 605 op/s; 1399/136632 objects misplaced (1.024%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:52:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:09 smithi177 ceph-mon[195797]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:09 smithi177 ceph-mon[195797]: osdmap e394: 8 total, 8 up, 4 in 2023-10-09T04:52:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:09 smithi177 ceph-mon[195797]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:09 smithi177 ceph-mon[195797]: failed to encode map e394 with expected crc 2023-10-09T04:52:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:09 smithi029 conmon[216878]: 2023-10-09T04:52:09.488+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e394 definitely_dead 0 2023-10-09T04:52:10.165 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.2. 2023-10-09T04:52:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3566584640' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:52:10.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:52:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3566584640' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:52:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:52:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3566584640' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:52:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T04:52:10.612 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:10.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:10.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:52:10.677 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:52:10.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:10.776 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:10.893 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:52:10.919 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:52:10.934 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:10.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:10.946 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:52:10.972 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:52:10.996 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:52:11.024 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:52:11.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:11.049 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:52:11.065 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:11.066 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:11.071 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:52:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.072 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.073 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:52:11.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:11.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:11.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:11.242 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: pgmap v1460: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 541/132546 objects misplaced (0.408%); 26 MiB/s, 412 objects/s recovering 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: osdmap e395: 8 total, 8 up, 5 in 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: osdmap e396: 8 total, 8 up, 5 in 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/120271762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: pgmap v1460: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 541/132546 objects misplaced (0.408%); 26 MiB/s, 412 objects/s recovering 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T04:52:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: osdmap e395: 8 total, 8 up, 5 in 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: osdmap e396: 8 total, 8 up, 5 in 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/120271762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: pgmap v1460: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.7 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 541/132546 objects misplaced (0.408%); 26 MiB/s, 412 objects/s recovering 2023-10-09T04:52:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: osdmap e395: 8 total, 8 up, 5 in 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e395 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: osdmap e396: 8 total, 8 up, 5 in 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: failed to encode map e396 with expected crc 2023-10-09T04:52:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/120271762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:11.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:11.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:11.764 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:11.799 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:11.839 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:11.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:11.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:11.979 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:12.290 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e395 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e396 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e396 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.19 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8137 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.19 continuing backfill to osd.7 from (350'2400,394'8137] 13:9f6a65a4:::benchmark_data_smithi029_495570_object90476:head to 394'8137 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: osdmap e397: 8 total, 8 up, 5 in 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.6 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8434 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.17 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8218 2023-10-09T04:52:12.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.12 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8310 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.9 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8143 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.8 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8277 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[216903]: 13.b starting backfill to osd.2 from (0'0,0'0] MAX to 394'9279 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e395 with expected crc 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e396 with expected crc 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e396 with expected crc 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.19 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8137 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.19 continuing backfill to osd.7 from (350'2400,394'8137] 13:9f6a65a4:::benchmark_data_smithi029_495570_object90476:head to 394'8137 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.292 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: osdmap e397: 8 total, 8 up, 5 in 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.6 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8434 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.17 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8218 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.12 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8310 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.9 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8143 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.8 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8277 2023-10-09T04:52:12.293 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:12 smithi029 ceph-mon[223858]: 13.b starting backfill to osd.2 from (0'0,0'0] MAX to 394'9279 2023-10-09T04:52:12.313 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:12.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:12.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:12.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:12.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e395 with expected crc 2023-10-09T04:52:12.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e396 with expected crc 2023-10-09T04:52:12.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e396 with expected crc 2023-10-09T04:52:12.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.19 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8137 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.19 continuing backfill to osd.7 from (350'2400,394'8137] 13:9f6a65a4:::benchmark_data_smithi029_495570_object90476:head to 394'8137 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: osdmap e397: 8 total, 8 up, 5 in 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.6 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8434 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.17 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8218 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.12 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8310 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.9 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8143 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.8 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8277 2023-10-09T04:52:12.496 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:12 smithi177 ceph-mon[195797]: 13.b starting backfill to osd.2 from (0'0,0'0] MAX to 394'9279 2023-10-09T04:52:12.499 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:12.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:12.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:12.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:12.787 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:13.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:13.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:13.169 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:52:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:52:12] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-10-09T04:52:13.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:13.267 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:13.350 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:13.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: pgmap v1463: 99 pgs: 13 peering, 1 active+remapped+backfilling, 85 active+clean; 2.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.11k op/s; 321/122385 objects misplaced (0.262%); 14 MiB/s, 219 objects/s recovering 2023-10-09T04:52:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.10 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8289 2023-10-09T04:52:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.11 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8287 2023-10-09T04:52:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.c starting backfill to osd.2 from (0'0,0'0] MAX to 394'9362 2023-10-09T04:52:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.a starting backfill to osd.2 from (0'0,0'0] MAX to 394'9317 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.4 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8156 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.1 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8239 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.16 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8417 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.f starting backfill to osd.2 from (0'0,0'0] MAX to 394'9302 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.18 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8210 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.13 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8165 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: 13.0 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8227 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: osdmap e398: 8 total, 8 up, 5 in 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[216903]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: pgmap v1463: 99 pgs: 13 peering, 1 active+remapped+backfilling, 85 active+clean; 2.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.11k op/s; 321/122385 objects misplaced (0.262%); 14 MiB/s, 219 objects/s recovering 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.10 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8289 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.11 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8287 2023-10-09T04:52:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.c starting backfill to osd.2 from (0'0,0'0] MAX to 394'9362 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.a starting backfill to osd.2 from (0'0,0'0] MAX to 394'9317 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.4 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8156 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.1 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8239 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.16 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8417 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.f starting backfill to osd.2 from (0'0,0'0] MAX to 394'9302 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.18 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8210 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.13 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8165 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: 13.0 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8227 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: osdmap e398: 8 total, 8 up, 5 in 2023-10-09T04:52:13.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:13 smithi029 ceph-mon[223858]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: pgmap v1463: 99 pgs: 13 peering, 1 active+remapped+backfilling, 85 active+clean; 2.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.11k op/s; 321/122385 objects misplaced (0.262%); 14 MiB/s, 219 objects/s recovering 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.10 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8289 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.11 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8287 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.c starting backfill to osd.2 from (0'0,0'0] MAX to 394'9362 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.a starting backfill to osd.2 from (0'0,0'0] MAX to 394'9317 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.4 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8156 2023-10-09T04:52:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.1 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8239 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.16 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8417 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.f starting backfill to osd.2 from (0'0,0'0] MAX to 394'9302 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.18 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8210 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.13 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8165 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: 13.0 starting backfill to osd.2 from (0'0,0'0] MAX to 394'8227 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e397 with expected crc 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: osdmap e398: 8 total, 8 up, 5 in 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:13 smithi177 ceph-mon[195797]: failed to encode map e398 with expected crc 2023-10-09T04:52:13.545 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:13.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:13.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:13.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:13.798 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:13.866 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:13.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:13.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:14.058 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:14.106 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:14.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:14.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:14.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:14.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:14.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:14.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:14.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:14.523 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T04:52:14.551 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:14.639 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:14.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:14.649 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:14.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:14.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: pgmap v1466: 99 pgs: 13 peering, 1 active+remapped+backfilling, 85 active+clean; 2.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 846 op/s; 321/122385 objects misplaced (0.262%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: osdmap e399: 8 total, 8 up, 5 in 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:14 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: pgmap v1466: 99 pgs: 13 peering, 1 active+remapped+backfilling, 85 active+clean; 2.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 846 op/s; 321/122385 objects misplaced (0.262%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:52:14.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:14.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:14.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: osdmap e399: 8 total, 8 up, 5 in 2023-10-09T04:52:14.780 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: pgmap v1466: 99 pgs: 13 peering, 1 active+remapped+backfilling, 85 active+clean; 2.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 846 op/s; 321/122385 objects misplaced (0.262%); 2.6 MiB/s, 42 objects/s recovering 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: osdmap e399: 8 total, 8 up, 5 in 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.781 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.782 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.782 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.782 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.782 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:14 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:14.783 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:14.846 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:14.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:14.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:15.017 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:15.132 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:15.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:15.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:15.305 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:15.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:15.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:15.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:15.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:15.571 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:15.642 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:15.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:15.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:15.779 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:15.829 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:15.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:15.933 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:15.961 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:16.008 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:16.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:16.112 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:16.139 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:16.189 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:16.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:16.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: failed to encode map e399 with expected crc 2023-10-09T04:52:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: osdmap e400: 8 total, 8 up, 5 in 2023-10-09T04:52:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.388 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.389 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:16 smithi177 ceph-mon[195797]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.389 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:16.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:16.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:16.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:52:16.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: failed to encode map e399 with expected crc 2023-10-09T04:52:16.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: osdmap e400: 8 total, 8 up, 5 in 2023-10-09T04:52:16.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[223858]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: failed to encode map e399 with expected crc 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: osdmap e400: 8 total, 8 up, 5 in 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:16 smithi029 ceph-mon[216903]: failed to encode map e400 with expected crc 2023-10-09T04:52:16.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:16.691 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:16.821 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:16.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:16.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:16.929 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:17.004 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:17.155 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:17.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:17.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:17.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:17.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:17 smithi177 ceph-mon[195797]: pgmap v1469: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 904 op/s; 24371/111531 objects misplaced (21.851%); 23 MiB/s, 2 keys/s, 377 objects/s recovering 2023-10-09T04:52:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:17 smithi177 ceph-mon[195797]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:17 smithi177 ceph-mon[195797]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:17 smithi177 ceph-mon[195797]: osdmap e401: 8 total, 8 up, 5 in 2023-10-09T04:52:17.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:17 smithi177 ceph-mon[195797]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.329 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:17.425 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:17.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[223858]: pgmap v1469: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 904 op/s; 24371/111531 objects misplaced (21.851%); 23 MiB/s, 2 keys/s, 377 objects/s recovering 2023-10-09T04:52:17.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[223858]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[223858]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[223858]: osdmap e401: 8 total, 8 up, 5 in 2023-10-09T04:52:17.435 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[223858]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:17.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:17.436 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[216903]: pgmap v1469: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 904 op/s; 24371/111531 objects misplaced (21.851%); 23 MiB/s, 2 keys/s, 377 objects/s recovering 2023-10-09T04:52:17.436 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[216903]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.436 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[216903]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[216903]: osdmap e401: 8 total, 8 up, 5 in 2023-10-09T04:52:17.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:17 smithi029 ceph-mon[216903]: failed to encode map e401 with expected crc 2023-10-09T04:52:17.453 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:17.537 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:17.586 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:17.868 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:17.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:17.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:18.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:18.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:18.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:18.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:18.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:18.344 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:18.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:18.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:18.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:18.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:18.631 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:18.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:18.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:18.887 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:18.967 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:19.084 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:19.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:19.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:19.205 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:52:19.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:19 smithi177 ceph-mon[195797]: pgmap v1471: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 771 op/s; 24371/111531 objects misplaced (21.851%); 20 MiB/s, 2 keys/s, 322 objects/s recovering 2023-10-09T04:52:19.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/364454454' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:19.270 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:19.522 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:19.524 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 2] out_osds: [6, 3, 0] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:52:19.524 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:52:19.525 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.7511137852003409 2023-10-09T04:52:19.525 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.7511137852003409 2023-10-09T04:52:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:19.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:19.559 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:19 smithi029 ceph-mon[223858]: pgmap v1471: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 771 op/s; 24371/111531 objects misplaced (21.851%); 20 MiB/s, 2 keys/s, 322 objects/s recovering 2023-10-09T04:52:19.560 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/364454454' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:19.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:19 smithi029 ceph-mon[216903]: pgmap v1471: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 771 op/s; 24371/111531 objects misplaced (21.851%); 20 MiB/s, 2 keys/s, 322 objects/s recovering 2023-10-09T04:52:19.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/364454454' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:19.629 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:19.684 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:19.863 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:19.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:19.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:20.002 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:20.095 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:52:20.095 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/364454454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: osdmap e402: 8 total, 8 up, 5 in 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[216903]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/364454454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: osdmap e402: 8 total, 8 up, 5 in 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.399 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:20 smithi029 ceph-mon[223858]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/364454454' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: osdmap e402: 8 total, 8 up, 5 in 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: failed to encode map e402 with expected crc 2023-10-09T04:52:20.532 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:20 smithi177 ceph-mon[195797]: failed to encode map e402 with expected crc 2023-10-09T04:52:21.073 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:21.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:21.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:21.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:21.230 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[223858]: pgmap v1473: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 649 op/s; 23105/107733 objects misplaced (21.447%); 1.8 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[223858]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[223858]: osdmap e403: 8 total, 8 up, 5 in 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[223858]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[223858]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[223858]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[216903]: pgmap v1473: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 649 op/s; 23105/107733 objects misplaced (21.447%); 1.8 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T04:52:21.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[216903]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[216903]: osdmap e403: 8 total, 8 up, 5 in 2023-10-09T04:52:21.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[216903]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[216903]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:21 smithi029 ceph-mon[216903]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.510 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:21.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:21.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:21 smithi177 ceph-mon[195797]: pgmap v1473: 99 pgs: 1 peering, 1 active+remapped+backfilling, 16 active+remapped+backfill_wait, 81 active+clean; 2.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 649 op/s; 23105/107733 objects misplaced (21.447%); 1.8 MiB/s, 1 keys/s, 32 objects/s recovering 2023-10-09T04:52:21.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:21 smithi177 ceph-mon[195797]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:21 smithi177 ceph-mon[195797]: osdmap e403: 8 total, 8 up, 5 in 2023-10-09T04:52:21.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:21 smithi177 ceph-mon[195797]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:21 smithi177 ceph-mon[195797]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:21 smithi177 ceph-mon[195797]: failed to encode map e403 with expected crc 2023-10-09T04:52:21.608 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:21.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:21.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:52:21.708 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:21.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:21.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:21.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:21.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:22.013 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:52:22.042 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:52:22.061 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:22.070 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:52:22.097 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:52:22.143 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:52:22.188 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:52:22.203 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:22.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:22.218 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:52:22.251 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:52:22.262 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.263 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:52:22.287 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/508241367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:22.288 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/508241367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:22.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:22.395 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:22.471 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:22.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:22.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/508241367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:22.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:22.620 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:22.666 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:22.786 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:22.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:22.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:22.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:22.918 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:22.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:52:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:52:22] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-10-09T04:52:22.961 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:22.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:23.065 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:23.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:23.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:23.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:23.418 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:23.441 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:23 smithi177 ceph-mon[195797]: pgmap v1475: 99 pgs: 1 active+remapped+backfilling, 1 peering, 15 active+remapped+backfill_wait, 82 active+clean; 2.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 749 op/s; 20061/98034 objects misplaced (20.463%); 15 MiB/s, 241 objects/s recovering 2023-10-09T04:52:23.441 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 04:52:23 smithi177 conmon[174387]: logger=cleanup t=2023-10-09T04:52:23.43360944Z level=info msg="Completed cleanup jobs" duration=250.301831ms 2023-10-09T04:52:23.459 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:23.505 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:52:23.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:23 smithi029 ceph-mon[216903]: pgmap v1475: 99 pgs: 1 active+remapped+backfilling, 1 peering, 15 active+remapped+backfill_wait, 82 active+clean; 2.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 749 op/s; 20061/98034 objects misplaced (20.463%); 15 MiB/s, 241 objects/s recovering 2023-10-09T04:52:23.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:23 smithi029 ceph-mon[223858]: pgmap v1475: 99 pgs: 1 active+remapped+backfilling, 1 peering, 15 active+remapped+backfill_wait, 82 active+clean; 2.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 749 op/s; 20061/98034 objects misplaced (20.463%); 15 MiB/s, 241 objects/s recovering 2023-10-09T04:52:24.187 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:24.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:24.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:24.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.320 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:24.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:24.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:24.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:24.571 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:24.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:24.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:24.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:24.848 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:24.918 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:24.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:24.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:25.028 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:25.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:25.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:25.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:25.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:25.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:25.322 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:52:25.322 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:52:25.322 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:52:25.322 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:52:25.323 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:52:25.397 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:25.478 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[216903]: pgmap v1476: 99 pgs: 1 active+remapped+backfilling, 1 peering, 15 active+remapped+backfill_wait, 82 active+clean; 2.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 20061/98034 objects misplaced (20.463%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[216903]: osdmap e404: 8 total, 8 up, 5 in 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[216903]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[216903]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[216903]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[223858]: pgmap v1476: 99 pgs: 1 active+remapped+backfilling, 1 peering, 15 active+remapped+backfill_wait, 82 active+clean; 2.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 20061/98034 objects misplaced (20.463%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[223858]: osdmap e404: 8 total, 8 up, 5 in 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[223858]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[223858]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.479 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:25 smithi029 ceph-mon[223858]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:25 smithi177 ceph-mon[195797]: pgmap v1476: 99 pgs: 1 active+remapped+backfilling, 1 peering, 15 active+remapped+backfill_wait, 82 active+clean; 2.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 20061/98034 objects misplaced (20.463%); 13 MiB/s, 200 objects/s recovering 2023-10-09T04:52:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:25 smithi177 ceph-mon[195797]: osdmap e404: 8 total, 8 up, 5 in 2023-10-09T04:52:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:25 smithi177 ceph-mon[195797]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:25 smithi177 ceph-mon[195797]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.531 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:25 smithi177 ceph-mon[195797]: failed to encode map e404 with expected crc 2023-10-09T04:52:25.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:25.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:25.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:25.729 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:25.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:25.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:25.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:25.951 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:26.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:26.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:26.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:26.133 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:26.184 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:26.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:26.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:26.236 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 0.751114 (c048) 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: osdmap e405: 8 total, 8 up, 5 in 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2789359397' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]: dispatch 2023-10-09T04:52:26.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]: dispatch 2023-10-09T04:52:26.288 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:26.314 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:26.481 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:26.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:26.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:26.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:52:26.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:26.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: osdmap e405: 8 total, 8 up, 5 in 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2789359397' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]: dispatch 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]: dispatch 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: osdmap e405: 8 total, 8 up, 5 in 2023-10-09T04:52:26.614 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.614 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.614 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: failed to encode map e405 with expected crc 2023-10-09T04:52:26.614 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2789359397' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]: dispatch 2023-10-09T04:52:26.614 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]: dispatch 2023-10-09T04:52:26.616 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:26.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:52:26.898 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:26.916 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:26.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:27.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:27.048 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:27.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:27.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:27.250 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:27.288 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:27.509 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:27 smithi177 ceph-mon[195797]: pgmap v1479: 99 pgs: 1 active+remapped+backfilling, 14 active+remapped+backfill_wait, 84 active+clean; 1.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.18k op/s; 17002/86535 objects misplaced (19.648%); 22 MiB/s, 352 objects/s recovering 2023-10-09T04:52:27.509 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:27 smithi177 ceph-mon[195797]: failed to encode map e406 with expected crc 2023-10-09T04:52:27.509 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]': finished 2023-10-09T04:52:27.509 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:27 smithi177 ceph-mon[195797]: osdmap e406: 8 total, 8 up, 5 in 2023-10-09T04:52:27.534 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:27.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:27.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[216903]: pgmap v1479: 99 pgs: 1 active+remapped+backfilling, 14 active+remapped+backfill_wait, 84 active+clean; 1.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.18k op/s; 17002/86535 objects misplaced (19.648%); 22 MiB/s, 352 objects/s recovering 2023-10-09T04:52:27.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[216903]: failed to encode map e406 with expected crc 2023-10-09T04:52:27.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]': finished 2023-10-09T04:52:27.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[216903]: osdmap e406: 8 total, 8 up, 5 in 2023-10-09T04:52:27.564 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[223858]: pgmap v1479: 99 pgs: 1 active+remapped+backfilling, 14 active+remapped+backfill_wait, 84 active+clean; 1.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.18k op/s; 17002/86535 objects misplaced (19.648%); 22 MiB/s, 352 objects/s recovering 2023-10-09T04:52:27.565 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[223858]: failed to encode map e406 with expected crc 2023-10-09T04:52:27.565 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7511137852003409}]': finished 2023-10-09T04:52:27.565 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:27 smithi029 ceph-mon[223858]: osdmap e406: 8 total, 8 up, 5 in 2023-10-09T04:52:27.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:27.667 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:27.713 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:27.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:27.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:27.847 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:27.963 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:27.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:27.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:28.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:28.094 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:28.139 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:28.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:28.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:28.276 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:28.360 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:28.513 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:28.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: osdmap e407: 8 total, 8 up, 5 in 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.521 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:28 smithi177 ceph-mon[195797]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:28.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: osdmap e407: 8 total, 8 up, 5 in 2023-10-09T04:52:28.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[216903]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: osdmap e407: 8 total, 8 up, 5 in 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:28 smithi029 ceph-mon[223858]: failed to encode map e407 with expected crc 2023-10-09T04:52:28.676 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:28.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:28.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:28.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:28.864 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:28.899 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:29.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:29.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:29.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:29.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:29.142 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:29.190 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:29.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122043 12.e 12:7d817187:::benchmark_data_smithi029_442458_object26793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.141234+0000", 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 365.07136245200002, 2023-10-09T04:52:29.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.199009717, 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122043 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141234+0000", 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141234+0000", 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141233+0000", 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141240+0000", 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2409999999999998e-06 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141256+0000", 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5911000000000001e-05 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141263+0000", 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4829999999999996e-06 2023-10-09T04:52:29.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141316+0000", 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3458999999999998e-05 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141466+0000", 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000150142 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:52:29.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141607+0000", 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014059700000000001 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.144527+0000", 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029195950000000001 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.144845+0000", 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031833499999999999 2023-10-09T04:52:29.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.166192+0000", 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0213472 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.166203+0000", 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0761000000000001e-05 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156356+0000", 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99015257800000001 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156367+0000", 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1296e-05 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340172+0000", 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1838049820000001 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340244+0000", 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2359000000000003e-05 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122063 12.e 12:7ad9492b:::benchmark_data_smithi029_442458_object26813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.243785+0000", 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 364.96881163, 2023-10-09T04:52:29.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1204199130000001, 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122063 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243785+0000", 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243785+0000", 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243784+0000", 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243791+0000", 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9959999999999996e-06 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243802+0000", 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1204e-05 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243808+0000", 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1279999999999996e-06 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243873+0000", 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4492000000000005e-05 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243882+0000", 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0259999999999996e-06 2023-10-09T04:52:29.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156389+0000", 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91250751799999996 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156396+0000", 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1330000000000001e-06 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340270+0000", 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.183874235 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340384+0000", 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113696 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340955+0000", 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057121199999999998 2023-10-09T04:52:29.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.352099+0000", 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011143922000000001 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.360393+0000", 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082938590000000006 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.362792+0000", 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023990779999999998 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.364037+0000", 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012447669999999999 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.364097+0000", 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9707999999999998e-05 2023-10-09T04:52:29.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.364205+0000", 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010851 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122096 12.e 12:70f0f7d5:::benchmark_data_smithi029_442458_object26846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:25.693094+0000", 2023-10-09T04:52:29.233 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 363.519502977, 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.67435742700000001, 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122096 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693094+0000", 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693094+0000", 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693093+0000", 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693099+0000", 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8000000000000001e-06 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693119+0000", 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9911e-05 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693126+0000", 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4289999999999998e-06 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693187+0000", 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1753000000000003e-05 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.693197+0000", 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7939999999999997e-06 2023-10-09T04:52:29.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.348358+0000", 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.655160415 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.348504+0000", 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000146527 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:52:29.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.348663+0000", 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015914299999999999 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.360220+0000", 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011556993 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.360425+0000", 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020419599999999999 2023-10-09T04:52:29.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.362800+0000", 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023758799999999999 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.367264+0000", 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044633549999999996 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.367328+0000", 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4362999999999993e-05 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.367451+0000", 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123194 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.241 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131604 12.0 12:03c61d3f:::benchmark_data_smithi029_442458_object36354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:52.669176+0000", 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 336.54342140599999, 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0355348680000001, 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131604 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669176+0000", 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669176+0000", 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669174+0000", 2023-10-09T04:52:29.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669179+0000", 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1549999999999998e-06 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880049+0000", 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21087008099999999 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.244 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880059+0000", 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0141e-05 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880645+0000", 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00058601599999999997 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880657+0000", 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1181e-05 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.680619+0000", 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79996239199999997 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.680757+0000", 2023-10-09T04:52:29.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000137754 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.680880+0000", 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123087 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.687147+0000", 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062670360000000001 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696233+0000", 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0090863739999999995 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704607+0000", 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083739330000000001 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704665+0000", 2023-10-09T04:52:29.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7516000000000003e-05 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704710+0000", 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5606999999999999e-05 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131623 12.0 12:0846b0cf:::benchmark_data_smithi029_442458_object36373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:52.926170+0000", 2023-10-09T04:52:29.249 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 336.28642704800001, 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.77871051700000005, 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131623 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926170+0000", 2023-10-09T04:52:29.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926170+0000", 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926168+0000", 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926175+0000", 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6449999999999999e-06 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926189+0000", 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4137e-05 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926195+0000", 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.144e-06 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926251+0000", 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5544000000000002e-05 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.926262+0000", 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1378e-05 2023-10-09T04:52:29.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.684197+0000", 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.75793424300000001 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.684321+0000", 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012399399999999999 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.684432+0000", 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111655 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696266+0000", 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011833728999999999 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696298+0000", 2023-10-09T04:52:29.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2277000000000003e-05 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704798+0000", 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0085002180000000004 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704841+0000", 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3043000000000001e-05 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704880+0000", 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8926e-05 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131639 12.0 12:0a013c69:::benchmark_data_smithi029_442458_object36389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:52.967845+0000", 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 336.244752398, 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73711237600000001, 2023-10-09T04:52:29.257 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131639 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967845+0000", 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967845+0000", 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967843+0000", 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967854+0000", 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1609e-05 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967871+0000", 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6663e-05 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967879+0000", 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1079999999999993e-06 2023-10-09T04:52:29.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967942+0000", 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3348000000000004e-05 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.967954+0000", 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1351e-05 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.687174+0000", 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71922045400000001 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.687293+0000", 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118236 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.687425+0000", 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000132526 2023-10-09T04:52:29.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696329+0000", 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0089038889999999999 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696360+0000", 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0851e-05 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.263 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704889+0000", 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0085290829999999998 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704926+0000", 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7419999999999997e-05 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704957+0000", 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0608999999999998e-05 2023-10-09T04:52:29.264 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131707 12.0 12:00765af0:::benchmark_data_smithi029_442458_object36457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.112624+0000", 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 336.09997275000001, 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.59240891900000003, 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.265 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131707 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112624+0000", 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112624+0000", 2023-10-09T04:52:29.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112621+0000", 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999967 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112640+0000", 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9364e-05 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112656+0000", 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5627000000000001e-05 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112662+0000", 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7309999999999998e-06 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112717+0000", 2023-10-09T04:52:29.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5748000000000002e-05 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.112727+0000", 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4690000000000003e-06 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.689965+0000", 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.57723852200000003 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.690098+0000", 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000132581 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.690228+0000", 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000130295 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696394+0000", 2023-10-09T04:52:29.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061652590000000002 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.701687+0000", 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052933930000000004 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704964+0000", 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032770830000000001 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.705001+0000", 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6965000000000003e-05 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.705033+0000", 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2063000000000001e-05 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131715 12.0 12:048e2158:::benchmark_data_smithi029_442458_object36465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.121982+0000", 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 336.09061461300001, 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.58312790000000003, 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131715 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.273 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.121982+0000", 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.121982+0000", 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.121981+0000", 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.121988+0000", 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6059999999999999e-06 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.121998+0000", 2023-10-09T04:52:29.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.058e-05 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.122004+0000", 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5969999999999999e-06 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.122064+0000", 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9809000000000003e-05 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.122072+0000", 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3839999999999997e-06 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.693389+0000", 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.57131657599999996 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.693523+0000", 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013435000000000001 2023-10-09T04:52:29.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.693660+0000", 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000136904 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.701721+0000", 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080610079999999997 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.701792+0000", 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1421999999999995e-05 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.705040+0000", 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003247757 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.705076+0000", 2023-10-09T04:52:29.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6350000000000003e-05 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.705110+0000", 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3822000000000002e-05 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131725 12.0 12:0702dff6:::benchmark_data_smithi029_442458_object36475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:53.145279+0000", 2023-10-09T04:52:29.280 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 336.067318124, 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.562931081, 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131725 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.281 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145279+0000", 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145279+0000", 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145277+0000", 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:52:29.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145284+0000", 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0210000000000002e-06 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145296+0000", 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1718e-05 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145302+0000", 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.562e-06 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145347+0000", 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4391000000000002e-05 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.145357+0000", 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0196e-05 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696416+0000", 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55105971499999995 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696544+0000", 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012707500000000001 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696682+0000", 2023-10-09T04:52:29.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013811499999999999 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.705116+0000", 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0084346609999999995 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.708061+0000", 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029443210000000002 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.286 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.708121+0000", 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0613999999999999e-05 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.708163+0000", 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1603999999999997e-05 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.287 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.708210+0000", 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7024999999999998e-05 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138063 13.15 13:a870998f:::benchmark_data_smithi029_495570_object43803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.123001+0000", 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 32.089596268000001, 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95806695900000005, 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.288 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138063 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123001+0000", 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.289 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123001+0000", 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123000+0000", 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123003+0000", 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5519999999999999e-06 2023-10-09T04:52:29.290 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123008+0000", 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6169999999999998e-06 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123012+0000", 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8010000000000001e-06 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.291 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.124906+0000", 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001893573 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.124914+0000", 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6179999999999995e-06 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.077118+0000", 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95220400199999999 2023-10-09T04:52:29.292 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.077203+0000", 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5028e-05 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.077393+0000", 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018976600000000001 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.293 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.080895+0000", 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035016520000000001 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.080938+0000", 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3207999999999997e-05 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.080988+0000", 2023-10-09T04:52:29.294 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0454999999999997e-05 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081030+0000", 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1866000000000002e-05 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081068+0000", 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7231999999999999e-05 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.295 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138128 13.15 13:a984223b:::benchmark_data_smithi029_495570_object43868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.169301+0000", 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 32.043296353999999, 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91213407700000004, 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.296 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138128 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169301+0000", 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169301+0000", 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.297 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169300+0000", 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169305+0000", 2023-10-09T04:52:29.298 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8510000000000004e-06 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169315+0000", 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9660000000000008e-06 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169321+0000", 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4909999999999996e-06 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.299 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169365+0000", 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3952999999999997e-05 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169370+0000", 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5099999999999998e-06 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.078581+0000", 2023-10-09T04:52:29.300 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90921049899999995 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.078671+0000", 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0488000000000005e-05 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.078846+0000", 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017483399999999999 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.301 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081151+0000", 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023050940000000002 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081201+0000", 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9378000000000001e-05 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081329+0000", 2023-10-09T04:52:29.302 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012866599999999999 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081392+0000", 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2657999999999995e-05 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081435+0000", 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2656e-05 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.303 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138146 13.15 13:afaa51d5:::benchmark_data_smithi029_495570_object43886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.182639+0000", 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 32.029958350999998, 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89888632300000004, 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.304 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138146 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182639+0000", 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182639+0000", 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.305 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182637+0000", 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182642+0000", 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2939999999999999e-06 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.306 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182650+0000", 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5760000000000006e-06 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182655+0000", 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0300000000000001e-06 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.183838+0000", 2023-10-09T04:52:29.307 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011824730000000001 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.183844+0000", 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4110000000000001e-06 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.079602+0000", 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89575800400000005 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.308 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.079707+0000", 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000104713 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.079892+0000", 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018569699999999999 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.309 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081285+0000", 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013926349999999999 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081296+0000", 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0651e-05 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081480+0000", 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018418300000000001 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.310 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081505+0000", 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4831999999999999e-05 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081525+0000", 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0157e-05 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.311 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138191 13.15 13:ae58c5b5:::benchmark_data_smithi029_495570_object43931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.219678+0000", 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.992918454000002, 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.86589176400000001, 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138191 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.312 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.219678+0000", 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.219678+0000", 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.313 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.219678+0000", 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.219686+0000", 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7390000000000005e-06 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.219701+0000", 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4515000000000001e-05 2023-10-09T04:52:29.314 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.219707+0000", 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1920000000000002e-06 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221795+0000", 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002088178 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.315 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221800+0000", 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6240000000000001e-06 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081559+0000", 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85975926199999997 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081653+0000", 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4208000000000003e-05 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.316 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081820+0000", 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016666599999999999 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085410+0000", 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003590074 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085451+0000", 2023-10-09T04:52:29.317 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1325999999999997e-05 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085492+0000", 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0612999999999999e-05 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085535+0000", 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2870999999999997e-05 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.318 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085570+0000", 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5442999999999999e-05 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138194 13.15 13:af3aa515:::benchmark_data_smithi029_495570_object43934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.221442+0000", 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.991155080999999, 2023-10-09T04:52:29.319 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.86662757899999998, 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138194 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221442+0000", 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.320 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221442+0000", 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221441+0000", 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.321 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221444+0000", 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0369999999999999e-06 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221451+0000", 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2759999999999997e-06 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221454+0000", 2023-10-09T04:52:29.322 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3189999999999999e-06 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221817+0000", 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036278100000000001 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.221819+0000", 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7130000000000001e-06 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.323 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.082811+0000", 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.86099217299999997 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.082903+0000", 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1613999999999994e-05 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.083063+0000", 2023-10-09T04:52:29.324 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000159994 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085445+0000", 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023818609999999999 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085456+0000", 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0987e-05 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.325 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088005+0000", 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025493690000000001 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088037+0000", 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2051999999999997e-05 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088069+0000", 2023-10-09T04:52:29.326 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2033000000000003e-05 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138204 13.15 13:af26a55c:::benchmark_data_smithi029_495570_object43944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.229038+0000", 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.983558727999998, 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85915439400000004, 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.327 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138204 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229038+0000", 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.328 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229038+0000", 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229037+0000", 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229042+0000", 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2889999999999999e-06 2023-10-09T04:52:29.329 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229053+0000", 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0988000000000001e-05 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229057+0000", 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4920000000000004e-06 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.330 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229104+0000", 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7039000000000001e-05 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.229111+0000", 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3209999999999999e-06 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.084101+0000", 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85499002800000001 2023-10-09T04:52:29.331 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.084196+0000", 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4467000000000002e-05 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.084364+0000", 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016848500000000001 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.332 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.086901+0000", 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025370430000000001 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.087971+0000", 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001069403 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088117+0000", 2023-10-09T04:52:29.333 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014574299999999999 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088158+0000", 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1873999999999999e-05 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088193+0000", 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4100000000000002e-05 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.334 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138219 13.15 13:aa276d17:::benchmark_data_smithi029_495570_object43959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.242268+0000", 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.970328822999999, 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85360191200000002, 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.335 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138219 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242268+0000", 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242268+0000", 2023-10-09T04:52:29.336 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242267+0000", 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242273+0000", 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2770000000000002e-06 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.337 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242289+0000", 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5815e-05 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242294+0000", 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4140000000000002e-06 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242342+0000", 2023-10-09T04:52:29.338 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7352999999999999e-05 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.242349+0000", 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4309999999999997e-06 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085595+0000", 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.84324589900000002 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.339 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085696+0000", 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100517 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.085875+0000", 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000179061 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.340 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088078+0000", 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022027969999999998 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.092357+0000", 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042796049999999997 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095789+0000", 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003431917 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.341 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095838+0000", 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8921999999999997e-05 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095870+0000", 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2014999999999998e-05 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.342 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138234 13.15 13:ac30b8c6:::benchmark_data_smithi029_495570_object43974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.255497+0000", 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.957099545999998, 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.840464663, 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138234 2023-10-09T04:52:29.343 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.255497+0000", 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.255497+0000", 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.344 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.255496+0000", 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.255500+0000", 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3200000000000001e-06 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.345 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.255509+0000", 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0280000000000006e-06 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.255514+0000", 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7469999999999996e-06 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.256720+0000", 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001205795 2023-10-09T04:52:29.346 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.256728+0000", 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5070000000000003e-06 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.086927+0000", 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.83019899500000005 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.347 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.087025+0000", 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7985000000000005e-05 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.087195+0000", 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017086900000000001 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.089781+0000", 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025857549999999999 2023-10-09T04:52:29.348 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.092408+0000", 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002626682 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095897+0000", 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034890759999999998 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.349 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095932+0000", 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4907000000000003e-05 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095962+0000", 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0213999999999999e-05 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.350 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138252 13.15 13:a9d003e1:::benchmark_data_smithi029_495570_object43992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.269740+0000", 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.942856840000001, 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.82629420399999998, 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138252 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.351 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269740+0000", 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269740+0000", 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.352 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269739+0000", 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269744+0000", 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6229999999999996e-06 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269754+0000", 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9520000000000006e-06 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.353 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269759+0000", 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4539999999999998e-06 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269804+0000", 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5290999999999997e-05 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.354 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.269814+0000", 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1910000000000006e-06 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088233+0000", 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.818419275 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088320+0000", 2023-10-09T04:52:29.355 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7008000000000004e-05 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.088490+0000", 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000170364 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.092368+0000", 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038779890000000001 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.356 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.092442+0000", 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3837000000000005e-05 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.095981+0000", 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003538701 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096005+0000", 2023-10-09T04:52:29.357 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4496000000000001e-05 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096034+0000", 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9003e-05 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138301 13.15 13:acdd7e44:::benchmark_data_smithi029_495570_object44041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.311204+0000", 2023-10-09T04:52:29.358 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.901392438999999, 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.78577052999999997, 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138301 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.359 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311204+0000", 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311204+0000", 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311203+0000", 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.360 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311214+0000", 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0750999999999999e-05 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311228+0000", 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3991999999999999e-05 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.361 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311235+0000", 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7089999999999996e-06 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311266+0000", 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1192999999999998e-05 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.311271+0000", 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5060000000000003e-06 2023-10-09T04:52:29.362 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.089807+0000", 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.77853603999999998 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.089908+0000", 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100865 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.363 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.090088+0000", 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018032899999999999 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.092484+0000", 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023951570000000002 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096052+0000", 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035683999999999998 2023-10-09T04:52:29.364 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096886+0000", 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083353500000000005 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096936+0000", 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0498999999999999e-05 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.365 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096975+0000", 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8927999999999997e-05 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138335 13.15 13:ac8fc5e4:::benchmark_data_smithi029_495570_object44075:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.343667+0000", 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 31.868929454, 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.75338669400000002, 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:52:29.366 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138335 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343667+0000", 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.367 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343667+0000", 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343666+0000", 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343675+0000", 2023-10-09T04:52:29.368 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3499999999999997e-06 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343685+0000", 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0607e-05 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343691+0000", 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8289999999999996e-06 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.369 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343732+0000", 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0938e-05 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.343739+0000", 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1110000000000002e-06 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:52:29.370 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.091101+0000", 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.74736195299999997 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.091228+0000", 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000126575 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.091437+0000", 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000209815 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.371 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.094604+0000", 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031663910000000002 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096068+0000", 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014642959999999999 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:52:29.372 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.096983+0000", 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00091531100000000003 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.097021+0000", 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7845000000000001e-05 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.097054+0000", 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2852999999999999e-05 2023-10-09T04:52:29.373 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.374 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.374 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.374 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:52:29.374 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:52:29.374 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:52:29.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:29.375 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:52:29.415 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: pgmap v1482: 99 pgs: 1 active+remapped+backfilling, 14 active+remapped+backfill_wait, 84 active+clean; 1.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 958 op/s; 17002/86535 objects misplaced (19.648%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: osdmap e408: 8 total, 8 up, 5 in 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: 13.13 starting backfill to osd.1 from (350'2500,403'8501] MIN to 405'8532 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:29.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1977171138' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: pgmap v1482: 99 pgs: 1 active+remapped+backfilling, 14 active+remapped+backfill_wait, 84 active+clean; 1.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 958 op/s; 17002/86535 objects misplaced (19.648%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: osdmap e408: 8 total, 8 up, 5 in 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: 13.13 starting backfill to osd.1 from (350'2500,403'8501] MIN to 405'8532 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:29.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1977171138' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:52:29.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:29 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:52:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: pgmap v1482: 99 pgs: 1 active+remapped+backfilling, 14 active+remapped+backfill_wait, 84 active+clean; 1.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 958 op/s; 17002/86535 objects misplaced (19.648%); 10 MiB/s, 165 objects/s recovering 2023-10-09T04:52:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: osdmap e408: 8 total, 8 up, 5 in 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: failed to encode map e408 with expected crc 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: 13.13 starting backfill to osd.1 from (350'2500,403'8501] MIN to 405'8532 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1977171138' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:52:29.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:29 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:52:29.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:29.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:29.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:29.697 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:30.299 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:30.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:30.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:52:30.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:30.438 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: failed to encode map e408 with expected crc 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: osdmap e409: 8 total, 8 up, 5 in 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:30 smithi177 ceph-mon[195797]: 13.2 scrub starts 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: failed to encode map e408 with expected crc 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: osdmap e409: 8 total, 8 up, 5 in 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[216903]: 13.2 scrub starts 2023-10-09T04:52:30.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: failed to encode map e408 with expected crc 2023-10-09T04:52:30.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:30.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:52:30.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: osdmap e409: 8 total, 8 up, 5 in 2023-10-09T04:52:30.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: failed to encode map e409 with expected crc 2023-10-09T04:52:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:30 smithi029 ceph-mon[223858]: 13.2 scrub starts 2023-10-09T04:52:31.041 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:31.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:31.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:31.127 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:31.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:31.237 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:31 smithi029 ceph-mon[216903]: pgmap v1485: 99 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 86 active+clean; 1.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 144 op/s; 13972/84801 objects misplaced (16.476%) 2023-10-09T04:52:31.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:31 smithi029 ceph-mon[223858]: pgmap v1485: 99 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 86 active+clean; 1.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 144 op/s; 13972/84801 objects misplaced (16.476%) 2023-10-09T04:52:31.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:31.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:31.480 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:31 smithi177 ceph-mon[195797]: pgmap v1485: 99 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 86 active+clean; 1.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 144 op/s; 13972/84801 objects misplaced (16.476%) 2023-10-09T04:52:31.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:31.658 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:31.767 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:52:31.797 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:52:31.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:31.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:31.825 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:52:31.855 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:52:31.895 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:52:31.918 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:31.929 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:52:31.961 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:52:31.996 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:31.996 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:52:31.998 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:52:32.116 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:32.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:32.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:32.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:32.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:32 smithi029 ceph-mon[216903]: 13.11 scrub starts 2023-10-09T04:52:32.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:32 smithi029 ceph-mon[216903]: 13.9 scrub starts 2023-10-09T04:52:32.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1207539036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:32.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:32 smithi029 ceph-mon[223858]: 13.11 scrub starts 2023-10-09T04:52:32.279 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:32 smithi029 ceph-mon[223858]: 13.9 scrub starts 2023-10-09T04:52:32.279 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1207539036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:32.288 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:32.351 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:32.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:32.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:32.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:32 smithi177 ceph-mon[195797]: 13.11 scrub starts 2023-10-09T04:52:32.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:32 smithi177 ceph-mon[195797]: 13.9 scrub starts 2023-10-09T04:52:32.498 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1207539036' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:32.503 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:32.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:32.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:32.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:32.686 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:32.738 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:32.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:32.841 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:32.883 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:32.951 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:32.953 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:52:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:52:32] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-10-09T04:52:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:32.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:33.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:33.081 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:33.089 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 5, 7, 2] out_osds: [6, 3, 0] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:52:33.089 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:52:33.090 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 4, 5, 7, 2] 2023-10-09T04:52:33.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T04:52:33.158 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:33.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:33.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:52:33.262 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:33.302 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:33.499 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:33.500 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: pgmap v1486: 99 pgs: 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 12094/81000 objects misplaced (14.931%); 27 MiB/s, 427 objects/s recovering 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: 13.2 scrub starts 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: osdmap e410: 8 total, 8 up, 5 in 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:33 smithi177 ceph-mon[195797]: 13.3 scrub starts 2023-10-09T04:52:33.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:33.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:33.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:33.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: pgmap v1486: 99 pgs: 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 12094/81000 objects misplaced (14.931%); 27 MiB/s, 427 objects/s recovering 2023-10-09T04:52:33.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: 13.2 scrub starts 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: osdmap e410: 8 total, 8 up, 5 in 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[223858]: 13.3 scrub starts 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: pgmap v1486: 99 pgs: 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 12094/81000 objects misplaced (14.931%); 27 MiB/s, 427 objects/s recovering 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: 13.2 scrub starts 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.644 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: osdmap e410: 8 total, 8 up, 5 in 2023-10-09T04:52:33.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: failed to encode map e410 with expected crc 2023-10-09T04:52:33.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:33 smithi029 ceph-mon[216903]: 13.3 scrub starts 2023-10-09T04:52:33.647 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:33.807 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:52:33.808 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:52:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: 13.11 scrub starts 2023-10-09T04:52:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: 13.11 scrub ok 2023-10-09T04:52:34.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: failed to encode map e410 with expected crc 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: osdmap e411: 8 total, 8 up, 5 in 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[216903]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: 13.11 scrub starts 2023-10-09T04:52:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: 13.11 scrub ok 2023-10-09T04:52:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: failed to encode map e410 with expected crc 2023-10-09T04:52:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: osdmap e411: 8 total, 8 up, 5 in 2023-10-09T04:52:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:34 smithi029 ceph-mon[223858]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: 13.11 scrub starts 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: 13.11 scrub ok 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: failed to encode map e410 with expected crc 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: osdmap e411: 8 total, 8 up, 5 in 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:34 smithi177 ceph-mon[195797]: failed to encode map e411 with expected crc 2023-10-09T04:52:34.832 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:34.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:34.936 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:34.977 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:35.046 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:35.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:52:35.053 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:35.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:35.078 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:35.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:35.188 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:35.310 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: pgmap v1489: 99 pgs: 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 12094/81000 objects misplaced (14.931%); 27 MiB/s, 428 objects/s recovering 2023-10-09T04:52:35.310 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: 13.14 scrub starts 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: osdmap e412: 8 total, 8 up, 5 in 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.311 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:35 smithi177 ceph-mon[195797]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.348 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:35.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:35.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:35.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:35.480 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: pgmap v1489: 99 pgs: 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 12094/81000 objects misplaced (14.931%); 27 MiB/s, 428 objects/s recovering 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: 13.14 scrub starts 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: osdmap e412: 8 total, 8 up, 5 in 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[223858]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.481 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: pgmap v1489: 99 pgs: 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 87 active+clean; 1.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 356 op/s; 12094/81000 objects misplaced (14.931%); 27 MiB/s, 428 objects/s recovering 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: 13.14 scrub starts 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: osdmap e412: 8 total, 8 up, 5 in 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.482 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:35 smithi029 ceph-mon[216903]: failed to encode map e412 with expected crc 2023-10-09T04:52:35.520 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:35.522 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:35.643 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:35.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:35.751 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:35.810 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:35.904 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:35.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:36.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:36.041 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:36.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:36.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:36.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:36.390 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: osdmap e413: 8 total, 8 up, 5 in 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:36 smithi177 ceph-mon[195797]: osdmap e414: 8 total, 8 up, 5 in 2023-10-09T04:52:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: osdmap e413: 8 total, 8 up, 5 in 2023-10-09T04:52:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[223858]: osdmap e414: 8 total, 8 up, 5 in 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: osdmap e413: 8 total, 8 up, 5 in 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: failed to encode map e413 with expected crc 2023-10-09T04:52:36.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:36 smithi029 ceph-mon[216903]: osdmap e414: 8 total, 8 up, 5 in 2023-10-09T04:52:36.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:52:37.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[216903]: pgmap v1492: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 1 active+degraded, 88 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/69864 objects degraded (0.001%); 8590/69864 objects misplaced (12.295%) 2023-10-09T04:52:37.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[216903]: failed to encode map e414 with expected crc 2023-10-09T04:52:37.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 1/69864 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:52:37.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[216903]: failed to encode map e414 with expected crc 2023-10-09T04:52:37.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[216903]: 13.5 scrub starts 2023-10-09T04:52:37.642 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:37 smithi177 ceph-mon[195797]: pgmap v1492: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 1 active+degraded, 88 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/69864 objects degraded (0.001%); 8590/69864 objects misplaced (12.295%) 2023-10-09T04:52:37.642 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:37 smithi177 ceph-mon[195797]: failed to encode map e414 with expected crc 2023-10-09T04:52:37.642 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:37 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 1/69864 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:52:37.642 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:37 smithi177 ceph-mon[195797]: failed to encode map e414 with expected crc 2023-10-09T04:52:37.642 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:37 smithi177 ceph-mon[195797]: 13.5 scrub starts 2023-10-09T04:52:37.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[223858]: pgmap v1492: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 1 active+degraded, 88 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/69864 objects degraded (0.001%); 8590/69864 objects misplaced (12.295%) 2023-10-09T04:52:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[223858]: failed to encode map e414 with expected crc 2023-10-09T04:52:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 1/69864 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:52:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[223858]: failed to encode map e414 with expected crc 2023-10-09T04:52:37.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:37 smithi029 ceph-mon[223858]: 13.5 scrub starts 2023-10-09T04:52:37.991 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:52:37.991 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:52:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:38 smithi029 ceph-mon[216903]: 13.9 scrub starts 2023-10-09T04:52:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:38 smithi029 ceph-mon[216903]: pgmap v1494: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 1 active+degraded, 88 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/69864 objects degraded (0.001%); 8590/69864 objects misplaced (12.295%) 2023-10-09T04:52:38.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:38 smithi029 ceph-mon[216903]: 13.15 scrub starts 2023-10-09T04:52:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:38 smithi029 ceph-mon[223858]: 13.9 scrub starts 2023-10-09T04:52:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:38 smithi029 ceph-mon[223858]: pgmap v1494: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 1 active+degraded, 88 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/69864 objects degraded (0.001%); 8590/69864 objects misplaced (12.295%) 2023-10-09T04:52:38.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:38 smithi029 ceph-mon[223858]: 13.15 scrub starts 2023-10-09T04:52:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:38 smithi177 ceph-mon[195797]: 13.9 scrub starts 2023-10-09T04:52:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:38 smithi177 ceph-mon[195797]: pgmap v1494: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 1 active+degraded, 88 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/69864 objects degraded (0.001%); 8590/69864 objects misplaced (12.295%) 2023-10-09T04:52:38.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:38 smithi177 ceph-mon[195797]: 13.15 scrub starts 2023-10-09T04:52:39.206 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:39.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:39.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:39.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:39.383 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:39.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:52:39.482 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:39.489 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:39 smithi029 ceph-mon[223858]: 13.2 scrub starts 2023-10-09T04:52:39.489 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:39 smithi029 ceph-mon[223858]: 13.2 scrub ok 2023-10-09T04:52:39.489 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:39 smithi029 ceph-mon[216903]: 13.2 scrub starts 2023-10-09T04:52:39.490 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:39 smithi029 ceph-mon[216903]: 13.2 scrub ok 2023-10-09T04:52:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:39.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:39.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:39.602 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:39.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:39 smithi177 ceph-mon[195797]: 13.2 scrub starts 2023-10-09T04:52:39.669 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:39 smithi177 ceph-mon[195797]: 13.2 scrub ok 2023-10-09T04:52:39.681 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:39.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:39.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:39.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:39.920 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:39.967 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:40.047 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:40.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:40.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:40.194 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:40.207 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:40.378 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.1. 2023-10-09T04:52:40.434 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:40.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:40.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:40.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:40.573 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:40.644 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:40 smithi177 ceph-mon[195797]: pgmap v1495: 99 pgs: 9 active+remapped+backfill_wait, 1 active+degraded, 89 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/66723 objects degraded (0.001%); 7340/66723 objects misplaced (11.001%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:52:40.644 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:40 smithi177 ceph-mon[195797]: 13.12 scrub starts 2023-10-09T04:52:40.644 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1551051864' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T04:52:40.644 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:40 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T04:52:40.644 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:40 smithi177 ceph-mon[195797]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[223858]: pgmap v1495: 99 pgs: 9 active+remapped+backfill_wait, 1 active+degraded, 89 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/66723 objects degraded (0.001%); 7340/66723 objects misplaced (11.001%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[223858]: 13.12 scrub starts 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1551051864' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[223858]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:40 smithi029 conmon[216878]: 2023-10-09T04:52:40.327+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e414 definitely_dead 0 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[216903]: pgmap v1495: 99 pgs: 9 active+remapped+backfill_wait, 1 active+degraded, 89 active+clean; 1.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 1/66723 objects degraded (0.001%); 7340/66723 objects misplaced (11.001%); 1.3 MiB/s, 21 objects/s recovering 2023-10-09T04:52:40.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[216903]: 13.12 scrub starts 2023-10-09T04:52:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1551051864' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T04:52:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T04:52:40.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:40 smithi029 ceph-mon[216903]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: osdmap e415: 8 total, 8 up, 4 in 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: osdmap e416: 8 total, 8 up, 4 in 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[223858]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: osdmap e415: 8 total, 8 up, 4 in 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: osdmap e416: 8 total, 8 up, 4 in 2023-10-09T04:52:41.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:41 smithi029 ceph-mon[216903]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: osdmap e415: 8 total, 8 up, 4 in 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e415 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: osdmap e416: 8 total, 8 up, 4 in 2023-10-09T04:52:41.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e416 with expected crc 2023-10-09T04:52:41.781 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:41 smithi177 ceph-mon[195797]: failed to encode map e416 with expected crc 2023-10-09T04:52:42.666 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:52:42.666 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.10 continuing backfill to osd.2 from (351'2600,414'8940] MIN to 414'8940 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.4 continuing backfill to osd.2 from (353'2900,414'8860] MIN to 414'8860 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: pgmap v1498: 99 pgs: 1 remapped+peering, 16 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 74 active+clean; 1.2 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 4969/58404 objects misplaced (8.508%); 18 MiB/s, 294 objects/s recovering 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.19 scrub starts 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: osdmap e417: 8 total, 8 up, 4 in 2023-10-09T04:52:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.a starting backfill to osd.5 from (0'0,0'0] MAX to 414'10588 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.5 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8933 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.12 starting backfill to osd.7 from (351'2600,401'8529] MIN to 414'8967 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.15 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8932 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.8 continuing backfill to osd.2 from (351'2600,414'8922] MIN to 414'8922 2023-10-09T04:52:42.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.16 starting backfill to osd.4 from (351'2700,412'8965] MIN to 414'9106 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.13 starting backfill to osd.5 from (350'2500,380'7635] MIN to 414'8795 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.14 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8738 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.11 starting backfill to osd.7 from (351'2600,374'7683] MIN to 414'8956 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.0 continuing backfill to osd.2 from (351'2600,414'8906] MIN to 414'8906 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.c continuing backfill to osd.2 from (362'6672,414'10691] 13:39fc7aa0:::benchmark_data_smithi029_495570_object93600:head to 414'10691 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[216903]: 13.d starting backfill to osd.5 from (0'0,0'0] MAX to 414'10696 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.10 continuing backfill to osd.2 from (351'2600,414'8940] MIN to 414'8940 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.4 continuing backfill to osd.2 from (353'2900,414'8860] MIN to 414'8860 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: pgmap v1498: 99 pgs: 1 remapped+peering, 16 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 74 active+clean; 1.2 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 4969/58404 objects misplaced (8.508%); 18 MiB/s, 294 objects/s recovering 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.19 scrub starts 2023-10-09T04:52:42.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: osdmap e417: 8 total, 8 up, 4 in 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.a starting backfill to osd.5 from (0'0,0'0] MAX to 414'10588 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.5 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8933 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.12 starting backfill to osd.7 from (351'2600,401'8529] MIN to 414'8967 2023-10-09T04:52:42.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.15 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8932 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.8 continuing backfill to osd.2 from (351'2600,414'8922] MIN to 414'8922 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.16 starting backfill to osd.4 from (351'2700,412'8965] MIN to 414'9106 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.13 starting backfill to osd.5 from (350'2500,380'7635] MIN to 414'8795 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.14 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8738 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.11 starting backfill to osd.7 from (351'2600,374'7683] MIN to 414'8956 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.0 continuing backfill to osd.2 from (351'2600,414'8906] MIN to 414'8906 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.c continuing backfill to osd.2 from (362'6672,414'10691] 13:39fc7aa0:::benchmark_data_smithi029_495570_object93600:head to 414'10691 2023-10-09T04:52:42.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:42 smithi029 ceph-mon[223858]: 13.d starting backfill to osd.5 from (0'0,0'0] MAX to 414'10696 2023-10-09T04:52:42.691 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.10 continuing backfill to osd.2 from (351'2600,414'8940] MIN to 414'8940 2023-10-09T04:52:42.691 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.4 continuing backfill to osd.2 from (353'2900,414'8860] MIN to 414'8860 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: pgmap v1498: 99 pgs: 1 remapped+peering, 16 peering, 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 74 active+clean; 1.2 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 4969/58404 objects misplaced (8.508%); 18 MiB/s, 294 objects/s recovering 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.19 scrub starts 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: osdmap e417: 8 total, 8 up, 4 in 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.692 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.a starting backfill to osd.5 from (0'0,0'0] MAX to 414'10588 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.5 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8933 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.12 starting backfill to osd.7 from (351'2600,401'8529] MIN to 414'8967 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.15 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8932 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.8 continuing backfill to osd.2 from (351'2600,414'8922] MIN to 414'8922 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.16 starting backfill to osd.4 from (351'2700,412'8965] MIN to 414'9106 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.13 starting backfill to osd.5 from (350'2500,380'7635] MIN to 414'8795 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.14 starting backfill to osd.2 from (0'0,0'0] MAX to 414'8738 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.11 starting backfill to osd.7 from (351'2600,374'7683] MIN to 414'8956 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.0 continuing backfill to osd.2 from (351'2600,414'8906] MIN to 414'8906 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.c continuing backfill to osd.2 from (362'6672,414'10691] 13:39fc7aa0:::benchmark_data_smithi029_495570_object93600:head to 414'10691 2023-10-09T04:52:42.693 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:42 smithi177 ceph-mon[195797]: 13.d starting backfill to osd.5 from (0'0,0'0] MAX to 414'10696 2023-10-09T04:52:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:52:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:52:42] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-10-09T04:52:43.316 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:43.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:43.423 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:43.453 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/66723 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: osdmap e418: 8 total, 8 up, 4 in 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[223858]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/66723 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: osdmap e418: 8 total, 8 up, 4 in 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:43 smithi029 ceph-mon[216903]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:52:43.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/66723 objects degraded (0.001%), 1 pg degraded) 2023-10-09T04:52:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: osdmap e418: 8 total, 8 up, 4 in 2023-10-09T04:52:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: failed to encode map e418 with expected crc 2023-10-09T04:52:43.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:43 smithi177 ceph-mon[195797]: failed to encode map e418 with expected crc 2023-10-09T04:52:44.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:52:44.573 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:44.605 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:44.607 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e417 with expected crc 2023-10-09T04:52:44.607 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: pgmap v1501: 99 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 74 active+clean; 1.2 GiB data, 8.4 GiB used, 349 GiB / 358 GiB avail; 4315/58296 objects misplaced (7.402%); 25 MiB/s, 397 objects/s recovering 2023-10-09T04:52:44.607 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:44.607 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:44.607 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: osdmap e419: 8 total, 8 up, 4 in 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[223858]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.608 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e417 with expected crc 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: pgmap v1501: 99 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 74 active+clean; 1.2 GiB data, 8.4 GiB used, 349 GiB / 358 GiB avail; 4315/58296 objects misplaced (7.402%); 25 MiB/s, 397 objects/s recovering 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: osdmap e419: 8 total, 8 up, 4 in 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.610 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.610 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.610 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.610 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:44 smithi029 ceph-mon[216903]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:44.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:44.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:44.743 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:44.811 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e417 with expected crc 2023-10-09T04:52:44.811 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: pgmap v1501: 99 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 74 active+clean; 1.2 GiB data, 8.4 GiB used, 349 GiB / 358 GiB avail; 4315/58296 objects misplaced (7.402%); 25 MiB/s, 397 objects/s recovering 2023-10-09T04:52:44.811 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:44.811 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:44.811 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: osdmap e419: 8 total, 8 up, 4 in 2023-10-09T04:52:44.811 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:44 smithi177 ceph-mon[195797]: failed to encode map e419 with expected crc 2023-10-09T04:52:44.849 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:52:44.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:44.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:52:44.958 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:45.014 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:45.180 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:45.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:45.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:45.266 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:52:45.284 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:45.289 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:52:45.312 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:52:45.319 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:45.333 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:52:45.356 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:52:45.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:45.378 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:52:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:45.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:45.399 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:52:45.418 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:52:45.419 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.419 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.419 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.419 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.419 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.419 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.420 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:52:45.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:45.513 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:45.663 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:45.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:45.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:45.703 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:52:45.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:45.798 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:45.861 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:45.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:45.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:45.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: osdmap e420: 8 total, 8 up, 4 in 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3063576542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/158077681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3063576542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:45.993 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[223858]: osdmap e421: 8 total, 8 up, 4 in 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: osdmap e420: 8 total, 8 up, 4 in 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: failed to encode map e420 with expected crc 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3063576542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/158077681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3063576542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:45.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:45 smithi029 ceph-mon[216903]: osdmap e421: 8 total, 8 up, 4 in 2023-10-09T04:52:45.998 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: failed to encode map e420 with expected crc 2023-10-09T04:52:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: osdmap e420: 8 total, 8 up, 4 in 2023-10-09T04:52:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: failed to encode map e420 with expected crc 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: failed to encode map e420 with expected crc 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: failed to encode map e420 with expected crc 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3063576542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/158077681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3063576542' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:52:46.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:45 smithi177 ceph-mon[195797]: osdmap e421: 8 total, 8 up, 4 in 2023-10-09T04:52:46.057 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:46.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:46.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:46.193 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:46.696 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:52:46.696 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:52:46.696 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:52:46.696 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:52:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: pgmap v1504: 99 pgs: 1 active+clean+remapped, 1 peering, 3 active+remapped+backfilling, 12 active+remapped+backfill_wait, 82 active+clean; 1.0 GiB data, 8.0 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 583 op/s; 8308/51405 objects misplaced (16.162%); 41 MiB/s, 657 objects/s recovering 2023-10-09T04:52:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: failed to encode map e421 with expected crc 2023-10-09T04:52:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: failed to encode map e421 with expected crc 2023-10-09T04:52:46.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: failed to encode map e421 with expected crc 2023-10-09T04:52:46.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: failed to encode map e421 with expected crc 2023-10-09T04:52:46.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:52:46.780 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:46 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:52:47.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: pgmap v1504: 99 pgs: 1 active+clean+remapped, 1 peering, 3 active+remapped+backfilling, 12 active+remapped+backfill_wait, 82 active+clean; 1.0 GiB data, 8.0 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 583 op/s; 8308/51405 objects misplaced (16.162%); 41 MiB/s, 657 objects/s recovering 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:52:47.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: pgmap v1504: 99 pgs: 1 active+clean+remapped, 1 peering, 3 active+remapped+backfilling, 12 active+remapped+backfill_wait, 82 active+clean; 1.0 GiB data, 8.0 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 583 op/s; 8308/51405 objects misplaced (16.162%); 41 MiB/s, 657 objects/s recovering 2023-10-09T04:52:47.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: failed to encode map e421 with expected crc 2023-10-09T04:52:47.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:52:47.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:46 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:52:47.553 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:47.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:47.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:47.696 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:47.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[216903]: osdmap e422: 8 total, 8 up, 4 in 2023-10-09T04:52:47.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[216903]: failed to encode map e422 with expected crc 2023-10-09T04:52:47.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[216903]: failed to encode map e422 with expected crc 2023-10-09T04:52:47.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[216903]: failed to encode map e422 with expected crc 2023-10-09T04:52:47.807 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[223858]: osdmap e422: 8 total, 8 up, 4 in 2023-10-09T04:52:47.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[223858]: failed to encode map e422 with expected crc 2023-10-09T04:52:47.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[223858]: failed to encode map e422 with expected crc 2023-10-09T04:52:47.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:47 smithi029 ceph-mon[223858]: failed to encode map e422 with expected crc 2023-10-09T04:52:47.821 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:47.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:47.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:47.965 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:48.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:47 smithi177 ceph-mon[195797]: osdmap e422: 8 total, 8 up, 4 in 2023-10-09T04:52:48.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:47 smithi177 ceph-mon[195797]: failed to encode map e422 with expected crc 2023-10-09T04:52:48.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:47 smithi177 ceph-mon[195797]: failed to encode map e422 with expected crc 2023-10-09T04:52:48.018 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:47 smithi177 ceph-mon[195797]: failed to encode map e422 with expected crc 2023-10-09T04:52:48.024 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:48.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:48.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:52:48.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:52:48.164 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:48.223 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:52:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:48.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:52:48.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:48.364 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:48.425 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:48.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:48.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:48.566 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:48.622 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:48.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:48.733 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:48.804 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:48.833 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:48.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:48.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:49.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:49.034 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:48 smithi177 ceph-mon[195797]: pgmap v1507: 99 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 85 active+clean; 1.0 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 616 op/s; 8122/50898 objects misplaced (15.957%); 36 MiB/s, 575 objects/s recovering 2023-10-09T04:52:49.112 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:48 smithi029 ceph-mon[216903]: pgmap v1507: 99 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 85 active+clean; 1.0 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 616 op/s; 8122/50898 objects misplaced (15.957%); 36 MiB/s, 575 objects/s recovering 2023-10-09T04:52:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:48 smithi029 ceph-mon[223858]: pgmap v1507: 99 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 85 active+clean; 1.0 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 616 op/s; 8122/50898 objects misplaced (15.957%); 36 MiB/s, 575 objects/s recovering 2023-10-09T04:52:49.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:49.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:49.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:49.413 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:49.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:49.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:49.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:49.634 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:49.671 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:49.730 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:49.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:49.742 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2] out_osds: [6, 3, 0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:52:49.742 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:52:49.742 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-09T04:52:49.742 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:52:49.742 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:52:49.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-09T04:52:49.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:52:49.823 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:52:49.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:49.906 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:49.981 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:49.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:50.089 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:50.145 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:50.268 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:50.315 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:50.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: pgmap v1508: 99 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 85 active+clean; 1.0 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 403 op/s; 8122/50898 objects misplaced (15.957%); 16 MiB/s, 249 objects/s recovering 2023-10-09T04:52:50.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: osdmap e423: 8 total, 8 up, 4 in 2023-10-09T04:52:50.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[216903]: osdmap e424: 8 total, 8 up, 4 in 2023-10-09T04:52:50.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: pgmap v1508: 99 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 85 active+clean; 1.0 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 403 op/s; 8122/50898 objects misplaced (15.957%); 16 MiB/s, 249 objects/s recovering 2023-10-09T04:52:50.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: osdmap e423: 8 total, 8 up, 4 in 2023-10-09T04:52:50.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: failed to encode map e423 with expected crc 2023-10-09T04:52:50.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:50 smithi029 ceph-mon[223858]: osdmap e424: 8 total, 8 up, 4 in 2023-10-09T04:52:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: pgmap v1508: 99 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 85 active+clean; 1.0 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 403 op/s; 8122/50898 objects misplaced (15.957%); 16 MiB/s, 249 objects/s recovering 2023-10-09T04:52:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: osdmap e423: 8 total, 8 up, 4 in 2023-10-09T04:52:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: failed to encode map e423 with expected crc 2023-10-09T04:52:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: failed to encode map e423 with expected crc 2023-10-09T04:52:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: failed to encode map e423 with expected crc 2023-10-09T04:52:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: failed to encode map e423 with expected crc 2023-10-09T04:52:51.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:50 smithi177 ceph-mon[195797]: osdmap e424: 8 total, 8 up, 4 in 2023-10-09T04:52:51.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:51.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:51.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:51.218 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:51.251 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:51.722 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:51.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:51.733 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:52:51.733 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:52:51.733 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:52:51.733 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:52:51.733 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:52:51.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:51.918 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:51 smithi029 ceph-mon[216903]: failed to encode map e424 with expected crc 2023-10-09T04:52:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:51 smithi029 ceph-mon[216903]: failed to encode map e424 with expected crc 2023-10-09T04:52:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:51 smithi029 ceph-mon[216903]: osdmap e425: 8 total, 8 up, 4 in 2023-10-09T04:52:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:51 smithi029 ceph-mon[223858]: failed to encode map e424 with expected crc 2023-10-09T04:52:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:51 smithi029 ceph-mon[223858]: failed to encode map e424 with expected crc 2023-10-09T04:52:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:51 smithi029 ceph-mon[223858]: osdmap e425: 8 total, 8 up, 4 in 2023-10-09T04:52:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:51 smithi177 ceph-mon[195797]: failed to encode map e424 with expected crc 2023-10-09T04:52:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:51 smithi177 ceph-mon[195797]: failed to encode map e424 with expected crc 2023-10-09T04:52:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:51 smithi177 ceph-mon[195797]: osdmap e425: 8 total, 8 up, 4 in 2023-10-09T04:52:52.216 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:52.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:52.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:52.352 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:52 smithi177 ceph-mon[195797]: pgmap v1511: 99 pgs: 1 active+remapped+backfilling, 1 peering, 11 active+remapped+backfill_wait, 86 active+clean; 861 MiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 582 op/s; 5377/41394 objects misplaced (12.990%); 17 MiB/s, 278 objects/s recovering 2023-10-09T04:52:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:52 smithi177 ceph-mon[195797]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:52 smithi177 ceph-mon[195797]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:52 smithi177 ceph-mon[195797]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:52 smithi177 ceph-mon[195797]: osdmap e426: 8 total, 8 up, 4 in 2023-10-09T04:52:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:52:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:52:52] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-10-09T04:52:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[216903]: pgmap v1511: 99 pgs: 1 active+remapped+backfilling, 1 peering, 11 active+remapped+backfill_wait, 86 active+clean; 861 MiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 582 op/s; 5377/41394 objects misplaced (12.990%); 17 MiB/s, 278 objects/s recovering 2023-10-09T04:52:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[216903]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[216903]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[216903]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[216903]: osdmap e426: 8 total, 8 up, 4 in 2023-10-09T04:52:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[223858]: pgmap v1511: 99 pgs: 1 active+remapped+backfilling, 1 peering, 11 active+remapped+backfill_wait, 86 active+clean; 861 MiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 582 op/s; 5377/41394 objects misplaced (12.990%); 17 MiB/s, 278 objects/s recovering 2023-10-09T04:52:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[223858]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[223858]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[223858]: failed to encode map e425 with expected crc 2023-10-09T04:52:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:52 smithi029 ceph-mon[223858]: osdmap e426: 8 total, 8 up, 4 in 2023-10-09T04:52:53.213 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:53.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:53.317 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:53.356 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:53.735 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:53.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:53.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:53.876 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:53.933 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:53.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:53.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[216903]: osdmap e427: 8 total, 8 up, 4 in 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.975 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.976 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.976 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.976 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.976 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: failed to encode map e426 with expected crc 2023-10-09T04:52:53.976 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:53 smithi029 ceph-mon[223858]: osdmap e427: 8 total, 8 up, 4 in 2023-10-09T04:52:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: failed to encode map e426 with expected crc 2023-10-09T04:52:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: failed to encode map e426 with expected crc 2023-10-09T04:52:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: failed to encode map e426 with expected crc 2023-10-09T04:52:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: failed to encode map e426 with expected crc 2023-10-09T04:52:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: failed to encode map e426 with expected crc 2023-10-09T04:52:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: failed to encode map e426 with expected crc 2023-10-09T04:52:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:53 smithi177 ceph-mon[195797]: osdmap e427: 8 total, 8 up, 4 in 2023-10-09T04:52:54.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:54.070 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:54.120 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:54.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:54.180 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:52:54.200 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:52:54.220 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:52:54.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:54.241 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:52:54.255 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:54.264 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:52:54.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:52:54.308 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:52:54.330 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:52:54.331 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.331 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.332 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:52:54.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:54.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:54.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:54.909 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: pgmap v1514: 99 pgs: 1 active+remapped+backfilling, 1 peering, 11 active+remapped+backfill_wait, 86 active+clean; 842 MiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 871 op/s; 5377/40443 objects misplaced (13.295%); 21 MiB/s, 338 objects/s recovering 2023-10-09T04:52:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2233020387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: osdmap e428: 8 total, 8 up, 4 in 2023-10-09T04:52:54.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3701149524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:54.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: pgmap v1514: 99 pgs: 1 active+remapped+backfilling, 1 peering, 11 active+remapped+backfill_wait, 86 active+clean; 842 MiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 871 op/s; 5377/40443 objects misplaced (13.295%); 21 MiB/s, 338 objects/s recovering 2023-10-09T04:52:54.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: failed to encode map e427 with expected crc 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2233020387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: osdmap e428: 8 total, 8 up, 4 in 2023-10-09T04:52:54.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3701149524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:54.957 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:54.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: pgmap v1514: 99 pgs: 1 active+remapped+backfilling, 1 peering, 11 active+remapped+backfill_wait, 86 active+clean; 842 MiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 871 op/s; 5377/40443 objects misplaced (13.295%); 21 MiB/s, 338 objects/s recovering 2023-10-09T04:52:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: failed to encode map e427 with expected crc 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: failed to encode map e427 with expected crc 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: failed to encode map e427 with expected crc 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: failed to encode map e427 with expected crc 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: failed to encode map e427 with expected crc 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2233020387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: osdmap e428: 8 total, 8 up, 4 in 2023-10-09T04:52:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3701149524' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:52:55.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:55.120 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:55.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:55.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:55.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:52:55.327 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:55.475 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:52:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:55.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:52:55.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:55.608 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:55.654 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:55.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:55.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:55.709 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:52:55.757 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:55.788 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:55.946 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:55.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:55.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:56.006 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3701149524' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:56.007 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:55 smithi177 ceph-mon[195797]: osdmap e429: 8 total, 8 up, 4 in 2023-10-09T04:52:56.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:52:56.050 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:56.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:56.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:56.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3701149524' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[216903]: osdmap e429: 8 total, 8 up, 4 in 2023-10-09T04:52:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: failed to encode map e428 with expected crc 2023-10-09T04:52:56.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:52:56.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3701149524' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:52:56.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:55 smithi029 ceph-mon[223858]: osdmap e429: 8 total, 8 up, 4 in 2023-10-09T04:52:56.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:56.282 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:56.327 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:56.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:56.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:56.461 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:56.511 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:56.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:56.538 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:52:56.538 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2023-10-09T04:52:56.538 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:52:56.552 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:52:56.616 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.644 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:56.766 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:56.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:56.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: pgmap v1517: 99 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 5 peering, 90 active+clean; 739 MiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 489 op/s; 756/35523 objects misplaced (2.128%); 51 MiB/s, 817 objects/s recovering 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.849 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:56 smithi177 ceph-mon[195797]: 13.18 scrub starts 2023-10-09T04:52:56.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:56.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: pgmap v1517: 99 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 5 peering, 90 active+clean; 739 MiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 489 op/s; 756/35523 objects misplaced (2.128%); 51 MiB/s, 817 objects/s recovering 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[223858]: 13.18 scrub starts 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: pgmap v1517: 99 pgs: 1 active+clean+remapped, 3 active+remapped+backfilling, 5 peering, 90 active+clean; 739 MiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 489 op/s; 756/35523 objects misplaced (2.128%); 51 MiB/s, 817 objects/s recovering 2023-10-09T04:52:56.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: failed to encode map e429 with expected crc 2023-10-09T04:52:56.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:56 smithi029 ceph-mon[216903]: 13.18 scrub starts 2023-10-09T04:52:56.901 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:56.951 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:56.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:57.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:52:57.085 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:57.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:52:57.906 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:57.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: osdmap e430: 8 total, 8 up, 4 in 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[223858]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: osdmap e430: 8 total, 8 up, 4 in 2023-10-09T04:52:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: failed to encode map e430 with expected crc 2023-10-09T04:52:57.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:57 smithi029 ceph-mon[216903]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:52:58.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:58.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: osdmap e430: 8 total, 8 up, 4 in 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:57 smithi177 ceph-mon[195797]: failed to encode map e430 with expected crc 2023-10-09T04:52:58.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:58.325 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:58.422 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:58.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:58.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:52:58.561 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:58.616 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:52:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:58.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:52:58.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:58.749 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:58.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:52:58.801 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:58.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:58.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:58 smithi177 ceph-mon[195797]: pgmap v1520: 99 pgs: 3 active+remapped+backfilling, 5 peering, 91 active+clean; 730 MiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 445 op/s; 756/35094 objects misplaced (2.154%); 51 MiB/s, 817 objects/s recovering 2023-10-09T04:52:58.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:58 smithi177 ceph-mon[195797]: failed to encode map e431 with expected crc 2023-10-09T04:52:58.825 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:58 smithi177 ceph-mon[195797]: osdmap e431: 8 total, 8 up, 4 in 2023-10-09T04:52:58.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:58 smithi177 ceph-mon[195797]: failed to encode map e431 with expected crc 2023-10-09T04:52:58.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:58.826 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:52:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:58.905 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:52:58.939 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:59.002 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:52:59.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[223858]: pgmap v1520: 99 pgs: 3 active+remapped+backfilling, 5 peering, 91 active+clean; 730 MiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 445 op/s; 756/35094 objects misplaced (2.154%); 51 MiB/s, 817 objects/s recovering 2023-10-09T04:52:59.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[223858]: failed to encode map e431 with expected crc 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[223858]: osdmap e431: 8 total, 8 up, 4 in 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[223858]: failed to encode map e431 with expected crc 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[216903]: pgmap v1520: 99 pgs: 3 active+remapped+backfilling, 5 peering, 91 active+clean; 730 MiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 445 op/s; 756/35094 objects misplaced (2.154%); 51 MiB/s, 817 objects/s recovering 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[216903]: failed to encode map e431 with expected crc 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[216903]: osdmap e431: 8 total, 8 up, 4 in 2023-10-09T04:52:59.005 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[216903]: failed to encode map e431 with expected crc 2023-10-09T04:52:59.006 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:52:59.006 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:52:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:52:59.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:59.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:52:59.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:52:59.196 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:59.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:52:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:59.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:52:59.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:52:59.465 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:52:59.466 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:52:59.501 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:52:59.521 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:52:59.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:52:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:52:59.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:52:59.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:52:59.908 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:00.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:00.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:00.051 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:53:00.082 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:53:00.105 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:53:00.117 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:00.128 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:53:00.150 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:00.153 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:53:00.190 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:53:00.216 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:53:00.253 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:53:00.254 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.254 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.255 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.255 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.255 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.255 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.257 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.257 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:53:00.361 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:00.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:00.467 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:00.540 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:00.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:53:00.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:00.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:00.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:00.824 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:01.086 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:53:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:00 smithi029 ceph-mon[216903]: pgmap v1522: 99 pgs: 3 active+remapped+backfilling, 5 peering, 91 active+clean; 730 MiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 245 op/s; 756/35094 objects misplaced (2.154%); 33 MiB/s, 530 objects/s recovering 2023-10-09T04:53:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3120023985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:00 smithi029 ceph-mon[223858]: pgmap v1522: 99 pgs: 3 active+remapped+backfilling, 5 peering, 91 active+clean; 730 MiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 245 op/s; 756/35094 objects misplaced (2.154%); 33 MiB/s, 530 objects/s recovering 2023-10-09T04:53:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3120023985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:00 smithi177 ceph-mon[195797]: pgmap v1522: 99 pgs: 3 active+remapped+backfilling, 5 peering, 91 active+clean; 730 MiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 245 op/s; 756/35094 objects misplaced (2.154%); 33 MiB/s, 530 objects/s recovering 2023-10-09T04:53:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3120023985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:02.340 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:02.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:02.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:02.498 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:02.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:02.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:02.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:02.999 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:03.033 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:02 smithi177 ceph-mon[195797]: pgmap v1523: 99 pgs: 99 active+clean; 593 MiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 389 op/s; 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:53:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:02 smithi029 ceph-mon[216903]: pgmap v1523: 99 pgs: 99 active+clean; 593 MiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 389 op/s; 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:53:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:53:02] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:53:03.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:02 smithi029 ceph-mon[223858]: pgmap v1523: 99 pgs: 99 active+clean; 593 MiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 389 op/s; 7.8 MiB/s, 124 objects/s recovering 2023-10-09T04:53:03.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:03.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:03.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:03.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:03.435 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:182250 13.13 13:cb50fbd1:::benchmark_data_smithi029_495570_object87990:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.492662+0000", 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0013811139999999999, 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014063120000000001, 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:53:03.496 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 182250 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492662+0000", 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492662+0000", 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492660+0000", 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492668+0000", 2023-10-09T04:53:03.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8089999999999993e-06 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492684+0000", 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6232999999999999e-05 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492691+0000", 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0859999999999996e-06 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492737+0000", 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6288000000000001e-05 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492869+0000", 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013160599999999999 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.493191+0000", 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032207299999999998 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:182232 13.e 13:73539ab4:::benchmark_data_smithi029_495570_object87972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.472777+0000", 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.021266027, 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021344107000000001, 2023-10-09T04:53:03.501 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 182232 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472777+0000", 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472777+0000", 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472775+0000", 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472783+0000", 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7190000000000001e-06 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472797+0000", 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4202000000000001e-05 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472804+0000", 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2380000000000002e-06 2023-10-09T04:53:03.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472852+0000", 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8488000000000001e-05 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.472977+0000", 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012543600000000001 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.473112+0000", 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013483599999999999 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.475449+0000", 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023364739999999998 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:53:03.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486350+0000", 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010901239 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:182239 13.0 13:07c6c9a3:::benchmark_data_smithi029_495570_object87979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.478392+0000", 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.015650699000000001, 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015779535000000001, 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.507 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 182239 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478392+0000", 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478392+0000", 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478391+0000", 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478397+0000", 2023-10-09T04:53:03.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.55e-06 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478408+0000", 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0641e-05 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478414+0000", 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3160000000000002e-06 2023-10-09T04:53:03.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.481040+0000", 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002625741 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.481141+0000", 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000101267 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.481468+0000", 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032645999999999999 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.484841+0000", 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033734139999999999 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486438+0000", 2023-10-09T04:53:03.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015971620000000001 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:182236 13.10 e431/419 13:0c8b026e:::benchmark_data_smithi029_495570_object87976:head v 431'9344, mlcod=431'9341)", 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.478453+0000", 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.015589373, 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015768799, 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:53:03.513 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478453+0000", 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478453+0000", 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478453+0000", 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478460+0000", 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4440000000000002e-06 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478469+0000", 2023-10-09T04:53:03.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7779999999999996e-06 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478474+0000", 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6970000000000001e-06 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478772+0000", 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029786500000000002 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.478789+0000", 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7000999999999999e-05 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:182241 13.e 13:726692d0:::benchmark_data_smithi029_495570_object87981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.480655+0000", 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.013388061999999999, 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01360337, 2023-10-09T04:53:03.517 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 182241 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480655+0000", 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480655+0000", 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480654+0000", 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480660+0000", 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.93e-06 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480671+0000", 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1875999999999999e-05 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480676+0000", 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7149999999999997e-06 2023-10-09T04:53:03.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480710+0000", 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3606999999999999e-05 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480817+0000", 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107508 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.480921+0000", 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103723 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.483325+0000", 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002404322 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486380+0000", 2023-10-09T04:53:03.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003054668 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:182244 13.0 13:071f21a2:::benchmark_data_smithi029_495570_object87984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.486352+0000", 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0076905369999999999, 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079659139999999993, 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for sub ops", 2023-10-09T04:53:03.523 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 182244 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486352+0000", 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486352+0000", 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486351+0000", 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486357+0000", 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.331e-06 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486367+0000", 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7149999999999993e-06 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486373+0000", 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9699999999999996e-06 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486461+0000", 2023-10-09T04:53:03.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8098999999999998e-05 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486568+0000", 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000107161 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.486944+0000", 2023-10-09T04:53:03.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00037587999999999999 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492902+0000", 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059579389999999998 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492920+0000", 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8355e-05 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:182245 13.8 e431/428 13:115bfe83:::benchmark_data_smithi029_495570_object87985:head v 431'9318, mlcod=431'9316)", 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.488081+0000", 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0059620970000000004, 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062950239999999998, 2023-10-09T04:53:03.531 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488081+0000", 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488081+0000", 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488080+0000", 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488085+0000", 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1540000000000001e-06 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488096+0000", 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0412e-05 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488102+0000", 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1179999999999997e-06 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.491010+0000", 2023-10-09T04:53:03.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029080540000000002 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.491027+0000", 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7041000000000002e-05 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:182248 13.10 e431/419 13:09149a14:::benchmark_data_smithi029_495570_object87988:head v 431'9345, mlcod=431'9342)", 2023-10-09T04:53:03.535 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.488107+0000", 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0059356579999999999, 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063027700000000001, 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488107+0000", 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488107+0000", 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488107+0000", 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967296 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488110+0000", 2023-10-09T04:53:03.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1329999999999999e-06 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488115+0000", 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0849999999999996e-06 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.488118+0000", 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.993e-06 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492092+0000", 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003974198 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.492111+0000", 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8454999999999999e-05 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.46418.0:182249 13.d e431/428 13:bd574302:::benchmark_data_smithi029_495570_object87989:head v 431'11441, mlcod=431'11437)", 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:53:03.490218+0000", 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.0038248449999999999, 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042439299999999999, 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.490218+0000", 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.490218+0000", 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.490217+0000", 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.490223+0000", 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8069999999999996e-06 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.490232+0000", 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2909999999999996e-06 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:53:03.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.490237+0000", 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.882e-06 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.491354+0000", 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00111652 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:53:03.491372+0000", 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7757e-05 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.543 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:53:03.544 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.544 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:53:03.544 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T04:53:03.544 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 9 2023-10-09T04:53:03.544 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:53:03.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:03.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:03.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:03.601 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2] out_osds: [6, 3, 0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:53:03.602 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:53:03.602 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:53:03.602 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2023-10-09T04:53:03.602 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0 2023-10-09T04:53:03.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:03.695 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:03.755 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:03.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:03.820 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:53:03.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:03.889 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/57718481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:03.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/57718481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:03.894 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/57718481' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:04.056 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:04.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:04.160 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:04.188 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:04.233 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:04.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:04.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:04.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:04.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:53:04.373 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:04.426 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:04.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:04.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:04.530 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:04.558 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:04.606 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:04.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:04.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:04.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:53:04.710 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:04.799 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:04.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:04.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:04.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:04 smithi177 ceph-mon[195797]: pgmap v1524: 99 pgs: 99 active+clean; 564 MiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 363 op/s; 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:53:04.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:04 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:04.983 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:04 smithi177 ceph-mon[195797]: failed to encode map e432 with expected crc 2023-10-09T04:53:04.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/57718481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:04.984 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:04 smithi177 ceph-mon[195797]: osdmap e432: 8 total, 8 up, 4 in 2023-10-09T04:53:05.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:05.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[216903]: pgmap v1524: 99 pgs: 99 active+clean; 564 MiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 363 op/s; 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:53:05.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:05.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[216903]: failed to encode map e432 with expected crc 2023-10-09T04:53:05.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/57718481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:05.099 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[216903]: osdmap e432: 8 total, 8 up, 4 in 2023-10-09T04:53:05.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[223858]: pgmap v1524: 99 pgs: 99 active+clean; 564 MiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 363 op/s; 6.1 MiB/s, 97 objects/s recovering 2023-10-09T04:53:05.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:05.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[223858]: failed to encode map e432 with expected crc 2023-10-09T04:53:05.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/57718481' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:05.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:04 smithi029 ceph-mon[223858]: osdmap e432: 8 total, 8 up, 4 in 2023-10-09T04:53:05.149 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:05.156 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:05.191 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:05.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:05.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:05.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:05.397 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:05.458 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:53:05.505 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:05.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:05.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:05.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:05.641 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:05.692 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:05.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:05.797 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:05.831 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:07.040 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:07.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:07.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:07.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:06 smithi029 ceph-mon[216903]: pgmap v1526: 99 pgs: 99 active+clean; 337 MiB data, 6.1 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 786 op/s; 5.9 MiB/s, 93 objects/s recovering 2023-10-09T04:53:07.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:06 smithi029 ceph-mon[223858]: pgmap v1526: 99 pgs: 99 active+clean; 337 MiB data, 6.1 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 786 op/s; 5.9 MiB/s, 93 objects/s recovering 2023-10-09T04:53:07.102 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:53:07.146 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:07.174 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:06 smithi177 ceph-mon[195797]: pgmap v1526: 99 pgs: 99 active+clean; 337 MiB data, 6.1 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 786 op/s; 5.9 MiB/s, 93 objects/s recovering 2023-10-09T04:53:07.676 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:07.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:07.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:07.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:07.826 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:07.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:07 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:07.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:07 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:08.018 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:08.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:08.023 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:08.123 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:08.186 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:53:08.186 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:53:08.186 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:53:08.186 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:53:08.187 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:53:08.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:07 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:53:08.222 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:08.542 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:53:08.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:08.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:08.569 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:53:08.598 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:53:08.625 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:53:08.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:08.666 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:53:08.698 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:53:08.722 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:53:08.738 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:08.750 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:53:08.752 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.752 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.752 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.752 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.752 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.752 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.753 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.753 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:53:08.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:08.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:08.854 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.7 primary-affinity to 0 (802) 2023-10-09T04:53:08.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:08.916 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:08.958 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:08.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:09.062 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:09.093 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:09.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[216903]: pgmap v1527: 99 pgs: 99 active+clean; 302 MiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 710 op/s; 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:53:09.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4013316804' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T04:53:09.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T04:53:09.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1159230869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:09.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[223858]: pgmap v1527: 99 pgs: 99 active+clean; 302 MiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 710 op/s; 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:53:09.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4013316804' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T04:53:09.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T04:53:09.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1159230869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:08 smithi177 ceph-mon[195797]: pgmap v1527: 99 pgs: 99 active+clean; 302 MiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 710 op/s; 4.9 MiB/s, 77 objects/s recovering 2023-10-09T04:53:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4013316804' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T04:53:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-09T04:53:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1159230869' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:09.457 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:53:10.121 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[216903]: failed to encode map e433 with expected crc 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[216903]: osdmap e433: 8 total, 8 up, 4 in 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[216903]: failed to encode map e433 with expected crc 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[223858]: failed to encode map e433 with expected crc 2023-10-09T04:53:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T04:53:10.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[223858]: osdmap e433: 8 total, 8 up, 4 in 2023-10-09T04:53:10.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[223858]: failed to encode map e433 with expected crc 2023-10-09T04:53:10.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:10.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:09 smithi177 ceph-mon[195797]: failed to encode map e433 with expected crc 2023-10-09T04:53:10.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-09T04:53:10.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:09 smithi177 ceph-mon[195797]: osdmap e433: 8 total, 8 up, 4 in 2023-10-09T04:53:10.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:09 smithi177 ceph-mon[195797]: failed to encode map e433 with expected crc 2023-10-09T04:53:10.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:10.397 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:10.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:10.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:10.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:10.995 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:10.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:10.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: pgmap v1529: 99 pgs: 99 active+clean; 302 MiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: osdmap e434: 8 total, 8 up, 4 in 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:10 smithi177 ceph-mon[195797]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.100 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:11.167 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:11.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: pgmap v1529: 99 pgs: 99 active+clean; 302 MiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: osdmap e434: 8 total, 8 up, 4 in 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[216903]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: pgmap v1529: 99 pgs: 99 active+clean; 302 MiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 581 op/s 2023-10-09T04:53:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: osdmap e434: 8 total, 8 up, 4 in 2023-10-09T04:53:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:10 smithi029 ceph-mon[223858]: failed to encode map e434 with expected crc 2023-10-09T04:53:11.771 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 94257 objects 2023-10-09T04:53:11.771 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :132.754 2023-10-09T04:53:12.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:12.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:12.141 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:12.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:11 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:53:12.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:11 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:53:12.176 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:12.227 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:12.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:12.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:11 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:53:12.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:12.361 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:12.513 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:12.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:12.549 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:53:12.550 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:53:12.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:12.663 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:12.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:12 smithi177 ceph-mon[195797]: pgmap v1531: 99 pgs: 8 peering, 91 active+clean; 136 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:53:12.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1305765178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:12.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:12.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:12.920 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:53:13.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:13.042 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:13.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:13.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:53:12] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-10-09T04:53:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:12 smithi029 ceph-mon[216903]: pgmap v1531: 99 pgs: 8 peering, 91 active+clean; 136 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:53:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1305765178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:12 smithi029 ceph-mon[223858]: pgmap v1531: 99 pgs: 8 peering, 91 active+clean; 136 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 902 op/s 2023-10-09T04:53:13.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1305765178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:53:13.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:13.268 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:13.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:13.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:13.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:53:13.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:13.764 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:13.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:53:13.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:13.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:14.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:14.039 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:14.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:14.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1305765178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:53:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[216903]: osdmap e435: 8 total, 8 up, 4 in 2023-10-09T04:53:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:14.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1305765178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:53:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[223858]: osdmap e435: 8 total, 8 up, 4 in 2023-10-09T04:53:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:14.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:14.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:14.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:13 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:14.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1305765178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:53:14.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:13 smithi177 ceph-mon[195797]: osdmap e435: 8 total, 8 up, 4 in 2023-10-09T04:53:14.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:14.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:14.266 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:14.313 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:14.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:14.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:14.330 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:14.379 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:14.417 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:14.443 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:14.486 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:14.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:14.594 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:14.619 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:14.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:14.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:14.707 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-10-09T04:53:14.707 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-10-09T04:53:14.769 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:14.796 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:14.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:14.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:14.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:14.981 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:15.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: pgmap v1533: 99 pgs: 8 peering, 91 active+clean; 82 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T04:53:15.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: failed to encode map e435 with expected crc 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: failed to encode map e435 with expected crc 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: pgmap v1533: 99 pgs: 8 peering, 91 active+clean; 82 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T04:53:15.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: failed to encode map e435 with expected crc 2023-10-09T04:53:15.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: failed to encode map e435 with expected crc 2023-10-09T04:53:15.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.186 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:15.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: pgmap v1533: 99 pgs: 8 peering, 91 active+clean; 82 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 611 op/s 2023-10-09T04:53:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: failed to encode map e435 with expected crc 2023-10-09T04:53:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: failed to encode map e435 with expected crc 2023-10-09T04:53:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:15.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:15.323 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:15.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:15.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:15.424 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2] out_osds: [6, 3, 0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:53:15.424 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:53:15.424 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T04:53:15.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T04:53:15.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:15.502 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:15.604 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:15.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:15.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:15.736 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: 13.3 scrub starts 2023-10-09T04:53:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: 13.3 scrub ok 2023-10-09T04:53:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: 13.1 scrub starts 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[216903]: 13.1 scrub ok 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: 13.3 scrub starts 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: 13.3 scrub ok 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:16.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: 13.1 scrub starts 2023-10-09T04:53:16.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:15 smithi029 ceph-mon[223858]: 13.1 scrub ok 2023-10-09T04:53:16.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:53:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: 13.3 scrub starts 2023-10-09T04:53:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: 13.3 scrub ok 2023-10-09T04:53:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:53:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:53:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: 13.1 scrub starts 2023-10-09T04:53:16.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:15 smithi177 ceph-mon[195797]: 13.1 scrub ok 2023-10-09T04:53:16.870 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:16.904 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:16.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:16.949 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:17.009 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:17.035 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:16 smithi029 ceph-mon[216903]: pgmap v1534: 99 pgs: 99 active+clean; 22 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:17.036 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:16 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-09T04:53:17.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:16 smithi029 ceph-mon[223858]: pgmap v1534: 99 pgs: 99 active+clean; 22 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:17.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:16 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-09T04:53:17.038 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:17.151 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:17.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:17.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:16 smithi177 ceph-mon[195797]: pgmap v1534: 99 pgs: 99 active+clean; 22 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:16 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-09T04:53:17.285 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:17.304 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:53:17.304 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:53:17.584 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:17.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:17.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:17.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:17.721 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:17.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:17.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:17.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:17.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:17.903 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:17 smithi177 ceph-mon[195797]: 13.6 scrub starts 2023-10-09T04:53:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:17 smithi177 ceph-mon[195797]: 13.6 scrub ok 2023-10-09T04:53:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:17 smithi177 ceph-mon[195797]: 13.10 scrub starts 2023-10-09T04:53:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:17 smithi177 ceph-mon[195797]: 13.10 scrub ok 2023-10-09T04:53:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:17 smithi177 ceph-mon[195797]: 13.9 scrub starts 2023-10-09T04:53:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:17 smithi177 ceph-mon[195797]: 13.9 scrub ok 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[216903]: 13.6 scrub starts 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[216903]: 13.6 scrub ok 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[216903]: 13.10 scrub starts 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[216903]: 13.10 scrub ok 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[216903]: 13.9 scrub starts 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[216903]: 13.9 scrub ok 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[223858]: 13.6 scrub starts 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[223858]: 13.6 scrub ok 2023-10-09T04:53:18.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[223858]: 13.10 scrub starts 2023-10-09T04:53:18.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[223858]: 13.10 scrub ok 2023-10-09T04:53:18.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[223858]: 13.9 scrub starts 2023-10-09T04:53:18.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:17 smithi029 ceph-mon[223858]: 13.9 scrub ok 2023-10-09T04:53:18.276 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:18.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:18.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:53:18.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:18.448 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:18.575 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:18.663 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:53:18.686 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:53:18.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:53:18.726 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:53:18.745 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:53:18.765 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:53:18.784 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:53:18.804 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:53:18.849 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:18.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:18.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:18.916 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:18.955 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:18.986 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:19.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:19.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:19.140 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:19.168 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: pgmap v1535: 99 pgs: 99 active+clean; 19 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: 13.5 scrub starts 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: 13.5 scrub ok 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: 13.4 scrub starts 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: 13.4 scrub ok 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: 13.17 scrub starts 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: 13.17 scrub ok 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1012110776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: pgmap v1535: 99 pgs: 99 active+clean; 19 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: 13.5 scrub starts 2023-10-09T04:53:19.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: 13.5 scrub ok 2023-10-09T04:53:19.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: 13.4 scrub starts 2023-10-09T04:53:19.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: 13.4 scrub ok 2023-10-09T04:53:19.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: 13.17 scrub starts 2023-10-09T04:53:19.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: 13.17 scrub ok 2023-10-09T04:53:19.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1012110776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:19.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:19.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: pgmap v1535: 99 pgs: 99 active+clean; 19 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: 13.5 scrub starts 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: 13.5 scrub ok 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: 13.4 scrub starts 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: 13.4 scrub ok 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: 13.17 scrub starts 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: 13.17 scrub ok 2023-10-09T04:53:19.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1012110776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:19.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:19.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:19.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:19.344 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:19.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:19.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:19.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:19.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:19.517 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:19.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:19.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:19.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:19.692 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:19.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:19.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:19.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:19.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:19.875 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:20.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:19 smithi177 ceph-mon[195797]: 13.8 scrub starts 2023-10-09T04:53:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:19 smithi177 ceph-mon[195797]: 13.8 scrub ok 2023-10-09T04:53:20.284 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:19 smithi029 ceph-mon[216903]: 13.8 scrub starts 2023-10-09T04:53:20.284 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:19 smithi029 ceph-mon[216903]: 13.8 scrub ok 2023-10-09T04:53:20.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:19 smithi029 ceph-mon[223858]: 13.8 scrub starts 2023-10-09T04:53:20.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:19 smithi029 ceph-mon[223858]: 13.8 scrub ok 2023-10-09T04:53:20.293 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:20.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:20.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:20.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:20.473 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:20 smithi029 ceph-mon[216903]: pgmap v1536: 99 pgs: 99 active+clean; 19 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:20 smithi029 ceph-mon[223858]: pgmap v1536: 99 pgs: 99 active+clean; 19 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:20 smithi177 ceph-mon[195797]: pgmap v1536: 99 pgs: 99 active+clean; 19 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:21.425 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:21.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:21.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:21.562 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:22.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:21 smithi177 ceph-mon[195797]: 13.19 scrub starts 2023-10-09T04:53:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:21 smithi177 ceph-mon[195797]: 13.19 scrub ok 2023-10-09T04:53:22.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:21 smithi029 ceph-mon[216903]: 13.19 scrub starts 2023-10-09T04:53:22.352 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:21 smithi029 ceph-mon[216903]: 13.19 scrub ok 2023-10-09T04:53:22.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:21 smithi029 ceph-mon[223858]: 13.19 scrub starts 2023-10-09T04:53:22.352 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:21 smithi029 ceph-mon[223858]: 13.19 scrub ok 2023-10-09T04:53:22.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:53:23.108 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:22 smithi029 ceph-mon[216903]: pgmap v1537: 99 pgs: 99 active+clean; 18 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:23.109 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:22 smithi029 ceph-mon[216903]: 13.16 scrub starts 2023-10-09T04:53:23.109 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:22 smithi029 ceph-mon[223858]: pgmap v1537: 99 pgs: 99 active+clean; 18 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:23.109 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:22 smithi029 ceph-mon[223858]: 13.16 scrub starts 2023-10-09T04:53:23.109 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:53:22] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.43.0" 2023-10-09T04:53:23.162 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:23.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:23.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:23.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:22 smithi177 ceph-mon[195797]: pgmap v1537: 99 pgs: 99 active+clean; 18 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:22 smithi177 ceph-mon[195797]: 13.16 scrub starts 2023-10-09T04:53:23.302 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:23.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:23.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:23.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:23.493 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:23.582 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:23.592 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:23.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:23.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:23.701 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:23.728 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:23.953 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_8' removed 2023-10-09T04:53:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:23 smithi029 ceph-mon[223858]: 13.16 scrub ok 2023-10-09T04:53:24.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4110311763' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:23 smithi029 ceph-mon[216903]: 13.16 scrub ok 2023-10-09T04:53:24.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4110311763' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:23 smithi177 ceph-mon[195797]: 13.16 scrub ok 2023-10-09T04:53:24.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4110311763' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T04:53:24.489 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:24.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:24.593 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:24.628 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:24.680 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:24.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:24.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:24.811 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:24.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:24.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:24.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:24.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:24.987 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:25.030 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:25.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:25.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: pgmap v1538: 99 pgs: 99 active+clean; 17 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: 13.15 scrub starts 2023-10-09T04:53:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: 13.15 scrub ok 2023-10-09T04:53:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4110311763' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:53:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: osdmap e436: 8 total, 8 up, 4 in 2023-10-09T04:53:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4161122811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: pgmap v1538: 99 pgs: 99 active+clean; 17 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: 13.15 scrub starts 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: 13.15 scrub ok 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4110311763' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: osdmap e436: 8 total, 8 up, 4 in 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4161122811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:25.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: pgmap v1538: 99 pgs: 99 active+clean; 17 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:25.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: 13.15 scrub starts 2023-10-09T04:53:25.214 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: 13.15 scrub ok 2023-10-09T04:53:25.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4110311763' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T04:53:25.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: osdmap e436: 8 total, 8 up, 4 in 2023-10-09T04:53:25.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:25.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4161122811' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:25.216 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:25.270 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:53:25.270 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:53:25.270 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:53:25.270 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:53:25.271 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:53:25.287 INFO:teuthology.orchestra.run.smithi029.stdout:moved 35 / 219 (15.9817%) 2023-10-09T04:53:25.287 INFO:teuthology.orchestra.run.smithi029.stdout:avg 54.75 2023-10-09T04:53:25.287 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 9.44391 -> 3.03109 (expected baseline 6.408) 2023-10-09T04:53:25.287 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 39 -> 57 pgs (0.712329 -> 1.0411 * mean) 2023-10-09T04:53:25.287 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.7 with 63 -> 50 pgs (1.15068 -> 0.913242 * mean) 2023-10-09T04:53:25.287 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0138 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0152 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.7511 -> 1.0000 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 1.0000 -> 0.8390 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 weight 1.0000 -> 0.8585 2023-10-09T04:53:25.288 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 1.0000 -> 0.8615 2023-10-09T04:53:25.290 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:53:25.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:25.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:25.330 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-09T04:53:25.331 INFO:tasks.radosbench:Beginning radosbench... 2023-10-09T04:53:25.331 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-10-09T04:53:25.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-10-09T04:53:25.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:25.461 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: 13.12 scrub starts 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: from='client.46781 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536','4':'56262','5':'56461','7':'54986'}"}]: dispatch 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: failed to encode map e437 with expected crc 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4161122811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','4':'56262','5':'56461','7':'54986'}"}]': finished 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: osdmap e437: 8 total, 8 up, 4 in 2023-10-09T04:53:26.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:25 smithi177 ceph-mon[195797]: osdmap e438: 8 total, 8 up, 4 in 2023-10-09T04:53:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: 13.12 scrub starts 2023-10-09T04:53:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: from='client.46781 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536','4':'56262','5':'56461','7':'54986'}"}]: dispatch 2023-10-09T04:53:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: failed to encode map e437 with expected crc 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4161122811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','4':'56262','5':'56461','7':'54986'}"}]': finished 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: osdmap e437: 8 total, 8 up, 4 in 2023-10-09T04:53:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[223858]: osdmap e438: 8 total, 8 up, 4 in 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: 13.12 scrub starts 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: from='client.46781 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536','4':'56262','5':'56461','7':'54986'}"}]: dispatch 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: failed to encode map e436 with expected crc 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T04:53:26.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: failed to encode map e437 with expected crc 2023-10-09T04:53:26.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4161122811' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:26.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536','4':'56262','5':'56461','7':'54986'}"}]': finished 2023-10-09T04:53:26.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: osdmap e437: 8 total, 8 up, 4 in 2023-10-09T04:53:26.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:25 smithi029 ceph-mon[216903]: osdmap e438: 8 total, 8 up, 4 in 2023-10-09T04:53:26.624 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:26.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:26.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:26.643 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T04:53:26.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-09T04:53:26.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:26.755 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:26.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:26.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:26.907 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:26.933 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:26.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:26.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:26.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: pgmap v1541: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:26 smithi177 ceph-mon[195797]: osdmap e439: 8 total, 8 up, 4 in 2023-10-09T04:53:27.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: pgmap v1541: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:27.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[216903]: osdmap e439: 8 total, 8 up, 4 in 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: pgmap v1541: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: failed to encode map e438 with expected crc 2023-10-09T04:53:27.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:26 smithi029 ceph-mon[223858]: osdmap e439: 8 total, 8 up, 4 in 2023-10-09T04:53:27.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:27.111 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:27.657 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:27.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:27.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:27.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:27.793 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:27.840 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:27.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:27.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:27.974 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:28.018 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:28.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:28.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:28.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:28.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:53:28.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:28.212 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:28.214 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:27 smithi029 ceph-mon[223858]: failed to encode map e439 with expected crc 2023-10-09T04:53:28.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:28.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:28.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:27 smithi029 ceph-mon[216903]: failed to encode map e439 with expected crc 2023-10-09T04:53:28.244 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:27 smithi177 ceph-mon[195797]: failed to encode map e439 with expected crc 2023-10-09T04:53:28.298 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:28.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:28.346 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:28.388 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:28.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:28.493 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:28.524 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:28.826 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:28.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:28.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:28.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:28.958 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:29.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:29.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:29.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:29.105 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:29.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[216903]: pgmap v1544: 73 pgs: 2 active+clean+remapped, 71 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:29.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[216903]: failed to encode map e439 with expected crc 2023-10-09T04:53:29.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:29.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:29.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[223858]: pgmap v1544: 73 pgs: 2 active+clean+remapped, 71 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:29.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[223858]: failed to encode map e439 with expected crc 2023-10-09T04:53:29.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:29.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:29.134 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:28 smithi177 ceph-mon[195797]: pgmap v1544: 73 pgs: 2 active+clean+remapped, 71 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:28 smithi177 ceph-mon[195797]: failed to encode map e439 with expected crc 2023-10-09T04:53:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:29.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:53:29.736 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:29.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:29.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:29.878 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:29.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:29.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:29.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:30.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:30.063 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:30.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:30.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:30.214 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:30.241 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:30.283 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:30.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:30.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:30.420 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:30.474 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:30.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:30.537 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:30.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:30.578 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:53:30.599 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:53:30.604 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:30.620 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:53:30.644 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:53:30.646 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:30.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:30.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:30.667 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:53:30.689 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:53:30.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:53:30.734 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:53:30.736 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.736 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.736 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:53:30.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:30.786 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:31.009 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:30 smithi029 ceph-mon[216903]: pgmap v1545: 73 pgs: 2 active+clean+remapped, 71 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1883919000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:30 smithi029 ceph-mon[223858]: pgmap v1545: 73 pgs: 2 active+clean+remapped, 71 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1883919000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:30 smithi177 ceph-mon[195797]: pgmap v1545: 73 pgs: 2 active+clean+remapped, 71 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail 2023-10-09T04:53:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1883919000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:31.970 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_9' created 2023-10-09T04:53:31.990 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:31.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:32.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:32.126 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1998795296' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T04:53:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T04:53:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1998795296' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T04:53:32.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T04:53:32.174 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:32.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1998795296' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T04:53:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-09T04:53:32.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:32.315 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 267 (0%) 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:avg 66.75 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 4.33734 -> 4.33734 (expected baseline 7.07549) 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.7 with 59 -> 59 pgs (0.883895 -> 0.883895 * mean) 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.2 with 70 -> 70 pgs (1.04869 -> 1.04869 * mean) 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T04:53:32.480 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2023-10-09T04:53:32.481 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:53:32.481 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0137 2023-10-09T04:53:32.481 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.1373 2023-10-09T04:53:32.481 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:53:33.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:53:32] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: pgmap v1546: 73 pgs: 3 active+clean+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 90 B/s, 0 objects/s recovering 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: osdmap e440: 8 total, 8 up, 4 in 2023-10-09T04:53:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[216903]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: pgmap v1546: 73 pgs: 3 active+clean+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 90 B/s, 0 objects/s recovering 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: osdmap e440: 8 total, 8 up, 4 in 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:32 smithi029 ceph-mon[223858]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: pgmap v1546: 73 pgs: 3 active+clean+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 90 B/s, 0 objects/s recovering 2023-10-09T04:53:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-09T04:53:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: osdmap e440: 8 total, 8 up, 4 in 2023-10-09T04:53:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:32 smithi177 ceph-mon[195797]: failed to encode map e440 with expected crc 2023-10-09T04:53:33.728 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:33.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:33.832 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:33.861 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:33.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:53:34.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[216903]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:34.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[216903]: osdmap e441: 8 total, 8 up, 4 in 2023-10-09T04:53:34.013 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[216903]: failed to encode map e441 with expected crc 2023-10-09T04:53:34.013 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[216903]: failed to encode map e441 with expected crc 2023-10-09T04:53:34.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[223858]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:34.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[223858]: osdmap e441: 8 total, 8 up, 4 in 2023-10-09T04:53:34.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[223858]: failed to encode map e441 with expected crc 2023-10-09T04:53:34.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:33 smithi029 ceph-mon[223858]: failed to encode map e441 with expected crc 2023-10-09T04:53:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:33 smithi177 ceph-mon[195797]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:33 smithi177 ceph-mon[195797]: osdmap e441: 8 total, 8 up, 4 in 2023-10-09T04:53:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:33 smithi177 ceph-mon[195797]: failed to encode map e441 with expected crc 2023-10-09T04:53:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:33 smithi177 ceph-mon[195797]: failed to encode map e441 with expected crc 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: pgmap v1549: 89 pgs: 5 creating+peering, 11 unknown, 3 active+clean+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 82 B/s, 0 objects/s recovering 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: osdmap e442: 8 total, 8 up, 4 in 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[223858]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: pgmap v1549: 89 pgs: 5 creating+peering, 11 unknown, 3 active+clean+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 82 B/s, 0 objects/s recovering 2023-10-09T04:53:35.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: osdmap e442: 8 total, 8 up, 4 in 2023-10-09T04:53:35.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:34 smithi029 ceph-mon[216903]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.073 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:53:35.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-10-09T04:53:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: pgmap v1549: 89 pgs: 5 creating+peering, 11 unknown, 3 active+clean+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 82 B/s, 0 objects/s recovering 2023-10-09T04:53:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: osdmap e442: 8 total, 8 up, 4 in 2023-10-09T04:53:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:34 smithi177 ceph-mon[195797]: failed to encode map e442 with expected crc 2023-10-09T04:53:35.415 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:35.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:35.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:35.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-10-09T04:53:35.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:35.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:35.641 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:35.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:35.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:35.781 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:35.993 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:36.034 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:36.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:36.041 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:36.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:36.173 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:36.361 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:36.414 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:53:36.414 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:53:36.479 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:36.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:36.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:36.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:36.615 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:36.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:36.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:36.771 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:36.799 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:36.842 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:36.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:36.947 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:36.979 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:53:36.982 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:37.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:36 smithi177 ceph-mon[195797]: pgmap v1551: 89 pgs: 5 creating+peering, 3 active+clean+remapped, 81 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 92 B/s, 0 objects/s recovering 2023-10-09T04:53:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/731829845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:53:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:36 smithi029 ceph-mon[216903]: pgmap v1551: 89 pgs: 5 creating+peering, 3 active+clean+remapped, 81 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 92 B/s, 0 objects/s recovering 2023-10-09T04:53:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/731829845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:53:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:36 smithi029 ceph-mon[223858]: pgmap v1551: 89 pgs: 5 creating+peering, 3 active+clean+remapped, 81 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%); 92 B/s, 0 objects/s recovering 2023-10-09T04:53:37.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/731829845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:53:37.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:53:37.635 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:37.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:37.740 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:37.768 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/731829845' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:53:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: osdmap e443: 8 total, 8 up, 4 in 2023-10-09T04:53:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:37 smithi177 ceph-mon[195797]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/731829845' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: osdmap e443: 8 total, 8 up, 4 in 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[216903]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.396 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:38.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/731829845' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:53:38.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: osdmap e443: 8 total, 8 up, 4 in 2023-10-09T04:53:38.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: failed to encode map e443 with expected crc 2023-10-09T04:53:38.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:37 smithi029 ceph-mon[223858]: failed to encode map e443 with expected crc 2023-10-09T04:53:39.022 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:39.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:39.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:39.130 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:39.165 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:39.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:39.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:39.275 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:38 smithi029 ceph-mon[216903]: pgmap v1553: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:39.275 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:38 smithi029 ceph-mon[223858]: pgmap v1553: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:38 smithi177 ceph-mon[195797]: pgmap v1553: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:39.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:39.347 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:39.391 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:39.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:39.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:39.525 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:39.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:39.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:39.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:39.707 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:40.746 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:40.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:40.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:40.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:40.882 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:40.939 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:40.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:41.035 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:41.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:41.077 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:41.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:53:41.099 INFO:teuthology.orchestra.run.smithi029.stdout:moved 30 / 267 (11.236%) 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:avg 66.75 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 4.33734 -> 11.5027 (expected baseline 7.07549) 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.7 with 59 -> 49 pgs (0.883895 -> 0.734082 * mean) 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.2 with 70 -> 77 pgs (1.04869 -> 1.15356 * mean) 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 1 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0137 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0151 2023-10-09T04:53:41.100 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 0.8390 -> 0.7108 2023-10-09T04:53:41.101 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 0.8615 -> 0.7379 2023-10-09T04:53:41.101 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 weight 0.8585 -> 0.7359 2023-10-09T04:53:41.103 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:53:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:40 smithi029 ceph-mon[216903]: pgmap v1554: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:41.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:41 smithi029 conmon[202582]: 2023-10-09T04:53:41.098+0000 7f6049533700 -1 mgr.server reply reply (3) No such process no change 2023-10-09T04:53:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:40 smithi029 ceph-mon[223858]: pgmap v1554: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:40 smithi177 ceph-mon[195797]: pgmap v1554: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:41.997 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-10-09T04:53:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:41 smithi177 ceph-mon[195797]: from='client.46811 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:42.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4080809139' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:53:42.402 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:42.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:41 smithi029 ceph-mon[216903]: from='client.46811 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:42.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4080809139' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:53:42.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:41 smithi029 ceph-mon[223858]: from='client.46811 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:42.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4080809139' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-09T04:53:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:42.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:42.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:42.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:42.588 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:42.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:42.682 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T04:53:42.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T04:53:42.694 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:42.722 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:42.768 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:42.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:42.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:42.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:53:42.905 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:42.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:53:42] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2023-10-09T04:53:43.088 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:53:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:43.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:53:43.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:53:43.222 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[216903]: pgmap v1555: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[216903]: failed to encode map e444 with expected crc 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4080809139' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[216903]: osdmap e444: 8 total, 8 up, 4 in 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[223858]: pgmap v1555: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[223858]: failed to encode map e444 with expected crc 2023-10-09T04:53:43.254 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4080809139' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:53:43.255 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:42 smithi029 ceph-mon[223858]: osdmap e444: 8 total, 8 up, 4 in 2023-10-09T04:53:43.268 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:53:43.271 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:42 smithi177 ceph-mon[195797]: pgmap v1555: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:43.272 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:42 smithi177 ceph-mon[195797]: failed to encode map e444 with expected crc 2023-10-09T04:53:43.272 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4080809139' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-09T04:53:43.272 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:42 smithi177 ceph-mon[195797]: osdmap e444: 8 total, 8 up, 4 in 2023-10-09T04:53:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:43.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:53:43.373 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:43.407 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:43.606 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:43.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:43.710 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:43.739 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:53:44.068 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.069 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.070 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:53:44.142 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-10-09T04:53:44.145 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-09T04:53:44.145 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-09T04:53:44.193 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2023-10-09T04:53:44.193 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-09T04:53:44.193 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_550670 2023-10-09T04:53:44.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:53:44.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2023-10-09T04:53:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2693192984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:44.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:44.362 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2693192984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:44.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:44.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:44.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2693192984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:53:44.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:44.363 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:44.742 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:44.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:44.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:44.847 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:44.896 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:45.000 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:45.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:45.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:45.168 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:45.193 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 2668 2652 165.742 165.75 0.00398329 0.00601535 2023-10-09T04:53:45.254 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:45.295 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:45.401 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:45.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:45.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:45 smithi029 ceph-mon[216903]: pgmap v1557: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:45.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:45 smithi029 ceph-mon[223858]: pgmap v1557: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:45.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:45.555 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:45.659 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:53:45.659 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:53:45.660 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:53:45.660 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:53:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:45 smithi177 ceph-mon[195797]: pgmap v1557: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 582 KiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 1/72 objects misplaced (1.389%) 2023-10-09T04:53:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:46.014 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:46.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:46.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:46.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 5851 5835 182.332 198.938 0.00293 0.00547615 2023-10-09T04:53:46.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:46.264 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:46.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:46.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:46.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:46.502 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:47.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:53:47.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 8476 8460 176.237 164.062 0.00677284 0.00566515 2023-10-09T04:53:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:47 smithi029 ceph-mon[216903]: pgmap v1558: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 56 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail; 6.6 MiB/s wr, 105 op/s; 1/2754 objects misplaced (0.036%) 2023-10-09T04:53:47.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:47 smithi029 ceph-mon[223858]: pgmap v1558: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 56 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail; 6.6 MiB/s wr, 105 op/s; 1/2754 objects misplaced (0.036%) 2023-10-09T04:53:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:47 smithi177 ceph-mon[195797]: pgmap v1558: 89 pgs: 1 active+undersized, 3 active+clean+remapped, 85 active+clean; 56 MiB data, 5.0 GiB used, 353 GiB / 358 GiB avail; 6.6 MiB/s wr, 105 op/s; 1/2754 objects misplaced (0.036%) 2023-10-09T04:53:48.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 11036 11020 172.175 160 0.00713902 0.00579918 2023-10-09T04:53:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:48 smithi029 ceph-mon[216903]: pgmap v1559: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 233 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 23 MiB/s wr, 371 op/s; 537/11217 objects degraded (4.787%); 1/11217 objects misplaced (0.009%) 2023-10-09T04:53:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:48 smithi029 ceph-mon[216903]: Health check failed: Degraded data redundancy: 537/11217 objects degraded (4.787%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:53:48.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:48 smithi029 ceph-mon[223858]: pgmap v1559: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 233 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 23 MiB/s wr, 371 op/s; 537/11217 objects degraded (4.787%); 1/11217 objects misplaced (0.009%) 2023-10-09T04:53:48.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:48 smithi029 ceph-mon[223858]: Health check failed: Degraded data redundancy: 537/11217 objects degraded (4.787%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:53:48.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:48 smithi177 ceph-mon[195797]: pgmap v1559: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 233 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 23 MiB/s wr, 371 op/s; 537/11217 objects degraded (4.787%); 1/11217 objects misplaced (0.009%) 2023-10-09T04:53:48.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:48 smithi177 ceph-mon[195797]: Health check failed: Degraded data redundancy: 537/11217 objects degraded (4.787%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:53:49.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 13876 13860 173.237 177.5 0.00490884 0.00576818 2023-10-09T04:53:50.118 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:50.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:50.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:53:50.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 16418 16402 170.84 158.875 0.00704019 0.0058465 2023-10-09T04:53:50.229 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:53:50.290 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:50.409 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:53:50.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:50.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:53:50.521 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:50.575 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:50 smithi177 ceph-mon[195797]: pgmap v1560: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 233 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 23 MiB/s wr, 371 op/s; 537/11217 objects degraded (4.787%); 1/11217 objects misplaced (0.009%) 2023-10-09T04:53:50.586 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:50.669 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:50.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:50.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:50.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:50.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:50 smithi029 ceph-mon[216903]: pgmap v1560: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 233 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 23 MiB/s wr, 371 op/s; 537/11217 objects degraded (4.787%); 1/11217 objects misplaced (0.009%) 2023-10-09T04:53:50.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:50 smithi029 ceph-mon[223858]: pgmap v1560: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 233 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 23 MiB/s wr, 371 op/s; 537/11217 objects degraded (4.787%); 1/11217 objects misplaced (0.009%) 2023-10-09T04:53:50.818 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:50.886 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:50.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:50.990 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:51.027 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:51.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 18733 18717 167.103 144.688 0.00471352 0.00597931 2023-10-09T04:53:51.506 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:51.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:51.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:51.658 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:51.748 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:51.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:51.845 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:51.856 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:53:51.887 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:53:51.913 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:52.026 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:53:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:52.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:53:52.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:52.185 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:52.198 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 20718 20702 161.721 124.062 0.00612655 0.00617683 2023-10-09T04:53:52.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:52.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:52.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:52.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:52.519 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:52.739 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:53:52.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:52 smithi029 ceph-mon[216903]: pgmap v1561: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 836 MiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s; 537/40191 objects degraded (1.336%); 1/40191 objects misplaced (0.002%) 2023-10-09T04:53:52.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2387376909' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:52.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:52 smithi029 ceph-mon[223858]: pgmap v1561: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 836 MiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s; 537/40191 objects degraded (1.336%); 1/40191 objects misplaced (0.002%) 2023-10-09T04:53:52.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2387376909' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:52.946 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 267 (0%) 2023-10-09T04:53:52.946 INFO:teuthology.orchestra.run.smithi029.stdout:avg 66.75 2023-10-09T04:53:52.946 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 4.33734 -> 4.33734 (expected baseline 7.07549) 2023-10-09T04:53:52.946 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.7 with 59 -> 59 pgs (0.883895 -> 0.883895 * mean) 2023-10-09T04:53:52.946 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.2 with 70 -> 70 pgs (1.04869 -> 1.04869 * mean) 2023-10-09T04:53:52.946 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:53:52.947 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T04:53:52.947 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 1 2023-10-09T04:53:52.947 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:53:52.947 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0218 2023-10-09T04:53:52.947 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.2181 2023-10-09T04:53:52.951 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:53:53.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:52 smithi177 ceph-mon[195797]: pgmap v1561: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 836 MiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 84 MiB/s wr, 1.34k op/s; 537/40191 objects degraded (1.336%); 1/40191 objects misplaced (0.002%) 2023-10-09T04:53:53.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2387376909' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:53:53.053 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:53:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:53:52] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-10-09T04:53:53.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 22816 22800 158.32 131.125 0.011571 0.0063115 2023-10-09T04:53:54.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:53 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:53 smithi177 ceph-mon[195797]: failed to encode map e445 with expected crc 2023-10-09T04:53:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2387376909' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:53 smithi177 ceph-mon[195797]: osdmap e445: 8 total, 8 up, 4 in 2023-10-09T04:53:54.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:53 smithi177 ceph-mon[195797]: from='client.36960 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[223858]: failed to encode map e445 with expected crc 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2387376909' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[223858]: osdmap e445: 8 total, 8 up, 4 in 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[223858]: from='client.36960 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[216903]: failed to encode map e445 with expected crc 2023-10-09T04:53:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2387376909' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:53:54.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[216903]: osdmap e445: 8 total, 8 up, 4 in 2023-10-09T04:53:54.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:53 smithi029 ceph-mon[216903]: from='client.36960 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:53:54.194 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 15 24955 24940 155.862 133.75 0.0109825 0.00641157 2023-10-09T04:53:54.456 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:54.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:54.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:53:54.596 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:54.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:53:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:54.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:53:54.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:54.786 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:55.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:54 smithi177 ceph-mon[195797]: pgmap v1563: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.0 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 107 MiB/s wr, 1.71k op/s; 1276/51369 objects degraded (2.484%); 1/51369 objects misplaced (0.002%) 2023-10-09T04:53:55.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:54 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1276/51369 objects degraded (2.484%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:53:55.100 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:55.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:54 smithi029 ceph-mon[216903]: pgmap v1563: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.0 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 107 MiB/s wr, 1.71k op/s; 1276/51369 objects degraded (2.484%); 1/51369 objects misplaced (0.002%) 2023-10-09T04:53:55.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:54 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1276/51369 objects degraded (2.484%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:53:55.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:54 smithi029 ceph-mon[223858]: pgmap v1563: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.0 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 107 MiB/s wr, 1.71k op/s; 1276/51369 objects degraded (2.484%); 1/51369 objects misplaced (0.002%) 2023-10-09T04:53:55.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:54 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1276/51369 objects degraded (2.484%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:53:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:55.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:55.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 26674 26658 151.453 107.375 0.00678557 0.00659604 2023-10-09T04:53:55.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:55.242 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:56.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 28403 28387 147.836 108.062 0.00774083 0.00675974 2023-10-09T04:53:56.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:56 smithi029 ceph-mon[216903]: pgmap v1564: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.5 GiB data, 9.2 GiB used, 348 GiB / 358 GiB avail; 147 MiB/s wr, 2.35k op/s; 1276/73137 objects degraded (1.745%); 1/73137 objects misplaced (0.001%) 2023-10-09T04:53:56.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:56 smithi029 ceph-mon[223858]: pgmap v1564: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.5 GiB data, 9.2 GiB used, 348 GiB / 358 GiB avail; 147 MiB/s wr, 2.35k op/s; 1276/73137 objects degraded (1.745%); 1/73137 objects misplaced (0.001%) 2023-10-09T04:53:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:56 smithi177 ceph-mon[195797]: pgmap v1564: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.5 GiB data, 9.2 GiB used, 348 GiB / 358 GiB avail; 147 MiB/s wr, 2.35k op/s; 1276/73137 objects degraded (1.745%); 1/73137 objects misplaced (0.001%) 2023-10-09T04:53:57.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 15 30241 30226 145.305 114.938 0.0179929 0.00687613 2023-10-09T04:53:57.309 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:57.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:57.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:57.451 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:58.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 31974 31958 142.657 108.25 0.00485321 0.00700528 2023-10-09T04:53:58.307 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T04:53:58.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T04:53:58.417 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:53:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:58.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:53:58.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:53:58.616 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:58.687 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:53:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:58.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:53:58.792 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:53:58.838 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:58 smithi177 ceph-mon[195797]: pgmap v1565: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.7 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 148 MiB/s wr, 2.37k op/s; 1883/82278 objects degraded (2.289%); 1/82278 objects misplaced (0.001%) 2023-10-09T04:53:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:59.072 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:53:59.073 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:53:59.073 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:53:59.073 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:53:59.073 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:53:59.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:58 smithi029 ceph-mon[216903]: pgmap v1565: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.7 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 148 MiB/s wr, 2.37k op/s; 1883/82278 objects degraded (2.289%); 1/82278 objects misplaced (0.001%) 2023-10-09T04:53:59.095 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:59.095 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:59.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:58 smithi029 ceph-mon[223858]: pgmap v1565: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.7 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 148 MiB/s wr, 2.37k op/s; 1883/82278 objects degraded (2.289%); 1/82278 objects misplaced (0.001%) 2023-10-09T04:53:59.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:53:59.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:53:59.110 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:53:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:53:59.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:53:59.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 33711 33695 140.384 108.562 0.013911 0.00711857 2023-10-09T04:53:59.215 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:53:59.254 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:53:59.780 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.781 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:53:59.782 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:54:00.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:53:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1476018450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:53:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1476018450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:53:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1476018450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:00.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:54:00.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 35421 35405 138.289 106.875 0.00773077 0.00722596 2023-10-09T04:54:00.841 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:00.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:00.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:00.945 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:01.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:00 smithi177 ceph-mon[195797]: pgmap v1566: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.7 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 148 MiB/s wr, 2.37k op/s; 1883/82278 objects degraded (2.289%); 1/82278 objects misplaced (0.001%) 2023-10-09T04:54:01.010 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1883/82278 objects degraded (2.289%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:01.017 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:01.083 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:01.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:00 smithi029 ceph-mon[216903]: pgmap v1566: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.7 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 148 MiB/s wr, 2.37k op/s; 1883/82278 objects degraded (2.289%); 1/82278 objects misplaced (0.001%) 2023-10-09T04:54:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1883/82278 objects degraded (2.289%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:00 smithi029 ceph-mon[223858]: pgmap v1566: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 1.7 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 148 MiB/s wr, 2.37k op/s; 1883/82278 objects degraded (2.289%); 1/82278 objects misplaced (0.001%) 2023-10-09T04:54:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1883/82278 objects degraded (2.289%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:01.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:01.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 36769 36753 135.109 84.25 0.00215029 0.00736891 2023-10-09T04:54:01.217 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:01.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:01.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:01.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:01.411 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:01.782 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:01.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:01.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:01.887 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:01.964 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:02.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:54:02.081 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:02.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:02.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:02.130 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:02.176 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:02.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:02.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 38059 38043 132.082 80.625 0.0082207 0.00756676 2023-10-09T04:54:02.258 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:02.337 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:02.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:02.442 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:02.512 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:02.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:02.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:02.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:02.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:02 smithi177 ceph-mon[195797]: pgmap v1567: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 125 MiB/s wr, 2.00k op/s; 1883/100299 objects degraded (1.877%); 1/100299 objects misplaced (0.001%) 2023-10-09T04:54:02.790 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:02 smithi029 ceph-mon[216903]: pgmap v1567: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 125 MiB/s wr, 2.00k op/s; 1883/100299 objects degraded (1.877%); 1/100299 objects misplaced (0.001%) 2023-10-09T04:54:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:02 smithi029 ceph-mon[223858]: pgmap v1567: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 125 MiB/s wr, 2.00k op/s; 1883/100299 objects degraded (1.877%); 1/100299 objects misplaced (0.001%) 2023-10-09T04:54:02.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:54:02] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-10-09T04:54:02.922 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:02.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:03.036 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:03.105 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:03.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 39601 39585 130.202 96.375 0.0114163 0.00767564 2023-10-09T04:54:03.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:03.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:03.335 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:03.371 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:04.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:54:04.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:54:04.196022+0000 min lat: 0.00111616 max lat: 0.257386 avg lat: 0.00776703 2023-10-09T04:54:04.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:54:04.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 41194 41178 128.67 99.5625 0.00652648 0.00776703 2023-10-09T04:54:04.241 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:04.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:04.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:04.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:04.383 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:04 smithi029 ceph-mon[216903]: pgmap v1568: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 109 MiB/s wr, 1.75k op/s; 2378/107490 objects degraded (2.212%); 1/107490 objects misplaced (0.001%) 2023-10-09T04:54:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:04 smithi029 ceph-mon[223858]: pgmap v1568: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 109 MiB/s wr, 1.75k op/s; 2378/107490 objects degraded (2.212%); 1/107490 objects misplaced (0.001%) 2023-10-09T04:54:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:04 smithi177 ceph-mon[195797]: pgmap v1568: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.2 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 109 MiB/s wr, 1.75k op/s; 2378/107490 objects degraded (2.212%); 1/107490 objects misplaced (0.001%) 2023-10-09T04:54:05.195 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:05.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 42399 42383 126.129 75.3125 0.0067448 0.0078964 2023-10-09T04:54:05.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:05.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:05.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:05.336 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:05.341 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:05.827 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:05.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:05.895 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:05.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:05.998 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2378/107490 objects degraded (2.212%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:06.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2378/107490 objects degraded (2.212%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:06.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2378/107490 objects degraded (2.212%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:06.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:06.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:06.195 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 43812 43796 124.41 88.3125 0.00897517 0.00803299 2023-10-09T04:54:06.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:06.293 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:06.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:06.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:06.520 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:06.590 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:06.691 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:06.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:06.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:06.854 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:06.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:06 smithi029 ceph-mon[216903]: pgmap v1569: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 124 MiB/s wr, 1.98k op/s; 2378/122661 objects degraded (1.939%); 1/122661 objects misplaced (0.001%) 2023-10-09T04:54:06.997 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:06 smithi029 ceph-mon[223858]: pgmap v1569: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 124 MiB/s wr, 1.98k op/s; 2378/122661 objects degraded (1.939%); 1/122661 objects misplaced (0.001%) 2023-10-09T04:54:06.997 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:06 smithi029 conmon[202582]: 2023-10-09T04:54:06.857+0000 7f6049533700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-10-09T04:54:07.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:06 smithi177 ceph-mon[195797]: pgmap v1569: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.5 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 124 MiB/s wr, 1.98k op/s; 2378/122661 objects degraded (1.939%); 1/122661 objects misplaced (0.001%) 2023-10-09T04:54:07.104 INFO:teuthology.orchestra.run.smithi029.stdout:moved 4 / 267 (1.49813%) 2023-10-09T04:54:07.104 INFO:teuthology.orchestra.run.smithi029.stdout:avg 66.75 2023-10-09T04:54:07.104 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 4.33734 -> 2.07666 (expected baseline 7.07549) 2023-10-09T04:54:07.104 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.7 with 59 -> 63 pgs (0.883895 -> 0.94382 * mean) 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.2 with 70 -> 67 pgs (1.04869 -> 1.00375 * mean) 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0341 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.3413 2023-10-09T04:54:07.105 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 0.8390 -> 0.8854 2023-10-09T04:54:07.106 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T04:54:07.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 45027 45011 122.302 75.9375 0.00340476 0.00817087 2023-10-09T04:54:07.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:07 smithi029 ceph-mon[216903]: from='client.46856 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:54:07.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:07 smithi029 ceph-mon[223858]: from='client.46856 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:54:08.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 46318 46302 120.568 80.6875 0.0125687 0.00828946 2023-10-09T04:54:08.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:07 smithi177 ceph-mon[195797]: from='client.46856 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:54:08.479 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:08.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:08.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:08.622 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:08.814 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:54:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:08 smithi029 ceph-mon[216903]: pgmap v1570: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.6 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 98 MiB/s wr, 1.56k op/s; 2828/129357 objects degraded (2.186%); 1/129357 objects misplaced (0.001%) 2023-10-09T04:54:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3920585268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:08.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:08.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:08 smithi029 ceph-mon[223858]: pgmap v1570: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.6 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 98 MiB/s wr, 1.56k op/s; 2828/129357 objects degraded (2.186%); 1/129357 objects misplaced (0.001%) 2023-10-09T04:54:08.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3920585268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:08.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:09.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 47471 47455 118.627 72.0625 0.0139543 0.00842554 2023-10-09T04:54:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:08 smithi177 ceph-mon[195797]: pgmap v1570: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.6 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 98 MiB/s wr, 1.56k op/s; 2828/129357 objects degraded (2.186%); 1/129357 objects misplaced (0.001%) 2023-10-09T04:54:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3920585268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:09.331 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:09.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:09.441 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:09.467 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:09.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:09.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:09.617 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:09.682 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:09.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:09.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:09.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:09.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:09.945 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: osdmap e446: 8 total, 8 up, 4 in 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[216903]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: osdmap e446: 8 total, 8 up, 4 in 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.104 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:09 smithi029 ceph-mon[223858]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 48563 48547 116.689 68.25 0.0052391 0.00856443 2023-10-09T04:54:10.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:10.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: osdmap e446: 8 total, 8 up, 4 in 2023-10-09T04:54:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:09 smithi177 ceph-mon[195797]: failed to encode map e446 with expected crc 2023-10-09T04:54:10.786 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:10.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:10.891 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:10.961 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:11.070 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:11.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:11.082 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:10 smithi177 ceph-mon[195797]: pgmap v1572: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.6 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 98 MiB/s wr, 1.57k op/s; 2828/129357 objects degraded (2.186%); 1/129357 objects misplaced (0.001%) 2023-10-09T04:54:11.082 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 2828/129357 objects degraded (2.186%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:10 smithi029 ceph-mon[216903]: pgmap v1572: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.6 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 98 MiB/s wr, 1.57k op/s; 2828/129357 objects degraded (2.186%); 1/129357 objects misplaced (0.001%) 2023-10-09T04:54:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 2828/129357 objects degraded (2.186%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:10 smithi029 ceph-mon[223858]: pgmap v1572: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.6 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 98 MiB/s wr, 1.57k op/s; 2828/129357 objects degraded (2.186%); 1/129357 objects misplaced (0.001%) 2023-10-09T04:54:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 2828/129357 objects degraded (2.186%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:11.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:11.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 49939 49923 115.553 86 0.0148606 0.00864936 2023-10-09T04:54:11.243 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:54:11.244 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:54:11.254 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:11.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:11.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:11.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:11.557 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:11.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:11.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:11.771 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:11.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:12.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 51269 51253 114.394 83.125 0.0109882 0.00873786 2023-10-09T04:54:12.449 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.450 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.451 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.451 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.451 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.451 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.451 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:54:12.485 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:12.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:12.555 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:54:12.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:12.663 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:12.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:12.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:12.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:12.954 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:13.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:12 smithi177 ceph-mon[195797]: pgmap v1573: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s; 2828/141519 objects degraded (1.998%); 1/141519 objects misplaced (0.001%) 2023-10-09T04:54:13.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:12 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3823626165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:54:12] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-10-09T04:54:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:12 smithi029 ceph-mon[223858]: pgmap v1573: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s; 2828/141519 objects degraded (1.998%); 1/141519 objects misplaced (0.001%) 2023-10-09T04:54:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:12 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3823626165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:12 smithi029 ceph-mon[216903]: pgmap v1573: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 2.9 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s; 2828/141519 objects degraded (1.998%); 1/141519 objects misplaced (0.001%) 2023-10-09T04:54:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:12 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3823626165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:13.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 52383 52367 112.85 69.625 0.00296046 0.00884273 2023-10-09T04:54:13.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:54:13.894 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:13.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:13.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:14.045 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:14.103 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:14.143 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:14.181 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:14.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:14.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:14.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 53412 53396 111.232 64.3125 0.0112365 0.00898555 2023-10-09T04:54:14.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:14.284 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:14.320 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:14.748 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:14.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:14.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:14.853 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:14.936 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:15.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:15.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:15.124 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2] out_osds: [6, 3, 0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:54:15.124 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:54:15.124 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-09T04:54:15.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-09T04:54:15.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:14 smithi029 ceph-mon[216903]: pgmap v1574: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s; 3220/147534 objects degraded (2.183%); 1/147534 objects misplaced (0.001%) 2023-10-09T04:54:15.161 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:14 smithi029 ceph-mon[223858]: pgmap v1574: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s; 3220/147534 objects degraded (2.183%); 1/147534 objects misplaced (0.001%) 2023-10-09T04:54:15.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:15.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 54687 54671 110.214 79.6875 0.0124596 0.00906713 2023-10-09T04:54:15.197 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:15.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:14 smithi177 ceph-mon[195797]: pgmap v1574: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.0 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s; 3220/147534 objects degraded (2.183%); 1/147534 objects misplaced (0.001%) 2023-10-09T04:54:16.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:54:16.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:16.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:15 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3220/147534 objects degraded (2.183%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:16.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:15 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3220/147534 objects degraded (2.183%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:16.196 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 55745 55729 108.836 66.125 0.0144721 0.00918181 2023-10-09T04:54:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:54:16.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:15 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3220/147534 objects degraded (2.183%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:17.037 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:17.412 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:16 smithi177 ceph-mon[195797]: pgmap v1575: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 78 MiB/s wr, 1.25k op/s; 3220/160026 objects degraded (2.012%); 1/160026 objects misplaced (0.001%) 2023-10-09T04:54:17.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:17.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:17.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:16 smithi029 ceph-mon[216903]: pgmap v1575: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 78 MiB/s wr, 1.25k op/s; 3220/160026 objects degraded (2.012%); 1/160026 objects misplaced (0.001%) 2023-10-09T04:54:17.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:16 smithi029 ceph-mon[223858]: pgmap v1575: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 78 MiB/s wr, 1.25k op/s; 3220/160026 objects degraded (2.012%); 1/160026 objects misplaced (0.001%) 2023-10-09T04:54:17.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 15 56585 56570 107.123 52.5625 0.011809 0.0093307 2023-10-09T04:54:17.415 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:17.416 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:17.515 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:17.640 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:17.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:18.281 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:54:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:54:18.281 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:54:18.281 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:54:18.281 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 57698 57682 106.017 69.5 0.0137613 0.00942707 2023-10-09T04:54:18.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:18.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:18.420 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:18.864 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:54:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:18 smithi029 ceph-mon[216903]: pgmap v1576: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s; 3550/165117 objects degraded (2.150%); 1/165117 objects misplaced (0.001%) 2023-10-09T04:54:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3119341986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:18 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:18 smithi029 ceph-mon[223858]: pgmap v1576: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s; 3550/165117 objects degraded (2.150%); 1/165117 objects misplaced (0.001%) 2023-10-09T04:54:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3119341986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:18 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 58412 58396 104.263 44.625 0.0134223 0.00958677 2023-10-09T04:54:19.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:18 smithi177 ceph-mon[195797]: pgmap v1576: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s; 3550/165117 objects degraded (2.150%); 1/165117 objects misplaced (0.001%) 2023-10-09T04:54:19.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3119341986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:18 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:19.503 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:19.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:19.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:19.644 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:19.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:54:19.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:19.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[216903]: osdmap e447: 8 total, 8 up, 4 in 2023-10-09T04:54:19.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[216903]: failed to encode map e447 with expected crc 2023-10-09T04:54:19.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[216903]: failed to encode map e447 with expected crc 2023-10-09T04:54:19.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[216903]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:54:20.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:20.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[223858]: osdmap e447: 8 total, 8 up, 4 in 2023-10-09T04:54:20.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[223858]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[223858]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.110 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:19 smithi029 ceph-mon[223858]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 59503 59487 103.26 68.1875 0.0225397 0.00967827 2023-10-09T04:54:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:19 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:54:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:19 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:19 smithi177 ceph-mon[195797]: osdmap e447: 8 total, 8 up, 4 in 2023-10-09T04:54:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:19 smithi177 ceph-mon[195797]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:19 smithi177 ceph-mon[195797]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:19 smithi177 ceph-mon[195797]: failed to encode map e447 with expected crc 2023-10-09T04:54:20.431 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:54:20.667 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:20.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:20.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:20.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:20.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:20.924 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:20.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:21.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:21.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:20 smithi029 ceph-mon[216903]: pgmap v1578: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s; 3550/165117 objects degraded (2.150%); 1/165117 objects misplaced (0.001%) 2023-10-09T04:54:21.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3550/165117 objects degraded (2.150%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:21.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:20 smithi029 ceph-mon[223858]: pgmap v1578: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s; 3550/165117 objects degraded (2.150%); 1/165117 objects misplaced (0.001%) 2023-10-09T04:54:21.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3550/165117 objects degraded (2.150%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:21.087 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:21.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 60610 60594 102.339 69.1875 0.0213575 0.0097665 2023-10-09T04:54:21.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:21.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:20 smithi177 ceph-mon[195797]: pgmap v1578: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.4 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s; 3550/165117 objects degraded (2.150%); 1/165117 objects misplaced (0.001%) 2023-10-09T04:54:21.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3550/165117 objects degraded (2.150%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:21.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:21.374 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:22.200 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 61291 61275 100.766 42.5625 0.00764585 0.00992009 2023-10-09T04:54:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:22.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:22.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:22.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:22.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:54:22.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:22.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:23.094 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:54:22] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-10-09T04:54:23.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 62280 62264 99.7674 61.8125 0.0128629 0.0100187 2023-10-09T04:54:23.315 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:23.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:23.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:23 smithi029 ceph-mon[223858]: pgmap v1579: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s; 3550/175194 objects degraded (2.026%); 1/175194 objects misplaced (0.001%) 2023-10-09T04:54:23.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:23.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:23 smithi029 ceph-mon[216903]: pgmap v1579: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s; 3550/175194 objects degraded (2.026%); 1/175194 objects misplaced (0.001%) 2023-10-09T04:54:23.466 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:23.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:23 smithi177 ceph-mon[195797]: pgmap v1579: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.6 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s; 3550/175194 objects degraded (2.026%); 1/175194 objects misplaced (0.001%) 2023-10-09T04:54:23.604 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:23.814 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:23.876 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:23.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:23.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:23.930 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:54:23.965 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:54:23.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:23.993 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:54:24.020 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:54:24.049 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:54:24.078 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:54:24.089 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:24.117 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:54:24.156 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:54:24.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3979501993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:24.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3979501993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.161 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:54:24.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:54:24.200017+0000 min lat: 0.00111616 max lat: 0.39433 avg lat: 0.010095 2023-10-09T04:54:24.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:54:24.200 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 63398 63382 99.0201 69.875 0.0145176 0.010095 2023-10-09T04:54:24.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:24.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:24.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:24.395 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:24.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3979501993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:24.610 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:24.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:24.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:24.715 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:24.751 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:24.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:24 smithi029 conmon[216878]: 2023-10-09T04:54:24.495+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e447 definitely_dead 0 2023-10-09T04:54:25.124 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.3. 2023-10-09T04:54:25.199 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 64468 64452 98.2359 66.875 0.0452693 0.0101599 2023-10-09T04:54:25.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:25 smithi029 ceph-mon[216903]: pgmap v1580: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 3836/179700 objects degraded (2.135%); 1/179700 objects misplaced (0.001%) 2023-10-09T04:54:25.416 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3369358896' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:54:25.416 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:25 smithi029 ceph-mon[223858]: pgmap v1580: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 3836/179700 objects degraded (2.135%); 1/179700 objects misplaced (0.001%) 2023-10-09T04:54:25.416 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3369358896' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:54:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:25 smithi177 ceph-mon[195797]: pgmap v1580: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 3836/179700 objects degraded (2.135%); 1/179700 objects misplaced (0.001%) 2023-10-09T04:54:25.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3369358896' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-09T04:54:26.200 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 64470 64454 95.9 0.125 0.178254 0.0101648 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[216903]: failed to encode map e448 with expected crc 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3369358896' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[216903]: osdmap e448: 8 total, 8 up, 5 in 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 3836/179700 objects degraded (2.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[216903]: osdmap e449: 8 total, 8 up, 5 in 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[223858]: failed to encode map e448 with expected crc 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3369358896' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:54:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[223858]: osdmap e448: 8 total, 8 up, 5 in 2023-10-09T04:54:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 3836/179700 objects degraded (2.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:26.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:26 smithi029 ceph-mon[223858]: osdmap e449: 8 total, 8 up, 5 in 2023-10-09T04:54:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:26 smithi177 ceph-mon[195797]: failed to encode map e448 with expected crc 2023-10-09T04:54:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3369358896' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-09T04:54:26.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:26 smithi177 ceph-mon[195797]: osdmap e448: 8 total, 8 up, 5 in 2023-10-09T04:54:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 3836/179700 objects degraded (2.135%), 1 pg degraded (PG_DEGRADED) 2023-10-09T04:54:26.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:26 smithi177 ceph-mon[195797]: osdmap e449: 8 total, 8 up, 5 in 2023-10-09T04:54:26.532 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:26.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:26.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:26.637 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:26.666 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:26.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:26.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:26.815 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:26.848 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: pgmap v1582: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 65 MiB/s wr, 1.03k op/s; 3836/189966 objects degraded (2.019%); 1/189966 objects misplaced (0.001%) 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.127 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: failed to encode map e448 with expected crc 2023-10-09T04:54:27.201 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 64473 64457 93.6706 0.1875 0.0243645 0.0102281 2023-10-09T04:54:27.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: pgmap v1582: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 65 MiB/s wr, 1.03k op/s; 3836/189966 objects degraded (2.019%); 1/189966 objects misplaced (0.001%) 2023-10-09T04:54:27.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: failed to encode map e448 with expected crc 2023-10-09T04:54:27.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[223858]: osdmap e450: 8 total, 8 up, 5 in 2023-10-09T04:54:27.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:27 smithi029 ceph-mon[216903]: osdmap e450: 8 total, 8 up, 5 in 2023-10-09T04:54:27.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: pgmap v1582: 89 pgs: 1 active+undersized+degraded, 3 active+clean+remapped, 85 active+clean; 3.9 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 65 MiB/s wr, 1.03k op/s; 3836/189966 objects degraded (2.019%); 1/189966 objects misplaced (0.001%) 2023-10-09T04:54:27.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e449 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: failed to encode map e448 with expected crc 2023-10-09T04:54:27.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:27 smithi177 ceph-mon[195797]: osdmap e450: 8 total, 8 up, 5 in 2023-10-09T04:54:28.143 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:28.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:28.201 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 65241 65225 92.6326 48 0.0154788 0.0107909 2023-10-09T04:54:28.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:28.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[216903]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[216903]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[216903]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[216903]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[216903]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[216903]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[223858]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[223858]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[223858]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[223858]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[223858]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:28 smithi029 ceph-mon[223858]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.381 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:28.400 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-09T04:54:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:28 smithi177 ceph-mon[195797]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:28 smithi177 ceph-mon[195797]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:28 smithi177 ceph-mon[195797]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:28 smithi177 ceph-mon[195797]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:28 smithi177 ceph-mon[195797]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:28 smithi177 ceph-mon[195797]: failed to encode map e450 with expected crc 2023-10-09T04:54:28.550 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:28.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:28.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:28.744 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:29.203 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 65999 65983 91.6239 47.375 0.0175449 0.0109082 2023-10-09T04:54:29.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:29 smithi029 ceph-mon[223858]: pgmap v1585: 89 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 82 active+clean; 3.9 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 62 MiB/s wr, 992 op/s; 12118/193056 objects degraded (6.277%); 8076/193056 objects misplaced (4.183%); 32 KiB/s, 0 objects/s recovering 2023-10-09T04:54:29.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:29.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:29 smithi029 ceph-mon[216903]: pgmap v1585: 89 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 82 active+clean; 3.9 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 62 MiB/s wr, 992 op/s; 12118/193056 objects degraded (6.277%); 8076/193056 objects misplaced (4.183%); 32 KiB/s, 0 objects/s recovering 2023-10-09T04:54:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:29.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:29.472 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:29.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:29.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:29.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:29 smithi177 ceph-mon[195797]: pgmap v1585: 89 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 82 active+clean; 3.9 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 62 MiB/s wr, 992 op/s; 12118/193056 objects degraded (6.277%); 8076/193056 objects misplaced (4.183%); 32 KiB/s, 0 objects/s recovering 2023-10-09T04:54:29.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:29.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:29.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:29.630 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:29.720 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:29.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:29.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:29.921 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:29.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:54:30.204 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 66732 66716 90.6279 45.8125 0.010701 0.0110277 2023-10-09T04:54:30.347 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:30.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:30.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:30.505 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:30.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:30.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:30.688 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:30.812 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:30.838 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:54:30.838 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:54:31.082 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:31.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:31.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:31.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:31.203 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 67405 67389 89.5947 42.0625 0.0121949 0.0111559 2023-10-09T04:54:31.266 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:31.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:31.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[216903]: pgmap v1586: 89 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 83 active+clean; 3.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 46 MiB/s wr, 741 op/s; 12118/193056 objects degraded (6.277%); 8076/193056 objects misplaced (4.183%); 32 KiB/s, 3 objects/s recovering 2023-10-09T04:54:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 12118/193056 objects degraded (6.277%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/353257551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:54:31.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:54:31.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[223858]: pgmap v1586: 89 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 83 active+clean; 3.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 46 MiB/s wr, 741 op/s; 12118/193056 objects degraded (6.277%); 8076/193056 objects misplaced (4.183%); 32 KiB/s, 3 objects/s recovering 2023-10-09T04:54:31.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 12118/193056 objects degraded (6.277%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:31.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/353257551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:54:31.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:54:31.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:31 smithi177 ceph-mon[195797]: pgmap v1586: 89 pgs: 2 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 83 active+clean; 3.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 46 MiB/s wr, 741 op/s; 12118/193056 objects degraded (6.277%); 8076/193056 objects misplaced (4.183%); 32 KiB/s, 3 objects/s recovering 2023-10-09T04:54:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 12118/193056 objects degraded (6.277%), 3 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/353257551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:54:31.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:54:31.547 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:31.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:31.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:31.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:31.742 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:54:31.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:31.772 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:31.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:31.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:31.920 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:31.986 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:32.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:32.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:32.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:32.204 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 68105 68089 88.6396 43.75 0.0312145 0.011275 2023-10-09T04:54:32.266 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:32.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:32 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:32.536 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:32 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:54:32.537 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:32 smithi177 ceph-mon[195797]: osdmap e451: 8 total, 8 up, 5 in 2023-10-09T04:54:32.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:32 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:32.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:32 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:54:32.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:32 smithi029 ceph-mon[216903]: osdmap e451: 8 total, 8 up, 5 in 2023-10-09T04:54:32.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:32 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:32.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:32 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:54:32.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:32 smithi029 ceph-mon[223858]: osdmap e451: 8 total, 8 up, 5 in 2023-10-09T04:54:32.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:54:32] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2023-10-09T04:54:33.178 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:33.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:33.203 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 68749 68733 87.6521 40.25 0.0209377 0.0114039 2023-10-09T04:54:33.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:54:33.285 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:33.367 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:33.402 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3] out_osds: [6, 0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:54:33.402 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:54:33.402 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T04:54:33.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T04:54:33.508 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:33 smithi029 ceph-mon[216903]: pgmap v1587: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 23 MiB/s wr, 374 op/s; 32063/197082 objects degraded (16.269%); 11994/197082 objects misplaced (6.086%); 869 KiB/s, 16 objects/s recovering 2023-10-09T04:54:33.508 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:33 smithi029 ceph-mon[216903]: failed to encode map e451 with expected crc 2023-10-09T04:54:33.508 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:33 smithi029 ceph-mon[216903]: failed to encode map e451 with expected crc 2023-10-09T04:54:33.509 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:33 smithi029 ceph-mon[223858]: pgmap v1587: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 23 MiB/s wr, 374 op/s; 32063/197082 objects degraded (16.269%); 11994/197082 objects misplaced (6.086%); 869 KiB/s, 16 objects/s recovering 2023-10-09T04:54:33.509 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:33 smithi029 ceph-mon[223858]: failed to encode map e451 with expected crc 2023-10-09T04:54:33.509 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:33 smithi029 ceph-mon[223858]: failed to encode map e451 with expected crc 2023-10-09T04:54:33.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:33 smithi177 ceph-mon[195797]: pgmap v1587: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 23 MiB/s wr, 374 op/s; 32063/197082 objects degraded (16.269%); 11994/197082 objects misplaced (6.086%); 869 KiB/s, 16 objects/s recovering 2023-10-09T04:54:33.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:33 smithi177 ceph-mon[195797]: failed to encode map e451 with expected crc 2023-10-09T04:54:33.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:33 smithi177 ceph-mon[195797]: failed to encode map e451 with expected crc 2023-10-09T04:54:33.703 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:33.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:33.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:33.868 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:33.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:33.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:33.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:34.054 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:34.089 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:34.137 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:34.205 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 69385 69369 86.6916 39.75 0.0230694 0.0115293 2023-10-09T04:54:34.324 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:34.335 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:34.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:34.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:34.439 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:34.483 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:34.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:34.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:34.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:34.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:34.718 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:34.811 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:34.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:34.900 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:34.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:34.971 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:35.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:35.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:35.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:35.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:35.205 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 70079 70063 85.8423 43.375 0.00821125 0.0116447 2023-10-09T04:54:35.217 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:35.322 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:35.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:35.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:35 smithi029 ceph-mon[216903]: pgmap v1589: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 32 MiB/s wr, 515 op/s; 32063/201921 objects degraded (15.879%); 11994/201921 objects misplaced (5.940%); 712 KiB/s, 13 objects/s recovering 2023-10-09T04:54:35.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:35 smithi029 ceph-mon[223858]: pgmap v1589: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 32 MiB/s wr, 515 op/s; 32063/201921 objects degraded (15.879%); 11994/201921 objects misplaced (5.940%); 712 KiB/s, 13 objects/s recovering 2023-10-09T04:54:35.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:35.494 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:35 smithi177 ceph-mon[195797]: pgmap v1589: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 32 MiB/s wr, 515 op/s; 32063/201921 objects degraded (15.879%); 11994/201921 objects misplaced (5.940%); 712 KiB/s, 13 objects/s recovering 2023-10-09T04:54:35.499 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:35.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:35.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:35.711 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:35.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:54:35.780 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:35.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:35.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:35.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:35.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:36.024 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:36.205 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 70731 70715 84.9751 40.75 0.0215616 0.0117632 2023-10-09T04:54:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 32063/201921 objects degraded (15.879%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:36.632 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 32063/201921 objects degraded (15.879%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:36.632 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 32063/201921 objects degraded (15.879%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:37.208 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 71322 71306 84.0651 36.9375 0.017393 0.011889 2023-10-09T04:54:37.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:37 smithi029 ceph-mon[216903]: pgmap v1590: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 42 MiB/s wr, 674 op/s; 32063/207618 objects degraded (15.443%); 11823/207618 objects misplaced (5.695%); 1.8 MiB/s, 31 objects/s recovering 2023-10-09T04:54:37.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:37 smithi029 ceph-mon[223858]: pgmap v1590: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 42 MiB/s wr, 674 op/s; 32063/207618 objects degraded (15.443%); 11823/207618 objects misplaced (5.695%); 1.8 MiB/s, 31 objects/s recovering 2023-10-09T04:54:37.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:37 smithi177 ceph-mon[195797]: pgmap v1590: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 42 MiB/s wr, 674 op/s; 32063/207618 objects degraded (15.443%); 11823/207618 objects misplaced (5.695%); 1.8 MiB/s, 31 objects/s recovering 2023-10-09T04:54:37.781 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:37.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:37.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:37.891 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:37.942 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:38.025 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:38.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:38.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:38.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 72016 72000 83.3087 43.375 0.0159237 0.0119978 2023-10-09T04:54:38.215 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:38.480 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:54:38.551 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:54:38.587 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:54:38.638 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:54:38.681 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:54:38.725 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:54:38.737 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:38.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:38.767 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:54:38.810 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:54:38.827 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.827 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.827 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.827 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.827 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.828 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.828 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.828 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:54:38.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:38.932 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:39.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 72572 72556 82.426 34.75 0.0158452 0.0121255 2023-10-09T04:54:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[216903]: pgmap v1591: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 30 MiB/s wr, 484 op/s; 32063/207618 objects degraded (15.443%); 11823/207618 objects misplaced (5.695%); 1.6 MiB/s, 27 objects/s recovering 2023-10-09T04:54:39.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:39.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:39.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1025209755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:39.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[223858]: pgmap v1591: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 30 MiB/s wr, 484 op/s; 32063/207618 objects degraded (15.443%); 11823/207618 objects misplaced (5.695%); 1.6 MiB/s, 27 objects/s recovering 2023-10-09T04:54:39.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:39.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:39.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1025209755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:39 smithi177 ceph-mon[195797]: pgmap v1591: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 30 MiB/s wr, 484 op/s; 32063/207618 objects degraded (15.443%); 11823/207618 objects misplaced (5.695%); 1.6 MiB/s, 27 objects/s recovering 2023-10-09T04:54:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:39.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1025209755' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:40.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 73196 73180 81.6506 39 0.0260333 0.0122421 2023-10-09T04:54:40.793 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:40.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:40.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:40.945 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:41.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:41.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:41.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[216903]: pgmap v1592: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 39 MiB/s wr, 620 op/s; 32063/211689 objects degraded (15.146%); 11823/211689 objects misplaced (5.585%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 32063/211689 objects degraded (15.146%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[223858]: pgmap v1592: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 39 MiB/s wr, 620 op/s; 32063/211689 objects degraded (15.146%); 11823/211689 objects misplaced (5.585%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:54:41.052 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:41.053 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:41.053 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:41.053 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 32063/211689 objects degraded (15.146%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:41.073 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:41.116 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:54:41.116 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:54:41.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:54:41.116 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:54:41.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122040 12.6 12:61901a80:::benchmark_data_smithi029_442458_object26790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.129024+0000", 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 496.98305304299998, 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3996381749999998, 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122040 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.117 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129024+0000", 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129024+0000", 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129023+0000", 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129031+0000", 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0590000000000003e-06 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129048+0000", 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7487e-05 2023-10-09T04:54:41.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129059+0000", 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0854e-05 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129213+0000", 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154024 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129339+0000", 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000126391 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.129452+0000", 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000112485 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.133565+0000", 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041133230000000003 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.158583+0000", 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025017377 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.158595+0000", 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2877000000000001e-05 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132748+0000", 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.97415217899999995 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132763+0000", 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5218999999999999e-05 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528607+0000", 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3958441239999999 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528663+0000", 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5662999999999999e-05 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36225.0:122080 12.6 12:68c22d6d:::benchmark_data_smithi029_442458_object26830:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:46:24.317667+0000", 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 496.79441041799998, 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.227455092, 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36225", 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 122080 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.125 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317667+0000", 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317667+0000", 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317666+0000", 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317672+0000", 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3090000000000001e-06 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317685+0000", 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3329999999999999e-05 2023-10-09T04:54:41.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317692+0000", 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.477e-06 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317740+0000", 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8185e-05 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:24.317748+0000", 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9640000000000003e-06 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132836+0000", 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81508806 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:25.132846+0000", 2023-10-09T04:54:41.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4609999999999997e-06 2023-10-09T04:54:41.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528868+0000", 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3960220569999999 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.528955+0000", 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7470000000000001e-05 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.529238+0000", 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000283262 2023-10-09T04:54:41.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.538506+0000", 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092673530000000007 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.541036+0000", 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025307760000000002 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.542769+0000", 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001732305 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.544942+0000", 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021731960000000001 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.545004+0000", 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2064999999999996e-05 2023-10-09T04:54:41.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:46:26.545122+0000", 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011807900000000001 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:123965 13.10 13:0a5f630d:::benchmark_data_smithi029_495570_object29705:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.134 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.919443+0000", 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.19263481300001, 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1698377679999998, 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 123965 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919443+0000", 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919443+0000", 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919441+0000", 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:41.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919448+0000", 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9059999999999996e-06 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919464+0000", 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5885000000000001e-05 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919472+0000", 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919516+0000", 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4830999999999998e-05 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919525+0000", 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0880000000000001e-06 2023-10-09T04:54:41.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915497+0000", 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99597117800000001 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915511+0000", 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4846e-05 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089166+0000", 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1736544769999999 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089280+0000", 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011445 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.140 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:123983 13.10 13:0fa29277:::benchmark_data_smithi029_495570_object29723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.922458+0000", 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.18961922700001, 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.171047722, 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 123983 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922458+0000", 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922458+0000", 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922457+0000", 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922463+0000", 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1689999999999997e-06 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922476+0000", 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2351000000000001e-05 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922485+0000", 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4579999999999999e-06 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922519+0000", 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3288999999999999e-05 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922525+0000", 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3319999999999997e-06 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915545+0000", 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99301996000000003 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915565+0000", 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9817e-05 2023-10-09T04:54:41.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089535+0000", 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.173970545 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089674+0000", 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139209 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089925+0000", 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025076100000000001 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093194+0000", 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032688420000000001 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093256+0000", 2023-10-09T04:54:41.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1579000000000001e-05 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093295+0000", 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9700000000000003e-05 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093382+0000", 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6839e-05 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093417+0000", 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5318999999999998e-05 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093506+0000", 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8469999999999998e-05 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:123990 13.10 13:0c5d2d0a:::benchmark_data_smithi029_495570_object29730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.929298+0000", 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.182779705, 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1687639679999999, 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 123990 2023-10-09T04:54:41.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929298+0000", 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929298+0000", 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929297+0000", 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929302+0000", 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.668e-06 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929316+0000", 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4443999999999999e-05 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929324+0000", 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4930000000000001e-06 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929380+0000", 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6214000000000001e-05 2023-10-09T04:54:41.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929393+0000", 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3568e-05 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915590+0000", 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98619684699999999 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915610+0000", 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9402000000000002e-05 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.090958+0000", 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1753482129999999 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.091054+0000", 2023-10-09T04:54:41.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6655000000000002e-05 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.091229+0000", 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000174396 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093302+0000", 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00207311 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093345+0000", 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3010000000000003e-05 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095725+0000", 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002379595 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.097880+0000", 2023-10-09T04:54:41.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021552210000000001 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.097933+0000", 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3149999999999998e-05 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098062+0000", 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012882599999999999 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124023 13.10 13:0b027d7d:::benchmark_data_smithi029_495570_object29763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.957144+0000", 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.154933465, 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1411683950000002, 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.159 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124023 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957144+0000", 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957144+0000", 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957143+0000", 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957148+0000", 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1379999999999998e-06 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957163+0000", 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4810999999999999e-05 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957171+0000", 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6420000000000007e-06 2023-10-09T04:54:41.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957219+0000", 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.816e-05 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957226+0000", 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.844e-06 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915638+0000", 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95841202299999995 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915646+0000", 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5019999999999992e-06 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.091976+0000", 2023-10-09T04:54:41.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1763292949999999 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.092125+0000", 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014919600000000001 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.092343+0000", 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000218345 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093306+0000", 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00096309800000000001 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095671+0000", 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023646850000000001 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095776+0000", 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010521799999999999 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098140+0000", 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002363876 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098194+0000", 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4024000000000003e-05 2023-10-09T04:54:41.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098312+0000", 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118243 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124034 13.10 13:0f418103:::benchmark_data_smithi029_495570_object29774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.969963+0000", 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.14211441, 2023-10-09T04:54:41.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1372428700000001, 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124034 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969963+0000", 2023-10-09T04:54:41.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969963+0000", 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969961+0000", 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:54:41.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969975+0000", 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4773000000000001e-05 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970035+0000", 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9525999999999999e-05 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970058+0000", 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3067000000000001e-05 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970195+0000", 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137358 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970223+0000", 2023-10-09T04:54:41.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7881999999999998e-05 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915671+0000", 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94544759700000003 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915678+0000", 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7260000000000002e-06 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093551+0000", 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.177872775 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093682+0000", 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013097200000000001 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093902+0000", 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000220268 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098082+0000", 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004179332 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099560+0000", 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001477843 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101720+0000", 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021601229999999999 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107021+0000", 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053014940000000003 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107077+0000", 2023-10-09T04:54:41.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6048999999999998e-05 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107206+0000", 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128582 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124111 13.10 13:0cb9c5a2:::benchmark_data_smithi029_495570_object29851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.177 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.07776321, 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0730542339999998, 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124111 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:41.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034317+0000", 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7620000000000001e-06 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034326+0000", 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8629999999999997e-06 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034331+0000", 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9269999999999996e-06 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034373+0000", 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1601999999999999e-05 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034382+0000", 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0690000000000008e-06 2023-10-09T04:54:41.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915702+0000", 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.88132001599999998 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915716+0000", 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4725999999999999e-05 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.094617+0000", 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1789001459999999 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.094748+0000", 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131156 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.094974+0000", 2023-10-09T04:54:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000226016 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098090+0000", 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003116438 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099588+0000", 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001498227 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101786+0000", 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021974389999999998 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107256+0000", 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054705459999999997 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107291+0000", 2023-10-09T04:54:41.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4193000000000003e-05 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107368+0000", 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7820999999999999e-05 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124120 13.10 13:0906ebd1:::benchmark_data_smithi029_495570_object29860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.186 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.039809+0000", 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.07226854999999, 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.067902353, 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124120 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039809+0000", 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039809+0000", 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039808+0000", 2023-10-09T04:54:41.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039813+0000", 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.13e-06 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039827+0000", 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4053000000000001e-05 2023-10-09T04:54:41.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039835+0000", 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5580000000000003e-06 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039895+0000", 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0350999999999999e-05 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039908+0000", 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3132000000000001e-05 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915749+0000", 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.87584070400000003 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915758+0000", 2023-10-09T04:54:41.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7099999999999996e-06 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095804+0000", 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1800463809999999 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095929+0000", 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012519199999999999 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.096163+0000", 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023405100000000001 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098433+0000", 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002269573 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099642+0000", 2023-10-09T04:54:41.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001209031 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107450+0000", 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078077040000000004 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107545+0000", 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5366999999999997e-05 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107600+0000", 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5290000000000001e-05 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107711+0000", 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110988 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124129 13.10 13:0b5bf9cf:::benchmark_data_smithi029_495570_object29869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.063078026, 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0589564650000001, 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.196 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124129 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049002+0000", 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5130000000000001e-06 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049015+0000", 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.254e-05 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049019+0000", 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4880000000000001e-06 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049059+0000", 2023-10-09T04:54:41.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9400999999999998e-05 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049067+0000", 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9170000000000006e-06 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915784+0000", 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86671685399999998 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915792+0000", 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7450000000000001e-06 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.096835+0000", 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1810427530000001 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.096968+0000", 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013281400000000001 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.097195+0000", 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022751100000000001 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099508+0000", 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023125870000000001 2023-10-09T04:54:41.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103860+0000", 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043520130000000001 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107490+0000", 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003630365 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107778+0000", 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028744000000000003 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107827+0000", 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9236000000000001e-05 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107956+0000", 2023-10-09T04:54:41.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012891700000000001 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124138 13.10 13:0d79deba:::benchmark_data_smithi029_495570_object29878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.056682026, 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0585424350000001, 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.205 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124138 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055399+0000", 2023-10-09T04:54:41.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7149999999999997e-06 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055410+0000", 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1024e-05 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055417+0000", 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9179999999999997e-06 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055458+0000", 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0586000000000001e-05 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055464+0000", 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5459999999999997e-06 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915818+0000", 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86035340100000002 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915827+0000", 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7239999999999998e-06 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098460+0000", 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1826330460000001 2023-10-09T04:54:41.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098589+0000", 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012895300000000001 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098800+0000", 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021192199999999999 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101727+0000", 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029267469999999999 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108023+0000", 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062958520000000002 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108412+0000", 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038892199999999999 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113753+0000", 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053406740000000001 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113810+0000", 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7272000000000003e-05 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113938+0000", 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012794 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124143 13.10 13:0fbfbcd2:::benchmark_data_smithi029_495570_object29883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.059887+0000", 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.05219077000001, 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0542333460000002, 2023-10-09T04:54:41.214 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124143 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059887+0000", 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059887+0000", 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059886+0000", 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059891+0000", 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8040000000000001e-06 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059900+0000", 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4399999999999994e-06 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059907+0000", 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8650000000000003e-06 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059951+0000", 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3905000000000001e-05 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059961+0000", 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9750000000000002e-06 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915850+0000", 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85588862200000004 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915859+0000", 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0329999999999997e-06 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099666+0000", 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.183807608 2023-10-09T04:54:41.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099748+0000", 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2101999999999994e-05 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099925+0000", 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000177116 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102757+0000", 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028320070000000001 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108084+0000", 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053261519999999998 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108465+0000", 2023-10-09T04:54:41.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038118400000000002 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113987+0000", 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055226390000000002 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114037+0000", 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9552999999999998e-05 2023-10-09T04:54:41.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114120+0000", 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3090000000000006e-05 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124173 13.10 13:0c5be9df:::benchmark_data_smithi029_495570_object29913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.085066+0000", 2023-10-09T04:54:41.223 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 189.02701136499999, 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0292819039999999, 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124173 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085066+0000", 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085066+0000", 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085065+0000", 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:41.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085069+0000", 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9770000000000002e-06 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085079+0000", 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7419999999999992e-06 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085084+0000", 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8659999999999997e-06 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085131+0000", 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6662000000000001e-05 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085138+0000", 2023-10-09T04:54:41.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9120000000000001e-06 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915882+0000", 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83074457800000001 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915893+0000", 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0726e-05 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.100647+0000", 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1847542309999999 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.100789+0000", 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014199300000000001 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101022+0000", 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023304200000000001 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103867+0000", 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002845237 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108138+0000", 2023-10-09T04:54:41.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042705249999999998 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108521+0000", 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000383373 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114177+0000", 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056552 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114229+0000", 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2278000000000003e-05 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114348+0000", 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011919 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124226 13.10 13:0ca3c500:::benchmark_data_smithi029_495570_object29966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.131197+0000", 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.98088033600001, 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9834688599999999, 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124226 2023-10-09T04:54:41.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131197+0000", 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131197+0000", 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131196+0000", 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131201+0000", 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2409999999999997e-06 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.235 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131212+0000", 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1579999999999999e-05 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131221+0000", 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5250000000000005e-06 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131262+0000", 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1656999999999998e-05 2023-10-09T04:54:41.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131270+0000", 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5569999999999998e-06 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915917+0000", 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78464664799999995 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915925+0000", 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9119999999999998e-06 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101819+0000", 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1858948819999999 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101906+0000", 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6304000000000006e-05 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102073+0000", 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016688400000000001 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.106960+0000", 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048878719999999997 2023-10-09T04:54:41.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108190+0000", 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001229562 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108573+0000", 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000383011 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114500+0000", 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059266700000000002 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114551+0000", 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1011999999999999e-05 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114666+0000", 2023-10-09T04:54:41.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115226 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124272 13.10 13:0b31e4b8:::benchmark_data_smithi029_495570_object30012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.185036+0000", 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.92704147500001, 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.929835937, 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:41.244 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.244 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.244 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124272 2023-10-09T04:54:41.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.244 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185036+0000", 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185036+0000", 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185035+0000", 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185040+0000", 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6419999999999998e-06 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185052+0000", 2023-10-09T04:54:41.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2128e-05 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185058+0000", 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185096+0000", 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8209e-05 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185104+0000", 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8529999999999992e-06 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915950+0000", 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73084573500000005 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915955+0000", 2023-10-09T04:54:41.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0930000000000002e-06 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102798+0000", 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1868433789999999 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102941+0000", 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014232200000000001 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103162+0000", 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022179900000000001 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107378+0000", 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042155780000000002 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108258+0000", 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00087970400000000003 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108607+0000", 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034909900000000001 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114707+0000", 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061005299999999998 2023-10-09T04:54:41.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114757+0000", 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9990000000000001e-05 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114872+0000", 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114527 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.252 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124309 13.10 13:096f70c8:::benchmark_data_smithi029_495570_object30049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.228544+0000", 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.88353333800001, 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.886540643, 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.253 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124309 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228544+0000", 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228544+0000", 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228543+0000", 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228548+0000", 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4449999999999998e-06 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228559+0000", 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1073999999999999e-05 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228564+0000", 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3850000000000001e-06 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228624+0000", 2023-10-09T04:54:41.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9373999999999998e-05 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228637+0000", 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3105000000000001e-05 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915980+0000", 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.68734309599999999 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915988+0000", 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3829999999999992e-06 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103909+0000", 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.187921037 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.104053+0000", 2023-10-09T04:54:41.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014338099999999999 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.104268+0000", 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021500299999999999 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107383+0000", 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031151859999999998 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108309+0000", 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00092590800000000003 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108663+0000", 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035403000000000002 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114912+0000", 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062496920000000003 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114963+0000", 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0989000000000003e-05 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115085+0000", 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000121316 2023-10-09T04:54:41.261 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124317 13.10 13:098473ff:::benchmark_data_smithi029_495570_object30057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.240369+0000", 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.871708818, 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.874924493, 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.262 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124317 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240369+0000", 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240369+0000", 2023-10-09T04:54:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240368+0000", 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240373+0000", 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1200000000000001e-06 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240385+0000", 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1861e-05 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240391+0000", 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9109999999999998e-06 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.265 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240434+0000", 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3297000000000002e-05 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240443+0000", 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7900000000000005e-06 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.916024+0000", 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.675581608 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.916032+0000", 2023-10-09T04:54:41.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9799999999999998e-06 2023-10-09T04:54:41.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.104947+0000", 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1889144270000001 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.105088+0000", 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140989 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.105302+0000", 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021454699999999999 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107388+0000", 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020854229999999999 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108363+0000", 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00097550399999999998 2023-10-09T04:54:41.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114429+0000", 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060662110000000002 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115131+0000", 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00070187500000000002 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115182+0000", 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0986999999999998e-05 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115293+0000", 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110894 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124366 13.11 13:8cf34284:::benchmark_data_smithi029_495570_object30106:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:34.089772+0000", 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 187.02230584200001, 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8917853280000001, 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124366 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089772+0000", 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089772+0000", 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089770+0000", 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089777+0000", 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6850000000000004e-06 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089791+0000", 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4003e-05 2023-10-09T04:54:41.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089801+0000", 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0427e-05 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089853+0000", 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2114999999999998e-05 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089861+0000", 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6159999999999996e-06 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925469+0000", 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83560847900000002 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925484+0000", 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4106e-05 2023-10-09T04:54:41.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.977705+0000", 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0522217629999999 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.977842+0000", 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013667 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.978091+0000", 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024873499999999998 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.979457+0000", 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001366183 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.979466+0000", 2023-10-09T04:54:41.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1379999999999996e-06 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981363+0000", 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001896589 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981394+0000", 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1341999999999999e-05 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981431+0000", 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7092999999999999e-05 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981557+0000", 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012592500000000001 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.280 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124543 13.11 13:8cb72e80:::benchmark_data_smithi029_495570_object30283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:34.255796+0000", 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 186.85628155200001, 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.910879671, 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124543 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255796+0000", 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255796+0000", 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255795+0000", 2023-10-09T04:54:41.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255800+0000", 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8890000000000001e-06 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255811+0000", 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1471000000000001e-05 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255817+0000", 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4820000000000002e-06 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255865+0000", 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8118999999999998e-05 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255874+0000", 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0999999999999993e-06 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925795+0000", 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.66992072000000003 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925803+0000", 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4989999999999994e-06 2023-10-09T04:54:41.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.991295+0000", 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0654915330000001 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.991388+0000", 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3405000000000004e-05 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.991539+0000", 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015072100000000001 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.994220+0000", 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002680956 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.995297+0000", 2023-10-09T04:54:41.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010776519999999999 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.999638+0000", 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043407749999999998 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.001196+0000", 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015575929999999999 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.001226+0000", 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9669999999999999e-05 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166676+0000", 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.165450035 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124583 13.11 13:8a76f3cb:::benchmark_data_smithi029_495570_object30323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 186.81698913100001, 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.871790938, 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124583 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.290 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:41.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295092+0000", 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3189999999999996e-06 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295102+0000", 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0542e-05 2023-10-09T04:54:41.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295107+0000", 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9490000000000004e-06 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295154+0000", 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6842999999999999e-05 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295162+0000", 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0860000000000002e-06 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925832+0000", 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.63066938500000003 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925840+0000", 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7269999999999996e-06 2023-10-09T04:54:41.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.992437+0000", 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.066596439 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.992547+0000", 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109912 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.992733+0000", 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018629900000000001 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.995172+0000", 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024386080000000001 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.997259+0000", 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020876649999999998 2023-10-09T04:54:41.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.999711+0000", 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024519519999999999 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166727+0000", 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16701532499999999 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166770+0000", 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3504000000000002e-05 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166879+0000", 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109095 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:41.298 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:54:41.299 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 73824 73808 80.906 39.25 0.0369994 0.0123541 2023-10-09T04:54:41.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:41 smithi177 ceph-mon[195797]: pgmap v1592: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 39 MiB/s wr, 620 op/s; 32063/211689 objects degraded (15.146%); 11823/211689 objects misplaced (5.585%); 1.6 MiB/s, 25 objects/s recovering 2023-10-09T04:54:41.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:54:41.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:54:41.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:41.300 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 32063/211689 objects degraded (15.146%), 9 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:41.300 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:41.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:41.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:41.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:41.501 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:41.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:42.210 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 74320 74304 80.0456 31 0.00744848 0.0124649 2023-10-09T04:54:42.338 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:42.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:42.444 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:42.513 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:42.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:54:42.744 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:42.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:42.850 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:42.924 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:54:42] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2023-10-09T04:54:42.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:54:43.029 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:43.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 74992 74976 79.4008 42 0.0200711 0.0125889 2023-10-09T04:54:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:43 smithi029 ceph-mon[216903]: pgmap v1593: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 41 MiB/s wr, 660 op/s; 32063/216906 objects degraded (14.782%); 11616/216906 objects misplaced (5.355%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:54:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:43 smithi029 ceph-mon[223858]: pgmap v1593: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 41 MiB/s wr, 660 op/s; 32063/216906 objects degraded (14.782%); 11616/216906 objects misplaced (5.355%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:54:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:43 smithi177 ceph-mon[195797]: pgmap v1593: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 41 MiB/s wr, 660 op/s; 32063/216906 objects degraded (14.782%); 11616/216906 objects misplaced (5.355%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T04:54:43.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:43 smithi029 conmon[216878]: 2023-10-09T04:54:43.867+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e451 definitely_dead 0 2023-10-09T04:54:44.217 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:54:44.211387+0000 min lat: 0.00111616 max lat: 2.15434 avg lat: 0.0127122 2023-10-09T04:54:44.217 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:54:44.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 75527 75511 78.6348 33.4375 0.0185845 0.0127122 2023-10-09T04:54:44.363 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:44.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:44.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:44.505 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:44.554 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:44.579 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:44.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:44.617 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.6. 2023-10-09T04:54:44.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[216903]: pgmap v1594: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 42 MiB/s wr, 676 op/s; 32063/220872 objects degraded (14.517%); 11616/220872 objects misplaced (5.259%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:54:44.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:44.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:44.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2195408323' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:54:44.639 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:54:44.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[223858]: pgmap v1594: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 42 MiB/s wr, 676 op/s; 32063/220872 objects degraded (14.517%); 11616/220872 objects misplaced (5.259%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:54:44.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:44.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:44.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2195408323' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:54:44.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:44 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:54:44.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:44.727 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:44.737 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:44 smithi177 ceph-mon[195797]: pgmap v1594: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 42 MiB/s wr, 676 op/s; 32063/220872 objects degraded (14.517%); 11616/220872 objects misplaced (5.259%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T04:54:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:54:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2195408323' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:54:44.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:44 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:54:45.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 75785 75769 77.6101 16.125 0.00470421 0.0127536 2023-10-09T04:54:45.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:54:45.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[216903]: osdmap e452: 8 total, 8 up, 6 in 2023-10-09T04:54:45.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[216903]: failed to encode map e452 with expected crc 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[216903]: failed to encode map e452 with expected crc 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[216903]: failed to encode map e452 with expected crc 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[223858]: osdmap e452: 8 total, 8 up, 6 in 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[223858]: failed to encode map e452 with expected crc 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[223858]: failed to encode map e452 with expected crc 2023-10-09T04:54:45.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:45 smithi029 ceph-mon[223858]: failed to encode map e452 with expected crc 2023-10-09T04:54:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:45 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:54:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:45 smithi177 ceph-mon[195797]: osdmap e452: 8 total, 8 up, 6 in 2023-10-09T04:54:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:45 smithi177 ceph-mon[195797]: failed to encode map e452 with expected crc 2023-10-09T04:54:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:45 smithi177 ceph-mon[195797]: failed to encode map e452 with expected crc 2023-10-09T04:54:46.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:45 smithi177 ceph-mon[195797]: failed to encode map e452 with expected crc 2023-10-09T04:54:46.053 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:46.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:46.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:46.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:46.172 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T04:54:46.186 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:46.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 75785 75769 76.3586 0 - 0.0127536 2023-10-09T04:54:46.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:46.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:46.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:46.334 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:46.361 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:46.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:46.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:46.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:46.538 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:46.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:46.589 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:54:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:46.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:46.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:46.793 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: pgmap v1596: 89 pgs: 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 58 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 37 MiB/s wr, 586 op/s; 24113/225210 objects degraded (10.707%); 11405/225210 objects misplaced (5.064%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: osdmap e453: 8 total, 8 up, 6 in 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.794 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 24113/225210 objects degraded (10.707%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:46.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4114291805' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:46.795 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:46.797 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: pgmap v1596: 89 pgs: 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 58 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 37 MiB/s wr, 586 op/s; 24113/225210 objects degraded (10.707%); 11405/225210 objects misplaced (5.064%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:54:46.797 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: osdmap e453: 8 total, 8 up, 6 in 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: failed to encode map e453 with expected crc 2023-10-09T04:54:46.798 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 24113/225210 objects degraded (10.707%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:46.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4114291805' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:46.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:46 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:46.800 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: pgmap v1596: 89 pgs: 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 58 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 37 MiB/s wr, 586 op/s; 24113/225210 objects degraded (10.707%); 11405/225210 objects misplaced (5.064%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: osdmap e453: 8 total, 8 up, 6 in 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: failed to encode map e453 with expected crc 2023-10-09T04:54:47.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 24113/225210 objects degraded (10.707%), 7 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:47.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4114291805' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:47.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:46 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:54:47.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 75785 75769 75.1468 0 - 0.0127536 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[223858]: failed to encode map e454 with expected crc 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[223858]: osdmap e454: 8 total, 8 up, 6 in 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[223858]: failed to encode map e454 with expected crc 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[223858]: failed to encode map e454 with expected crc 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[216903]: failed to encode map e454 with expected crc 2023-10-09T04:54:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[216903]: osdmap e454: 8 total, 8 up, 6 in 2023-10-09T04:54:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[216903]: failed to encode map e454 with expected crc 2023-10-09T04:54:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:47 smithi029 ceph-mon[216903]: failed to encode map e454 with expected crc 2023-10-09T04:54:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:47 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:47 smithi177 ceph-mon[195797]: failed to encode map e454 with expected crc 2023-10-09T04:54:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:54:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:47 smithi177 ceph-mon[195797]: osdmap e454: 8 total, 8 up, 6 in 2023-10-09T04:54:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:47 smithi177 ceph-mon[195797]: failed to encode map e454 with expected crc 2023-10-09T04:54:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:47 smithi177 ceph-mon[195797]: failed to encode map e454 with expected crc 2023-10-09T04:54:48.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 75840 75824 74.0265 1.14583 0.00337619 0.0132911 2023-10-09T04:54:48.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:48 smithi029 ceph-mon[223858]: failed to encode map e454 with expected crc 2023-10-09T04:54:48.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:48 smithi029 ceph-mon[223858]: pgmap v1599: 89 pgs: 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 58 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 29 MiB/s wr, 461 op/s; 24113/225210 objects degraded (10.707%); 11405/225210 objects misplaced (5.064%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:54:48.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:48 smithi029 ceph-mon[216903]: failed to encode map e454 with expected crc 2023-10-09T04:54:48.585 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:48 smithi029 ceph-mon[216903]: pgmap v1599: 89 pgs: 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 58 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 29 MiB/s wr, 461 op/s; 24113/225210 objects degraded (10.707%); 11405/225210 objects misplaced (5.064%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:54:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:48 smithi177 ceph-mon[195797]: failed to encode map e454 with expected crc 2023-10-09T04:54:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:48 smithi177 ceph-mon[195797]: pgmap v1599: 89 pgs: 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 58 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 29 MiB/s wr, 461 op/s; 24113/225210 objects degraded (10.707%); 11405/225210 objects misplaced (5.064%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:54:49.139 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:49.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:49.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:49.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 75840 75824 72.8879 0 - 0.0132911 2023-10-09T04:54:49.244 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:49.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:49.334 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:49.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:49.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:49.438 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:49.465 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:49.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:49.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:49.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:49.646 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:49.846 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:49.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:49.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:49.976 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:50.020 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:50.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:50.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:50.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:50.164 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:50.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 75840 75824 71.7837 0 - 0.0132911 2023-10-09T04:54:50.427 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:54:50.457 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:54:50.483 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:50.486 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:54:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:50.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:50.591 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:50.619 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:50.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:50.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:50.703 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:54:50.703 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75829 14.1 14.f579d4d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.571726+0000", 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1301765389999998, 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1302059560000002, 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.704 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75829 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571726+0000", 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571726+0000", 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571724+0000", 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571762+0000", 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8056000000000003e-05 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571776+0000", 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.349e-05 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571786+0000", 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8719999999999997e-06 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.707 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75832 14.9 e454/449)", 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.582541+0000", 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1193620979999999, 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.119460085, 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582541+0000", 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582541+0000", 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582540+0000", 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582578+0000", 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8028000000000003e-05 2023-10-09T04:54:50.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582594+0000", 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6560999999999999e-05 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582603+0000", 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3489999999999992e-06 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.710 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75834 14.9 e454/449)", 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.585970+0000", 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.115933311, 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1160990659999999, 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585970+0000", 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585970+0000", 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585969+0000", 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:54:50.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586023+0000", 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4001e-05 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586033+0000", 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0575e-05 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586040+0000", 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0829999999999998e-06 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75837 14.1 14.54eaa031 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.601333+0000", 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.10056976, 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1007917749999998, 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75837 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601333+0000", 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601333+0000", 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601332+0000", 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601361+0000", 2023-10-09T04:54:50.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9600000000000001e-05 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601376+0000", 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4403000000000001e-05 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601386+0000", 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0416e-05 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.717 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75840 14.9 e454/449)", 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.614574+0000", 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.0873290739999999, 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0876106299999999, 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614574+0000", 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614574+0000", 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614573+0000", 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:50.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614607+0000", 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3590999999999997e-05 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614616+0000", 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.081e-06 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614624+0000", 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4540000000000007e-06 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75800 14.3 14.2205bb53 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.518105+0000", 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1837975549999999, 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.184133477, 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.721 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75800 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.518105+0000", 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.518105+0000", 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.518104+0000", 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.518133+0000", 2023-10-09T04:54:50.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9271000000000001e-05 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.518152+0000", 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8686e-05 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.518161+0000", 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5999999999999996e-06 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.724 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75803 14.3 14.901fe9c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.520374+0000", 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.181528804, 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1819242339999998, 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.725 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75803 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520374+0000", 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520374+0000", 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520372+0000", 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520399+0000", 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7089000000000001e-05 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520412+0000", 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2784e-05 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520422+0000", 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6099999999999995e-06 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75806 14.9 e454/449)", 2023-10-09T04:54:50.728 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.525967+0000", 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1759356689999998, 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1763904030000001, 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525967+0000", 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525967+0000", 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525966+0000", 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526032+0000", 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6188999999999999e-05 2023-10-09T04:54:50.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526047+0000", 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4789e-05 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526056+0000", 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6300000000000004e-06 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75808 14.9 e454/449)", 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.529907+0000", 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1719957220000001, 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1725005259999999, 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529907+0000", 2023-10-09T04:54:50.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529907+0000", 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529906+0000", 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529941+0000", 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4762999999999999e-05 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529954+0000", 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3297e-05 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529963+0000", 2023-10-09T04:54:50.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3389999999999999e-06 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75811 14.9 e454/449)", 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.544466+0000", 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1574371179999998, 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.157991349, 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544466+0000", 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544466+0000", 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544465+0000", 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544531+0000", 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5989999999999997e-05 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544543+0000", 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1668000000000001e-05 2023-10-09T04:54:50.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544550+0000", 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8689999999999998e-06 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.738 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75815 14.1 14.451bbc1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.548948+0000", 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1529548429999998, 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1535639409999998, 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75815 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548948+0000", 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548948+0000", 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548946+0000", 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548972+0000", 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5953999999999999e-05 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548991+0000", 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8706e-05 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.741 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.549004+0000", 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2743e-05 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75820 14.3 14.f564c943 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.560540+0000", 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.1413632339999999, 2023-10-09T04:54:50.742 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1420335000000001, 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75820 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.560540+0000", 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.743 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.560540+0000", 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.560538+0000", 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.560565+0000", 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6835e-05 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.560591+0000", 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5803e-05 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.560604+0000", 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2947e-05 2023-10-09T04:54:50.745 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75824 14.1 14.aac81761 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.566240+0000", 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.135663128, 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1363930369999999, 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.746 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75824 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566240+0000", 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566240+0000", 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566238+0000", 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566260+0000", 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1810999999999998e-05 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566270+0000", 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6870000000000006e-06 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566277+0000", 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.481e-06 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.749 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75825 14.3 14.e9c7e63 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.568242+0000", 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 3.133660645, 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1344529909999999, 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75825 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.568242+0000", 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.568242+0000", 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.568240+0000", 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:54:50.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.568281+0000", 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1357999999999999e-05 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.568301+0000", 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9063e-05 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.568317+0000", 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6339000000000001e-05 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75783 14.3 14.b6c69443 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581890+0000", 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.1200124989999996, 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1208658040000001, 2023-10-09T04:54:50.753 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75783 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581890+0000", 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581890+0000", 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581889+0000", 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581929+0000", 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9944999999999998e-05 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581939+0000", 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0885e-05 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581949+0000", 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.149e-06 2023-10-09T04:54:50.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582080+0000", 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013105699999999999 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582090+0000", 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.079e-05 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75785 14.1 14.44ec6ad1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581965+0000", 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.1199379040000004, 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1208638610000001, 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75785 2023-10-09T04:54:50.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581965+0000", 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581965+0000", 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581964+0000", 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581992+0000", 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7560999999999999e-05 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582001+0000", 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2410000000000001e-06 2023-10-09T04:54:50.760 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582014+0000", 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2964e-05 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582129+0000", 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115244 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:54:50.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582138+0000", 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0499999999999997e-06 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 16 2023-10-09T04:54:50.762 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:54:50.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:50 smithi029 ceph-mon[216903]: pgmap v1600: 89 pgs: 1 activating+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 54 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 21 MiB/s wr, 339 op/s; 42344/226983 objects degraded (18.655%); 16803/226983 objects misplaced (7.403%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:54:50.763 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2120284850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:50.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:50 smithi029 ceph-mon[223858]: pgmap v1600: 89 pgs: 1 activating+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 54 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 21 MiB/s wr, 339 op/s; 42344/226983 objects degraded (18.655%); 16803/226983 objects misplaced (7.403%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:54:50.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2120284850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:50.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:50.815 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:50.974 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:50.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:50 smithi177 ceph-mon[195797]: pgmap v1600: 89 pgs: 1 activating+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 20 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 54 active+clean; 4.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 21 MiB/s wr, 339 op/s; 42344/226983 objects degraded (18.655%); 16803/226983 objects misplaced (7.403%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T04:54:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2120284850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:54:51.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:51.111 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:51.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:54:51.157 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:51.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:51.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:51.172 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6] out_osds: [0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:54:51.173 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:54:51.173 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-09T04:54:51.173 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-10-09T04:54:51.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T04:54:51.211 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 75840 75824 70.7125 0 - 0.0132911 2023-10-09T04:54:51.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:51.303 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:51.671 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:51.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:51.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:54:51.775 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:51.815 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:51.897 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:51.897 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:51.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 42344/226983 objects degraded (18.655%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:51.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:51.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 42344/226983 objects degraded (18.655%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:51.939 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:52.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 42344/226983 objects degraded (18.655%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:52.033 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:52.077 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:52.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:52.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:52.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 75840 75824 69.6728 0 - 0.0132911 2023-10-09T04:54:52.226 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:52.284 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:52.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:52.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:52.423 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:52 smithi177 ceph-mon[195797]: pgmap v1601: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 19 MiB/s wr, 298 op/s; 55113/227037 objects degraded (24.275%); 21742/227037 objects misplaced (9.576%); 6.2 MiB/s, 100 objects/s recovering 2023-10-09T04:54:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:52 smithi029 ceph-mon[216903]: pgmap v1601: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 19 MiB/s wr, 298 op/s; 55113/227037 objects degraded (24.275%); 21742/227037 objects misplaced (9.576%); 6.2 MiB/s, 100 objects/s recovering 2023-10-09T04:54:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:54:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:54:52] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2023-10-09T04:54:53.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:52 smithi029 ceph-mon[223858]: pgmap v1601: 89 pgs: 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 3 activating+undersized+degraded+remapped, 2 active+recovery_wait+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 19 MiB/s wr, 298 op/s; 55113/227037 objects degraded (24.275%); 21742/227037 objects misplaced (9.576%); 6.2 MiB/s, 100 objects/s recovering 2023-10-09T04:54:53.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 75840 75824 68.6632 0 - 0.0132911 2023-10-09T04:54:53.488 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:53.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:53.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:53.627 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:54.109 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:54.132 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:54.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:54.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 75840 75824 67.6825 0 - 0.0132911 2023-10-09T04:54:54.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:54.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:54.615 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:54.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:54.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:54.747 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:54.790 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:54.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:54.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:54 smithi029 ceph-mon[223858]: pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 5.0 MiB/s wr, 79 op/s; 55111/227115 objects degraded (24.266%); 21742/227115 objects misplaced (9.573%); 3.7 MiB/s, 60 objects/s recovering 2023-10-09T04:54:54.882 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:54 smithi029 ceph-mon[216903]: pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 5.0 MiB/s wr, 79 op/s; 55111/227115 objects degraded (24.266%); 21742/227115 objects misplaced (9.573%); 3.7 MiB/s, 60 objects/s recovering 2023-10-09T04:54:54.895 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:54.939 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:54 smithi177 ceph-mon[195797]: pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 5.0 MiB/s wr, 79 op/s; 55111/227115 objects degraded (24.266%); 21742/227115 objects misplaced (9.573%); 3.7 MiB/s, 60 objects/s recovering 2023-10-09T04:54:55.053 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:55.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:55.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:55.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:55.191 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:55.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 75840 75824 66.7293 0 - 0.0132911 2023-10-09T04:54:55.354 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:55.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:55.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:55.458 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:54:55.489 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:55.547 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:54:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:55.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:54:55.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:55.679 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:55.723 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:55.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:55.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1590557201' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:54:55.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/724932589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:55.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1590557201' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:54:55.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/724932589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:55.760 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:54:55.762 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T04:54:55.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:55.855 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:55.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:55.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:55.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:56.007 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:54:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1590557201' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T04:54:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/724932589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:54:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 55111/227115 objects degraded (24.266%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:56.033 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:56.078 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:54:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:56.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:54:56.111 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 55111/227115 objects degraded (24.266%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:56.112 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 55111/227115 objects degraded (24.266%), 13 pgs degraded (PG_DEGRADED) 2023-10-09T04:54:56.185 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:56.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 75840 75824 65.8027 0 - 0.0132911 2023-10-09T04:54:56.219 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:56.684 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:56.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:56.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:56.815 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:56.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:56.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:56.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:56.967 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:56.997 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[216903]: pgmap v1603: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.0 MiB/s wr, 64 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:54:56.997 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:56.997 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1590557201' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/724932589' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[216903]: osdmap e455: 8 total, 8 up, 6 in 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[223858]: pgmap v1603: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.0 MiB/s wr, 64 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1590557201' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/724932589' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:56.998 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:56 smithi029 ceph-mon[223858]: osdmap e455: 8 total, 8 up, 6 in 2023-10-09T04:54:57.001 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:56 smithi177 ceph-mon[195797]: pgmap v1603: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.0 MiB/s wr, 64 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T04:54:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:56 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:54:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1590557201' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T04:54:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/724932589' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:54:57.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:56 smithi177 ceph-mon[195797]: osdmap e455: 8 total, 8 up, 6 in 2023-10-09T04:54:57.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 75840 75824 64.9014 0 - 0.0132911 2023-10-09T04:54:57.505 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:57.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:57.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:57.636 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:57.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:57.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:57.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:57.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:57.815 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:57.859 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:57.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:57.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:57 smithi029 ceph-mon[216903]: failed to encode map e455 with expected crc 2023-10-09T04:54:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:57 smithi029 ceph-mon[223858]: failed to encode map e455 with expected crc 2023-10-09T04:54:57.940 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:54:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:54:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:54:57.940 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:54:57.940 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:54:57.966 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:54:57.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:57 smithi177 ceph-mon[195797]: failed to encode map e455 with expected crc 2023-10-09T04:54:57.992 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:58.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:54:58.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:58.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:54:58.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:58.190 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:58.214 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 75878 75862 64.0566 0.2375 0.00732728 0.01402 2023-10-09T04:54:58.259 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:58.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:58.364 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:58.399 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:58.454 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:58.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:58.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:58.559 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:58.589 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:58.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:58.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:58.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:54:58.789 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:59.013 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:54:59.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:58 smithi029 ceph-mon[216903]: pgmap v1605: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.0 MiB/s wr, 63 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:54:59.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:54:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:59.015 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:58 smithi029 ceph-mon[223858]: pgmap v1605: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.0 MiB/s wr, 63 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:54:59.015 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:54:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:59.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:59.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:54:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:58 smithi177 ceph-mon[195797]: pgmap v1605: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.0 MiB/s wr, 63 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:54:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:54:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:54:59.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:54:59.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:54:59.144 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:59.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:54:59.188 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:54:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:59.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:54:59.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 75982 75966 63.2893 6.5 0.00577408 0.0148865 2023-10-09T04:54:59.294 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:54:59.332 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:59.409 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:54:59.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:59.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:54:59.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:59.553 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:59.556 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:59.631 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:54:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:59.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:54:59.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:54:59.764 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:54:59.772 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:54:59.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:54:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:54:59.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:54:59.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:54:59.949 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:00.067 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:00.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:00.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:00.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:00.199 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:00.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 75982 75966 62.4567 0 - 0.0148865 2023-10-09T04:55:00.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:00.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:00.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:00.378 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:00.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:00.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:00.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:00.531 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:00.561 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:00.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:00 smithi029 ceph-mon[216903]: pgmap v1606: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 282 KiB/s wr, 4 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:55:00.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:00 smithi029 ceph-mon[223858]: pgmap v1606: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 282 KiB/s wr, 4 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:55:00.865 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:00.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:00.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:01.003 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:00 smithi177 ceph-mon[195797]: pgmap v1606: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 282 KiB/s wr, 4 op/s; 55111/227115 objects degraded (24.266%); 21180/227115 objects misplaced (9.326%); 6.5 MiB/s, 104 objects/s recovering 2023-10-09T04:55:01.212 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 75982 75966 61.6457 0 - 0.0148865 2023-10-09T04:55:01.219 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:55:01.241 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:55:01.262 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:55:01.292 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:55:01.315 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:55:01.324 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.324 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.324 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.324 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.324 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.324 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.325 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.325 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:55:01.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:01.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:01.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:01.580 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 68658/227748 objects degraded (30.146%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:02.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 68658/227748 objects degraded (30.146%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:02.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 76146 76130 60.9868 3.41667 0.00503531 0.0162148 2023-10-09T04:55:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 68658/227748 objects degraded (30.146%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:02.330 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:02.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:02.434 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:02.460 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:02.503 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:02.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:02.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:02.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:02.618 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6] out_osds: [0, 1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:55:02.618 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:55:02.618 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T04:55:02.618 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T04:55:02.655 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:02 smithi029 ceph-mon[216903]: pgmap v1607: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 1 active, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 1.5 MiB/s wr, 23 op/s; 68658/227748 objects degraded (30.146%); 16538/227748 objects misplaced (7.262%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:55:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:02 smithi029 ceph-mon[223858]: pgmap v1607: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 1 active, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 1.5 MiB/s wr, 23 op/s; 68658/227748 objects degraded (30.146%); 16538/227748 objects misplaced (7.262%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:55:02.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:55:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:55:02] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2023-10-09T04:55:02.981 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:02.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:02.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:03.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:03.111 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:02 smithi177 ceph-mon[195797]: pgmap v1607: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped, 1 active, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 1.5 MiB/s wr, 23 op/s; 68658/227748 objects degraded (30.146%); 16538/227748 objects misplaced (7.262%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:55:03.114 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:03.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:03.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:03.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 76377 76361 60.3977 14.4375 0.00886543 0.0165018 2023-10-09T04:55:03.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:03.334 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:03.576 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:03.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:03.626 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:55:03.627 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:55:03.683 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:03.745 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:03.805 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:03.946 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:03.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:04.051 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:04.081 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:04.126 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:04.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:04.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:55:04.213295+0000 min lat: 0.00111616 max lat: 16.5782 avg lat: 0.0166741 2023-10-09T04:55:04.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:55:04.213 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 76549 76533 59.7772 10.75 0.187443 0.0166741 2023-10-09T04:55:04.233 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:04.262 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:04.339 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:04.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:04.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:04.495 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:04.805 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:04.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:04.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:04.857 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:55:04.887 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:55:04.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:04.949 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:05.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:55:05.073 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:04 smithi029 ceph-mon[216903]: pgmap v1608: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 1 active, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 2.2 MiB/s wr, 35 op/s; 68658/228183 objects degraded (30.089%); 16538/228183 objects misplaced (7.248%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:55:05.073 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2809648219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:05.073 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:04 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:05.073 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:04 smithi029 ceph-mon[223858]: pgmap v1608: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 1 active, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 2.2 MiB/s wr, 35 op/s; 68658/228183 objects degraded (30.089%); 16538/228183 objects misplaced (7.248%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:55:05.074 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2809648219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:05.074 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:04 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:05.133 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:05.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:05.214 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 76791 76775 59.2255 15.125 0.00553065 0.0168166 2023-10-09T04:55:05.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:05.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:04 smithi177 ceph-mon[195797]: pgmap v1608: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 1 active, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 2.2 MiB/s wr, 35 op/s; 68658/228183 objects degraded (30.089%); 16538/228183 objects misplaced (7.248%); 7.3 MiB/s, 116 objects/s recovering 2023-10-09T04:55:05.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2809648219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:05.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:04 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:05.268 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:05.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:05.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:05.347 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:05.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:05.471 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:06.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:06.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:06.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: osdmap e456: 8 total, 8 up, 6 in 2023-10-09T04:55:06.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[216903]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:06.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:06.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: osdmap e456: 8 total, 8 up, 6 in 2023-10-09T04:55:06.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:05 smithi029 ceph-mon[223858]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.214 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 76970 76954 58.6398 11.1875 0.00812072 0.0170172 2023-10-09T04:55:06.288 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: osdmap e456: 8 total, 8 up, 6 in 2023-10-09T04:55:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:05 smithi177 ceph-mon[195797]: failed to encode map e456 with expected crc 2023-10-09T04:55:06.787 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:06.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:06.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:06.892 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:06.927 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:06.992 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:06.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:07.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:07.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:06 smithi029 ceph-mon[216903]: pgmap v1610: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 1 active, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.8 MiB/s wr, 77 op/s; 68658/229368 objects degraded (29.934%); 15992/229368 objects misplaced (6.972%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T04:55:07.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:06 smithi029 ceph-mon[223858]: pgmap v1610: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 1 active, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.8 MiB/s wr, 77 op/s; 68658/229368 objects degraded (29.934%); 15992/229368 objects misplaced (6.972%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T04:55:07.149 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:07.165 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:07.215 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 77200 77184 58.1052 14.375 0.0106682 0.0171553 2023-10-09T04:55:07.264 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:07.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:07.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:06 smithi177 ceph-mon[195797]: pgmap v1610: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 1 active, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 4.8 MiB/s wr, 77 op/s; 68658/229368 objects degraded (29.934%); 15992/229368 objects misplaced (6.972%); 7.4 MiB/s, 118 objects/s recovering 2023-10-09T04:55:07.370 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:07.417 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:07.702 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:07.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:07.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:07.861 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:08.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:07 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 63873/229755 objects degraded (27.800%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:08.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:07 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 63873/229755 objects degraded (27.800%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:08.150 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:08.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:08.215 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 77425 77409 57.581 14.0625 0.00623681 0.017304 2023-10-09T04:55:08.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:08.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:07 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 63873/229755 objects degraded (27.800%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:08.289 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:08.352 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:08.353 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:08 smithi029 conmon[216878]: 2023-10-09T04:55:08.094+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e456 definitely_dead 0 2023-10-09T04:55:08.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:08.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:08.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:08.483 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:08.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:08.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:08.629 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:08.672 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:08.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:08.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:08.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:08.873 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:08.929 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.0. 2023-10-09T04:55:09.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:55:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:08 smithi177 ceph-mon[195797]: pgmap v1611: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 5.5 MiB/s wr, 87 op/s; 63873/229755 objects degraded (27.800%); 20713/229755 objects misplaced (9.015%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T04:55:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3407697006' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:55:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:55:09.069 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:09.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:08 smithi029 ceph-mon[216903]: pgmap v1611: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 5.5 MiB/s wr, 87 op/s; 63873/229755 objects degraded (27.800%); 20713/229755 objects misplaced (9.015%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T04:55:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3407697006' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:55:09.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:55:09.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:08 smithi029 ceph-mon[223858]: pgmap v1611: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 5.5 MiB/s wr, 87 op/s; 63873/229755 objects degraded (27.800%); 20713/229755 objects misplaced (9.015%); 7.4 MiB/s, 117 objects/s recovering 2023-10-09T04:55:09.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3407697006' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:55:09.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T04:55:09.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:09.207 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:09.215 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 77537 77521 56.986 7 0.610605 0.0174781 2023-10-09T04:55:09.839 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:09.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:09.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:09.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:09.987 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:10.038 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:10.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:10.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:10.142 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:10.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:55:10.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:09 smithi177 ceph-mon[195797]: osdmap e457: 8 total, 8 up, 7 in 2023-10-09T04:55:10.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:09 smithi177 ceph-mon[195797]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:09 smithi177 ceph-mon[195797]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:09 smithi177 ceph-mon[195797]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.166 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:09 smithi177 ceph-mon[195797]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.168 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:55:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[216903]: osdmap e457: 8 total, 8 up, 7 in 2023-10-09T04:55:10.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[216903]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[216903]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[216903]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[216903]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T04:55:10.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[223858]: osdmap e457: 8 total, 8 up, 7 in 2023-10-09T04:55:10.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[223858]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[223858]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[223858]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:09 smithi029 ceph-mon[223858]: failed to encode map e457 with expected crc 2023-10-09T04:55:10.211 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:10.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:10.217 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 77547 77531 56.3294 0.625 0.00603383 0.017522 2023-10-09T04:55:10.296 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T04:55:10.314 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:10.340 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:10.390 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:10.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:10.499 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:10.502 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:10.528 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:10.563 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:55:10.577 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:10.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:10.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:10.585 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:55:10.606 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:55:10.626 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:55:10.645 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:55:10.665 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:55:10.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:10.683 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:55:10.703 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.705 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:55:10.709 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:10.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:10.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:10.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:10.895 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: pgmap v1613: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 8.3 MiB/s wr, 133 op/s; 63873/230949 objects degraded (27.657%); 20713/230949 objects misplaced (8.969%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: osdmap e458: 8 total, 8 up, 7 in 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1459983321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:11.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: pgmap v1613: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 8.3 MiB/s wr, 133 op/s; 63873/230949 objects degraded (27.657%); 20713/230949 objects misplaced (8.969%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:55:11.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: osdmap e458: 8 total, 8 up, 7 in 2023-10-09T04:55:11.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1459983321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:11.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 77553 77537 55.6864 0.375 0.119755 0.0175577 2023-10-09T04:55:11.235 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:11.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: pgmap v1613: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 8.3 MiB/s wr, 133 op/s; 63873/230949 objects degraded (27.657%); 20713/230949 objects misplaced (8.969%); 4.5 MiB/s, 72 objects/s recovering 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: osdmap e458: 8 total, 8 up, 7 in 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: failed to encode map e458 with expected crc 2023-10-09T04:55:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1459983321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:11.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:11.370 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:11.417 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:11.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:11.522 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:11.564 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:11.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[223858]: osdmap e459: 8 total, 8 up, 7 in 2023-10-09T04:55:11.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[223858]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[223858]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[223858]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[223858]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[216903]: osdmap e459: 8 total, 8 up, 7 in 2023-10-09T04:55:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[216903]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[216903]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[216903]: failed to encode map e459 with expected crc 2023-10-09T04:55:11.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:11 smithi029 ceph-mon[216903]: failed to encode map e459 with expected crc 2023-10-09T04:55:12.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 77608 77592 55.0927 3.4375 0.00798296 0.0177413 2023-10-09T04:55:12.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:11 smithi177 ceph-mon[195797]: osdmap e459: 8 total, 8 up, 7 in 2023-10-09T04:55:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:11 smithi177 ceph-mon[195797]: failed to encode map e459 with expected crc 2023-10-09T04:55:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:11 smithi177 ceph-mon[195797]: failed to encode map e459 with expected crc 2023-10-09T04:55:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:11 smithi177 ceph-mon[195797]: failed to encode map e459 with expected crc 2023-10-09T04:55:12.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:11 smithi177 ceph-mon[195797]: failed to encode map e459 with expected crc 2023-10-09T04:55:12.521 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:12.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:12.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:12.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:12.653 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:12.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:12.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:12.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:12.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:12.836 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:12.884 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:12.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:12.987 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:13.016 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:13.063 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:13.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:13.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:13.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[216903]: pgmap v1616: 89 pgs: 2 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 10 peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 9.5 MiB/s wr, 152 op/s; 64791/232116 objects degraded (27.913%); 20140/232116 objects misplaced (8.677%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T04:55:13.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[216903]: failed to encode map e460 with expected crc 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[216903]: osdmap e460: 8 total, 8 up, 7 in 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[216903]: failed to encode map e460 with expected crc 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:55:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:55:12] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[223858]: pgmap v1616: 89 pgs: 2 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 10 peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 9.5 MiB/s wr, 152 op/s; 64791/232116 objects degraded (27.913%); 20140/232116 objects misplaced (8.677%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[223858]: failed to encode map e460 with expected crc 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[223858]: osdmap e460: 8 total, 8 up, 7 in 2023-10-09T04:55:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:12 smithi029 ceph-mon[223858]: failed to encode map e460 with expected crc 2023-10-09T04:55:13.195 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:13.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 77608 77592 54.4738 0 - 0.0177413 2023-10-09T04:55:13.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:13.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:13.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:12 smithi177 ceph-mon[195797]: pgmap v1616: 89 pgs: 2 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 10 peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 9.5 MiB/s wr, 152 op/s; 64791/232116 objects degraded (27.913%); 20140/232116 objects misplaced (8.677%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T04:55:13.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:12 smithi177 ceph-mon[195797]: failed to encode map e460 with expected crc 2023-10-09T04:55:13.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:12 smithi177 ceph-mon[195797]: osdmap e460: 8 total, 8 up, 7 in 2023-10-09T04:55:13.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:12 smithi177 ceph-mon[195797]: failed to encode map e460 with expected crc 2023-10-09T04:55:13.347 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:13.374 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:14.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 16 77608 77592 53.8687 0 - 0.0177413 2023-10-09T04:55:14.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[223858]: failed to encode map e461 with expected crc 2023-10-09T04:55:14.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[223858]: osdmap e461: 8 total, 8 up, 7 in 2023-10-09T04:55:14.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[223858]: failed to encode map e461 with expected crc 2023-10-09T04:55:14.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:14.265 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:14.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[216903]: failed to encode map e461 with expected crc 2023-10-09T04:55:14.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[216903]: osdmap e461: 8 total, 8 up, 7 in 2023-10-09T04:55:14.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[216903]: failed to encode map e461 with expected crc 2023-10-09T04:55:14.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:14.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:13 smithi177 ceph-mon[195797]: failed to encode map e461 with expected crc 2023-10-09T04:55:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:13 smithi177 ceph-mon[195797]: osdmap e461: 8 total, 8 up, 7 in 2023-10-09T04:55:14.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:13 smithi177 ceph-mon[195797]: failed to encode map e461 with expected crc 2023-10-09T04:55:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:14.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:14.327 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:14.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:14.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:14.472 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:14.523 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:14.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:14.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:14.656 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:14.704 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:14.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:14.807 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:14.841 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:14.896 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:14.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:15.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:15.029 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:15.085 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:15.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:15.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:14 smithi029 ceph-mon[216903]: pgmap v1619: 89 pgs: 2 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 10 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 8.3 MiB/s wr, 132 op/s; 74990/232536 objects degraded (32.249%); 20127/232536 objects misplaced (8.655%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T04:55:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:14 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 74990/232536 objects degraded (32.249%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:14 smithi029 ceph-mon[223858]: pgmap v1619: 89 pgs: 2 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 10 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 8.3 MiB/s wr, 132 op/s; 74990/232536 objects degraded (32.249%); 20127/232536 objects misplaced (8.655%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T04:55:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:14 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 74990/232536 objects degraded (32.249%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:15.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:15.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 91 16 77608 77592 53.2768 0 - 0.0177413 2023-10-09T04:55:15.218 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:15.270 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:15.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:15.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:14 smithi177 ceph-mon[195797]: pgmap v1619: 89 pgs: 2 activating+undersized+degraded+remapped, 1 activating, 1 active+recovering+undersized+remapped, 10 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 59 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 8.3 MiB/s wr, 132 op/s; 74990/232536 objects degraded (32.249%); 20127/232536 objects misplaced (8.655%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T04:55:15.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:14 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 74990/232536 objects degraded (32.249%), 17 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:15.297 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0] out_osds: [1] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:55:15.298 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:55:15.298 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T04:55:15.298 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T04:55:15.374 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:15.401 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:15.448 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:15.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:15.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:15.584 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:15.638 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:15.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:15.692 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:15.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:15.771 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:15.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:15.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:15.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:15.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:15.946 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:55:15.961 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:16.015 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:16.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:16.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3489107393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:15 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:16.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:16.150 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:16.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 92 16 77608 77592 52.6978 0 - 0.0177413 2023-10-09T04:55:16.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3489107393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:16.279 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:15 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:16.279 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3489107393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:16.279 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:15 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:16.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:55:16.300 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:16.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:16.404 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:16.432 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:16.481 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:16.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:16.586 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:16.615 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:17.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 93 16 77608 77592 52.1313 0 - 0.0177413 2023-10-09T04:55:17.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:16 smithi177 ceph-mon[195797]: pgmap v1620: 89 pgs: 1 activating, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 3.9 MiB/s wr, 61 op/s; 72349/232638 objects degraded (31.099%); 26926/232638 objects misplaced (11.574%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T04:55:17.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:16 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:17.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:16 smithi177 ceph-mon[195797]: failed to encode map e462 with expected crc 2023-10-09T04:55:17.253 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:16 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:17.254 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:16 smithi177 ceph-mon[195797]: osdmap e462: 8 total, 8 up, 7 in 2023-10-09T04:55:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[216903]: pgmap v1620: 89 pgs: 1 activating, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 3.9 MiB/s wr, 61 op/s; 72349/232638 objects degraded (31.099%); 26926/232638 objects misplaced (11.574%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T04:55:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[216903]: failed to encode map e462 with expected crc 2023-10-09T04:55:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[216903]: osdmap e462: 8 total, 8 up, 7 in 2023-10-09T04:55:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[223858]: pgmap v1620: 89 pgs: 1 activating, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 3.9 MiB/s wr, 61 op/s; 72349/232638 objects degraded (31.099%); 26926/232638 objects misplaced (11.574%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T04:55:17.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:17.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[223858]: failed to encode map e462 with expected crc 2023-10-09T04:55:17.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:17.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:16 smithi029 ceph-mon[223858]: osdmap e462: 8 total, 8 up, 7 in 2023-10-09T04:55:17.718 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:17.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:17.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:17.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:17.861 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:17.887 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:17.888 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:55:17.888 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:55:17.888 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:55:17.917 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:17.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:17.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:17.923 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:18.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.050 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:18.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:18.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:18.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 94 16 77608 77592 51.5768 0 - 0.0177413 2023-10-09T04:55:18.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:18.291 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:18.444 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:18.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:18.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:18.576 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:18.626 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:18.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:18 smithi029 conmon[216878]: 2023-10-09T04:55:18.583+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e462 definitely_dead 0 2023-10-09T04:55:18.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:18.758 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:18.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:18.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:18.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:18.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:18.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:18 smithi177 ceph-mon[195797]: pgmap v1622: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 71 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.6 MiB/s wr, 40 op/s; 77157/232851 objects degraded (33.136%); 22205/232851 objects misplaced (9.536%); 7.7 MiB/s, 124 objects/s recovering 2023-10-09T04:55:18.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1626321903' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T04:55:18.939 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:18.959 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.1. 2023-10-09T04:55:19.010 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:19.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:19.117 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:19.145 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:18 smithi029 ceph-mon[216903]: pgmap v1622: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 71 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.6 MiB/s wr, 40 op/s; 77157/232851 objects degraded (33.136%); 22205/232851 objects misplaced (9.536%); 7.7 MiB/s, 124 objects/s recovering 2023-10-09T04:55:19.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1626321903' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T04:55:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:18 smithi029 ceph-mon[223858]: pgmap v1622: 89 pgs: 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 71 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 2.6 MiB/s wr, 40 op/s; 77157/232851 objects degraded (33.136%); 22205/232851 objects misplaced (9.536%); 7.7 MiB/s, 124 objects/s recovering 2023-10-09T04:55:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1626321903' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T04:55:19.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:19.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:19.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:19.218 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 95 16 77608 77592 51.034 0 - 0.0177413 2023-10-09T04:55:19.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.322 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:19.369 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:19.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:19.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:19.473 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:19.505 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:19.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:55:19.708 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:19.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:19.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:19.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:19.839 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:19.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:19.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:19.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:19.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:20.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1626321903' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T04:55:20.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[216903]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T04:55:20.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[216903]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[216903]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[216903]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.090 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1626321903' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T04:55:20.090 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[223858]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T04:55:20.090 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[223858]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.090 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[223858]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.090 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:19 smithi029 ceph-mon[223858]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.092 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:20.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 96 16 77608 77592 50.5024 0 - 0.0177413 2023-10-09T04:55:20.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1626321903' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T04:55:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:19 smithi177 ceph-mon[195797]: osdmap e463: 8 total, 8 up, 8 in 2023-10-09T04:55:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:19 smithi177 ceph-mon[195797]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:19 smithi177 ceph-mon[195797]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:19 smithi177 ceph-mon[195797]: failed to encode map e463 with expected crc 2023-10-09T04:55:20.499 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:20.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:20.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:20.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:20.634 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:20.635 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T04:55:20.837 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:20.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:20.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:20.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:20.979 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:21.033 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:21.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:21.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:21.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:21.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:21.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: pgmap v1624: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 71 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 1.0 MiB/s wr, 16 op/s; 77157/232851 objects degraded (33.136%); 22205/232851 objects misplaced (9.536%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T04:55:21.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T04:55:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 77157/232851 objects degraded (33.136%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:21.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: pgmap v1624: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 71 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 1.0 MiB/s wr, 16 op/s; 77157/232851 objects degraded (33.136%); 22205/232851 objects misplaced (9.536%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T04:55:21.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T04:55:21.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 77157/232851 objects degraded (33.136%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:21.188 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:21.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 97 16 77608 77592 49.9819 0 - 0.0177413 2023-10-09T04:55:21.241 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:21.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: pgmap v1624: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 71 active+clean; 4.7 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 1.0 MiB/s wr, 16 op/s; 77157/232851 objects degraded (33.136%); 22205/232851 objects misplaced (9.536%); 7.0 MiB/s, 112 objects/s recovering 2023-10-09T04:55:21.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.244 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: osdmap e464: 8 total, 8 up, 8 in 2023-10-09T04:55:21.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: failed to encode map e464 with expected crc 2023-10-09T04:55:21.246 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 77157/232851 objects degraded (33.136%), 14 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:21.247 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:55:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:21.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:21.316 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:55:21.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:21.384 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:55:21.587 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:21.732 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:55:21.920 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:21.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:22.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:21 smithi177 ceph-mon[195797]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T04:55:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:21 smithi177 ceph-mon[195797]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:21 smithi177 ceph-mon[195797]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:21 smithi177 ceph-mon[195797]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:21 smithi177 ceph-mon[195797]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3663918555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:22.056 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:22.107 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:22.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:22.166 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:22.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[216903]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[216903]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[216903]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[216903]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[216903]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3663918555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[223858]: osdmap e465: 8 total, 8 up, 8 in 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[223858]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[223858]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[223858]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[223858]: failed to encode map e465 with expected crc 2023-10-09T04:55:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3663918555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:22.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:22.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 98 16 77608 77592 49.4719 0 - 0.0177413 2023-10-09T04:55:22.247 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:22.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:22.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:22.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:22.472 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:22.632 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:22.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:22.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:22.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:22.765 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:22.814 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:22.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:22.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:22.949 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:23.002 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:23.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:23.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:23.138 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:23.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:22 smithi029 ceph-mon[216903]: pgmap v1627: 89 pgs: 2 remapped+peering, 6 peering, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 66 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 821 KiB/s wr, 12 op/s; 67866/232851 objects degraded (29.146%); 26254/232851 objects misplaced (11.275%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T04:55:23.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:55:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:55:22] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2023-10-09T04:55:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:22 smithi029 ceph-mon[223858]: pgmap v1627: 89 pgs: 2 remapped+peering, 6 peering, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 66 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 821 KiB/s wr, 12 op/s; 67866/232851 objects degraded (29.146%); 26254/232851 objects misplaced (11.275%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T04:55:23.191 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:23.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:23.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:23.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 99 16 77608 77592 48.9723 0 - 0.0177413 2023-10-09T04:55:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:22 smithi177 ceph-mon[195797]: pgmap v1627: 89 pgs: 2 remapped+peering, 6 peering, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 66 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 821 KiB/s wr, 12 op/s; 67866/232851 objects degraded (29.146%); 26254/232851 objects misplaced (11.275%); 9.2 MiB/s, 146 objects/s recovering 2023-10-09T04:55:23.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:23.328 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:23.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:23.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:23.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:23.518 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:23.568 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:23.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:23.673 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:23.704 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:23.752 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:23.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:23.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:23.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:23.887 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:23.937 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:23.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:24.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:24.069 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:24.120 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:24.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:24.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:24.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:55:24.219650+0000 min lat: 0.00111616 max lat: 16.5782 avg lat: 0.0177413 2023-10-09T04:55:24.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:55:24.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 100 16 77608 77592 48.4827 0 - 0.0177413 2023-10-09T04:55:24.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:24.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:24.571 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:24.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:24.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:24.708 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:24.760 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:24.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:24.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:24.893 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:24.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:24.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:25.055 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:25.083 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:24 smithi029 ceph-mon[216903]: pgmap v1628: 89 pgs: 1 active+recovery_wait+remapped, 2 remapped+peering, 6 peering, 1 activating+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 66 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69757/232851 objects degraded (29.958%); 26254/232851 objects misplaced (11.275%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:55:25.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:24 smithi029 ceph-mon[223858]: pgmap v1628: 89 pgs: 1 active+recovery_wait+remapped, 2 remapped+peering, 6 peering, 1 activating+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 66 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69757/232851 objects degraded (29.958%); 26254/232851 objects misplaced (11.275%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:55:25.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 101 16 77608 77592 48.0027 0 - 0.0177413 2023-10-09T04:55:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:24 smithi177 ceph-mon[195797]: pgmap v1628: 89 pgs: 1 active+recovery_wait+remapped, 2 remapped+peering, 6 peering, 1 activating+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 66 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 69757/232851 objects degraded (29.958%); 26254/232851 objects misplaced (11.275%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:55:25.335 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:25.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:25.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:25.439 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:25.470 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:25.524 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:25.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:25.627 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:25.636 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:55:25.636 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:55:25.636 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-09T04:55:25.636 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:55:25.637 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:55:25.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-09T04:55:25.658 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:25.723 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:25.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:25.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:25.864 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:25.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:25.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:25.985 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:55:26.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:26.031 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:26.031 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:55:26.031 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:55:26.032 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:55:26.032 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:55:26.032 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:26.053 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:26.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:26.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:26.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:26.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 102 16 77608 77592 47.5322 0 - 0.0177413 2023-10-09T04:55:26.235 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3136514111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:26.235 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:26.235 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 69757/232851 objects degraded (29.958%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:26.247 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:26.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:26.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:26.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:26.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:26.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3136514111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:26.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:26.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 69757/232851 objects degraded (29.958%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3136514111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:26.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 69757/232851 objects degraded (29.958%), 11 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:26.449 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:26.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:26.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:26.604 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:26.638 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:26.989 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:26.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:27.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:27.129 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:27.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:27.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:27.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 103 16 77608 77592 47.0708 0 - 0.0177413 2023-10-09T04:55:27.222 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[216903]: pgmap v1629: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 3 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78968/232851 objects degraded (33.914%); 22372/232851 objects misplaced (9.608%); 14 MiB/s, 224 objects/s recovering 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[216903]: failed to encode map e466 with expected crc 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[216903]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[223858]: pgmap v1629: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 3 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78968/232851 objects degraded (33.914%); 22372/232851 objects misplaced (9.608%); 14 MiB/s, 224 objects/s recovering 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[223858]: failed to encode map e466 with expected crc 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:27.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:26 smithi029 ceph-mon[223858]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T04:55:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:26 smithi177 ceph-mon[195797]: pgmap v1629: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 3 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78968/232851 objects degraded (33.914%); 22372/232851 objects misplaced (9.608%); 14 MiB/s, 224 objects/s recovering 2023-10-09T04:55:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:26 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:26 smithi177 ceph-mon[195797]: failed to encode map e466 with expected crc 2023-10-09T04:55:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:27.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:26 smithi177 ceph-mon[195797]: osdmap e466: 8 total, 8 up, 8 in 2023-10-09T04:55:27.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:27.320 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:27.675 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:27.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:27.779 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:27.807 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:27.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:27.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:27.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:28.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:55:28.069 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:27 smithi177 ceph-mon[195797]: failed to encode map e466 with expected crc 2023-10-09T04:55:28.085 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:28.119 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:28.177 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:28.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:28.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 104 16 77608 77592 46.6182 0 - 0.0177413 2023-10-09T04:55:28.285 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:28.316 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:28.322 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:27 smithi029 ceph-mon[223858]: failed to encode map e466 with expected crc 2023-10-09T04:55:28.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:27 smithi029 ceph-mon[216903]: failed to encode map e466 with expected crc 2023-10-09T04:55:28.380 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:28.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:28.483 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:28.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:28.526 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:28.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:28.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:28.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:28.764 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:28.821 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:28.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:28.898 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:55:28.898 INFO:teuthology.orchestra.run.smithi029.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T04:55:28.899 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:55:28.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:28.962 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:29.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:55:29.124 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:29.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:29.220 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 105 16 77608 77592 46.1743 0 - 0.0177413 2023-10-09T04:55:29.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:29.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:28 smithi029 ceph-mon[216903]: pgmap v1631: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 3 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78968/232851 objects degraded (33.914%); 22372/232851 objects misplaced (9.608%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:55:29.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:29.258 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:29.258 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:28 smithi029 ceph-mon[223858]: pgmap v1631: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 3 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78968/232851 objects degraded (33.914%); 22372/232851 objects misplaced (9.608%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:55:29.258 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:29.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:29.261 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:28 smithi177 ceph-mon[195797]: pgmap v1631: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 3 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 69 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 78968/232851 objects degraded (33.914%); 22372/232851 objects misplaced (9.608%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:55:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:29.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:29.320 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:29.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:29.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:29.456 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:29.917 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:29.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:30.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:30.096 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:30.097 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:30.157 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:30.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:30.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:30.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 106 16 77608 77592 45.7388 0 - 0.0177413 2023-10-09T04:55:30.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:30.292 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:30.344 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:30.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:30.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:30.492 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:30.549 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:30.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:30.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:30.654 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:30.689 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:30.996 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:31.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:31.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:31.140 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:30 smithi029 ceph-mon[216903]: pgmap v1632: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 17653/232851 objects misplaced (7.581%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:55:31.140 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 88502/232851 objects degraded (38.008%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:31.140 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:30 smithi029 ceph-mon[223858]: pgmap v1632: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 17653/232851 objects misplaced (7.581%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:55:31.140 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 88502/232851 objects degraded (38.008%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:31.144 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:31.202 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:31.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:31.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:31.219 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 107 16 77608 77592 45.3114 0 - 0.0177413 2023-10-09T04:55:31.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:30 smithi177 ceph-mon[195797]: pgmap v1632: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 17653/232851 objects misplaced (7.581%); 4.8 MiB/s, 77 objects/s recovering 2023-10-09T04:55:31.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 88502/232851 objects degraded (38.008%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:31.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.340 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:31.447 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:31.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:31.554 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122043 12.e 12:7d817187:::benchmark_data_smithi029_442458_object26793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-10-09T04:55:31.564 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.141234+0000", 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 547.41890310099996, 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.199009717, 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122043 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141234+0000", 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141234+0000", 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141233+0000", 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141240+0000", 2023-10-09T04:55:31.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2409999999999998e-06 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141256+0000", 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5911000000000001e-05 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141263+0000", 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4829999999999996e-06 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141316+0000", 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3458999999999998e-05 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141466+0000", 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000150142 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.141607+0000", 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014059700000000001 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.144527+0000", 2023-10-09T04:55:31.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029195950000000001 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.144845+0000", 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031833499999999999 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.166192+0000", 2023-10-09T04:55:31.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0213472 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.166203+0000", 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0761000000000001e-05 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156356+0000", 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99015257800000001 2023-10-09T04:55:31.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156367+0000", 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1296e-05 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340172+0000", 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1838049820000001 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340244+0000", 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2359000000000003e-05 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:122063 12.e 12:7ad9492b:::benchmark_data_smithi029_442458_object26813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:24.243785+0000", 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 547.31635227899994, 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1204199130000001, 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 122063 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.575 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243785+0000", 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.576 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243785+0000", 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243784+0000", 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243791+0000", 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9959999999999996e-06 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243802+0000", 2023-10-09T04:55:31.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1204e-05 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243808+0000", 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1279999999999996e-06 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243873+0000", 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4492000000000005e-05 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:24.243882+0000", 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0259999999999996e-06 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156389+0000", 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91250751799999996 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.580 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:25.156396+0000", 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1330000000000001e-06 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340270+0000", 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.183874235 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340384+0000", 2023-10-09T04:55:31.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113696 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,5,7", 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.340955+0000", 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057121199999999998 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.352099+0000", 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011143922000000001 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.360393+0000", 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082938590000000006 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.362792+0000", 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023990779999999998 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.364037+0000", 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012447669999999999 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.364097+0000", 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9707999999999998e-05 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.584 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:26.364205+0000", 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010851 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:131604 12.0 12:03c61d3f:::benchmark_data_smithi029_442458_object36354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:46:52.669176+0000", 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 518.89096205500005, 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0355348680000001, 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.585 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131604 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669176+0000", 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669176+0000", 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669174+0000", 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.669179+0000", 2023-10-09T04:55:31.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1549999999999998e-06 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880049+0000", 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21087008099999999 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.589 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880059+0000", 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0141e-05 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880645+0000", 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00058601599999999997 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:52.880657+0000", 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1181e-05 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.680619+0000", 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.79996239199999997 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.680757+0000", 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000137754 2023-10-09T04:55:31.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.680880+0000", 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000123087 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.687147+0000", 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062670360000000001 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.696233+0000", 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0090863739999999995 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704607+0000", 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083739330000000001 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704665+0000", 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7516000000000003e-05 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:46:53.704710+0000", 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5606999999999999e-05 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.594 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138063 13.15 13:a870998f:::benchmark_data_smithi029_495570_object43803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.123001+0000", 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 214.437136917, 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95806695900000005, 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138063 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.595 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123001+0000", 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123001+0000", 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123000+0000", 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123003+0000", 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5519999999999999e-06 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123008+0000", 2023-10-09T04:55:31.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6169999999999998e-06 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.123012+0000", 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8010000000000001e-06 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.124906+0000", 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001893573 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.598 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.124914+0000", 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6179999999999995e-06 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.077118+0000", 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95220400199999999 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.599 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.077203+0000", 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5028e-05 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.077393+0000", 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018976600000000001 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.600 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.080895+0000", 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035016520000000001 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.080938+0000", 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3207999999999997e-05 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.601 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.080988+0000", 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0454999999999997e-05 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081030+0000", 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1866000000000002e-05 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081068+0000", 2023-10-09T04:55:31.602 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7231999999999999e-05 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138128 13.15 13:a984223b:::benchmark_data_smithi029_495570_object43868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.169301+0000", 2023-10-09T04:55:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 214.390837003, 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91213407700000004, 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138128 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.604 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169301+0000", 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169301+0000", 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169300+0000", 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:55:31.605 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169305+0000", 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8510000000000004e-06 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169315+0000", 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9660000000000008e-06 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.606 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169321+0000", 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4909999999999996e-06 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169365+0000", 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3952999999999997e-05 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.607 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.169370+0000", 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5099999999999998e-06 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.078581+0000", 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90921049899999995 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.608 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.078671+0000", 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0488000000000005e-05 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.078846+0000", 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017483399999999999 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081151+0000", 2023-10-09T04:55:31.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023050940000000002 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081201+0000", 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9378000000000001e-05 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.610 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081329+0000", 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012866599999999999 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081392+0000", 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2657999999999995e-05 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081435+0000", 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2656e-05 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:138146 13.15 13:afaa51d5:::benchmark_data_smithi029_495570_object43886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:51:57.182639+0000", 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 214.377499, 2023-10-09T04:55:31.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89888632300000004, 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 138146 2023-10-09T04:55:31.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182639+0000", 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182639+0000", 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182637+0000", 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182642+0000", 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2939999999999999e-06 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182650+0000", 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5760000000000006e-06 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.182655+0000", 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0300000000000001e-06 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.183838+0000", 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011824730000000001 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:57.183844+0000", 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4110000000000001e-06 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.079602+0000", 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89575800400000005 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.079707+0000", 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000104713 2023-10-09T04:55:31.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,7", 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.079892+0000", 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018569699999999999 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081285+0000", 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013926349999999999 2023-10-09T04:55:31.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081296+0000", 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0651e-05 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081480+0000", 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018418300000000001 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081505+0000", 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4831999999999999e-05 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:51:58.081525+0000", 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0157e-05 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:174814 13.5 13:a7f28997:::benchmark_data_smithi029_495570_object80554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:49.751891+0000", 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 161.808246397, 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97608966399999997, 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 174814 2023-10-09T04:55:31.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.751891+0000", 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.751891+0000", 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.751891+0000", 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.751894+0000", 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7960000000000002e-06 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.624 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.751901+0000", 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2149999999999998e-06 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.751904+0000", 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4860000000000002e-06 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.754863+0000", 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029593599999999999 2023-10-09T04:55:31.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.754942+0000", 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8712000000000001e-05 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,2,5", 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.755323+0000", 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038131299999999999 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.760311+0000", 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049872110000000001 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.763730+0000", 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034195179999999999 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.766898+0000", 2023-10-09T04:55:31.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003167736 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.766910+0000", 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2247e-05 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.727897+0000", 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.960986809 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.727981+0000", 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3910000000000001e-05 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:174834 13.5 13:a16988df:::benchmark_data_smithi029_495570_object80574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:49.786332+0000", 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 161.77380545700001, 2023-10-09T04:55:31.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95538497700000002, 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 174834 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786332+0000", 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786332+0000", 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786332+0000", 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786334+0000", 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5440000000000001e-06 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786340+0000", 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7740000000000001e-06 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786344+0000", 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0550000000000001e-06 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786884+0000", 2023-10-09T04:55:31.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00053965399999999998 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.786890+0000", 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3679999999999998e-06 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.728421+0000", 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94153114400000004 2023-10-09T04:55:31.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.728538+0000", 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116873 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.728895+0000", 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035684999999999997 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.737335+0000", 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008440276 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741479+0000", 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004143903 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.636 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741621+0000", 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014232200000000001 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741678+0000", 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6827999999999999e-05 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.637 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741717+0000", 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8784e-05 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:174857 13.5 13:a1520d86:::benchmark_data_smithi029_495570_object80597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:49.816535+0000", 2023-10-09T04:55:31.638 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 161.74360283499999, 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92532782999999996, 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 174857 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816535+0000", 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816535+0000", 2023-10-09T04:55:31.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816533+0000", 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816539+0000", 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4219999999999999e-06 2023-10-09T04:55:31.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816549+0000", 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0183999999999999e-05 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816559+0000", 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0593e-05 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816787+0000", 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00022754000000000001 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:49.816795+0000", 2023-10-09T04:55:31.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9640000000000003e-06 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.730563+0000", 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91376752900000002 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.730643+0000", 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0450000000000004e-05 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.730851+0000", 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020801399999999999 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741352+0000", 2023-10-09T04:55:31.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0105012 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741485+0000", 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013261800000000001 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741782+0000", 2023-10-09T04:55:31.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029700800000000003 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741824+0000", 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2632000000000001e-05 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.741863+0000", 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8124000000000003e-05 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175392 13.15 13:af70c9c6:::benchmark_data_smithi029_495570_object81132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-10-09T04:55:31.648 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:50.783021+0000", 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 160.77711644600001, 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96531572300000001, 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175392 2023-10-09T04:55:31.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.783021+0000", 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.783021+0000", 2023-10-09T04:55:31.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.783020+0000", 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.783025+0000", 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.417e-06 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.783038+0000", 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2381999999999999e-05 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.783044+0000", 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0430000000000004e-06 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.784906+0000", 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001862671 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.784914+0000", 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9219999999999997e-06 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.740830+0000", 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95591528299999995 2023-10-09T04:55:31.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.740968+0000", 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000138343 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.741434+0000", 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00046636099999999999 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.745562+0000", 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0041277099999999997 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.745606+0000", 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3927000000000001e-05 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748243+0000", 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026373600000000001 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748298+0000", 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4540999999999998e-05 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748337+0000", 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9082000000000003e-05 2023-10-09T04:55:31.656 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175447 13.15 13:af0fcaf4:::benchmark_data_smithi029_495570_object81187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:50.836450+0000", 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 160.72368774500001, 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.912030644, 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.657 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175447 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.836450+0000", 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.658 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.836450+0000", 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.836448+0000", 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.836455+0000", 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.849e-06 2023-10-09T04:55:31.659 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.836467+0000", 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2094e-05 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.836473+0000", 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1369999999999999e-06 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.660 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.838262+0000", 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001788726 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.838267+0000", 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3619999999999997e-06 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.743360+0000", 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90509244600000005 2023-10-09T04:55:31.661 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.743472+0000", 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000112612 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.743860+0000", 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038753000000000003 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.662 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748204+0000", 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043444160000000002 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748212+0000", 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5290000000000003e-06 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748407+0000", 2023-10-09T04:55:31.663 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000194785 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748449+0000", 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2407000000000002e-05 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748480+0000", 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1479000000000001e-05 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.664 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175451 13.15 13:a9ca056f:::benchmark_data_smithi029_495570_object81191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:50.840603+0000", 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 160.71953426499999, 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91207884100000003, 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.665 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175451 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840603+0000", 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.666 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840603+0000", 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840602+0000", 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840607+0000", 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7600000000000002e-06 2023-10-09T04:55:31.667 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840619+0000", 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2531000000000001e-05 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840626+0000", 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4500000000000001e-06 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.668 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840702+0000", 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6137999999999995e-05 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.840710+0000", 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2369999999999992e-06 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.669 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.745736+0000", 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90502566399999995 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.745853+0000", 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011755 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.670 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.746204+0000", 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00035056399999999999 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748373+0000", 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021690920000000001 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.671 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.750649+0000", 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022761370000000001 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752587+0000", 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019375989999999999 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.672 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752641+0000", 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4157999999999999e-05 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752682+0000", 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1121000000000002e-05 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.673 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175466 13.15 13:ab10d0de:::benchmark_data_smithi029_495570_object81206:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:50.854981+0000", 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 160.705156286, 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90240234500000005, 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.674 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175466 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.854981+0000", 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.854981+0000", 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.675 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.854980+0000", 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.854985+0000", 2023-10-09T04:55:31.676 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7380000000000002e-06 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.854994+0000", 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9989999999999997e-06 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.854998+0000", 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8899999999999998e-06 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.677 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.856972+0000", 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019736490000000001 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.856979+0000", 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5220000000000004e-06 2023-10-09T04:55:31.678 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748517+0000", 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89153876899999995 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748622+0000", 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010431 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.679 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.748951+0000", 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032928599999999998 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752836+0000", 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038853680000000002 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.680 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752846+0000", 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5000000000000005e-06 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757297+0000", 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044514000000000003 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757348+0000", 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0943000000000003e-05 2023-10-09T04:55:31.681 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757384+0000", 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5445999999999999e-05 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175482 13.15 13:ae237f76:::benchmark_data_smithi029_495570_object81222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2023-10-09T04:55:31.682 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:50.870489+0000", 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 160.68964903200001, 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.88701087199999995, 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175482 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.683 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870489+0000", 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870489+0000", 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.684 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870487+0000", 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870493+0000", 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4809999999999997e-06 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.685 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870505+0000", 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2140000000000001e-05 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870511+0000", 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8860000000000001e-06 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.686 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870667+0000", 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000156081 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:50.870674+0000", 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0589999999999997e-06 2023-10-09T04:55:31.687 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.750678+0000", 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.88000354599999997 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.688 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.750789+0000", 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000111023 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.751038+0000", 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024910399999999999 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.689 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752892+0000", 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018545619999999999 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757257+0000", 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043647679999999998 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757431+0000", 2023-10-09T04:55:31.690 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017375499999999999 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757465+0000", 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4285000000000001e-05 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.757499+0000", 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4249999999999999e-05 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.691 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175688 13.12 13:4c528d10:::benchmark_data_smithi029_495570_object81428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:51.752889+0000", 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 159.80724885999999, 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99572341200000003, 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.692 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175688 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752889+0000", 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752889+0000", 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.693 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752887+0000", 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752894+0000", 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.793e-06 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.694 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752904+0000", 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8430000000000005e-06 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752910+0000", 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8340000000000004e-06 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.695 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752951+0000", 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1598999999999999e-05 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.752958+0000", 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6270000000000002e-06 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.741704+0000", 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98874592100000003 2023-10-09T04:55:31.696 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.742049+0000", 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034448900000000001 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.742435+0000", 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00038672000000000001 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.697 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748343+0000", 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059081899999999998 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748482+0000", 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013826999999999999 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748517+0000", 2023-10-09T04:55:31.698 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4842999999999998e-05 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748572+0000", 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5016999999999999e-05 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748612+0000", 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0565000000000002e-05 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.699 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175689 13.12 13:4c0a0b09:::benchmark_data_smithi029_495570_object81429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:51.755785+0000", 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 159.80435223000001, 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99302170099999998, 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.700 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175689 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.755785+0000", 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.701 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.755785+0000", 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.755784+0000", 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.755791+0000", 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1049999999999997e-06 2023-10-09T04:55:31.702 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.755802+0000", 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1623e-05 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.755809+0000", 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7920000000000004e-06 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.703 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.759433+0000", 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036235180000000001 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.759441+0000", 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7450000000000001e-06 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.744355+0000", 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.984913339 2023-10-09T04:55:31.704 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.744449+0000", 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4452999999999998e-05 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.744723+0000", 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00027430400000000001 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.705 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748641+0000", 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039171400000000004 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748699+0000", 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8841999999999997e-05 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748725+0000", 2023-10-09T04:55:31.706 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5948999999999998e-05 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748770+0000", 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4635999999999998e-05 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748807+0000", 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7054000000000001e-05 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.707 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175693 13.12 13:49d2b0ec:::benchmark_data_smithi029_495570_object81433:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:51.768952+0000", 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 159.79118562799999, 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.98745096099999996, 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.708 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175693 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.768952+0000", 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.709 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.768952+0000", 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.768951+0000", 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.768957+0000", 2023-10-09T04:55:31.710 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8490000000000002e-06 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.768965+0000", 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2770000000000005e-06 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.768970+0000", 2023-10-09T04:55:31.711 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0839999999999999e-06 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.769017+0000", 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7216000000000003e-05 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.769024+0000", 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8419999999999999e-06 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.712 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.746461+0000", 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97743696300000005 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.746569+0000", 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108162 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.713 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.746860+0000", 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029054100000000001 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.752670+0000", 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058106629999999998 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.714 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.752736+0000", 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5102000000000001e-05 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756319+0000", 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035831629999999999 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756367+0000", 2023-10-09T04:55:31.715 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8015e-05 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756403+0000", 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6182e-05 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.716 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:175751 13.12 13:4a2d28db:::benchmark_data_smithi029_495570_object81491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:51.848543+0000", 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 159.711594088, 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90796976900000004, 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:55:31.717 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 175751 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848543+0000", 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.718 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848543+0000", 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848543+0000", 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.719 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848546+0000", 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6290000000000002e-06 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848552+0000", 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9020000000000004e-06 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.720 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848556+0000", 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7500000000000003e-06 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848675+0000", 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000118836 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.721 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:51.848681+0000", 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0800000000000002e-06 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748846+0000", 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.90016474800000001 2023-10-09T04:55:31.722 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.748943+0000", 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6849999999999996e-05 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.723 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.749208+0000", 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026468599999999998 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.752702+0000", 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0034944139999999999 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.724 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756416+0000", 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003714261 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756454+0000", 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7583999999999998e-05 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.725 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756481+0000", 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7206e-05 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:52.756513+0000", 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2255000000000002e-05 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.726 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75764 14.2 14:4b81030d:::benchmark_data_smithi029_550670_object75763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:44.579076+0000", 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 46.981061244999999, 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9303421090000001, 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:55:31.727 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75764 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579076+0000", 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579076+0000", 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.728 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579075+0000", 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579118+0000", 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3828000000000001e-05 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.729 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579137+0000", 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8119999999999999e-05 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579144+0000", 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7160000000000003e-06 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579204+0000", 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9382999999999997e-05 2023-10-09T04:55:31.730 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579352+0000", 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000148202 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.579547+0000", 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000195458 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.731 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.582871+0000", 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003324095 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.586797+0000", 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039258749999999997 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.622411+0000", 2023-10-09T04:55:31.732 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.035613626000000002 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.622420+0000", 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3449999999999995e-06 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.584863+0000", 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96244258800000004 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.733 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.584875+0000", 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2123999999999999e-05 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.509310+0000", 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9244345119999999 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.509418+0000", 2023-10-09T04:55:31.734 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010888299999999999 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75765 14.2 14:43f6c852:::benchmark_data_smithi029_550670_object75764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:44.583267+0000", 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 46.976870173000002, 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.929008225, 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:55:31.735 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75765 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583267+0000", 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.736 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583267+0000", 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583266+0000", 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583311+0000", 2023-10-09T04:55:31.737 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4484999999999998e-05 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583323+0000", 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2055000000000001e-05 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583327+0000", 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7190000000000001e-06 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.738 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583373+0000", 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5074999999999998e-05 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583537+0000", 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000164229 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:55:31.739 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.583731+0000", 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019393599999999999 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.586252+0000", 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025215289999999998 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.592006+0000", 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057538720000000002 2023-10-09T04:55:31.740 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.622451+0000", 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030444689 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.622458+0000", 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3089999999999998e-06 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.741 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.584942+0000", 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96248361100000002 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.584950+0000", 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0439999999999996e-06 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:55:31.742 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.512160+0000", 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9272101799999999 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.512276+0000", 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011566900000000001 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:55:31.743 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:55:31.744 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:55:31.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:31.799 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:31.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:31.904 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:31.935 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:31.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:31.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:32.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:32.124 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:32.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:32.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:32.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:32.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 108 16 77608 77592 44.8919 0 - 0.0177413 2023-10-09T04:55:32.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:32.328 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:32.385 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:32.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:32.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:32.521 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:32.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:32.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:32.680 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:32.708 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:32.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:32.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:32.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:32.897 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:32.946 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:32.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:32.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:32.997 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:55:33.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:33.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:32 smithi029 ceph-mon[216903]: pgmap v1633: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16976/232851 objects misplaced (7.290%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T04:55:33.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1373640015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:55:33.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:32 smithi029 ceph-mon[223858]: pgmap v1633: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16976/232851 objects misplaced (7.290%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T04:55:33.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1373640015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:55:33.082 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:55:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:55:32] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2023-10-09T04:55:33.086 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:33.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 109 16 77608 77592 44.4799 0 - 0.0177413 2023-10-09T04:55:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:32 smithi177 ceph-mon[195797]: pgmap v1633: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16976/232851 objects misplaced (7.290%); 8.4 MiB/s, 133 objects/s recovering 2023-10-09T04:55:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1373640015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:55:33.344 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:33.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:33.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:33.479 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:33.531 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:33.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:33.635 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:33.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:55:33.664 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:34.041 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:34.137 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:34.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:34.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:34.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 110 16 77608 77592 44.0756 0 - 0.0177413 2023-10-09T04:55:34.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:34.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:33 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:34.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:33 smithi177 ceph-mon[195797]: failed to encode map e467 with expected crc 2023-10-09T04:55:34.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1373640015' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:55:34.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:33 smithi177 ceph-mon[195797]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T04:55:34.275 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:33 smithi177 ceph-mon[195797]: failed to encode map e467 with expected crc 2023-10-09T04:55:34.276 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:34.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:34.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[223858]: failed to encode map e467 with expected crc 2023-10-09T04:55:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1373640015' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:55:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[223858]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T04:55:34.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[223858]: failed to encode map e467 with expected crc 2023-10-09T04:55:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[216903]: failed to encode map e467 with expected crc 2023-10-09T04:55:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1373640015' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:55:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[216903]: osdmap e467: 8 total, 8 up, 8 in 2023-10-09T04:55:34.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:33 smithi029 ceph-mon[216903]: failed to encode map e467 with expected crc 2023-10-09T04:55:34.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:34.489 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:34.544 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:34.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:34.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.680 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:34.731 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:34.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:34.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:34.868 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:34.918 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:34.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:34.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:34.993 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:34 smithi177 ceph-mon[195797]: pgmap v1635: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16976/232851 objects misplaced (7.290%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:55:35.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:35.024 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:55:35.024 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:55:35.025 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-10-09T04:55:35.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:55:35.059 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:35.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:35.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:35.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:34 smithi029 ceph-mon[223858]: pgmap v1635: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16976/232851 objects misplaced (7.290%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:55:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:34 smithi029 ceph-mon[216903]: pgmap v1635: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16976/232851 objects misplaced (7.290%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T04:55:35.216 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:35.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 111 16 77608 77592 43.6786 0 - 0.0177413 2023-10-09T04:55:35.257 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:35.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:35.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:35.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:35.443 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:35.450 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:35.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:35.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:35.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:35.645 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:35.700 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:35.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:35.805 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:35.838 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:35.890 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:35.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:35.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:36.026 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:36.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:36.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:36.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:55:36.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:36.220 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:36.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 112 16 77608 77592 43.2887 0 - 0.0177413 2023-10-09T04:55:36.282 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:36.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:36.386 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:36.418 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:36.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:36.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:36.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:36.599 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:36.761 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:36.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:36.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:36.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:36.907 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:36.969 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:36.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:37.022 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:37.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.110 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:37.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:36 smithi177 ceph-mon[195797]: pgmap v1636: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:55:37.169 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:36 smithi029 ceph-mon[216903]: pgmap v1636: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:55:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:36 smithi029 ceph-mon[223858]: pgmap v1636: 89 pgs: 1 active+recovery_wait+remapped, 1 activating+undersized+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:55:37.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:37.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:37.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 113 16 77608 77592 42.9057 0 - 0.0177413 2023-10-09T04:55:37.276 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:37.311 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:37.481 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:37.481 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:55:37.481 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:55:37.481 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:55:37.571 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:37.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:37.675 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:37.713 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:37.796 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:55:37.798 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:55:38.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:38 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 88502/232851 objects degraded (38.008%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:38.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:38 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 88502/232851 objects degraded (38.008%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:38.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 114 16 77608 77592 42.5294 0 - 0.0177413 2023-10-09T04:55:38.274 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:38.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:38.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:38.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:38.407 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:38.457 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:38.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:38.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:38 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 88502/232851 objects degraded (38.008%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:38.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:38.589 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:39.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:55:39.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 115 16 77608 77592 42.1596 0 - 0.0177413 2023-10-09T04:55:39.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:39 smithi029 ceph-mon[216903]: pgmap v1637: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:55:39.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:39 smithi029 ceph-mon[216903]: from='client.37074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:39.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:39 smithi029 ceph-mon[223858]: pgmap v1637: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:55:39.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:39 smithi029 ceph-mon[223858]: from='client.37074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:39.406 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1637,"stamp":"2023-10-09T04:55:37.480806+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5085730720,"num_objects":77617,"num_object_clones":0,"num_object_copies":232851,"num_objects_missing_on_primary":8759,"num_objects_missing":8759,"num_objects_degraded":88502,"num_objects_misplaced":16250,"num_objects_unfound":0,"num_objects_dirty":77617,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":77893,"num_write_kb":4968397,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7197,"num_bytes_recovered":468681736,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77809,"ondisk_log_size":77809,"up":267,"acting":253,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":364,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24117908,"kb_used_data":15197524,"kb_used_omap":20,"kb_used_meta":8920299,"kb_avail":725843308,"statfs":{"total":767960285184,"available":743263547392,"internally_reserved":0,"allocated":15562264576,"data_stored":15551024490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":9134386830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,0,0,0,0,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":4719,"num_objects_missing":4719,"num_objects_degraded":9534,"num_objects_misplaced":-6122,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1403,"num_bytes_recovered":91947008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002898"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":265,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.006033+0000","last_change":"2023-10-09T04:55:19.971914+0000","last_active":"2023-10-09T04:55:26.006033+0000","last_peered":"2023-10-09T04:55:26.006033+0000","last_clean":"2023-10-09T04:55:26.006033+0000","last_became_active":"2023-10-09T04:55:19.971411+0000","last_became_peered":"2023-10-09T04:55:19.971411+0000","last_unstale":"2023-10-09T04:55:26.006033+0000","last_undegraded":"2023-10-09T04:55:26.006033+0000","last_fullsized":"2023-10-09T04:55:26.006033+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:25:15.101373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.933254+0000","last_change":"2023-10-09T04:54:45.598246+0000","last_active":"2023-10-09T04:55:12.933254+0000","last_peered":"2023-10-09T04:55:12.933254+0000","last_clean":"2023-10-09T04:55:12.933254+0000","last_became_active":"2023-10-09T04:54:45.597729+0000","last_became_peered":"2023-10-09T04:54:45.597729+0000","last_unstale":"2023-10-09T04:55:12.933254+0000","last_undegraded":"2023-10-09T04:55:12.933254+0000","last_fullsized":"2023-10-09T04:55:12.933254+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:05:43.330367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.978508+0000","last_change":"2023-10-09T04:55:19.965205+0000","last_active":"2023-10-09T04:55:25.978508+0000","last_peered":"2023-10-09T04:55:25.978508+0000","last_clean":"2023-10-09T04:55:25.978508+0000","last_became_active":"2023-10-09T04:55:19.964820+0000","last_became_peered":"2023-10-09T04:55:19.964820+0000","last_unstale":"2023-10-09T04:55:25.978508+0000","last_undegraded":"2023-10-09T04:55:25.978508+0000","last_fullsized":"2023-10-09T04:55:25.978508+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:17:23.849401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.933790+0000","last_change":"2023-10-09T04:55:09.939257+0000","last_active":"2023-10-09T04:55:12.933790+0000","last_peered":"2023-10-09T04:55:12.933790+0000","last_clean":"2023-10-09T04:55:12.933790+0000","last_became_active":"2023-10-09T04:55:09.938721+0000","last_became_peered":"2023-10-09T04:55:09.938721+0000","last_unstale":"2023-10-09T04:55:12.933790+0000","last_undegraded":"2023-10-09T04:55:12.933790+0000","last_fullsized":"2023-10-09T04:55:12.933790+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:54.606342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.931561+0000","last_change":"2023-10-09T04:55:09.983822+0000","last_active":"2023-10-09T04:55:12.931561+0000","last_peered":"2023-10-09T04:55:12.931561+0000","last_clean":"2023-10-09T04:55:12.931561+0000","last_became_active":"2023-10-09T04:55:09.983575+0000","last_became_peered":"2023-10-09T04:55:09.983575+0000","last_unstale":"2023-10-09T04:55:12.931561+0000","last_undegraded":"2023-10-09T04:55:12.931561+0000","last_fullsized":"2023-10-09T04:55:12.931561+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:07:45.260872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":44,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.987457+0000","last_change":"2023-10-09T04:55:19.996059+0000","last_active":"2023-10-09T04:55:25.987457+0000","last_peered":"2023-10-09T04:55:25.987457+0000","last_clean":"2023-10-09T04:55:25.987457+0000","last_became_active":"2023-10-09T04:55:19.995700+0000","last_became_peered":"2023-10-09T04:55:19.995700+0000","last_unstale":"2023-10-09T04:55:25.987457+0000","last_undegraded":"2023-10-09T04:55:25.987457+0000","last_fullsized":"2023-10-09T04:55:25.987457+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:12:39.286242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":479,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.988998+0000","last_change":"2023-10-09T04:55:09.934764+0000","last_active":"2023-10-09T04:55:25.988998+0000","last_peered":"2023-10-09T04:55:25.988998+0000","last_clean":"2023-10-09T04:55:25.988998+0000","last_became_active":"2023-10-09T04:55:09.934361+0000","last_became_peered":"2023-10-09T04:55:09.934361+0000","last_unstale":"2023-10-09T04:55:25.988998+0000","last_undegraded":"2023-10-09T04:55:25.988998+0000","last_fullsized":"2023-10-09T04:55:25.988998+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:49.252585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.931469+0000","last_change":"2023-10-09T04:54:45.599244+0000","last_active":"2023-10-09T04:55:12.931469+0000","last_peered":"2023-10-09T04:55:12.931469+0000","last_clean":"2023-10-09T04:55:12.931469+0000","last_became_active":"2023-10-09T04:54:45.598784+0000","last_became_peered":"2023-10-09T04:54:45.598784+0000","last_unstale":"2023-10-09T04:55:12.931469+0000","last_undegraded":"2023-10-09T04:55:12.931469+0000","last_fullsized":"2023-10-09T04:55:12.931469+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:49:01.671147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":457,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.981898+0000","last_change":"2023-10-09T04:55:20.004951+0000","last_active":"2023-10-09T04:55:25.981898+0000","last_peered":"2023-10-09T04:55:25.981898+0000","last_clean":"2023-10-09T04:55:25.981898+0000","last_became_active":"2023-10-09T04:55:20.004659+0000","last_became_peered":"2023-10-09T04:55:20.004659+0000","last_unstale":"2023-10-09T04:55:25.981898+0000","last_undegraded":"2023-10-09T04:55:25.981898+0000","last_fullsized":"2023-10-09T04:55:25.981898+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:48.952780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.934114+0000","last_change":"2023-10-09T04:54:45.599846+0000","last_active":"2023-10-09T04:55:12.934114+0000","last_peered":"2023-10-09T04:55:12.934114+0000","last_clean":"2023-10-09T04:55:12.934114+0000","last_became_active":"2023-10-09T04:54:45.599492+0000","last_became_peered":"2023-10-09T04:54:45.599492+0000","last_unstale":"2023-10-09T04:55:12.934114+0000","last_undegraded":"2023-10-09T04:55:12.934114+0000","last_fullsized":"2023-10-09T04:55:12.934114+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:11.482933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":30,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.933179+0000","last_change":"2023-10-09T04:54:25.779875+0000","last_active":"2023-10-09T04:55:12.933179+0000","last_peered":"2023-10-09T04:55:12.933179+0000","last_clean":"2023-10-09T04:55:12.933179+0000","last_became_active":"2023-10-09T04:54:25.753020+0000","last_became_peered":"2023-10-09T04:54:25.753020+0000","last_unstale":"2023-10-09T04:55:12.933179+0000","last_undegraded":"2023-10-09T04:55:12.933179+0000","last_fullsized":"2023-10-09T04:55:12.933179+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:51:50.320050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.979549+0000","last_change":"2023-10-09T04:55:19.962112+0000","last_active":"2023-10-09T04:55:25.979549+0000","last_peered":"2023-10-09T04:55:25.979549+0000","last_clean":"2023-10-09T04:55:25.979549+0000","last_became_active":"2023-10-09T04:55:19.961771+0000","last_became_peered":"2023-10-09T04:55:19.961771+0000","last_unstale":"2023-10-09T04:55:25.979549+0000","last_undegraded":"2023-10-09T04:55:25.979549+0000","last_fullsized":"2023-10-09T04:55:25.979549+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:08.314935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":305,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005920+0000","last_change":"2023-10-09T04:54:45.663172+0000","last_active":"2023-10-09T04:55:26.005920+0000","last_peered":"2023-10-09T04:55:26.005920+0000","last_clean":"2023-10-09T04:55:26.005920+0000","last_became_active":"2023-10-09T04:54:45.606698+0000","last_became_peered":"2023-10-09T04:54:45.606698+0000","last_unstale":"2023-10-09T04:55:26.005920+0000","last_undegraded":"2023-10-09T04:55:26.005920+0000","last_fullsized":"2023-10-09T04:55:26.005920+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:03:13.293610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":467,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005564+0000","last_change":"2023-10-09T04:55:19.968550+0000","last_active":"2023-10-09T04:55:26.005564+0000","last_peered":"2023-10-09T04:55:26.005564+0000","last_clean":"2023-10-09T04:55:26.005564+0000","last_became_active":"2023-10-09T04:55:19.968191+0000","last_became_peered":"2023-10-09T04:55:19.968191+0000","last_unstale":"2023-10-09T04:55:26.005564+0000","last_undegraded":"2023-10-09T04:55:26.005564+0000","last_fullsized":"2023-10-09T04:55:26.005564+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:45:21.385429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":303,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005293+0000","last_change":"2023-10-09T04:53:09.937463+0000","last_active":"2023-10-09T04:55:26.005293+0000","last_peered":"2023-10-09T04:55:26.005293+0000","last_clean":"2023-10-09T04:55:26.005293+0000","last_became_active":"2023-10-09T04:53:09.937146+0000","last_became_peered":"2023-10-09T04:53:09.937146+0000","last_unstale":"2023-10-09T04:55:26.005293+0000","last_undegraded":"2023-10-09T04:55:26.005293+0000","last_fullsized":"2023-10-09T04:55:26.005293+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:44:31.290920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.931921+0000","last_change":"2023-10-09T04:54:25.750413+0000","last_active":"2023-10-09T04:55:12.931921+0000","last_peered":"2023-10-09T04:55:12.931921+0000","last_clean":"2023-10-09T04:55:12.931921+0000","last_became_active":"2023-10-09T04:54:25.749544+0000","last_became_peered":"2023-10-09T04:54:25.749544+0000","last_unstale":"2023-10-09T04:55:12.931921+0000","last_undegraded":"2023-10-09T04:55:12.931921+0000","last_fullsized":"2023-10-09T04:55:12.931921+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:43:30.409386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.932172+0000","last_change":"2023-10-09T04:55:09.936045+0000","last_active":"2023-10-09T04:55:12.932172+0000","last_peered":"2023-10-09T04:55:12.932172+0000","last_clean":"2023-10-09T04:55:12.932172+0000","last_became_active":"2023-10-09T04:55:09.935167+0000","last_became_peered":"2023-10-09T04:55:09.935167+0000","last_unstale":"2023-10-09T04:55:12.932172+0000","last_undegraded":"2023-10-09T04:55:12.932172+0000","last_fullsized":"2023-10-09T04:55:12.932172+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:03:47.013497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":466,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.982094+0000","last_change":"2023-10-09T04:54:25.766303+0000","last_active":"2023-10-09T04:55:25.982094+0000","last_peered":"2023-10-09T04:55:25.982094+0000","last_clean":"2023-10-09T04:55:25.982094+0000","last_became_active":"2023-10-09T04:54:25.765766+0000","last_became_peered":"2023-10-09T04:54:25.765766+0000","last_unstale":"2023-10-09T04:55:25.982094+0000","last_undegraded":"2023-10-09T04:55:25.982094+0000","last_fullsized":"2023-10-09T04:55:25.982094+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:48.044596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":466,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005082+0000","last_change":"2023-10-09T04:54:45.605119+0000","last_active":"2023-10-09T04:55:26.005082+0000","last_peered":"2023-10-09T04:55:26.005082+0000","last_clean":"2023-10-09T04:55:26.005082+0000","last_became_active":"2023-10-09T04:54:45.604413+0000","last_became_peered":"2023-10-09T04:54:45.604413+0000","last_unstale":"2023-10-09T04:55:26.005082+0000","last_undegraded":"2023-10-09T04:55:26.005082+0000","last_fullsized":"2023-10-09T04:55:26.005082+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:38:47.224557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":42,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.933726+0000","last_change":"2023-10-09T04:55:10.101213+0000","last_active":"2023-10-09T04:55:12.933726+0000","last_peered":"2023-10-09T04:55:12.933726+0000","last_clean":"2023-10-09T04:55:12.933726+0000","last_became_active":"2023-10-09T04:55:10.100870+0000","last_became_peered":"2023-10-09T04:55:10.100870+0000","last_unstale":"2023-10-09T04:55:12.933726+0000","last_undegraded":"2023-10-09T04:55:12.933726+0000","last_fullsized":"2023-10-09T04:55:12.933726+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:20.271892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":54,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.986395+0000","last_change":"2023-10-09T04:55:10.381918+0000","last_active":"2023-10-09T04:55:25.986395+0000","last_peered":"2023-10-09T04:55:25.986395+0000","last_clean":"2023-10-09T04:55:25.986395+0000","last_became_active":"2023-10-09T04:55:09.949433+0000","last_became_peered":"2023-10-09T04:55:09.949433+0000","last_unstale":"2023-10-09T04:55:25.986395+0000","last_undegraded":"2023-10-09T04:55:25.986395+0000","last_fullsized":"2023-10-09T04:55:25.986395+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:53:04.490020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.932490+0000","last_change":"2023-10-09T04:54:45.592746+0000","last_active":"2023-10-09T04:55:12.932490+0000","last_peered":"2023-10-09T04:55:12.932490+0000","last_clean":"2023-10-09T04:55:12.932490+0000","last_became_active":"2023-10-09T04:54:45.592451+0000","last_became_peered":"2023-10-09T04:54:45.592451+0000","last_unstale":"2023-10-09T04:55:12.932490+0000","last_undegraded":"2023-10-09T04:55:12.932490+0000","last_fullsized":"2023-10-09T04:55:12.932490+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:31:42.688977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"459'4815","reported_seq":4871,"reported_epoch":467,"state":"active+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:37.353516+0000","last_change":"2023-10-09T04:55:37.353516+0000","last_active":"2023-10-09T04:55:37.353516+0000","last_peered":"2023-10-09T04:55:37.353516+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T04:55:37.353516+0000","last_became_peered":"2023-10-09T04:55:37.353516+0000","last_unstale":"2023-10-09T04:55:37.353516+0000","last_undegraded":"2023-10-09T04:55:28.611539+0000","last_fullsized":"2023-10-09T04:55:28.600153+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4815,"log_dups_size":0,"ondisk_log_size":4815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:41:48.701915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315555840,"num_objects":4815,"num_object_clones":0,"num_object_copies":14445,"num_objects_missing_on_primary":4719,"num_objects_missing":4719,"num_objects_degraded":13533,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4815,"num_write_kb":308160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,3],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"3,4,5","objects":720},{"shards":"3,4,5,6","objects":96},{"shards":"4,5,7","objects":3999}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":325,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.006983+0000","last_change":"2023-10-09T04:55:10.384421+0000","last_active":"2023-10-09T04:55:26.006983+0000","last_peered":"2023-10-09T04:55:26.006983+0000","last_clean":"2023-10-09T04:55:26.006983+0000","last_became_active":"2023-10-09T04:55:09.941278+0000","last_became_peered":"2023-10-09T04:55:09.941278+0000","last_unstale":"2023-10-09T04:55:26.006983+0000","last_undegraded":"2023-10-09T04:55:26.006983+0000","last_fullsized":"2023-10-09T04:55:26.006983+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:54:28.047852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":449,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.981544+0000","last_change":"2023-10-09T04:54:45.624431+0000","last_active":"2023-10-09T04:55:25.981544+0000","last_peered":"2023-10-09T04:55:25.981544+0000","last_clean":"2023-10-09T04:55:25.981544+0000","last_became_active":"2023-10-09T04:54:45.623929+0000","last_became_peered":"2023-10-09T04:54:45.623929+0000","last_unstale":"2023-10-09T04:55:25.981544+0000","last_undegraded":"2023-10-09T04:55:25.981544+0000","last_fullsized":"2023-10-09T04:55:25.981544+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:04:50.579170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"456'4849","reported_seq":4891,"reported_epoch":462,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:16.266704+0000","last_change":"2023-10-09T04:55:16.241335+0000","last_active":"2023-10-09T04:55:09.018699+0000","last_peered":"2023-10-09T04:55:08.901655+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:55:01.114177+0000","last_became_peered":"2023-10-09T04:55:01.114177+0000","last_unstale":"2023-10-09T04:55:16.266704+0000","last_undegraded":"2023-10-09T04:55:16.206699+0000","last_fullsized":"2023-10-09T04:55:09.913661+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4849,"log_dups_size":0,"ondisk_log_size":4849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:02:23.717585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317784064,"num_objects":4849,"num_object_clones":0,"num_object_copies":14547,"num_objects_missing_on_primary":4040,"num_objects_missing":4040,"num_objects_degraded":8890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4849,"num_write_kb":310336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[{"shards":"3","objects":1},{"shards":"3,5","objects":808},{"shards":"5","objects":4040}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":312,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.012325+0000","last_change":"2023-10-09T04:55:20.015510+0000","last_active":"2023-10-09T04:55:26.012325+0000","last_peered":"2023-10-09T04:55:26.012325+0000","last_clean":"2023-10-09T04:55:26.012325+0000","last_became_active":"2023-10-09T04:55:19.971728+0000","last_became_peered":"2023-10-09T04:55:19.971728+0000","last_unstale":"2023-10-09T04:55:26.012325+0000","last_undegraded":"2023-10-09T04:55:26.012325+0000","last_fullsized":"2023-10-09T04:55:26.012325+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:54:39.847815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":514,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005024+0000","last_change":"2023-10-09T04:55:19.966211+0000","last_active":"2023-10-09T04:55:26.005024+0000","last_peered":"2023-10-09T04:55:26.005024+0000","last_clean":"2023-10-09T04:55:26.005024+0000","last_became_active":"2023-10-09T04:55:19.965889+0000","last_became_peered":"2023-10-09T04:55:19.965889+0000","last_unstale":"2023-10-09T04:55:26.005024+0000","last_undegraded":"2023-10-09T04:55:26.005024+0000","last_fullsized":"2023-10-09T04:55:26.005024+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:35.380069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"459'4863","reported_seq":4914,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:26.743683+0000","last_change":"2023-10-09T04:55:10.401440+0000","last_active":"2023-10-09T04:55:26.743683+0000","last_peered":"2023-10-09T04:55:26.743683+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:55:10.385934+0000","last_became_peered":"2023-10-09T04:55:10.385934+0000","last_unstale":"2023-10-09T04:55:26.743683+0000","last_undegraded":"2023-10-09T04:55:09.972268+0000","last_fullsized":"2023-10-09T04:55:09.913593+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4863,"log_dups_size":0,"ondisk_log_size":4863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:45:30.906975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318701568,"num_objects":4863,"num_object_clones":0,"num_object_copies":14589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4859,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4863,"num_write_kb":311232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":4741},{"shards":"2,5","objects":4},{"shards":"2,5,6","objects":118}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":85,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.982171+0000","last_change":"2023-10-09T04:54:45.674901+0000","last_active":"2023-10-09T04:55:25.982171+0000","last_peered":"2023-10-09T04:55:25.982171+0000","last_clean":"2023-10-09T04:55:25.982171+0000","last_became_active":"2023-10-09T04:54:45.624735+0000","last_became_peered":"2023-10-09T04:54:45.624735+0000","last_unstale":"2023-10-09T04:55:25.982171+0000","last_undegraded":"2023-10-09T04:55:25.982171+0000","last_fullsized":"2023-10-09T04:55:25.982171+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:52.045738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":33,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.635855+0000","last_change":"2023-10-09T04:55:19.978105+0000","last_active":"2023-10-09T04:55:28.635855+0000","last_peered":"2023-10-09T04:55:28.635855+0000","last_clean":"2023-10-09T04:55:28.635855+0000","last_became_active":"2023-10-09T04:55:19.977813+0000","last_became_peered":"2023-10-09T04:55:19.977813+0000","last_unstale":"2023-10-09T04:55:28.635855+0000","last_undegraded":"2023-10-09T04:55:28.635855+0000","last_fullsized":"2023-10-09T04:55:28.635855+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:09:37.633472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"459'4859","reported_seq":5017,"reported_epoch":465,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:21.692483+0000","last_change":"2023-10-09T04:55:21.692483+0000","last_active":"2023-10-09T04:55:21.692483+0000","last_peered":"2023-10-09T04:55:21.692483+0000","last_clean":"2023-10-09T04:53:31.951851+0000","last_became_active":"2023-10-09T04:55:21.682662+0000","last_became_peered":"2023-10-09T04:55:21.682662+0000","last_unstale":"2023-10-09T04:55:21.692483+0000","last_undegraded":"2023-10-09T04:55:21.259527+0000","last_fullsized":"2023-10-09T04:55:20.949398+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4859,"log_dups_size":0,"ondisk_log_size":4859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:01.900003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4859,"num_write_kb":310976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[2,3],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,3,5","objects":725},{"shards":"2,3,5,6","objects":37},{"shards":"2,3,6","objects":106},{"shards":"2,5","objects":3991}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":507,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.989288+0000","last_change":"2023-10-09T04:54:45.609058+0000","last_active":"2023-10-09T04:55:25.989288+0000","last_peered":"2023-10-09T04:55:25.989288+0000","last_clean":"2023-10-09T04:55:25.989288+0000","last_became_active":"2023-10-09T04:54:45.608739+0000","last_became_peered":"2023-10-09T04:54:45.608739+0000","last_unstale":"2023-10-09T04:55:25.989288+0000","last_undegraded":"2023-10-09T04:55:25.989288+0000","last_fullsized":"2023-10-09T04:55:25.989288+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:50:14.107586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.978851+0000","last_change":"2023-10-09T04:55:19.969173+0000","last_active":"2023-10-09T04:55:25.978851+0000","last_peered":"2023-10-09T04:55:25.978851+0000","last_clean":"2023-10-09T04:55:25.978851+0000","last_became_active":"2023-10-09T04:55:19.966424+0000","last_became_peered":"2023-10-09T04:55:19.966424+0000","last_unstale":"2023-10-09T04:55:25.978851+0000","last_undegraded":"2023-10-09T04:55:25.978851+0000","last_fullsized":"2023-10-09T04:55:25.978851+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:57.826783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"459'4716","reported_seq":4785,"reported_epoch":465,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:21.733475+0000","last_change":"2023-10-09T04:55:21.733475+0000","last_active":"2023-10-09T04:55:21.733475+0000","last_peered":"2023-10-09T04:55:21.733475+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:55:21.726305+0000","last_became_peered":"2023-10-09T04:55:21.726305+0000","last_unstale":"2023-10-09T04:55:21.733475+0000","last_undegraded":"2023-10-09T04:55:21.207573+0000","last_fullsized":"2023-10-09T04:55:20.955284+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4716,"log_dups_size":0,"ondisk_log_size":4716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:40.239676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309067776,"num_objects":4716,"num_object_clones":0,"num_object_copies":14148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9430,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4716,"num_write_kb":301824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":666},{"shards":"2,3,6","objects":115},{"shards":"2,4,5","objects":3933}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":31,"reported_epoch":462,"state":"active+degraded","last_fresh":"2023-10-09T04:55:16.260854+0000","last_change":"2023-10-09T04:55:16.260854+0000","last_active":"2023-10-09T04:55:16.260854+0000","last_peered":"2023-10-09T04:55:16.260854+0000","last_clean":"2023-10-09T04:55:04.877417+0000","last_became_active":"2023-10-09T04:55:16.260854+0000","last_became_peered":"2023-10-09T04:55:16.260854+0000","last_unstale":"2023-10-09T04:55:16.260854+0000","last_undegraded":"2023-10-09T04:55:09.913426+0000","last_fullsized":"2023-10-09T04:55:16.260854+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:29:49.739642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":265,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005993+0000","last_change":"2023-10-09T04:54:45.602986+0000","last_active":"2023-10-09T04:55:26.005993+0000","last_peered":"2023-10-09T04:55:26.005993+0000","last_clean":"2023-10-09T04:55:26.005993+0000","last_became_active":"2023-10-09T04:54:45.593439+0000","last_became_peered":"2023-10-09T04:54:45.593439+0000","last_unstale":"2023-10-09T04:55:26.005993+0000","last_undegraded":"2023-10-09T04:55:26.005993+0000","last_fullsized":"2023-10-09T04:55:26.005993+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:56:37.193549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"459'4887","reported_seq":4961,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:25.986904+0000","last_change":"2023-10-09T04:55:19.995386+0000","last_active":"2023-10-09T04:55:25.986904+0000","last_peered":"2023-10-09T04:55:25.986904+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:55:11.633213+0000","last_became_peered":"2023-10-09T04:55:11.633213+0000","last_unstale":"2023-10-09T04:55:25.986904+0000","last_undegraded":"2023-10-09T04:55:19.992161+0000","last_fullsized":"2023-10-09T04:55:10.936369+0000","mapping_epoch":458,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4887,"log_dups_size":0,"ondisk_log_size":4887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:42:48.919216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320274432,"num_objects":4887,"num_object_clones":0,"num_object_copies":14661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4867,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4887,"num_write_kb":312768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":4867},{"shards":"4,5","objects":20}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":33,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:28.635749+0000","last_change":"2023-10-09T04:55:19.977893+0000","last_active":"2023-10-09T04:55:28.635749+0000","last_peered":"2023-10-09T04:55:28.635749+0000","last_clean":"2023-10-09T04:55:28.635749+0000","last_became_active":"2023-10-09T04:55:19.977607+0000","last_became_peered":"2023-10-09T04:55:19.977607+0000","last_unstale":"2023-10-09T04:55:28.635749+0000","last_undegraded":"2023-10-09T04:55:28.635749+0000","last_fullsized":"2023-10-09T04:55:28.635749+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:56.758116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":252,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T04:55:20.955188+0000","last_change":"2023-10-09T04:55:19.968173+0000","last_active":"2023-10-09T04:55:20.955188+0000","last_peered":"2023-10-09T04:55:20.955188+0000","last_clean":"2023-10-09T04:55:20.955188+0000","last_became_active":"2023-10-09T04:55:19.967812+0000","last_became_peered":"2023-10-09T04:55:19.967812+0000","last_unstale":"2023-10-09T04:55:20.955188+0000","last_undegraded":"2023-10-09T04:55:20.955188+0000","last_fullsized":"2023-10-09T04:55:20.955188+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:21:12.839071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"459'4860","reported_seq":4914,"reported_epoch":465,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-09T04:55:20.958315+0000","last_change":"2023-10-09T04:54:47.489271+0000","last_active":"2023-10-09T04:55:20.958315+0000","last_peered":"2023-10-09T04:55:20.958315+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:54:47.478635+0000","last_became_peered":"2023-10-09T04:54:47.478635+0000","last_unstale":"2023-10-09T04:55:20.958315+0000","last_undegraded":"2023-10-09T04:54:46.818074+0000","last_fullsized":"2023-10-09T04:55:20.958315+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4860,"log_dups_size":0,"ondisk_log_size":4860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:38:48.018305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318504960,"num_objects":4860,"num_object_clones":0,"num_object_copies":14580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4035,"num_objects_misplaced":4749,"num_objects_unfound":0,"num_objects_dirty":4860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4860,"num_write_kb":311040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[2,7,3],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4,7","objects":714},{"shards":"2,3,7","objects":111},{"shards":"2,4,7","objects":4035}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":235,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.004923+0000","last_change":"2023-10-09T04:52:10.726288+0000","last_active":"2023-10-09T04:55:26.004923+0000","last_peered":"2023-10-09T04:55:26.004923+0000","last_clean":"2023-10-09T04:55:26.004923+0000","last_became_active":"2023-10-09T04:52:10.725933+0000","last_became_peered":"2023-10-09T04:52:10.725933+0000","last_unstale":"2023-10-09T04:55:26.004923+0000","last_undegraded":"2023-10-09T04:55:26.004923+0000","last_fullsized":"2023-10-09T04:55:26.004923+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:01:34.651488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":487,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.012455+0000","last_change":"2023-10-09T04:55:19.972575+0000","last_active":"2023-10-09T04:55:26.012455+0000","last_peered":"2023-10-09T04:55:26.012455+0000","last_clean":"2023-10-09T04:55:26.012455+0000","last_became_active":"2023-10-09T04:55:19.972109+0000","last_became_peered":"2023-10-09T04:55:19.972109+0000","last_unstale":"2023-10-09T04:55:26.012455+0000","last_undegraded":"2023-10-09T04:55:26.012455+0000","last_fullsized":"2023-10-09T04:55:26.012455+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:02:22.048092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"459'4861","reported_seq":6734,"reported_epoch":467,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:55:35.129786+0000","last_change":"2023-10-09T04:55:20.082816+0000","last_active":"2023-10-09T04:55:35.129786+0000","last_peered":"2023-10-09T04:55:35.129786+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T04:55:10.377609+0000","last_became_peered":"2023-10-09T04:55:10.377609+0000","last_unstale":"2023-10-09T04:55:35.129786+0000","last_undegraded":"2023-10-09T04:55:35.129786+0000","last_fullsized":"2023-10-09T04:55:09.922687+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:15:57.647533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318570496,"num_objects":4861,"num_object_clones":0,"num_object_copies":14583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3948,"num_objects_unfound":0,"num_objects_dirty":4861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4861,"num_write_kb":311104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":908,"num_bytes_recovered":59506688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,7],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"3,4,7","objects":821},{"shards":"4,5,7","objects":3127},{"shards":"4,7","objects":913}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.931622+0000","last_change":"2023-10-09T04:54:25.753723+0000","last_active":"2023-10-09T04:55:12.931622+0000","last_peered":"2023-10-09T04:55:12.931622+0000","last_clean":"2023-10-09T04:55:12.931622+0000","last_became_active":"2023-10-09T04:54:25.753401+0000","last_became_peered":"2023-10-09T04:54:25.753401+0000","last_unstale":"2023-10-09T04:55:12.931622+0000","last_undegraded":"2023-10-09T04:55:12.931622+0000","last_fullsized":"2023-10-09T04:55:12.931622+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:38:39.904800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980658+0000","last_change":"2023-10-09T04:55:19.966596+0000","last_active":"2023-10-09T04:55:25.980658+0000","last_peered":"2023-10-09T04:55:25.980658+0000","last_clean":"2023-10-09T04:55:25.980658+0000","last_became_active":"2023-10-09T04:55:19.966184+0000","last_became_peered":"2023-10-09T04:55:19.966184+0000","last_unstale":"2023-10-09T04:55:25.980658+0000","last_undegraded":"2023-10-09T04:55:25.980658+0000","last_fullsized":"2023-10-09T04:55:25.980658+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:18:27.439844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"459'4994","reported_seq":5067,"reported_epoch":465,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:21.707113+0000","last_change":"2023-10-09T04:55:21.707113+0000","last_active":"2023-10-09T04:55:21.707113+0000","last_peered":"2023-10-09T04:55:21.707113+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:55:21.697479+0000","last_became_peered":"2023-10-09T04:55:21.697479+0000","last_unstale":"2023-10-09T04:55:21.707113+0000","last_undegraded":"2023-10-09T04:55:21.226941+0000","last_fullsized":"2023-10-09T04:55:20.946231+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4994,"log_dups_size":0,"ondisk_log_size":4994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:13:58.652941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327286784,"num_objects":4994,"num_object_clones":0,"num_object_copies":14982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9981,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4994,"num_write_kb":319616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[2,0],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":7},{"shards":"2,3,5","objects":828},{"shards":"2,3,5,7","objects":4},{"shards":"2,5,7","objects":4155}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980624+0000","last_change":"2023-10-09T04:55:19.969401+0000","last_active":"2023-10-09T04:55:25.980624+0000","last_peered":"2023-10-09T04:55:25.980624+0000","last_clean":"2023-10-09T04:55:25.980624+0000","last_became_active":"2023-10-09T04:55:19.966626+0000","last_became_peered":"2023-10-09T04:55:19.966626+0000","last_unstale":"2023-10-09T04:55:25.980624+0000","last_undegraded":"2023-10-09T04:55:25.980624+0000","last_fullsized":"2023-10-09T04:55:25.980624+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:36.485960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":511,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.982026+0000","last_change":"2023-10-09T04:54:25.766092+0000","last_active":"2023-10-09T04:55:25.982026+0000","last_peered":"2023-10-09T04:55:25.982026+0000","last_clean":"2023-10-09T04:55:25.982026+0000","last_became_active":"2023-10-09T04:54:25.761061+0000","last_became_peered":"2023-10-09T04:54:25.761061+0000","last_unstale":"2023-10-09T04:55:25.982026+0000","last_undegraded":"2023-10-09T04:55:25.982026+0000","last_fullsized":"2023-10-09T04:55:25.982026+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:39.310415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.979468+0000","last_change":"2023-10-09T04:55:19.976859+0000","last_active":"2023-10-09T04:55:25.979468+0000","last_peered":"2023-10-09T04:55:25.979468+0000","last_clean":"2023-10-09T04:55:25.979468+0000","last_became_active":"2023-10-09T04:55:19.976509+0000","last_became_peered":"2023-10-09T04:55:19.976509+0000","last_unstale":"2023-10-09T04:55:25.979468+0000","last_undegraded":"2023-10-09T04:55:25.979468+0000","last_fullsized":"2023-10-09T04:55:25.979468+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:51.474909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"459'4792","reported_seq":8444,"reported_epoch":467,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T04:55:35.126077+0000","last_change":"2023-10-09T04:55:10.457091+0000","last_active":"2023-10-09T04:55:35.126077+0000","last_peered":"2023-10-09T04:55:35.126077+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T04:54:45.752443+0000","last_became_peered":"2023-10-09T04:54:45.752443+0000","last_unstale":"2023-10-09T04:55:35.126077+0000","last_undegraded":"2023-10-09T04:55:35.126077+0000","last_fullsized":"2023-10-09T04:54:45.586831+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4792,"log_dups_size":0,"ondisk_log_size":4792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:55.507174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314048512,"num_objects":4792,"num_object_clones":0,"num_object_copies":14376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2882,"num_objects_unfound":0,"num_objects_dirty":4792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4792,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1799,"num_bytes_recovered":117899264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,3,4","objects":730},{"shards":"2,4","objects":1910},{"shards":"2,4,7","objects":2152}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":501,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.990746+0000","last_change":"2023-10-09T04:55:20.005142+0000","last_active":"2023-10-09T04:55:25.990746+0000","last_peered":"2023-10-09T04:55:25.990746+0000","last_clean":"2023-10-09T04:55:25.990746+0000","last_became_active":"2023-10-09T04:55:20.004808+0000","last_became_peered":"2023-10-09T04:55:20.004808+0000","last_unstale":"2023-10-09T04:55:25.990746+0000","last_undegraded":"2023-10-09T04:55:25.990746+0000","last_fullsized":"2023-10-09T04:55:25.990746+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:08.384995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":275,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005321+0000","last_change":"2023-10-09T04:55:09.941824+0000","last_active":"2023-10-09T04:55:26.005321+0000","last_peered":"2023-10-09T04:55:26.005321+0000","last_clean":"2023-10-09T04:55:26.005321+0000","last_became_active":"2023-10-09T04:55:09.941249+0000","last_became_peered":"2023-10-09T04:55:09.941249+0000","last_unstale":"2023-10-09T04:55:26.005321+0000","last_undegraded":"2023-10-09T04:55:26.005321+0000","last_fullsized":"2023-10-09T04:55:26.005321+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:58.168170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980547+0000","last_change":"2023-10-09T04:55:19.970998+0000","last_active":"2023-10-09T04:55:25.980547+0000","last_peered":"2023-10-09T04:55:25.980547+0000","last_clean":"2023-10-09T04:55:25.980547+0000","last_became_active":"2023-10-09T04:55:19.970496+0000","last_became_peered":"2023-10-09T04:55:19.970496+0000","last_unstale":"2023-10-09T04:55:25.980547+0000","last_undegraded":"2023-10-09T04:55:25.980547+0000","last_fullsized":"2023-10-09T04:55:25.980547+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:38.579231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"459'4803","reported_seq":4871,"reported_epoch":465,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-09T04:55:20.958240+0000","last_change":"2023-10-09T04:55:10.402245+0000","last_active":"2023-10-09T04:55:20.958240+0000","last_peered":"2023-10-09T04:55:20.958240+0000","last_clean":"2023-10-09T04:54:44.521216+0000","last_became_active":"2023-10-09T04:54:47.456774+0000","last_became_peered":"2023-10-09T04:54:47.456774+0000","last_unstale":"2023-10-09T04:55:20.958240+0000","last_undegraded":"2023-10-09T04:55:20.958240+0000","last_fullsized":"2023-10-09T04:55:20.958240+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:34:33.703777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4671,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4803,"num_write_kb":307392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":118,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T04:55:20.951098+0000","last_change":"2023-10-09T04:55:10.146676+0000","last_active":"2023-10-09T04:55:20.951098+0000","last_peered":"2023-10-09T04:55:20.951098+0000","last_clean":"2023-10-09T04:55:20.951098+0000","last_became_active":"2023-10-09T04:55:10.146409+0000","last_became_peered":"2023-10-09T04:55:10.146409+0000","last_unstale":"2023-10-09T04:55:20.951098+0000","last_undegraded":"2023-10-09T04:55:20.951098+0000","last_fullsized":"2023-10-09T04:55:20.951098+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:19:13.896052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.978709+0000","last_change":"2023-10-09T04:55:19.965913+0000","last_active":"2023-10-09T04:55:25.978709+0000","last_peered":"2023-10-09T04:55:25.978709+0000","last_clean":"2023-10-09T04:55:25.978709+0000","last_became_active":"2023-10-09T04:55:19.964289+0000","last_became_peered":"2023-10-09T04:55:19.964289+0000","last_unstale":"2023-10-09T04:55:25.978709+0000","last_undegraded":"2023-10-09T04:55:25.978709+0000","last_fullsized":"2023-10-09T04:55:25.978709+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:25:52.119711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980736+0000","last_change":"2023-10-09T04:55:20.004927+0000","last_active":"2023-10-09T04:55:25.980736+0000","last_peered":"2023-10-09T04:55:25.980736+0000","last_clean":"2023-10-09T04:55:25.980736+0000","last_became_active":"2023-10-09T04:55:20.004617+0000","last_became_peered":"2023-10-09T04:55:20.004617+0000","last_unstale":"2023-10-09T04:55:25.980736+0000","last_undegraded":"2023-10-09T04:55:25.980736+0000","last_fullsized":"2023-10-09T04:55:25.980736+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:34.757781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"459'4779","reported_seq":6393,"reported_epoch":463,"state":"remapped+peering","last_fresh":"2023-10-09T04:55:18.971331+0000","last_change":"2023-10-09T04:55:18.972245+0000","last_active":"2023-10-09T04:55:18.971298+0000","last_peered":"2023-10-09T04:55:15.959193+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T04:54:27.195661+0000","last_became_peered":"2023-10-09T04:54:27.195661+0000","last_unstale":"2023-10-09T04:55:18.971331+0000","last_undegraded":"2023-10-09T04:55:18.971331+0000","last_fullsized":"2023-10-09T04:55:18.971331+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4779,"log_dups_size":0,"ondisk_log_size":4779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:30:35.057433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313196544,"num_objects":4779,"num_object_clones":0,"num_object_copies":14337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4779,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":786,"num_bytes_recovered":51511296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,3,7],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":468,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.988659+0000","last_change":"2023-10-09T04:52:10.727903+0000","last_active":"2023-10-09T04:55:25.988659+0000","last_peered":"2023-10-09T04:55:25.988659+0000","last_clean":"2023-10-09T04:55:25.988659+0000","last_became_active":"2023-10-09T04:52:10.727522+0000","last_became_peered":"2023-10-09T04:52:10.727522+0000","last_unstale":"2023-10-09T04:55:25.988659+0000","last_undegraded":"2023-10-09T04:55:25.988659+0000","last_fullsized":"2023-10-09T04:55:25.988659+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:22:45.141607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":938,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.986271+0000","last_change":"2023-10-09T04:55:12.934902+0000","last_active":"2023-10-09T04:55:25.986271+0000","last_peered":"2023-10-09T04:55:25.986271+0000","last_clean":"2023-10-09T04:55:25.986271+0000","last_became_active":"2023-10-09T04:55:12.934419+0000","last_became_peered":"2023-10-09T04:55:12.934419+0000","last_unstale":"2023-10-09T04:55:25.986271+0000","last_undegraded":"2023-10-09T04:55:25.986271+0000","last_fullsized":"2023-10-09T04:55:25.986271+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:44.774867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.931920+0000","last_change":"2023-10-09T04:54:45.604002+0000","last_active":"2023-10-09T04:55:12.931920+0000","last_peered":"2023-10-09T04:55:12.931920+0000","last_clean":"2023-10-09T04:55:12.931920+0000","last_became_active":"2023-10-09T04:54:45.603663+0000","last_became_peered":"2023-10-09T04:54:45.603663+0000","last_unstale":"2023-10-09T04:55:12.931920+0000","last_undegraded":"2023-10-09T04:55:12.931920+0000","last_fullsized":"2023-10-09T04:55:12.931920+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:00.742663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":24,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.987249+0000","last_change":"2023-10-09T04:54:45.603505+0000","last_active":"2023-10-09T04:55:25.987249+0000","last_peered":"2023-10-09T04:55:25.987249+0000","last_clean":"2023-10-09T04:55:25.987249+0000","last_became_active":"2023-10-09T04:54:45.602607+0000","last_became_peered":"2023-10-09T04:54:45.602607+0000","last_unstale":"2023-10-09T04:55:25.987249+0000","last_undegraded":"2023-10-09T04:55:25.987249+0000","last_fullsized":"2023-10-09T04:55:25.987249+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:59:19.361882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"459'4875","reported_seq":4928,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:26.008416+0000","last_change":"2023-10-09T04:55:10.370573+0000","last_active":"2023-10-09T04:55:26.008416+0000","last_peered":"2023-10-09T04:55:26.008416+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T04:55:10.363239+0000","last_became_peered":"2023-10-09T04:55:10.363239+0000","last_unstale":"2023-10-09T04:55:26.008416+0000","last_undegraded":"2023-10-09T04:55:09.965250+0000","last_fullsized":"2023-10-09T04:55:09.922399+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4875,"log_dups_size":0,"ondisk_log_size":4875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:25:42.401349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319488000,"num_objects":4875,"num_object_clones":0,"num_object_copies":14625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4875,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4","objects":2},{"shards":"2,4,5","objects":4780},{"shards":"2,4,6","objects":93}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.978592+0000","last_change":"2023-10-09T04:55:19.969877+0000","last_active":"2023-10-09T04:55:25.978592+0000","last_peered":"2023-10-09T04:55:25.978592+0000","last_clean":"2023-10-09T04:55:25.978592+0000","last_became_active":"2023-10-09T04:55:19.965583+0000","last_became_peered":"2023-10-09T04:55:19.965583+0000","last_unstale":"2023-10-09T04:55:25.978592+0000","last_undegraded":"2023-10-09T04:55:25.978592+0000","last_fullsized":"2023-10-09T04:55:25.978592+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:45:47.999230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.934192+0000","last_change":"2023-10-09T04:55:10.078107+0000","last_active":"2023-10-09T04:55:12.934192+0000","last_peered":"2023-10-09T04:55:12.934192+0000","last_clean":"2023-10-09T04:55:12.934192+0000","last_became_active":"2023-10-09T04:55:10.077508+0000","last_became_peered":"2023-10-09T04:55:10.077508+0000","last_unstale":"2023-10-09T04:55:12.934192+0000","last_undegraded":"2023-10-09T04:55:12.934192+0000","last_fullsized":"2023-10-09T04:55:12.934192+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:12:46.862869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":29,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.021452+0000","last_change":"2023-10-09T04:55:19.972272+0000","last_active":"2023-10-09T04:55:26.021452+0000","last_peered":"2023-10-09T04:55:26.021452+0000","last_clean":"2023-10-09T04:55:26.021452+0000","last_became_active":"2023-10-09T04:55:19.971888+0000","last_became_peered":"2023-10-09T04:55:19.971888+0000","last_unstale":"2023-10-09T04:55:26.021452+0000","last_undegraded":"2023-10-09T04:55:26.021452+0000","last_fullsized":"2023-10-09T04:55:26.021452+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:24:04.418917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"459'4889","reported_seq":6576,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:25.984366+0000","last_change":"2023-10-09T04:55:19.763559+0000","last_active":"2023-10-09T04:55:25.984366+0000","last_peered":"2023-10-09T04:55:25.984366+0000","last_clean":"2023-10-09T04:54:25.130996+0000","last_became_active":"2023-10-09T04:54:27.247980+0000","last_became_peered":"2023-10-09T04:54:27.247980+0000","last_unstale":"2023-10-09T04:55:25.984366+0000","last_undegraded":"2023-10-09T04:55:19.761575+0000","last_fullsized":"2023-10-09T04:54:26.738743+0000","mapping_epoch":449,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:33.079103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320405504,"num_objects":4889,"num_object_clones":0,"num_object_copies":14667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3234,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4889,"num_write_kb":312896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":820,"num_bytes_recovered":53739520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":3234},{"shards":"4,5","objects":1655}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":506,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005888+0000","last_change":"2023-10-09T04:55:09.966863+0000","last_active":"2023-10-09T04:55:26.005888+0000","last_peered":"2023-10-09T04:55:26.005888+0000","last_clean":"2023-10-09T04:55:26.005888+0000","last_became_active":"2023-10-09T04:55:09.966618+0000","last_became_peered":"2023-10-09T04:55:09.966618+0000","last_unstale":"2023-10-09T04:55:26.005888+0000","last_undegraded":"2023-10-09T04:55:26.005888+0000","last_fullsized":"2023-10-09T04:55:26.005888+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:40.381224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980471+0000","last_change":"2023-10-09T04:55:19.977673+0000","last_active":"2023-10-09T04:55:25.980471+0000","last_peered":"2023-10-09T04:55:25.980471+0000","last_clean":"2023-10-09T04:55:25.980471+0000","last_became_active":"2023-10-09T04:55:19.977334+0000","last_became_peered":"2023-10-09T04:55:19.977334+0000","last_unstale":"2023-10-09T04:55:25.980471+0000","last_undegraded":"2023-10-09T04:55:25.980471+0000","last_fullsized":"2023-10-09T04:55:25.980471+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:43.681570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":351,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.988513+0000","last_change":"2023-10-09T04:55:09.929827+0000","last_active":"2023-10-09T04:55:25.988513+0000","last_peered":"2023-10-09T04:55:25.988513+0000","last_clean":"2023-10-09T04:55:25.988513+0000","last_became_active":"2023-10-09T04:55:09.929542+0000","last_became_peered":"2023-10-09T04:55:09.929542+0000","last_unstale":"2023-10-09T04:55:25.988513+0000","last_undegraded":"2023-10-09T04:55:25.988513+0000","last_fullsized":"2023-10-09T04:55:25.988513+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:40.210231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"459'4771","reported_seq":10203,"reported_epoch":466,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:25.988002+0000","last_change":"2023-10-09T04:55:21.633244+0000","last_active":"2023-10-09T04:55:25.988002+0000","last_peered":"2023-10-09T04:55:25.988002+0000","last_clean":"2023-10-09T04:54:25.113773+0000","last_became_active":"2023-10-09T04:55:21.623063+0000","last_became_peered":"2023-10-09T04:55:21.623063+0000","last_unstale":"2023-10-09T04:55:25.988002+0000","last_undegraded":"2023-10-09T04:55:21.089953+0000","last_fullsized":"2023-10-09T04:55:20.961092+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4771,"log_dups_size":0,"ondisk_log_size":4771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:38:17.662506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312672256,"num_objects":4771,"num_object_clones":0,"num_object_copies":14313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5990,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4771,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2687,"num_bytes_recovered":176095232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","4","7"],"object_location_counts":[{"shards":"3,4,5","objects":865},{"shards":"3,4,5,7","objects":2687},{"shards":"4,5,7","objects":1219}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":41,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.932404+0000","last_change":"2023-10-09T04:55:10.386387+0000","last_active":"2023-10-09T04:55:12.932404+0000","last_peered":"2023-10-09T04:55:12.932404+0000","last_clean":"2023-10-09T04:55:12.932404+0000","last_became_active":"2023-10-09T04:55:09.941531+0000","last_became_peered":"2023-10-09T04:55:09.941531+0000","last_unstale":"2023-10-09T04:55:12.932404+0000","last_undegraded":"2023-10-09T04:55:12.932404+0000","last_fullsized":"2023-10-09T04:55:12.932404+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:08:39.623361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980967+0000","last_change":"2023-10-09T04:55:19.971236+0000","last_active":"2023-10-09T04:55:25.980967+0000","last_peered":"2023-10-09T04:55:25.980967+0000","last_clean":"2023-10-09T04:55:25.980967+0000","last_became_active":"2023-10-09T04:55:19.970925+0000","last_became_peered":"2023-10-09T04:55:19.970925+0000","last_unstale":"2023-10-09T04:55:25.980967+0000","last_undegraded":"2023-10-09T04:55:25.980967+0000","last_fullsized":"2023-10-09T04:55:25.980967+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:29:04.830639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":42,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.931647+0000","last_change":"2023-10-09T04:55:09.936167+0000","last_active":"2023-10-09T04:55:12.931647+0000","last_peered":"2023-10-09T04:55:12.931647+0000","last_clean":"2023-10-09T04:55:12.931647+0000","last_became_active":"2023-10-09T04:55:09.935352+0000","last_became_peered":"2023-10-09T04:55:09.935352+0000","last_unstale":"2023-10-09T04:55:12.931647+0000","last_undegraded":"2023-10-09T04:55:12.931647+0000","last_fullsized":"2023-10-09T04:55:12.931647+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:49.327007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"459'4980","reported_seq":5043,"reported_epoch":465,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:55:21.660342+0000","last_change":"2023-10-09T04:55:21.660342+0000","last_active":"2023-10-09T04:55:21.660342+0000","last_peered":"2023-10-09T04:55:21.660342+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T04:55:21.649068+0000","last_became_peered":"2023-10-09T04:55:21.649068+0000","last_unstale":"2023-10-09T04:55:21.660342+0000","last_undegraded":"2023-10-09T04:55:21.247283+0000","last_fullsized":"2023-10-09T04:55:20.948607+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4980,"log_dups_size":0,"ondisk_log_size":4980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:55:07.919930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326369280,"num_objects":4980,"num_object_clones":0,"num_object_copies":14940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4980,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":4854},{"shards":"2,6,7","objects":125}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":24,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.987525+0000","last_change":"2023-10-09T04:54:45.597574+0000","last_active":"2023-10-09T04:55:25.987525+0000","last_peered":"2023-10-09T04:55:25.987525+0000","last_clean":"2023-10-09T04:55:25.987525+0000","last_became_active":"2023-10-09T04:54:45.597207+0000","last_became_peered":"2023-10-09T04:54:45.597207+0000","last_unstale":"2023-10-09T04:55:25.987525+0000","last_undegraded":"2023-10-09T04:55:25.987525+0000","last_fullsized":"2023-10-09T04:55:25.987525+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:56:15.202450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":19,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.980996+0000","last_change":"2023-10-09T04:55:20.019585+0000","last_active":"2023-10-09T04:55:25.980996+0000","last_peered":"2023-10-09T04:55:25.980996+0000","last_clean":"2023-10-09T04:55:25.980996+0000","last_became_active":"2023-10-09T04:55:19.966133+0000","last_became_peered":"2023-10-09T04:55:19.966133+0000","last_unstale":"2023-10-09T04:55:25.980996+0000","last_undegraded":"2023-10-09T04:55:25.980996+0000","last_fullsized":"2023-10-09T04:55:25.980996+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:45:00.741529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.004871+0000","last_change":"2023-10-09T04:54:45.621644+0000","last_active":"2023-10-09T04:55:26.004871+0000","last_peered":"2023-10-09T04:55:26.004871+0000","last_clean":"2023-10-09T04:55:26.004871+0000","last_became_active":"2023-10-09T04:54:45.621388+0000","last_became_peered":"2023-10-09T04:54:45.621388+0000","last_unstale":"2023-10-09T04:55:26.004871+0000","last_undegraded":"2023-10-09T04:55:26.004871+0000","last_fullsized":"2023-10-09T04:55:26.004871+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:56:59.299440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":357,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005492+0000","last_change":"2023-10-09T04:55:09.941998+0000","last_active":"2023-10-09T04:55:26.005492+0000","last_peered":"2023-10-09T04:55:26.005492+0000","last_clean":"2023-10-09T04:55:26.005492+0000","last_became_active":"2023-10-09T04:55:09.941632+0000","last_became_peered":"2023-10-09T04:55:09.941632+0000","last_unstale":"2023-10-09T04:55:26.005492+0000","last_undegraded":"2023-10-09T04:55:26.005492+0000","last_fullsized":"2023-10-09T04:55:26.005492+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:41.723572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":66,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-09T04:55:20.960535+0000","last_change":"2023-10-09T04:55:19.967907+0000","last_active":"2023-10-09T04:55:20.960535+0000","last_peered":"2023-10-09T04:55:20.960535+0000","last_clean":"2023-10-09T04:55:20.960535+0000","last_became_active":"2023-10-09T04:55:19.967695+0000","last_became_peered":"2023-10-09T04:55:19.967695+0000","last_unstale":"2023-10-09T04:55:20.960535+0000","last_undegraded":"2023-10-09T04:55:20.960535+0000","last_fullsized":"2023-10-09T04:55:20.960535+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:09:26.626955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":499,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.989357+0000","last_change":"2023-10-09T04:55:20.028305+0000","last_active":"2023-10-09T04:55:25.989357+0000","last_peered":"2023-10-09T04:55:25.989357+0000","last_clean":"2023-10-09T04:55:25.989357+0000","last_became_active":"2023-10-09T04:55:20.005429+0000","last_became_peered":"2023-10-09T04:55:20.005429+0000","last_unstale":"2023-10-09T04:55:25.989357+0000","last_undegraded":"2023-10-09T04:55:25.989357+0000","last_fullsized":"2023-10-09T04:55:25.989357+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:31:38.602377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":275,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.005409+0000","last_change":"2023-10-09T04:55:19.966456+0000","last_active":"2023-10-09T04:55:26.005409+0000","last_peered":"2023-10-09T04:55:26.005409+0000","last_clean":"2023-10-09T04:55:26.005409+0000","last_became_active":"2023-10-09T04:55:19.966054+0000","last_became_peered":"2023-10-09T04:55:19.966054+0000","last_unstale":"2023-10-09T04:55:26.005409+0000","last_undegraded":"2023-10-09T04:55:26.005409+0000","last_fullsized":"2023-10-09T04:55:26.005409+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:54.937277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":468,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.981947+0000","last_change":"2023-10-09T04:54:45.684519+0000","last_active":"2023-10-09T04:55:25.981947+0000","last_peered":"2023-10-09T04:55:25.981947+0000","last_clean":"2023-10-09T04:55:25.981947+0000","last_became_active":"2023-10-09T04:54:45.628235+0000","last_became_peered":"2023-10-09T04:54:45.628235+0000","last_unstale":"2023-10-09T04:55:25.981947+0000","last_undegraded":"2023-10-09T04:55:25.981947+0000","last_fullsized":"2023-10-09T04:55:25.981947+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:32:23.058329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.932005+0000","last_change":"2023-10-09T04:55:09.936248+0000","last_active":"2023-10-09T04:55:12.932005+0000","last_peered":"2023-10-09T04:55:12.932005+0000","last_clean":"2023-10-09T04:55:12.932005+0000","last_became_active":"2023-10-09T04:55:09.935685+0000","last_became_peered":"2023-10-09T04:55:09.935685+0000","last_unstale":"2023-10-09T04:55:12.932005+0000","last_undegraded":"2023-10-09T04:55:12.932005+0000","last_fullsized":"2023-10-09T04:55:12.932005+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:00.974750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.933653+0000","last_change":"2023-10-09T04:54:45.598314+0000","last_active":"2023-10-09T04:55:12.933653+0000","last_peered":"2023-10-09T04:55:12.933653+0000","last_clean":"2023-10-09T04:55:12.933653+0000","last_became_active":"2023-10-09T04:54:45.597965+0000","last_became_peered":"2023-10-09T04:54:45.597965+0000","last_unstale":"2023-10-09T04:55:12.933653+0000","last_undegraded":"2023-10-09T04:55:12.933653+0000","last_fullsized":"2023-10-09T04:55:12.933653+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:28:28.920958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":33,"reported_epoch":461,"state":"active+clean","last_fresh":"2023-10-09T04:55:12.934235+0000","last_change":"2023-10-09T04:55:10.146164+0000","last_active":"2023-10-09T04:55:12.934235+0000","last_peered":"2023-10-09T04:55:12.934235+0000","last_clean":"2023-10-09T04:55:12.934235+0000","last_became_active":"2023-10-09T04:55:10.145673+0000","last_became_peered":"2023-10-09T04:55:10.145673+0000","last_unstale":"2023-10-09T04:55:12.934235+0000","last_undegraded":"2023-10-09T04:55:12.934235+0000","last_fullsized":"2023-10-09T04:55:12.934235+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:23:46.564734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":453,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:25.989151+0000","last_change":"2023-10-09T04:55:09.946170+0000","last_active":"2023-10-09T04:55:25.989151+0000","last_peered":"2023-10-09T04:55:25.989151+0000","last_clean":"2023-10-09T04:55:25.989151+0000","last_became_active":"2023-10-09T04:55:09.932957+0000","last_became_peered":"2023-10-09T04:55:09.932957+0000","last_unstale":"2023-10-09T04:55:25.989151+0000","last_undegraded":"2023-10-09T04:55:25.989151+0000","last_fullsized":"2023-10-09T04:55:25.989151+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:44:35.281964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":295,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-10-09T04:55:26.004861+0000","last_change":"2023-10-09T04:55:19.971261+0000","last_active":"2023-10-09T04:55:26.004861+0000","last_peered":"2023-10-09T04:55:26.004861+0000","last_clean":"2023-10-09T04:55:26.004861+0000","last_became_active":"2023-10-09T04:55:19.970986+0000","last_became_peered":"2023-10-09T04:55:19.970986+0000","last_unstale":"2023-10-09T04:55:26.004861+0000","last_undegraded":"2023-10-09T04:55:26.004861+0000","last_fullsized":"2023-10-09T04:55:26.004861+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:48:00.555950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":5085134848,"num_objects":77593,"num_object_clones":0,"num_object_copies":232779,"num_objects_missing_on_primary":8759,"num_objects_missing":8759,"num_objects_degraded":88501,"num_objects_misplaced":16250,"num_objects_unfound":0,"num_objects_dirty":77593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":77593,"num_write_kb":4965952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7070,"num_bytes_recovered":463339520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15567683584,"data_stored":15567683584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77593,"ondisk_log_size":77593,"up":48,"acting":34,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":28904,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234584,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4098284,"kb_used_data":2580836,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":89646868,"statfs":{"total":95995035648,"available":91798392832,"internally_reserved":0,"allocated":2642776064,"data_stored":2641816845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.0209999999999999,"15min":1.335},"min":{"1min":0.26500000000000001,"5min":0.247,"15min":0.21099999999999999},"max":{"1min":3.4420000000000002,"5min":6.968,"15min":14.970000000000001},"last":0.437},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.5669999999999999,"15min":2.0129999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.249},"max":{"1min":6.657,"5min":21.216000000000001,"15min":21.216000000000001},"last":0.33500000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:54:57 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":3.9929999999999999,"15min":3.5169999999999999},"min":{"1min":0.318,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":5.0990000000000002,"5min":122.56,"15min":128.601},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.89200000000000002,"5min":3.9049999999999998,"15min":3.4769999999999999},"min":{"1min":0.33500000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.879,"5min":122.666,"15min":128.381},"last":0.61399999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:55:30 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":6.0529999999999999,"15min":6.1059999999999999},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.27100000000000002},"max":{"1min":8.3030000000000008,"5min":207.96600000000001,"15min":207.96600000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.333,"5min":6.3579999999999997,"15min":6.4009999999999998},"min":{"1min":0.40000000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3179999999999996,"5min":207.97900000000001,"15min":207.97900000000001},"last":0.52700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":1.5369999999999999,"15min":1.9470000000000001},"min":{"1min":0.42099999999999999,"5min":0.39500000000000002,"15min":0.29499999999999998},"max":{"1min":11.756,"5min":11.756,"15min":20.469999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.633,"5min":1.829,"15min":2.4670000000000001},"min":{"1min":0.41299999999999998,"5min":0.36599999999999999,"15min":0.29799999999999999},"max":{"1min":6.6760000000000002,"5min":21.295999999999999,"15min":21.295999999999999},"last":0.55600000000000005}]},{"osd":4,"last update":"Mon Oct 9 04:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":2.0139999999999998,"15min":1.897},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.22600000000000001},"max":{"1min":0.99299999999999999,"5min":21.355,"15min":21.355},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.85199999999999998,"5min":2.2599999999999998,"15min":1.5860000000000001},"min":{"1min":0.312,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":4.5860000000000003,"5min":21.489999999999998,"15min":21.489999999999998},"last":4.4139999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":3.0369999999999999,"15min":1.927},"min":{"1min":0.35299999999999998,"5min":0.28000000000000003,"15min":0.26900000000000002},"max":{"1min":3.6789999999999998,"5min":131.84899999999999,"15min":131.84899999999999},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":3.6899999999999999,"15min":2.5099999999999998},"min":{"1min":0.317,"5min":0.311,"15min":0.26000000000000001},"max":{"1min":1.7290000000000001,"5min":135.74199999999999,"15min":135.74199999999999},"last":0.57899999999999996}]},{"osd":6,"last update":"Mon Oct 9 04:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":1.5389999999999999,"15min":1.3},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32200000000000001},"max":{"1min":3.7200000000000002,"5min":9.0410000000000004,"15min":11.388},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":1.6579999999999999,"15min":1.7549999999999999},"min":{"1min":0.38800000000000001,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":0.88300000000000001,"5min":21.321999999999999,"15min":21.321999999999999},"last":0.60199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496217,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1087324,"kb_used_data":185044,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":92657828,"statfs":{"total":95995035648,"available":94881615872,"internally_reserved":0,"allocated":189485056,"data_stored":188541752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:49 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.92600000000000005,"15min":1.4359999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.20499999999999999},"max":{"1min":2.6150000000000002,"5min":7.6619999999999999,"15min":75.308000000000007},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.4259999999999999,"15min":1.8200000000000001},"min":{"1min":0.29799999999999999,"5min":0.29599999999999999,"15min":0.26600000000000001},"max":{"1min":7.3129999999999997,"5min":16.574000000000002,"15min":75.432000000000002},"last":0.59999999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":9.6739999999999995,"15min":6.3810000000000002},"min":{"1min":0.309,"5min":0.27700000000000002,"15min":0.24399999999999999},"max":{"1min":6.7599999999999998,"5min":260.738,"15min":260.738},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.90300000000000002,"5min":9.3170000000000002,"15min":6.1639999999999997},"min":{"1min":0.34000000000000002,"5min":0.28999999999999998,"15min":0.26300000000000001},"max":{"1min":2.7330000000000001,"5min":261.26100000000002,"15min":261.26100000000002},"last":0.65200000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:55:25 2023","interfaces":[{"interface":"back","average":{"1min":8.0370000000000008,"5min":3.4860000000000002,"15min":6.1760000000000002},"min":{"1min":0.32500000000000001,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":133.535,"5min":133.535,"15min":209.44399999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":7.6459999999999999,"5min":3.2589999999999999,"15min":5.9790000000000001},"min":{"1min":0.40300000000000002,"5min":0.31,"15min":0.31},"max":{"1min":133.51900000000001,"5min":133.51900000000001,"15min":209.428},"last":0.621}]},{"osd":3,"last update":"Mon Oct 9 04:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.226,"15min":3.8650000000000002},"min":{"1min":0.38300000000000001,"5min":0.34899999999999998,"15min":0.28100000000000003},"max":{"1min":7.0330000000000004,"5min":7.9710000000000001,"15min":157.93100000000001},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.972,"5min":1.546,"15min":3.6099999999999999},"min":{"1min":0.439,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":11.928000000000001,"5min":16.501999999999999,"15min":157.95099999999999},"last":0.81899999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":2.411,"15min":2.1139999999999999},"min":{"1min":0.32400000000000001,"5min":0.311,"15min":0.27200000000000002},"max":{"1min":4.5099999999999998,"5min":70.795000000000002,"15min":112.691},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.3109999999999999,"5min":2.4340000000000002,"15min":2.0550000000000002},"min":{"1min":0.40100000000000002,"5min":0.30599999999999999,"15min":0.26500000000000001},"max":{"1min":6.992,"5min":70.491,"15min":109.024},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Oct 9 04:54:49 2023","interfaces":[{"interface":"back","average":{"1min":1.714,"5min":2.3029999999999999,"15min":2.1400000000000001},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.5839999999999996,"5min":66.995999999999995,"15min":83.168999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.298,"5min":2.5249999999999999,"15min":2.2109999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29599999999999999},"max":{"1min":13.241,"5min":67.189999999999998,"15min":83.126000000000005},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":1.9219999999999999,"15min":1.6040000000000001},"min":{"1min":0.39100000000000001,"5min":0.34799999999999998,"15min":0.29699999999999999},"max":{"1min":2.4489999999999998,"5min":36.204999999999998,"15min":36.204999999999998},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.5580000000000001,"15min":1.5669999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.32900000000000001},"max":{"1min":3.452,"5min":10.661,"15min":10.661},"last":0.748}]}]},{"osd":1,"up_from":118,"seq":506806141308,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1086964,"kb_used_data":5740,"kb_used_omap":4,"kb_used_meta":1081211,"kb_avail":92658188,"statfs":{"total":95995035648,"available":94881984512,"internally_reserved":0,"allocated":5877760,"data_stored":3679078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1107160974},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.72099999999999997,"15min":1.736},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.14799999999999999},"max":{"1min":4.3520000000000003,"5min":4.3789999999999996,"15min":75.620000000000005},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.77800000000000002,"15min":1.738},"min":{"1min":0.25600000000000001,"5min":0.214,"15min":0.193},"max":{"1min":6.117,"5min":6.117,"15min":27.321000000000002},"last":0.255}]},{"osd":2,"last update":"Mon Oct 9 04:55:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9399999999999999,"5min":4.4390000000000001,"15min":5.5019999999999998},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.20899999999999999},"max":{"1min":41.628999999999998,"5min":263.089,"15min":263.60500000000002},"last":0.55900000000000005},{"interface":"front","average":{"1min":3.2400000000000002,"5min":4.3819999999999997,"15min":5.7610000000000001},"min":{"1min":0.35699999999999998,"5min":0.222,"15min":0.222},"max":{"1min":39.276000000000003,"5min":254.50899999999999,"15min":263.65199999999999},"last":0.308}]},{"osd":3,"last update":"Mon Oct 9 04:54:52 2023","interfaces":[{"interface":"back","average":{"1min":5.4400000000000004,"5min":1.712,"15min":4.4729999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.23000000000000001},"max":{"1min":77.819999999999993,"5min":77.819999999999993,"15min":379.80900000000003},"last":0.42799999999999999},{"interface":"front","average":{"1min":5.0330000000000004,"5min":1.6080000000000001,"15min":4.9640000000000004},"min":{"1min":0.315,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":77.796000000000006,"5min":77.796000000000006,"15min":379.95800000000003},"last":0.34200000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":4.2229999999999999,"15min":2.9249999999999998},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":7.8040000000000003,"5min":163.94200000000001,"15min":163.94200000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.498,"5min":4.3029999999999999,"15min":2.9609999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":9.6899999999999995,"5min":164.00700000000001,"15min":164.00700000000001},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:55:12 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.54,"15min":1.903},"min":{"1min":0.42399999999999999,"5min":0.35199999999999998,"15min":0.28299999999999997},"max":{"1min":7.4980000000000002,"5min":7.4980000000000002,"15min":11.499000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.677,"5min":1.248,"15min":2.016},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.30499999999999999},"max":{"1min":7.633,"5min":7.633,"15min":20.021000000000001},"last":0.54700000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.3120000000000001,"15min":2.1800000000000002},"min":{"1min":0.46000000000000002,"5min":0.39500000000000002,"15min":0.32000000000000001},"max":{"1min":6.2069999999999999,"5min":6.2069999999999999,"15min":132.36099999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.2270000000000001,"15min":2.4239999999999999},"min":{"1min":0.435,"5min":0.39700000000000002,"15min":0.32400000000000001},"max":{"1min":7.734,"5min":7.734,"15min":132.17699999999999},"last":0.46100000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":2.46,"15min":3.0129999999999999},"min":{"1min":0.45600000000000002,"5min":0.38800000000000001,"15min":0.35899999999999999},"max":{"1min":7.7000000000000002,"5min":59.972000000000001,"15min":132.136},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.7310000000000001,"5min":2.5379999999999998,"15min":2.8980000000000001},"min":{"1min":0.45200000000000001,"5min":0.378,"15min":0.377},"max":{"1min":10.167999999999999,"5min":60.011000000000003,"15min":132.12},"last":0.52600000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024365,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012840,"kb_used_data":67936,"kb_used_omap":5,"kb_used_meta":944890,"kb_avail":92732312,"statfs":{"total":95995035648,"available":94957887488,"internally_reserved":0,"allocated":69566464,"data_stored":67790089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":967567994},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 04:54:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":8.5839999999999996,"15min":5.0490000000000004},"min":{"1min":0.223,"5min":0.19600000000000001,"15min":0.18099999999999999},"max":{"1min":1.556,"5min":247.68899999999999,"15min":247.68899999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.0569999999999999,"5min":8.766,"15min":5.4820000000000002},"min":{"1min":0.28899999999999998,"5min":0.24099999999999999,"15min":0.20999999999999999},"max":{"1min":6.9340000000000002,"5min":246.38,"15min":246.38},"last":0.50800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:55:14 2023","interfaces":[{"interface":"back","average":{"1min":4.2510000000000003,"5min":3.992,"15min":3.7930000000000001},"min":{"1min":0.27800000000000002,"5min":0.25,"15min":0.215},"max":{"1min":53.719999999999999,"5min":151.06399999999999,"15min":151.06399999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":4.3499999999999996,"5min":4.1230000000000002,"15min":3.8730000000000002},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":53.363999999999997,"5min":151.22399999999999,"15min":151.22399999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Mon Oct 9 04:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.804,"5min":0.83399999999999996,"15min":4.2190000000000003},"min":{"1min":0.30199999999999999,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":16.042999999999999,"5min":16.042999999999999,"15min":295.91699999999997},"last":0.372},{"interface":"front","average":{"1min":3.9329999999999998,"5min":1.212,"15min":3.7349999999999999},"min":{"1min":0.32500000000000001,"5min":0.27300000000000002,"15min":0.23000000000000001},"max":{"1min":55.883000000000003,"5min":55.883000000000003,"15min":161.40199999999999},"last":0.53700000000000003}]},{"osd":4,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.1899999999999999,"15min":2.2909999999999999},"min":{"1min":0.39200000000000002,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":3.3090000000000002,"5min":8.1609999999999996,"15min":212.523},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.2669999999999999,"15min":2.4399999999999999},"min":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.26700000000000002},"max":{"1min":3.6230000000000002,"5min":8.1029999999999998,"15min":212.53800000000001},"last":0.47699999999999998}]},{"osd":5,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.1990000000000001,"15min":2.0499999999999998},"min":{"1min":0.39000000000000001,"5min":0.34899999999999998,"15min":0.29199999999999998},"max":{"1min":5.8940000000000001,"5min":5.8940000000000001,"15min":136.322},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.748,"5min":0.97099999999999997,"15min":1.9079999999999999},"min":{"1min":0.40200000000000002,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":1.8180000000000001,"5min":3.948,"15min":136.15799999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.004,"15min":1.984},"min":{"1min":0.42999999999999999,"5min":0.34799999999999998,"15min":0.307},"max":{"1min":3.6899999999999999,"5min":4.4480000000000004,"15min":78.438999999999993},"last":0.441},{"interface":"front","average":{"1min":1.028,"5min":0.96799999999999997,"15min":1.831},"min":{"1min":0.42099999999999999,"5min":0.379,"15min":0.318},"max":{"1min":5.7409999999999997,"5min":5.7409999999999997,"15min":78.617999999999995},"last":0.59999999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:55:08 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.3420000000000001,"15min":2.1299999999999999},"min":{"1min":0.39700000000000002,"5min":0.39000000000000001,"15min":0.33300000000000002},"max":{"1min":3.476,"5min":9.8480000000000008,"15min":77.682000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.3939999999999999,"15min":2.056},"min":{"1min":0.49199999999999999,"5min":0.38200000000000001,"15min":0.35999999999999999},"max":{"1min":7.0229999999999997,"5min":9.8620000000000001,"15min":77.606999999999999},"last":0.54500000000000004}]}]},{"osd":2,"up_from":221,"seq":949187772640,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4686764,"kb_used_data":3937508,"kb_used_omap":7,"kb_used_meta":749240,"kb_avail":89058388,"statfs":{"total":95995035648,"available":91195789312,"internally_reserved":0,"allocated":4032008192,"data_stored":4030223066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":767221841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.50600000000000001,"15min":1.698},"min":{"1min":0.214,"5min":0.192,"15min":0.183},"max":{"1min":3.3769999999999998,"5min":3.3769999999999998,"15min":25.899999999999999},"last":0.22800000000000001},{"interface":"front","average":{"1min":1.1060000000000001,"5min":0.56599999999999995,"15min":1.387},"min":{"1min":0.27100000000000002,"5min":0.22600000000000001,"15min":0.19500000000000001},"max":{"1min":7.9820000000000002,"5min":7.9820000000000002,"15min":16.550000000000001},"last":0.38400000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.7829999999999999,"15min":4.2859999999999996},"min":{"1min":0.28100000000000003,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.359,"5min":98.968000000000004,"15min":170.172},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.68700000000000006,"5min":1.7989999999999999,"15min":4.4969999999999999},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.219},"max":{"1min":5.2169999999999996,"5min":101.491,"15min":170.511},"last":0.52200000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":1.1040000000000001,"15min":5.3499999999999996},"min":{"1min":0.27800000000000002,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":45.664000000000001,"5min":45.664000000000001,"15min":335.928},"last":0.34200000000000003},{"interface":"front","average":{"1min":3.419,"5min":1.1619999999999999,"15min":5.5410000000000004},"min":{"1min":0.28199999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":45.686999999999998,"5min":45.686999999999998,"15min":335.68599999999998},"last":0.41099999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.1040000000000001,"15min":2.6859999999999999},"min":{"1min":0.31,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.2599999999999998,"5min":6.4699999999999998,"15min":211.642},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.157,"5min":1.1799999999999999,"15min":2.673},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":4.7199999999999998,"5min":11.984,"15min":211.614},"last":0.51100000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":5.4379999999999997,"15min":4.1399999999999997},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.32600000000000001},"max":{"1min":4.4450000000000003,"5min":272.04700000000003,"15min":272.04700000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.411,"5min":5.3200000000000003,"15min":4.282},"min":{"1min":0.40200000000000002,"5min":0.33500000000000002,"15min":0.32200000000000001},"max":{"1min":10.029999999999999,"5min":271.89699999999999,"15min":271.89699999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.161,"15min":1.931},"min":{"1min":0.41799999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.4169999999999998,"5min":6.8730000000000002,"15min":34.753},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.0509999999999999,"15min":1.7290000000000001},"min":{"1min":0.41199999999999998,"5min":0.378,"15min":0.34699999999999998},"max":{"1min":4.4740000000000002,"5min":7.6120000000000001,"15min":34.796999999999997},"last":0.60599999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.387,"15min":2.2229999999999999},"min":{"1min":0.47799999999999998,"5min":0.42699999999999999,"15min":0.34300000000000003},"max":{"1min":3.4929999999999999,"5min":6.9569999999999999,"15min":28.126999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.371,"15min":2.0270000000000001},"min":{"1min":0.40799999999999997,"5min":0.376,"15min":0.373},"max":{"1min":4.4080000000000004,"5min":6.9699999999999998,"15min":28.106000000000002},"last":0.53700000000000003}]}]},{"osd":3,"up_from":142,"seq":609885356358,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1545728,"kb_used_data":809336,"kb_used_omap":2,"kb_used_meta":736381,"kb_avail":92199424,"statfs":{"total":95995035648,"available":94412210176,"internally_reserved":0,"allocated":828760064,"data_stored":827085006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":754054197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:54:47 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.46100000000000002,"15min":1.5469999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.19},"max":{"1min":5.2850000000000001,"5min":5.2850000000000001,"15min":75.132999999999996},"last":0.254},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.52300000000000002,"15min":2.7919999999999998},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.22},"max":{"1min":5.4089999999999998,"5min":5.4089999999999998,"15min":254.38499999999999},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":4.1210000000000004,"15min":3.774},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.21299999999999999},"max":{"1min":6.9710000000000001,"5min":125.233,"15min":125.233},"last":0.312},{"interface":"front","average":{"1min":0.92000000000000004,"5min":3.996,"15min":3.4969999999999999},"min":{"1min":0.33100000000000002,"5min":0.24299999999999999,"15min":0.21299999999999999},"max":{"1min":5.3129999999999997,"5min":126.102,"15min":126.102},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Oct 9 04:54:19 2023","interfaces":[{"interface":"back","average":{"1min":9.3539999999999992,"5min":2.2570000000000001,"15min":4.1130000000000004},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":203.36000000000001,"5min":203.36000000000001,"15min":203.36000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":8.9420000000000002,"5min":2.137,"15min":4.9660000000000002},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":194.125,"5min":194.125,"15min":255.05199999999999},"last":0.28599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:54:47 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.365,"15min":2.8450000000000002},"min":{"1min":0.35899999999999999,"5min":0.312,"15min":0.312},"max":{"1min":6.8979999999999997,"5min":12.493,"15min":254.072},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.903,"5min":1.3879999999999999,"15min":2.593},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.33100000000000002},"max":{"1min":9.6379999999999999,"5min":12.542,"15min":253.49700000000001},"last":0.55400000000000005}]},{"osd":5,"last update":"Mon Oct 9 04:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":1.7150000000000001,"15min":2.9910000000000001},"min":{"1min":0.372,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.0380000000000003,"5min":28.302,"15min":253.57900000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.5780000000000001,"5min":1.7270000000000001,"15min":3.0369999999999999},"min":{"1min":0.40500000000000003,"5min":0.33600000000000002,"15min":0.30199999999999999},"max":{"1min":4.5229999999999997,"5min":28.280000000000001,"15min":254.191},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Oct 9 04:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":0.97099999999999997,"15min":2.8290000000000002},"min":{"1min":0.42799999999999999,"5min":0.34899999999999998,"15min":0.32900000000000001},"max":{"1min":4.7089999999999996,"5min":4.7089999999999996,"15min":254.14699999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.276,"15min":2.1200000000000001},"min":{"1min":0.443,"5min":0.36199999999999999,"15min":0.33900000000000002},"max":{"1min":9.5180000000000007,"5min":9.5180000000000007,"15min":50.308},"last":0.48099999999999998}]},{"osd":7,"last update":"Mon Oct 9 04:54:47 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":1.4810000000000001,"15min":2.2559999999999998},"min":{"1min":0.48499999999999999,"5min":0.375,"15min":0.375},"max":{"1min":6.9960000000000004,"5min":10.398999999999999,"15min":43.835000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.306,"15min":1.833},"min":{"1min":0.45800000000000002,"5min":0.42599999999999999,"15min":0.38400000000000001},"max":{"1min":3.875,"5min":6.1349999999999998,"15min":43.781999999999996},"last":0.57599999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019492,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5150700,"kb_used_data":3651044,"kb_used_omap":0,"kb_used_meta":1499648,"kb_avail":88594452,"statfs":{"total":95995035648,"available":90720718848,"internally_reserved":0,"allocated":3738669056,"data_stored":3737710321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1535639552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7050000000000001,"5min":1.143,"15min":1.5780000000000001},"min":{"1min":0.28000000000000003,"5min":0.252,"15min":0.247},"max":{"1min":5.6040000000000001,"5min":18.585999999999999,"15min":23.114999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.728,"5min":1.548,"15min":1.6359999999999999},"min":{"1min":0.26200000000000001,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":6.6660000000000004,"5min":19.469000000000001,"15min":19.469000000000001},"last":0.75700000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:55:04 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":4.5759999999999996,"15min":5.5129999999999999},"min":{"1min":0.28499999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.5590000000000002,"5min":92.030000000000001,"15min":385.78199999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.6259999999999999,"5min":3.7810000000000001,"15min":5.2050000000000001},"min":{"1min":0.32300000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.633,"5min":91.619,"15min":386.09399999999999},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Oct 9 04:55:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":4.2880000000000003,"15min":6.9589999999999996},"min":{"1min":0.32200000000000001,"5min":0.25,"15min":0.25},"max":{"1min":4.3819999999999997,"5min":136.31200000000001,"15min":440.75599999999997},"last":0.44},{"interface":"front","average":{"1min":1.607,"5min":4.7359999999999998,"15min":7.0179999999999998},"min":{"1min":0.33700000000000002,"5min":0.32100000000000001,"15min":0.30599999999999999},"max":{"1min":6.633,"5min":134.32400000000001,"15min":440.61900000000003},"last":0.69799999999999995}]},{"osd":3,"last update":"Mon Oct 9 04:54:43 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":1.8120000000000001,"15min":4.5019999999999998},"min":{"1min":0.374,"5min":0.313,"15min":0.313},"max":{"1min":6.8310000000000004,"5min":19.381,"15min":357.125},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.052,"5min":2.0289999999999999,"15min":4.5389999999999997},"min":{"1min":0.375,"5min":0.33100000000000002,"15min":0.314},"max":{"1min":3.5150000000000001,"5min":18.681000000000001,"15min":359.02800000000002},"last":0.48899999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":4.6870000000000003,"15min":2.536},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.253},"max":{"1min":5.4299999999999997,"5min":169.44200000000001,"15min":169.44200000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.5510000000000002,"5min":4.04,"15min":2.2240000000000002},"min":{"1min":0.32800000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.6539999999999999,"5min":172.36600000000001,"15min":172.36600000000001},"last":0.78300000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:55:09 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":2.3290000000000002,"15min":1.8240000000000001},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.4889999999999999,"5min":19.664999999999999,"15min":22.920999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.3120000000000001,"15min":1.349},"min":{"1min":0.38700000000000001,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":11.335000000000001,"5min":18.789000000000001,"15min":18.789000000000001},"last":0.50600000000000001}]},{"osd":7,"last update":"Mon Oct 9 04:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.1819999999999999,"15min":1.8109999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.29799999999999999},"max":{"1min":2.7250000000000001,"5min":19.111000000000001,"15min":19.111000000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":2.52,"15min":2.2490000000000001},"min":{"1min":0.38500000000000001,"5min":0.32800000000000001,"15min":0.27100000000000002},"max":{"1min":0.99299999999999999,"5min":19.611000000000001,"15min":22.858000000000001},"last":0.52700000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757856,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5449304,"kb_used_data":3960080,"kb_used_omap":0,"kb_used_meta":1489216,"kb_avail":88295848,"statfs":{"total":95995035648,"available":90414948352,"internally_reserved":0,"allocated":4055121920,"data_stored":4054178333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1524957184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:55:04 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.258,"15min":2.4470000000000001},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":2.7799999999999998,"5min":10.430999999999999,"15min":163.70699999999999},"last":0.498},{"interface":"front","average":{"1min":1.173,"5min":1.3220000000000001,"15min":2.5910000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":4.915,"5min":8.6750000000000007,"15min":163.756},"last":0.34699999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:55:04 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":4.9340000000000002,"15min":5.8049999999999997},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.3600000000000003,"5min":122.45399999999999,"15min":211.85400000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.091,"5min":5.2960000000000003,"15min":6.0030000000000001},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":2.8199999999999998,"5min":122.239,"15min":211.435},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:55:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.7210000000000001,"15min":3.8380000000000001},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.28999999999999998},"max":{"1min":4.8689999999999998,"5min":66.579999999999998,"15min":86.488},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.1110000000000002,"15min":4.0129999999999999},"min":{"1min":0.34799999999999998,"5min":0.34599999999999997,"15min":0.29699999999999999},"max":{"1min":4.5199999999999996,"5min":66.557000000000002,"15min":86.510999999999996},"last":0.47299999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:54:39 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":1.5189999999999999,"15min":3.1819999999999999},"min":{"1min":0.375,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":4.9960000000000004,"5min":8.6419999999999995,"15min":226.43799999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.161,"15min":2.8039999999999998},"min":{"1min":0.374,"5min":0.32300000000000001,"15min":0.29799999999999999},"max":{"1min":4.9139999999999997,"5min":5.6769999999999996,"15min":226.63300000000001},"last":0.42299999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:54:39 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":2.5350000000000001,"15min":1.5089999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":4.9749999999999996,"5min":116.054,"15min":116.054},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.792,"5min":3.5459999999999998,"15min":2.073},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.253},"max":{"1min":7.7839999999999998,"5min":118.47,"15min":118.47},"last":0.443}]},{"osd":6,"last update":"Mon Oct 9 04:55:07 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":1.173,"15min":1.0740000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.23100000000000001},"max":{"1min":2.8330000000000002,"5min":9.1780000000000008,"15min":13.162000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.214,"5min":1.5529999999999999,"15min":1.296},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.28000000000000003},"max":{"1min":5.04,"5min":13.083,"15min":13.231999999999999},"last":0.622}]},{"osd":7,"last update":"Mon Oct 9 04:54:39 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":3.9169999999999998,"15min":2.5739999999999998},"min":{"1min":0.42199999999999999,"5min":0.33400000000000002,"15min":0.31900000000000001},"max":{"1min":5.0839999999999996,"5min":110.521,"15min":110.521},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.417,"5min":3.8210000000000002,"15min":2.5449999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.0259999999999998,"5min":110.746,"15min":110.746},"last":0.48199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65929216,"data_stored":65929216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4026007552,"data_stored":4026007552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":848166912,"data_stored":848166912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3733585920,"data_stored":3733585920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4050649088,"data_stored":4050649088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":205651968,"data_stored":205651968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2637692928,"data_stored":2637692928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:55:39.413 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:55:39.413 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-10-09T04:55:39.463 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:39.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:39 smithi177 ceph-mon[195797]: pgmap v1637: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:55:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:39 smithi177 ceph-mon[195797]: from='client.37074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:55:39.555 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:39.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:39.600 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:39.652 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:39.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:39.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:39.795 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:39.846 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:39.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:39.950 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:39.989 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:40.019 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:40.048 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:40.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:40.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:40.184 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:40.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 116 16 77608 77592 41.7962 0 - 0.0177413 2023-10-09T04:55:40.286 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:40.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:40.390 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:40.421 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:40.532 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:40.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:40.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:40.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:40.673 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:41.059 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:55:41.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 117 16 77608 77592 41.439 0 - 0.0177413 2023-10-09T04:55:41.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:41 smithi029 ceph-mon[216903]: pgmap v1638: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:55:41.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:41 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1513096543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:41.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:41 smithi029 ceph-mon[223858]: pgmap v1638: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:55:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:41 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1513096543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:41 smithi177 ceph-mon[195797]: pgmap v1638: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 16250/232851 objects misplaced (6.979%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:55:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:41 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1513096543' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:55:41.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:41.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:42.012 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:42.046 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:42.119 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:42.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:42.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 118 16 77608 77592 41.0879 0 - 0.0177413 2023-10-09T04:55:42.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:42.258 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:42 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:42 smithi177 ceph-mon[195797]: failed to encode map e468 with expected crc 2023-10-09T04:55:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1513096543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:42 smithi177 ceph-mon[195797]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T04:55:42.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:42 smithi177 ceph-mon[195797]: failed to encode map e468 with expected crc 2023-10-09T04:55:42.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:42.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[216903]: failed to encode map e468 with expected crc 2023-10-09T04:55:42.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1513096543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:42.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[216903]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T04:55:42.304 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[216903]: failed to encode map e468 with expected crc 2023-10-09T04:55:42.304 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:55:42.304 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[223858]: failed to encode map e468 with expected crc 2023-10-09T04:55:42.304 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1513096543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:55:42.304 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[223858]: osdmap e468: 8 total, 8 up, 8 in 2023-10-09T04:55:42.304 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:42 smithi029 ceph-mon[223858]: failed to encode map e468 with expected crc 2023-10-09T04:55:42.314 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:42.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:42.419 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:42.453 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:42.524 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:42.524 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:55:42.524 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:55:42.524 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:55:42.524 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:55:42.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:42.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:42.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:42.664 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:42.702 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:42.763 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:42.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:42.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:42.901 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:42.966 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:42.969 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:55:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:55:42] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2023-10-09T04:55:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:42.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:43.068 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pg_num to 11 2023-10-09T04:55:43.071 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:43.102 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:43.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 119 16 77608 77592 40.7427 0 - 0.0177413 2023-10-09T04:55:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:43 smithi029 ceph-mon[216903]: pgmap v1640: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 15401/232851 objects misplaced (6.614%); 12 MiB/s, 185 objects/s recovering 2023-10-09T04:55:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2076822695' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:55:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:43 smithi029 ceph-mon[223858]: pgmap v1640: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 15401/232851 objects misplaced (6.614%); 12 MiB/s, 185 objects/s recovering 2023-10-09T04:55:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2076822695' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:55:43.467 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:43.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:43 smithi177 ceph-mon[195797]: pgmap v1640: 89 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 15401/232851 objects misplaced (6.614%); 12 MiB/s, 185 objects/s recovering 2023-10-09T04:55:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2076822695' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-10-09T04:55:43.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:43.600 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:43.910 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:55:43.929 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:55:43.950 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:55:43.970 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:55:43.992 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:55:43.993 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.993 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.993 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:55:43.994 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:55:44.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2023-10-09T04:55:44.221917+0000 min lat: 0.00111616 max lat: 35.4059 avg lat: 0.0191084 2023-10-09T04:55:44.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-09T04:55:44.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 120 13 77608 77595 40.4047 0.00585938 35.3455 0.0191084 2023-10-09T04:55:44.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2076822695' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T04:55:44.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[216903]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T04:55:44.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[216903]: failed to encode map e469 with expected crc 2023-10-09T04:55:44.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T04:55:44.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:44.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:44.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2076822695' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T04:55:44.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[223858]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T04:55:44.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[223858]: failed to encode map e469 with expected crc 2023-10-09T04:55:44.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T04:55:44.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:44.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:44.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:55:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2076822695' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-10-09T04:55:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:44 smithi177 ceph-mon[195797]: osdmap e469: 8 total, 8 up, 8 in 2023-10-09T04:55:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:44 smithi177 ceph-mon[195797]: failed to encode map e469 with expected crc 2023-10-09T04:55:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-10-09T04:55:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:44.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:44.815 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:44.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:44.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:55:44.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:44.948 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:45.049 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:45.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:45.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:45 smithi177 ceph-mon[195797]: pgmap v1642: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 15401/232851 objects misplaced (6.614%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:55:45.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:45 smithi177 ceph-mon[195797]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:45 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T04:55:45.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:45 smithi177 ceph-mon[195797]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T04:55:45.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:45 smithi177 ceph-mon[195797]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.068 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:45 smithi177 ceph-mon[195797]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:45.187 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:45.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 120.676 2023-10-09T04:55:45.221 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 77608 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 40.1945 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 49.3914 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 198.938 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 643 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 790.275 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3183 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.02479 2023-10-09T04:55:45.222 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.526505 2023-10-09T04:55:45.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 35.413 2023-10-09T04:55:45.223 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00111616 2023-10-09T04:55:45.244 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:45.245 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2023-10-09T04:55:45.357 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[216903]: pgmap v1642: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 15401/232851 objects misplaced (6.614%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:55:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[216903]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T04:55:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[216903]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T04:55:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[216903]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[216903]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[223858]: pgmap v1642: 89 pgs: 1 active+recovery_wait+remapped, 1 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 72 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88502/232851 objects degraded (38.008%); 15401/232851 objects misplaced (6.614%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:55:45.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[223858]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-10-09T04:55:45.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[223858]: osdmap e470: 8 total, 8 up, 8 in 2023-10-09T04:55:45.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[223858]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:45 smithi029 ceph-mon[223858]: failed to encode map e470 with expected crc 2023-10-09T04:55:45.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:45.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:45.475 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:55:45.523 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:45.543 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:45.602 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:55:45.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:45.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:55:45.707 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:45.742 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:45.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:45.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:46.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:46.042 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:46.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:46 smithi177 ceph-mon[195797]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T04:55:46.139 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:46 smithi177 ceph-mon[195797]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:46 smithi177 ceph-mon[195797]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.140 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:46 smithi177 ceph-mon[195797]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[223858]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[223858]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[223858]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[223858]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[216903]: osdmap e471: 8 total, 8 up, 8 in 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[216903]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[216903]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:46 smithi029 ceph-mon[216903]: failed to encode map e471 with expected crc 2023-10-09T04:55:46.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:46.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:46.986 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:47.018 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:47.044 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:55:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:47 smithi029 ceph-mon[223858]: pgmap v1645: 99 pgs: 10 unknown, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 101345/232851 objects degraded (43.524%); 14385/232851 objects misplaced (6.178%); 14 MiB/s, 229 objects/s recovering 2023-10-09T04:55:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:47 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 101345/232851 objects degraded (43.524%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:47 smithi029 ceph-mon[216903]: pgmap v1645: 99 pgs: 10 unknown, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 101345/232851 objects degraded (43.524%); 14385/232851 objects misplaced (6.178%); 14 MiB/s, 229 objects/s recovering 2023-10-09T04:55:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:47 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 101345/232851 objects degraded (43.524%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:47 smithi177 ceph-mon[195797]: pgmap v1645: 99 pgs: 10 unknown, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 101345/232851 objects degraded (43.524%); 14385/232851 objects misplaced (6.178%); 14 MiB/s, 229 objects/s recovering 2023-10-09T04:55:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:47 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 101345/232851 objects degraded (43.524%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:55:47.924 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:47.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:48.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:48.073 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:48.176 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:48.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:48.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:48.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:48.381 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:48.544 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:48.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:48.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:48.691 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:55:48.691 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:55:48.692 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:48.807 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:48.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:48.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:48.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:48.955 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:49.076 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:49.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:49.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:49 smithi029 ceph-mon[216903]: pgmap v1646: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 101317/232824 objects degraded (43.517%); 14385/232824 objects misplaced (6.178%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:55:49.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3210131968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:49.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:49 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:49.104 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:55:49.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:55:49.209 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:49 smithi177 ceph-mon[195797]: pgmap v1646: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 101317/232824 objects degraded (43.517%); 14385/232824 objects misplaced (6.178%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:55:49.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3210131968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:49.210 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:49 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:49.214 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:49.319 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:55:49.320 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:55:49.320 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-09T04:55:49.320 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T04:55:49.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T04:55:49.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:49 smithi029 ceph-mon[223858]: pgmap v1646: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 101317/232824 objects degraded (43.517%); 14385/232824 objects misplaced (6.178%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:55:49.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3210131968' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:49.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:49 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:55:49.527 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:55:49 smithi177 systemd[1]: Stopping Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:55:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[216903]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T04:55:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[216903]: failed to encode map e472 with expected crc 2023-10-09T04:55:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[216903]: failed to encode map e472 with expected crc 2023-10-09T04:55:50.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:50.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:50.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[223858]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T04:55:50.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[223858]: failed to encode map e472 with expected crc 2023-10-09T04:55:50.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:50 smithi029 ceph-mon[223858]: failed to encode map e472 with expected crc 2023-10-09T04:55:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:50 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:55:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:55:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:50 smithi177 ceph-mon[195797]: osdmap e472: 8 total, 8 up, 8 in 2023-10-09T04:55:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:50 smithi177 ceph-mon[195797]: failed to encode map e472 with expected crc 2023-10-09T04:55:50.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:50 smithi177 ceph-mon[195797]: failed to encode map e472 with expected crc 2023-10-09T04:55:51.136 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:55:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:51.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:55:51.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:55:51.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:51 smithi029 ceph-mon[216903]: pgmap v1648: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 160 KiB/s wr, 22 op/s; 101187/232548 objects degraded (43.512%); 14376/232548 objects misplaced (6.182%); 11 MiB/s, 180 objects/s recovering 2023-10-09T04:55:51.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:51.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:51.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:51 smithi029 ceph-mon[223858]: pgmap v1648: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 160 KiB/s wr, 22 op/s; 101187/232548 objects degraded (43.512%); 14376/232548 objects misplaced (6.182%); 11 MiB/s, 180 objects/s recovering 2023-10-09T04:55:51.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:51.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:51.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:51.303 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:51.430 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:51.436 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:55:51 smithi177 conmon[114075]: 2023-10-09T04:55:51.271+0000 7fb7e3ce6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T04:55:51.436 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:55:51 smithi177 conmon[114075]: 2023-10-09T04:55:51.271+0000 7fb7e3ce6700 -1 osd.4 472 *** Got signal Terminated *** 2023-10-09T04:55:51.436 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:55:51 smithi177 conmon[114075]: 2023-10-09T04:55:51.271+0000 7fb7e3ce6700 -1 osd.4 472 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T04:55:51.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:51 smithi177 ceph-mon[195797]: pgmap v1648: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 160 KiB/s wr, 22 op/s; 101187/232548 objects degraded (43.512%); 14376/232548 objects misplaced (6.182%); 11 MiB/s, 180 objects/s recovering 2023-10-09T04:55:51.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:51.437 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:51.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:51.497 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:55:51.526 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:55:51.538 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:51.549 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:55:51.574 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:55:51.578 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:51.600 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:55:51.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:55:51.642 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:51.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:51.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:51.646 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:55:51.666 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:55:51.668 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.668 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.668 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.668 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.668 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.669 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.669 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.669 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:55:51.742 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:51.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:51.786 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:51.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:51.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:51.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:52.033 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:52.083 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:52.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[216903]: osd.4 marked itself down and dead 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2544207077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[223858]: osd.4 marked itself down and dead 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2544207077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:52.189 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:52.216 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:52.262 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:52.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:52.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:52.410 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:52.433 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:52 smithi177 ceph-mon[195797]: osd.4 marked itself down and dead 2023-10-09T04:55:52.433 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2544207077' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:55:52.434 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:52.434 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:52.763 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:52.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:52.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:52.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:52.902 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:52.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:55:53.026 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:55:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:55:52] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.43.0" 2023-10-09T04:55:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: pgmap v1649: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 138 B/s rd, 130 KiB/s wr, 34 op/s; 101187/232188 objects degraded (43.580%); 13394/232188 objects misplaced (5.769%); 17 MiB/s, 278 objects/s recovering 2023-10-09T04:55:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 101187/232188 objects degraded (43.580%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:55:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:55:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: failed to encode map e473 with expected crc 2023-10-09T04:55:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: osdmap e473: 8 total, 7 up, 8 in 2023-10-09T04:55:53.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:53.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:55:53.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:53 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: pgmap v1649: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 138 B/s rd, 130 KiB/s wr, 34 op/s; 101187/232188 objects degraded (43.580%); 13394/232188 objects misplaced (5.769%); 17 MiB/s, 278 objects/s recovering 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 101187/232188 objects degraded (43.580%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: failed to encode map e473 with expected crc 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: osdmap e473: 8 total, 7 up, 8 in 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:55:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: pgmap v1649: 99 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 138 B/s rd, 130 KiB/s wr, 34 op/s; 101187/232188 objects degraded (43.580%); 13394/232188 objects misplaced (5.769%); 17 MiB/s, 278 objects/s recovering 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 101187/232188 objects degraded (43.580%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: failed to encode map e473 with expected crc 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: osdmap e473: 8 total, 7 up, 8 in 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:55:53.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:53 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:53.498 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:53.606 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:53.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:53.714 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:53.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:53.778 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:55:53 smithi177 bash[362286]: d4b592bd132216859e1d8e51a90b6680ba191f07fc6583214c3a667e5d7cf495 2023-10-09T04:55:54.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:54.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:54.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:54.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:55:54.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: osdmap e474: 8 total, 7 up, 8 in 2023-10-09T04:55:54.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.397 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[223858]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.398 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: osdmap e474: 8 total, 7 up, 8 in 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.399 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.400 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:54 smithi029 ceph-mon[216903]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.401 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:54.487 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:55:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:54.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:55:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: osdmap e474: 8 total, 7 up, 8 in 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:54 smithi177 ceph-mon[195797]: failed to encode map e474 with expected crc 2023-10-09T04:55:54.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:54.625 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:54.676 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:54.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:54.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:54.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:55:54.823 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:54.828 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:55:54.828 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:55:54.828 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:55:54.834 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:55:55.039 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:55:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:55.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:55:55.107 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: pgmap v1652: 99 pgs: 1 remapped+peering, 2 remapped, 10 peering, 1 stale+active+recovering+undersized+remapped, 9 stale+active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 64 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 160 KiB/s wr, 73 op/s; 96081/231594 objects degraded (41.487%); 7531/231594 objects misplaced (3.252%); 13 MiB/s, 3 keys/s, 210 objects/s recovering 2023-10-09T04:55:55.107 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:55:55.107 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.107 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: osdmap e475: 8 total, 7 up, 8 in 2023-10-09T04:55:55.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[223858]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.142 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:55.188 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: pgmap v1652: 99 pgs: 1 remapped+peering, 2 remapped, 10 peering, 1 stale+active+recovering+undersized+remapped, 9 stale+active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 64 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 160 KiB/s wr, 73 op/s; 96081/231594 objects degraded (41.487%); 7531/231594 objects misplaced (3.252%); 13 MiB/s, 3 keys/s, 210 objects/s recovering 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: osdmap e475: 8 total, 7 up, 8 in 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:55 smithi029 ceph-mon[216903]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: pgmap v1652: 99 pgs: 1 remapped+peering, 2 remapped, 10 peering, 1 stale+active+recovering+undersized+remapped, 9 stale+active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 64 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 160 KiB/s wr, 73 op/s; 96081/231594 objects degraded (41.487%); 7531/231594 objects misplaced (3.252%); 13 MiB/s, 3 keys/s, 210 objects/s recovering 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: osdmap e475: 8 total, 7 up, 8 in 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: failed to encode map e475 with expected crc 2023-10-09T04:55:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:55 smithi177 ceph-mon[195797]: failed to encode map e475 with expected crc 2023-10-09T04:55:56.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:55:56.632 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:55:57.141 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:57.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:57.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:57.257 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:57 smithi177 ceph-mon[195797]: pgmap v1654: 99 pgs: 4 active+undersized, 2 remapped+peering, 2 remapped, 10 peering, 1 stale+active+recovering+undersized+remapped, 6 stale+active+clean, 4 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 58 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 97666/231345 objects degraded (42.217%); 15596/231345 objects misplaced (6.741%); 13 MiB/s, 3 keys/s, 208 objects/s recovering 2023-10-09T04:55:57.311 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:57.404 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:55:57.413 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:57.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:57 smithi029 ceph-mon[223858]: pgmap v1654: 99 pgs: 4 active+undersized, 2 remapped+peering, 2 remapped, 10 peering, 1 stale+active+recovering+undersized+remapped, 6 stale+active+clean, 4 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 58 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 97666/231345 objects degraded (42.217%); 15596/231345 objects misplaced (6.741%); 13 MiB/s, 3 keys/s, 208 objects/s recovering 2023-10-09T04:55:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:57 smithi029 ceph-mon[216903]: pgmap v1654: 99 pgs: 4 active+undersized, 2 remapped+peering, 2 remapped, 10 peering, 1 stale+active+recovering+undersized+remapped, 6 stale+active+clean, 4 active+undersized+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 58 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 97666/231345 objects degraded (42.217%); 15596/231345 objects misplaced (6.741%); 13 MiB/s, 3 keys/s, 208 objects/s recovering 2023-10-09T04:55:57.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:57.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:57.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:55:57.586 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:57.711 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:55:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:57.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:55:57.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:57.941 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:58.017 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:55:58.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:58.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:55:58.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:55:58.167 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:58.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:58 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 104362/231285 objects degraded (45.123%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:58.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:58 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 104362/231285 objects degraded (45.123%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:58 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 104362/231285 objects degraded (45.123%), 18 pgs degraded (PG_DEGRADED) 2023-10-09T04:55:58.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:55:58.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T04:55:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:58.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:55:59.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:55:59.078 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T04:55:59.078 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T04:55:59.114 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:59.206 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:55:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:59 smithi177 ceph-mon[195797]: pgmap v1655: 99 pgs: 3 active+clean+remapped, 11 active+undersized, 2 remapped+peering, 2 remapped, 10 peering, 5 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 50 op/s; 104362/231285 objects degraded (45.123%); 13562/231285 objects misplaced (5.864%); 4.3 MiB/s, 3 keys/s, 69 objects/s recovering 2023-10-09T04:55:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2630510230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:59 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:55:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:59.368 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:55:59.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:55:59.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:55:59.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[216903]: pgmap v1655: 99 pgs: 3 active+clean+remapped, 11 active+undersized, 2 remapped+peering, 2 remapped, 10 peering, 5 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 50 op/s; 104362/231285 objects degraded (45.123%); 13562/231285 objects misplaced (5.864%); 4.3 MiB/s, 3 keys/s, 69 objects/s recovering 2023-10-09T04:55:59.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2630510230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[223858]: pgmap v1655: 99 pgs: 3 active+clean+remapped, 11 active+undersized, 2 remapped+peering, 2 remapped, 10 peering, 5 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 50 op/s; 104362/231285 objects degraded (45.123%); 13562/231285 objects misplaced (5.864%); 4.3 MiB/s, 3 keys/s, 69 objects/s recovering 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2630510230' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:55:59.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:55:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:55:59.476 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:55:59.508 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:55:59.604 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:55:59.604 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:55:59.604 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:55:59.604 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:55:59.604 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:56:00.102 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:00.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:00.209 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:00.245 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:00.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:00.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:00.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:00.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:00.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[216903]: osdmap e476: 8 total, 7 up, 8 in 2023-10-09T04:56:00.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[216903]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[216903]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[216903]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:00.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:00.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[223858]: osdmap e476: 8 total, 7 up, 8 in 2023-10-09T04:56:00.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[223858]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[223858]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.524 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:00 smithi029 ceph-mon[223858]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:00 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:00 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:00 smithi177 ceph-mon[195797]: osdmap e476: 8 total, 7 up, 8 in 2023-10-09T04:56:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:00 smithi177 ceph-mon[195797]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:00 smithi177 ceph-mon[195797]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:00 smithi177 ceph-mon[195797]: failed to encode map e476 with expected crc 2023-10-09T04:56:00.560 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:00.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:56:00.699 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:00.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:00.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:00.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:00.843 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:56:00.867 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:00.874 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:56:00.908 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:56:00.937 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:56:00.947 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:00.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:00.962 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:56:00.988 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:56:01.010 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:56:01.010 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.011 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:56:01.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:01.083 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:01.103 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:56:01.255 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:01.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:01.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:01.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:01 smithi029 ceph-mon[216903]: pgmap v1657: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 123178/230580 objects degraded (53.421%); 19034/230580 objects misplaced (8.255%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:56:01.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:01 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:56:01.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4032646505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:01.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:01 smithi029 ceph-mon[223858]: pgmap v1657: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 123178/230580 objects degraded (53.421%); 19034/230580 objects misplaced (8.255%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:56:01.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:01 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:56:01.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4032646505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:01.388 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:01.463 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:01.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:01 smithi177 ceph-mon[195797]: pgmap v1657: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 123178/230580 objects degraded (53.421%); 19034/230580 objects misplaced (8.255%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:56:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:01 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-10-09T04:56:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4032646505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:01.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:01.599 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:01.667 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:01.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:01.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:01.808 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:01.867 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:01.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:01.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:02.002 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:02.193 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:02.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:02.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:56:02.262 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:56:02.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:02.334 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:02.393 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:02.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:02.497 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:02.524 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:56:02.528 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:02.648 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:02.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:02.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:02.790 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:02.819 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:56:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:56:02] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2023-10-09T04:56:02.857 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:02.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:02.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:02.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:03.018 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:03.019 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:03.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:03.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:03 smithi177 ceph-mon[195797]: pgmap v1658: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 123104/230280 objects degraded (53.458%); 19011/230280 objects misplaced (8.256%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:56:03.289 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:56:03.289 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:56:03.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:03.371 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:03.526 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:03 smithi029 ceph-mon[216903]: pgmap v1658: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 123104/230280 objects degraded (53.458%); 19011/230280 objects misplaced (8.256%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:56:03.528 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:03.528 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:03 smithi029 ceph-mon[223858]: pgmap v1658: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 54 op/s; 123104/230280 objects degraded (53.458%); 19011/230280 objects misplaced (8.256%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:56:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:03.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:03.632 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:03.665 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:03.716 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:03.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:03.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:03.849 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:03.898 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:03.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:03.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:04.002 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:04.041 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:04.079 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0, 1] out_osds: [] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T04:56:04.079 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:04.079 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-09T04:56:04.079 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T04:56:04.079 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T04:56:04.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-09T04:56:04.101 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:04.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:04.206 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:04.237 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:04.263 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:56:04.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:04.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:04.393 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:04.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:04 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 122714/229500 objects degraded (53.470%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:04.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:04 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 122714/229500 objects degraded (53.470%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:04.421 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:04 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 122714/229500 objects degraded (53.470%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:04.426 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:04.518 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:04.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:56:04.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:04.620 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:56:04.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:04.658 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:04.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:04.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:04.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:04.922 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:05.072 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:05.233 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:05.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:05.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:05.339 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:05.367 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:05 smithi177 ceph-mon[195797]: pgmap v1659: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 122714/229500 objects degraded (53.470%); 18990/229500 objects misplaced (8.275%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T04:56:05.370 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:05.412 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:05.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:05.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:05.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:05.489 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:05 smithi029 ceph-mon[216903]: pgmap v1659: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 122714/229500 objects degraded (53.470%); 18990/229500 objects misplaced (8.275%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T04:56:05.490 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:05 smithi029 ceph-mon[223858]: pgmap v1659: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 122714/229500 objects degraded (53.470%); 18990/229500 objects misplaced (8.275%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T04:56:05.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:05.552 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:05.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:05.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:05.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:05.703 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:05.747 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:05.989 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:05.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:06.094 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:06.125 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:06.185 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:06.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:06.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:06.330 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:06.726 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:56:06.726 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "" 2023-10-09T04:56:06.726 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:56:06.790 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:56:07.027 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:06 smithi029 ceph-mon[216903]: pgmap v1660: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 122587/229125 objects degraded (53.502%); 18959/229125 objects misplaced (8.275%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T04:56:07.027 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2316288655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:07.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:06 smithi029 ceph-mon[223858]: pgmap v1660: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 122587/229125 objects degraded (53.502%); 18959/229125 objects misplaced (8.275%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T04:56:07.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2316288655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:07.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:06 smithi177 ceph-mon[195797]: pgmap v1660: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 122587/229125 objects degraded (53.502%); 18959/229125 objects misplaced (8.275%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T04:56:07.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2316288655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:07.450 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:07.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:07.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:07.593 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2316288655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[216903]: osdmap e477: 8 total, 7 up, 8 in 2023-10-09T04:56:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[216903]: failed to encode map e477 with expected crc 2023-10-09T04:56:07.768 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[216903]: failed to encode map e477 with expected crc 2023-10-09T04:56:07.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:07.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:07 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2316288655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:07 smithi177 ceph-mon[195797]: osdmap e477: 8 total, 7 up, 8 in 2023-10-09T04:56:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:07 smithi177 ceph-mon[195797]: failed to encode map e477 with expected crc 2023-10-09T04:56:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:07 smithi177 ceph-mon[195797]: failed to encode map e477 with expected crc 2023-10-09T04:56:07.863 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:07 smithi177 ceph-mon[195797]: failed to encode map e477 with expected crc 2023-10-09T04:56:07.925 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:07.964 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:08.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[216903]: failed to encode map e477 with expected crc 2023-10-09T04:56:08.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:08.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2316288655' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:08.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[223858]: osdmap e477: 8 total, 7 up, 8 in 2023-10-09T04:56:08.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[223858]: failed to encode map e477 with expected crc 2023-10-09T04:56:08.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[223858]: failed to encode map e477 with expected crc 2023-10-09T04:56:08.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:07 smithi029 ceph-mon[223858]: failed to encode map e477 with expected crc 2023-10-09T04:56:08.233 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:08.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:08.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:08.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:08.365 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:08.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:08.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:08.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:08.518 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:08.547 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:08.593 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:08.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:08.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:08.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:08.740 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:08.861 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:08.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:08.965 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:08.996 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:08 smithi029 ceph-mon[216903]: pgmap v1662: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 122556/229032 objects degraded (53.510%); 18959/229032 objects misplaced (8.278%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:56:08.996 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:08 smithi029 ceph-mon[223858]: pgmap v1662: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 122556/229032 objects degraded (53.510%); 18959/229032 objects misplaced (8.278%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:56:08.998 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:08 smithi177 ceph-mon[195797]: pgmap v1662: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 122556/229032 objects degraded (53.510%); 18959/229032 objects misplaced (8.278%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T04:56:09.047 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:09.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:09.153 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:09.190 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:09.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:09.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:09.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:09.323 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:56:09.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:09.352 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:56:09.371 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:56:09.397 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:09.397 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:56:09.435 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:56:09.453 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:56:09.483 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:56:09.484 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.484 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.484 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.484 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.484 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T04:56:09.484 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.485 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.485 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:56:09.535 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:09.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:09.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:09.683 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:09.738 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:09.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:09.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:09.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:09.879 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:09.931 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:09.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3729765543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:09.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:10.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:10.066 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:56:10.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3729765543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:10.086 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3729765543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:10.090 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:10.343 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:56:10.343 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:56:10.343 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:56:10.343 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:56:10.848 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:10.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:56:10.918 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:56:10.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:10.995 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:11.057 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:11.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:10 smithi029 ceph-mon[216903]: pgmap v1663: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 122251/228387 objects degraded (53.528%); 18937/228387 objects misplaced (8.292%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T04:56:11.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:10 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 122251/228387 objects degraded (53.528%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:11.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:10 smithi029 ceph-mon[223858]: pgmap v1663: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 122251/228387 objects degraded (53.528%); 18937/228387 objects misplaced (8.292%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T04:56:11.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:10 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 122251/228387 objects degraded (53.528%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:11.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:11.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:11.169 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:11.197 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:10 smithi177 ceph-mon[195797]: pgmap v1663: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 73 op/s; 122251/228387 objects degraded (53.528%); 18937/228387 objects misplaced (8.292%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T04:56:11.197 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:10 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 122251/228387 objects degraded (53.528%), 23 pgs degraded (PG_DEGRADED) 2023-10-09T04:56:11.201 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:11.254 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:11.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:11.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:11.395 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:11.735 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:11.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:11.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:56:11.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:11.843 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:11.882 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:12.020 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:12.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:12.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:12.173 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:12.248 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:12.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:12.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:12.360 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.394 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:12.708 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:12.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:12.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:12.842 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:12.880 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0, 1] out_osds: [] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T04:56:12.880 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:12.880 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.6357627661625044 2023-10-09T04:56:12.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 0.6357627661625044 2023-10-09T04:56:12.887 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:12.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:12 smithi029 ceph-mon[216903]: pgmap v1664: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 122115/228006 objects degraded (53.558%); 18771/228006 objects misplaced (8.233%); 7.4 MiB/s, 122 objects/s recovering 2023-10-09T04:56:12.890 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:56:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:56:12] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2023-10-09T04:56:12.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:12 smithi029 ceph-mon[223858]: pgmap v1664: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 122115/228006 objects degraded (53.558%); 18771/228006 objects misplaced (8.233%); 7.4 MiB/s, 122 objects/s recovering 2023-10-09T04:56:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:12.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:12.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:56:12.992 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:13.021 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:13.070 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:13.077 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:12 smithi177 ceph-mon[195797]: pgmap v1664: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 122115/228006 objects degraded (53.558%); 18771/228006 objects misplaced (8.233%); 7.4 MiB/s, 122 objects/s recovering 2023-10-09T04:56:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:13.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:13.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:13.233 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:13.547 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:13.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:13.652 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:13.680 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:13.699 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:13.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:13.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:13.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:13.750 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:13.773 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:13.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:13.867 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:13.918 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:13.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:13.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:14.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:14.052 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:14.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:14.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:14.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:14.255 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:14.433 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:14.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:14.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:14.569 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:14.617 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:14.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:14.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:14.766 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:14 smithi029 ceph-mon[216903]: pgmap v1665: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 121803/227430 objects degraded (53.556%); 18730/227430 objects misplaced (8.236%); 7.2 MiB/s, 119 objects/s recovering 2023-10-09T04:56:14.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:14.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:14 smithi029 ceph-mon[223858]: pgmap v1665: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 121803/227430 objects degraded (53.556%); 18730/227430 objects misplaced (8.236%); 7.2 MiB/s, 119 objects/s recovering 2023-10-09T04:56:14.769 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:14.770 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:14.773 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:14 smithi177 ceph-mon[195797]: pgmap v1665: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 121803/227430 objects degraded (53.556%); 18730/227430 objects misplaced (8.236%); 7.2 MiB/s, 119 objects/s recovering 2023-10-09T04:56:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:15.586 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:56:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4140099457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4140099457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4140099457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: pgmap v1666: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 121701/227127 objects degraded (53.583%); 18294/227127 objects misplaced (8.055%); 7.7 MiB/s, 127 objects/s recovering 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 121701/227127 objects degraded (53.583%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4140099457' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: osdmap e478: 8 total, 7 up, 8 in 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: failed to encode map e478 with expected crc 2023-10-09T04:56:16.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: failed to encode map e478 with expected crc 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[216903]: failed to encode map e478 with expected crc 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: pgmap v1666: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 121701/227127 objects degraded (53.583%); 18294/227127 objects misplaced (8.055%); 7.7 MiB/s, 127 objects/s recovering 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 121701/227127 objects degraded (53.583%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4140099457' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: osdmap e478: 8 total, 7 up, 8 in 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: failed to encode map e478 with expected crc 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: failed to encode map e478 with expected crc 2023-10-09T04:56:16.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:16 smithi029 ceph-mon[223858]: failed to encode map e478 with expected crc 2023-10-09T04:56:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: pgmap v1666: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 66 op/s; 121701/227127 objects degraded (53.583%); 18294/227127 objects misplaced (8.055%); 7.7 MiB/s, 127 objects/s recovering 2023-10-09T04:56:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 121701/227127 objects degraded (53.583%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:56:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:56:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4140099457' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: osdmap e478: 8 total, 7 up, 8 in 2023-10-09T04:56:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: failed to encode map e478 with expected crc 2023-10-09T04:56:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: failed to encode map e478 with expected crc 2023-10-09T04:56:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:16 smithi177 ceph-mon[195797]: failed to encode map e478 with expected crc 2023-10-09T04:56:17.046 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:17.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:17.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:17.151 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:17.183 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:17.510 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:17.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:17.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:17.652 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:18.380 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:18.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:18.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:18.525 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:18.591 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:18.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:18.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:18.696 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:18.722 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:18.769 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:18.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:18.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:18.880 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:18.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:18 smithi029 ceph-mon[216903]: pgmap v1668: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 121680/227064 objects degraded (53.588%); 18294/227064 objects misplaced (8.057%); 7.7 MiB/s, 126 objects/s recovering 2023-10-09T04:56:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:18 smithi029 ceph-mon[223858]: pgmap v1668: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 121680/227064 objects degraded (53.588%); 18294/227064 objects misplaced (8.057%); 7.7 MiB/s, 126 objects/s recovering 2023-10-09T04:56:18.924 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:19.026 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:18 smithi177 ceph-mon[195797]: pgmap v1668: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 65 op/s; 121680/227064 objects degraded (53.588%); 18294/227064 objects misplaced (8.057%); 7.7 MiB/s, 126 objects/s recovering 2023-10-09T04:56:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:19.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:19.132 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:19.160 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:19.207 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:19.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:19.316 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:19.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:56:19.377 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:19.436 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:56:19.471 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:56:19.477 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:19.480 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:56:19.480 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:56:19.480 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:56:19.480 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:56:19.480 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:56:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:19.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:19.501 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:56:19.534 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:56:19.564 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:56:19.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:19.585 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.632 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:56:19.638 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 0.635763 (a2c1) 2023-10-09T04:56:19.672 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:19.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1033491913' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]: dispatch 2023-10-09T04:56:19.809 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:19 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]: dispatch 2023-10-09T04:56:19.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/741322037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:19.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1033491913' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]: dispatch 2023-10-09T04:56:19.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:19 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]: dispatch 2023-10-09T04:56:19.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/741322037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:19.836 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:19.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:19.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:19.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:19.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1033491913' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]: dispatch 2023-10-09T04:56:19.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:19 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]: dispatch 2023-10-09T04:56:19.968 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/741322037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:19.970 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:20.017 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:20.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:20.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:20.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:20.150 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:20.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:20.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:20.304 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:20.333 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: pgmap v1669: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 121386/226509 objects degraded (53.590%); 18251/226509 objects misplaced (8.058%); 7.7 MiB/s, 127 objects/s recovering 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]': finished 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: osdmap e479: 8 total, 7 up, 8 in 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[223858]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: pgmap v1669: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 121386/226509 objects degraded (53.590%); 18251/226509 objects misplaced (8.058%); 7.7 MiB/s, 127 objects/s recovering 2023-10-09T04:56:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]': finished 2023-10-09T04:56:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: osdmap e479: 8 total, 7 up, 8 in 2023-10-09T04:56:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: failed to encode map e479 with expected crc 2023-10-09T04:56:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:20 smithi029 ceph-mon[216903]: failed to encode map e479 with expected crc 2023-10-09T04:56:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: pgmap v1669: 99 pgs: 8 active+clean+remapped, 15 active+undersized, 7 active+undersized+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 53 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 121386/226509 objects degraded (53.590%); 18251/226509 objects misplaced (8.058%); 7.7 MiB/s, 127 objects/s recovering 2023-10-09T04:56:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: failed to encode map e479 with expected crc 2023-10-09T04:56:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: failed to encode map e479 with expected crc 2023-10-09T04:56:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.6357627661625044}]': finished 2023-10-09T04:56:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: osdmap e479: 8 total, 7 up, 8 in 2023-10-09T04:56:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: failed to encode map e479 with expected crc 2023-10-09T04:56:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:20 smithi177 ceph-mon[195797]: failed to encode map e479 with expected crc 2023-10-09T04:56:21.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:56:21.821 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:21.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: osdmap e480: 8 total, 7 up, 8 in 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 121386/226509 objects degraded (53.590%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[216903]: osdmap e481: 8 total, 7 up, 8 in 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: osdmap e480: 8 total, 7 up, 8 in 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: failed to encode map e480 with expected crc 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 121386/226509 objects degraded (53.590%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:56:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:21 smithi029 ceph-mon[223858]: osdmap e481: 8 total, 7 up, 8 in 2023-10-09T04:56:21.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:21.983 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: failed to encode map e480 with expected crc 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: osdmap e480: 8 total, 7 up, 8 in 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: failed to encode map e480 with expected crc 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: failed to encode map e480 with expected crc 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: failed to encode map e480 with expected crc 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: failed to encode map e480 with expected crc 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 121386/226509 objects degraded (53.590%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:56:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:21 smithi177 ceph-mon[195797]: osdmap e481: 8 total, 7 up, 8 in 2023-10-09T04:56:22.108 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:22.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:22.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:22.242 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:22.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:22.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:22.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:22.421 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[216903]: pgmap v1672: 99 pgs: 5 remapped+peering, 2 peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 50 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 91344/226191 objects degraded (40.384%); 17699/226191 objects misplaced (7.825%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[216903]: failed to encode map e481 with expected crc 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[216903]: failed to encode map e481 with expected crc 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[216903]: failed to encode map e481 with expected crc 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[223858]: pgmap v1672: 99 pgs: 5 remapped+peering, 2 peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 50 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 91344/226191 objects degraded (40.384%); 17699/226191 objects misplaced (7.825%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[223858]: failed to encode map e481 with expected crc 2023-10-09T04:56:22.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[223858]: failed to encode map e481 with expected crc 2023-10-09T04:56:22.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[223858]: failed to encode map e481 with expected crc 2023-10-09T04:56:22.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:22 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:56:22.845 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:22.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:22.931 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:22.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:56:22.970 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:56:23.020 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:22 smithi177 ceph-mon[195797]: pgmap v1672: 99 pgs: 5 remapped+peering, 2 peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 50 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 91344/226191 objects degraded (40.384%); 17699/226191 objects misplaced (7.825%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T04:56:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:22 smithi177 ceph-mon[195797]: failed to encode map e481 with expected crc 2023-10-09T04:56:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:22 smithi177 ceph-mon[195797]: failed to encode map e481 with expected crc 2023-10-09T04:56:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:22 smithi177 ceph-mon[195797]: failed to encode map e481 with expected crc 2023-10-09T04:56:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:22 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:56:23.090 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:23.093 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:56:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:56:22] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.43.0" 2023-10-09T04:56:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:23.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:23.128 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:23.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:23.228 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:23.386 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:23.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:23.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:23.517 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:23.562 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:23.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:23.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:23.696 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:23.745 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:23.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:23.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:23.891 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:24.004 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:24.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:24.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:24.141 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:24.187 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:24.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:24.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:24.293 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:24.318 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:24.361 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:24.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:24.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:24.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:24.504 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:24.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:24 smithi029 ceph-mon[216903]: pgmap v1674: 99 pgs: 5 remapped+peering, 2 peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 50 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 57 op/s; 91272/226032 objects degraded (40.380%); 17690/226032 objects misplaced (7.826%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:56:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:24 smithi029 ceph-mon[223858]: pgmap v1674: 99 pgs: 5 remapped+peering, 2 peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 50 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 57 op/s; 91272/226032 objects degraded (40.380%); 17690/226032 objects misplaced (7.826%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:56:24.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:56:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:24 smithi177 ceph-mon[195797]: pgmap v1674: 99 pgs: 5 remapped+peering, 2 peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 50 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 57 op/s; 91272/226032 objects degraded (40.380%); 17690/226032 objects misplaced (7.826%); 8.4 MiB/s, 134 objects/s recovering 2023-10-09T04:56:25.382 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:25.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:25.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:25.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:25.515 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:25.559 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:25.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:25.659 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:25.666 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:25.715 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:26.115 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:26.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:26.222 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:26.231 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:56:26.231 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:56:26.249 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:26.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:26.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:26.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:26.431 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:26.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:26.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:26.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:26.632 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:26.772 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 2, 3, 6, 0, 1] out_osds: [] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T04:56:26.772 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:26.772 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [4, 5, 7, 2, 3, 6, 0, 1] 2023-10-09T04:56:26.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-09T04:56:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:26 smithi029 ceph-mon[216903]: pgmap v1675: 99 pgs: 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 91272/226029 objects degraded (40.381%); 16923/226029 objects misplaced (7.487%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:56:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 91272/226029 objects degraded (40.381%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:26 smithi029 ceph-mon[223858]: pgmap v1675: 99 pgs: 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 91272/226029 objects degraded (40.381%); 16923/226029 objects misplaced (7.487%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:56:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 91272/226029 objects degraded (40.381%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:26 smithi177 ceph-mon[195797]: pgmap v1675: 99 pgs: 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 26 op/s; 91272/226029 objects degraded (40.381%); 16923/226029 objects misplaced (7.487%); 15 MiB/s, 233 objects/s recovering 2023-10-09T04:56:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 91272/226029 objects degraded (40.381%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:27.582 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:27.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:27.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:27.647 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:56:27.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.713 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:27.757 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:27.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:27.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:27.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:27.886 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3765228436' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:56:27.886 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:56:27.889 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3765228436' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:56:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:56:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3765228436' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:56:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:56:27.933 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:27.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:28.041 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:28.090 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:28.666 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:28.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: pgmap v1676: 99 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 87532/225987 objects degraded (38.733%); 16914/225987 objects misplaced (7.485%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:56:28.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:28.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: failed to encode map e482 with expected crc 2023-10-09T04:56:28.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:56:28.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: osdmap e482: 8 total, 7 up, 8 in 2023-10-09T04:56:28.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:28.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:28.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: pgmap v1676: 99 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 87532/225987 objects degraded (38.733%); 16914/225987 objects misplaced (7.485%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:56:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: failed to encode map e482 with expected crc 2023-10-09T04:56:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:56:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: osdmap e482: 8 total, 7 up, 8 in 2023-10-09T04:56:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:28.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:28.772 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:28.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:28.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:56:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: pgmap v1676: 99 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 87532/225987 objects degraded (38.733%); 16914/225987 objects misplaced (7.485%); 11 MiB/s, 179 objects/s recovering 2023-10-09T04:56:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: failed to encode map e482 with expected crc 2023-10-09T04:56:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:56:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: osdmap e482: 8 total, 7 up, 8 in 2023-10-09T04:56:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:29.223 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:29.444 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:29.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:29.560 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:29.605 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:29.738 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:29 smithi029 ceph-mon[216903]: failed to encode map e482 with expected crc 2023-10-09T04:56:29.739 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:29 smithi029 ceph-mon[223858]: failed to encode map e482 with expected crc 2023-10-09T04:56:29.767 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:56:29.768 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:29.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:29.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:29.900 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:29 smithi177 ceph-mon[195797]: failed to encode map e482 with expected crc 2023-10-09T04:56:29.903 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:29.944 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:29.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:29.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:30.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:30.081 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:30.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:30.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:30.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:30.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:30.273 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:30.836 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:30.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:30 smithi029 ceph-mon[216903]: pgmap v1678: 99 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 87532/225987 objects degraded (38.733%); 16914/225987 objects misplaced (7.485%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:56:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1403489421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:30 smithi029 ceph-mon[223858]: pgmap v1678: 99 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 87532/225987 objects degraded (38.733%); 16914/225987 objects misplaced (7.485%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:56:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1403489421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:30.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:30.983 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:30 smithi177 ceph-mon[195797]: pgmap v1678: 99 pgs: 1 activating+undersized+remapped, 1 activating+remapped, 5 remapped+peering, 8 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 87532/225987 objects degraded (38.733%); 16914/225987 objects misplaced (7.485%); 6.4 MiB/s, 102 objects/s recovering 2023-10-09T04:56:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1403489421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:56:31.112 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:31.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:31.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:31.245 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:31.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:56:31.290 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:31.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:31.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:31.443 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:31.562 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:31.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:31.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:31.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:31.695 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:31.743 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:31.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:31.848 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:31.874 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:31.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:31 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:56:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 87532/225987 objects degraded (38.733%), 19 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:31 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:56:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 87532/225987 objects degraded (38.733%), 19 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:31.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:31 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:56:31.923 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 87532/225987 objects degraded (38.733%), 19 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:31.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:32.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:32.053 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:32.105 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:32.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:32.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:32.244 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:32.318 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:32.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:32.425 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:32.465 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:32.580 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:32.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:32.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:32.685 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:32.736 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:32.849 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:32 smithi029 ceph-mon[223858]: pgmap v1679: 99 pgs: 1 activating+undersized+remapped, 5 remapped+peering, 9 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 87532/225987 objects degraded (38.733%); 16052/225987 objects misplaced (7.103%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:56:32.849 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:32 smithi029 ceph-mon[216903]: pgmap v1679: 99 pgs: 1 activating+undersized+remapped, 5 remapped+peering, 9 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 87532/225987 objects degraded (38.733%); 16052/225987 objects misplaced (7.103%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:56:32.859 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:56:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:56:32] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.43.0" 2023-10-09T04:56:32.986 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:32.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:32.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:32 smithi177 ceph-mon[195797]: pgmap v1679: 99 pgs: 1 activating+undersized+remapped, 5 remapped+peering, 9 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 87532/225987 objects degraded (38.733%); 16052/225987 objects misplaced (7.103%); 11 MiB/s, 170 objects/s recovering 2023-10-09T04:56:33.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:33.129 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:33 smithi029 conmon[216878]: 2023-10-09T04:56:33.046+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e482 definitely_dead 0 2023-10-09T04:56:33.135 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:33.166 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:56:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:56:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.7 482/474 [PushOp(14:e2386d21:::benchmark_data_smithi029_550670_object2880:head, version: 444'177, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:e2386d21:::benchmark_data_smithi029_550670_object2880:head@444'177, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:33.166 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:33.163174+0000", 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0023498820000000002, 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024023780000000002, 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163174+0000", 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163174+0000", 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163164+0000", 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999895 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163235+0000", 2023-10-09T04:56:33.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1645999999999998e-05 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163269+0000", 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3181e-05 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163290+0000", 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1494000000000001e-05 2023-10-09T04:56:33.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163404+0000", 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114187 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:33.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:33.163422+0000", 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7615999999999999e-05 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 1 2023-10-09T04:56:33.171 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:56:33.210 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:33.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:33.316 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:33.354 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:33.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2951936943' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:56:33.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:33 smithi029 ceph-mon[216903]: Client client.admin marked osd.4 out, after it was down for 40 seconds 2023-10-09T04:56:33.797 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.4. 2023-10-09T04:56:33.869 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:33.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:33.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:34.003 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2951936943' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:56:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:33 smithi177 ceph-mon[195797]: Client client.admin marked osd.4 out, after it was down for 40 seconds 2023-10-09T04:56:34.053 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:34.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:34.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:34.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2951936943' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-09T04:56:34.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:33 smithi029 ceph-mon[223858]: Client client.admin marked osd.4 out, after it was down for 40 seconds 2023-10-09T04:56:34.186 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:34.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:34.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:34.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:34.398 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:34.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:56:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: pgmap v1680: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 4 remapped+peering, 9 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 96278/225987 objects degraded (42.603%); 16873/225987 objects misplaced (7.466%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:56:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:56:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2951936943' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:56:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: osdmap e483: 8 total, 7 up, 7 in 2023-10-09T04:56:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:34 smithi177 ceph-mon[195797]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: pgmap v1680: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 4 remapped+peering, 9 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 96278/225987 objects degraded (42.603%); 16873/225987 objects misplaced (7.466%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2951936943' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: osdmap e483: 8 total, 7 up, 7 in 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[216903]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: pgmap v1680: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 4 remapped+peering, 9 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 96278/225987 objects degraded (42.603%); 16873/225987 objects misplaced (7.466%); 10 MiB/s, 164 objects/s recovering 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2951936943' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: osdmap e483: 8 total, 7 up, 7 in 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:35.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:34 smithi029 ceph-mon[223858]: failed to encode map e483 with expected crc 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: osdmap e484: 8 total, 7 up, 7 in 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 96278/225987 objects degraded (42.603%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: osdmap e484: 8 total, 7 up, 7 in 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:56:36.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 96278/225987 objects degraded (42.603%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: osdmap e484: 8 total, 7 up, 7 in 2023-10-09T04:56:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: failed to encode map e484 with expected crc 2023-10-09T04:56:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:56:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 96278/225987 objects degraded (42.603%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:37.012 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:37.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:37.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:37.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: pgmap v1683: 99 pgs: 4 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 remapped+peering, 8 active+clean+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 82928/225987 objects degraded (36.696%); 15381/225987 objects misplaced (6.806%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:56:37.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: osdmap e485: 8 total, 7 up, 7 in 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[216903]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: pgmap v1683: 99 pgs: 4 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 remapped+peering, 8 active+clean+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 82928/225987 objects degraded (36.696%); 15381/225987 objects misplaced (6.806%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: osdmap e485: 8 total, 7 up, 7 in 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:36 smithi029 ceph-mon[223858]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:37.205 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:37.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: pgmap v1683: 99 pgs: 4 peering, 1 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 7 remapped+peering, 8 active+clean+remapped, 12 active+undersized, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 82928/225987 objects degraded (36.696%); 15381/225987 objects misplaced (6.806%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:56:37.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: osdmap e485: 8 total, 7 up, 7 in 2023-10-09T04:56:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:36 smithi177 ceph-mon[195797]: failed to encode map e485 with expected crc 2023-10-09T04:56:37.285 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:37.364 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:37.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:37.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:37.535 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:37.752 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:37.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:37.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:37.929 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:38.079 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[216903]: osdmap e486: 8 total, 7 up, 7 in 2023-10-09T04:56:38.080 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[216903]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.080 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[216903]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.080 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[216903]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.081 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:38.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[223858]: osdmap e486: 8 total, 7 up, 7 in 2023-10-09T04:56:38.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[223858]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.086 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[223858]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.086 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:37 smithi029 ceph-mon[223858]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:38.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:38.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:38.271 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:37 smithi177 ceph-mon[195797]: osdmap e486: 8 total, 7 up, 7 in 2023-10-09T04:56:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:37 smithi177 ceph-mon[195797]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:37 smithi177 ceph-mon[195797]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:37 smithi177 ceph-mon[195797]: failed to encode map e486 with expected crc 2023-10-09T04:56:38.400 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:38.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:38.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:38.544 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:38.649 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:38.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:38.754 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:38.781 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:38.830 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:38.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:38.868 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:56:38.937 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:38.965 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:39.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:39.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:39.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:39.125 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: pgmap v1686: 99 pgs: 1 active+recovery_wait+degraded, 4 peering, 1 activating+undersized+degraded+remapped, 8 remapped+peering, 4 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 57 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 82928/225987 objects degraded (36.696%); 15380/225987 objects misplaced (6.806%); 9.8 MiB/s, 164 objects/s recovering 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: osdmap e487: 8 total, 7 up, 7 in 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4245360266' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:39.151 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:38 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:39.154 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: pgmap v1686: 99 pgs: 1 active+recovery_wait+degraded, 4 peering, 1 activating+undersized+degraded+remapped, 8 remapped+peering, 4 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 57 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 82928/225987 objects degraded (36.696%); 15380/225987 objects misplaced (6.806%); 9.8 MiB/s, 164 objects/s recovering 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: osdmap e487: 8 total, 7 up, 7 in 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4245360266' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: pgmap v1686: 99 pgs: 1 active+recovery_wait+degraded, 4 peering, 1 activating+undersized+degraded+remapped, 8 remapped+peering, 4 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 57 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 82928/225987 objects degraded (36.696%); 15380/225987 objects misplaced (6.806%); 9.8 MiB/s, 164 objects/s recovering 2023-10-09T04:56:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: osdmap e487: 8 total, 7 up, 7 in 2023-10-09T04:56:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: failed to encode map e487 with expected crc 2023-10-09T04:56:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4245360266' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:38 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:56:39.201 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:39.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:39.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:39.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:39.337 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:39.386 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:39.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:39.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:39.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:39.522 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:39.570 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:39.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:39.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:39.720 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:40.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:40.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:40.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[216903]: osdmap e488: 8 total, 7 up, 7 in 2023-10-09T04:56:40.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[216903]: failed to encode map e488 with expected crc 2023-10-09T04:56:40.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[216903]: failed to encode map e488 with expected crc 2023-10-09T04:56:40.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:40.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:40.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[223858]: osdmap e488: 8 total, 7 up, 7 in 2023-10-09T04:56:40.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[223858]: failed to encode map e488 with expected crc 2023-10-09T04:56:40.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:39 smithi029 ceph-mon[223858]: failed to encode map e488 with expected crc 2023-10-09T04:56:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:39 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:39 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:56:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:39 smithi177 ceph-mon[195797]: osdmap e488: 8 total, 7 up, 7 in 2023-10-09T04:56:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:39 smithi177 ceph-mon[195797]: failed to encode map e488 with expected crc 2023-10-09T04:56:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:39 smithi177 ceph-mon[195797]: failed to encode map e488 with expected crc 2023-10-09T04:56:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:40 smithi029 ceph-mon[216903]: pgmap v1689: 99 pgs: 2 active+recovery_wait+degraded, 5 peering, 1 activating+undersized+degraded+remapped, 8 remapped+peering, 4 active+clean+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 78971/225984 objects degraded (34.945%); 14439/225984 objects misplaced (6.389%); 1.5 MiB/s, 2 keys/s, 24 objects/s recovering 2023-10-09T04:56:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 78971/225984 objects degraded (34.945%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:40 smithi029 ceph-mon[223858]: pgmap v1689: 99 pgs: 2 active+recovery_wait+degraded, 5 peering, 1 activating+undersized+degraded+remapped, 8 remapped+peering, 4 active+clean+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 78971/225984 objects degraded (34.945%); 14439/225984 objects misplaced (6.389%); 1.5 MiB/s, 2 keys/s, 24 objects/s recovering 2023-10-09T04:56:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 78971/225984 objects degraded (34.945%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:40 smithi177 ceph-mon[195797]: pgmap v1689: 99 pgs: 2 active+recovery_wait+degraded, 5 peering, 1 activating+undersized+degraded+remapped, 8 remapped+peering, 4 active+clean+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 78971/225984 objects degraded (34.945%); 14439/225984 objects misplaced (6.389%); 1.5 MiB/s, 2 keys/s, 24 objects/s recovering 2023-10-09T04:56:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 78971/225984 objects degraded (34.945%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:42.051 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 6, 0, 1] out_osds: [4] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T04:56:42.051 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:56:42.051 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [5, 7, 2, 3, 6, 0, 1] 2023-10-09T04:56:42.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-09T04:56:42.741 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:42.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:42.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:42.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:42.933 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:43.043 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:42 smithi029 ceph-mon[216903]: pgmap v1690: 99 pgs: 2 active+recovery_wait+degraded, 1 peering, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 4 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 82687/225984 objects degraded (36.590%); 15276/225984 objects misplaced (6.760%); 3.1 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-09T04:56:43.043 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:56:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:56:42] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.43.0" 2023-10-09T04:56:43.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:42 smithi029 ceph-mon[223858]: pgmap v1690: 99 pgs: 2 active+recovery_wait+degraded, 1 peering, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 4 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 82687/225984 objects degraded (36.590%); 15276/225984 objects misplaced (6.760%); 3.1 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-09T04:56:43.120 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:43.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:43.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:43.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:42 smithi177 ceph-mon[195797]: pgmap v1690: 99 pgs: 2 active+recovery_wait+degraded, 1 peering, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 4 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 82687/225984 objects degraded (36.590%); 15276/225984 objects misplaced (6.760%); 3.1 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-09T04:56:43.341 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:43.392 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:43.451 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:56:43.451 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:56:43.452 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:56:43.452 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:56:43.452 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:56:43.907 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:43.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:44.021 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:56:44.051 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:44.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:56:44.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:44.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:56:44.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:44.239 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:44.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:44.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:44.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:44.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:44.435 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:44.514 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:44.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:44.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:44.650 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:44.700 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:44.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:44.807 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:44.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: pgmap v1691: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77578/225981 objects degraded (34.329%); 19658/225981 objects misplaced (8.699%); 5.4 MiB/s, 1 keys/s, 86 objects/s recovering 2023-10-09T04:56:44.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:44.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:44.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: osdmap e489: 8 total, 7 up, 7 in 2023-10-09T04:56:44.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:44 smithi177 ceph-mon[195797]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.837 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:44.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:56:44.891 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:44.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: pgmap v1691: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77578/225981 objects degraded (34.329%); 19658/225981 objects misplaced (8.699%); 5.4 MiB/s, 1 keys/s, 86 objects/s recovering 2023-10-09T04:56:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: osdmap e489: 8 total, 7 up, 7 in 2023-10-09T04:56:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[216903]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: pgmap v1691: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77578/225981 objects degraded (34.329%); 19658/225981 objects misplaced (8.699%); 5.4 MiB/s, 1 keys/s, 86 objects/s recovering 2023-10-09T04:56:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: osdmap e489: 8 total, 7 up, 7 in 2023-10-09T04:56:44.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:44 smithi029 ceph-mon[223858]: failed to encode map e489 with expected crc 2023-10-09T04:56:44.998 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:45.046 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:45.133 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:45.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:45.238 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:45.278 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:45.975 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:45.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[216903]: osdmap e490: 8 total, 7 up, 7 in 2023-10-09T04:56:45.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[216903]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[216903]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[216903]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[216903]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 77578/225981 objects degraded (34.329%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[223858]: osdmap e490: 8 total, 7 up, 7 in 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[223858]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[223858]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[223858]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[223858]: failed to encode map e490 with expected crc 2023-10-09T04:56:45.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 77578/225981 objects degraded (34.329%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:45.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:46.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:46.107 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:46.154 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:46.158 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:45 smithi177 ceph-mon[195797]: osdmap e490: 8 total, 7 up, 7 in 2023-10-09T04:56:46.158 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:45 smithi177 ceph-mon[195797]: failed to encode map e490 with expected crc 2023-10-09T04:56:46.159 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:45 smithi177 ceph-mon[195797]: failed to encode map e490 with expected crc 2023-10-09T04:56:46.159 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:45 smithi177 ceph-mon[195797]: failed to encode map e490 with expected crc 2023-10-09T04:56:46.159 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:45 smithi177 ceph-mon[195797]: failed to encode map e490 with expected crc 2023-10-09T04:56:46.159 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 77578/225981 objects degraded (34.329%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:46.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:46.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:46.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:46.304 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:46 smithi029 ceph-mon[223858]: pgmap v1694: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77578/225981 objects degraded (34.329%); 19357/225981 objects misplaced (8.566%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:56:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:46 smithi029 ceph-mon[216903]: pgmap v1694: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77578/225981 objects degraded (34.329%); 19357/225981 objects misplaced (8.566%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:56:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:46 smithi177 ceph-mon[195797]: pgmap v1694: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77578/225981 objects degraded (34.329%); 19357/225981 objects misplaced (8.566%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:56:48.078 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:56:48.276 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:48.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:48.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:48.465 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:48.574 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:48.590 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:48.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:48.716 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:48.789 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:49.162 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:56:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:48 smithi029 ceph-mon[216903]: pgmap v1695: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77512/225981 objects degraded (34.300%); 19292/225981 objects misplaced (8.537%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:56:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:48 smithi029 ceph-mon[223858]: pgmap v1695: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77512/225981 objects degraded (34.300%); 19292/225981 objects misplaced (8.537%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:56:49.240 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:49.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:49.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:48 smithi177 ceph-mon[195797]: pgmap v1695: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77512/225981 objects degraded (34.300%); 19292/225981 objects misplaced (8.537%); 6.6 MiB/s, 106 objects/s recovering 2023-10-09T04:56:49.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:49.413 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:49.579 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:49.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:49.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:49.710 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:49.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:49.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:49.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:49.869 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:49 smithi029 conmon[216878]: 2023-10-09T04:56:49.601+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e490 definitely_dead 0 2023-10-09T04:56:49.911 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.0. 2023-10-09T04:56:49.930 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:49.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:49.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3835526265' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:56:49.982 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:49 smithi177 ceph-mon[195797]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:56:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:49.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:50.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:50.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3835526265' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:56:50.120 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:49 smithi029 ceph-mon[216903]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:56:50.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3835526265' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-09T04:56:50.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:49 smithi029 ceph-mon[223858]: Client client.admin marked osd.0 out, while it was still marked up 2023-10-09T04:56:50.126 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:50.194 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:50.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:50.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:56:50.335 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: pgmap v1696: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77251/225966 objects degraded (34.187%); 19035/225966 objects misplaced (8.424%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3835526265' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: osdmap e491: 8 total, 7 up, 6 in 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 77251/225966 objects degraded (34.187%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[223858]: osdmap e492: 8 total, 7 up, 6 in 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: pgmap v1696: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77251/225966 objects degraded (34.187%); 19035/225966 objects misplaced (8.424%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3835526265' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: osdmap e491: 8 total, 7 up, 6 in 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 77251/225966 objects degraded (34.187%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:51.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:50 smithi029 ceph-mon[216903]: osdmap e492: 8 total, 7 up, 6 in 2023-10-09T04:56:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: pgmap v1696: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 77251/225966 objects degraded (34.187%); 19035/225966 objects misplaced (8.424%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T04:56:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3835526265' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-09T04:56:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: osdmap e491: 8 total, 7 up, 6 in 2023-10-09T04:56:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: failed to encode map e491 with expected crc 2023-10-09T04:56:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 77251/225966 objects degraded (34.187%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:50 smithi177 ceph-mon[195797]: osdmap e492: 8 total, 7 up, 6 in 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[216903]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[216903]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[216903]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[216903]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[216903]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[216903]: osdmap e493: 8 total, 7 up, 6 in 2023-10-09T04:56:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[223858]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[223858]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[223858]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[223858]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[223858]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:51 smithi029 ceph-mon[223858]: osdmap e493: 8 total, 7 up, 6 in 2023-10-09T04:56:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:51 smithi177 ceph-mon[195797]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:51 smithi177 ceph-mon[195797]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:51 smithi177 ceph-mon[195797]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:51 smithi177 ceph-mon[195797]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:51 smithi177 ceph-mon[195797]: failed to encode map e492 with expected crc 2023-10-09T04:56:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:51 smithi177 ceph-mon[195797]: osdmap e493: 8 total, 7 up, 6 in 2023-10-09T04:56:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:56:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:56:52] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.43.0" 2023-10-09T04:56:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[216903]: pgmap v1699: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 77251/225966 objects degraded (34.187%); 18706/225966 objects misplaced (8.278%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T04:56:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[216903]: failed to encode map e493 with expected crc 2023-10-09T04:56:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[223858]: pgmap v1699: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 77251/225966 objects degraded (34.187%); 18706/225966 objects misplaced (8.278%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T04:56:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[223858]: failed to encode map e493 with expected crc 2023-10-09T04:56:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:52 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:52 smithi177 ceph-mon[195797]: pgmap v1699: 99 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 remapped+peering, 3 active+clean+remapped, 1 active+recovering+undersized+remapped, 8 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 77251/225966 objects degraded (34.187%); 18706/225966 objects misplaced (8.278%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T04:56:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:52 smithi177 ceph-mon[195797]: failed to encode map e493 with expected crc 2023-10-09T04:56:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:56:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:52 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:53.315 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:56:53.338 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:56:54.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:53 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:56:54.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:53 smithi029 ceph-mon[216903]: osdmap e494: 8 total, 7 up, 6 in 2023-10-09T04:56:54.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:53 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:56:54.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:53 smithi029 ceph-mon[223858]: osdmap e494: 8 total, 7 up, 6 in 2023-10-09T04:56:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:53 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T04:56:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:53 smithi177 ceph-mon[195797]: osdmap e494: 8 total, 7 up, 6 in 2023-10-09T04:56:54.412 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:56:54.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:54.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:56:54.516 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:54.547 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:55.066 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:55.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:55.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:55.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: pgmap v1701: 99 pgs: 16 peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 7 remapped+peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 62353/225957 objects degraded (27.595%); 14604/225957 objects misplaced (6.463%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T04:56:55.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.203 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:55.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:55.204 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:54 smithi177 ceph-mon[195797]: osdmap e495: 8 total, 7 up, 6 in 2023-10-09T04:56:55.206 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:55.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:55.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: pgmap v1701: 99 pgs: 16 peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 7 remapped+peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 62353/225957 objects degraded (27.595%); 14604/225957 objects misplaced (6.463%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:55.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[216903]: osdmap e495: 8 total, 7 up, 6 in 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: pgmap v1701: 99 pgs: 16 peering, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 7 remapped+peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 62353/225957 objects degraded (27.595%); 14604/225957 objects misplaced (6.463%); 8.7 MiB/s, 139 objects/s recovering 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: failed to encode map e494 with expected crc 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:55.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:54 smithi029 ceph-mon[223858]: osdmap e495: 8 total, 7 up, 6 in 2023-10-09T04:56:55.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:56:55.422 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:56:55.461 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:55.471 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:56:55.503 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:56:55.567 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:56:55.567 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.567 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.567 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.567 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.567 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T04:56:55.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.568 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:56:55.762 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:56:55.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:55.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:56:55.774 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:138978 12.8 e317/307 12:1765ab08:::benchmark_data_smithi029_442458_object43728:head v 317'8713, mlcod=317'8709)", 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:10.864771+0000", 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 584.90706490299999, 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022326006999999998, 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.775 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864771+0000", 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864771+0000", 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864770+0000", 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864779+0000", 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5960000000000004e-06 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864799+0000", 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0324999999999999e-05 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.864808+0000", 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1339999999999993e-06 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.866430+0000", 2023-10-09T04:56:55.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016219019999999999 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.866460+0000", 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9974000000000001e-05 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.887047+0000", 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.02058714 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:10.887097+0000", 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9845000000000001e-05 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:149161 12.8 e320/307 12:1a4a23ee:::benchmark_data_smithi029_442458_object53911:head v 320'9364, mlcod=320'9361)", 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:47:33.441480+0000", 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 562.33035651299997, 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023034631, 2023-10-09T04:56:55.780 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441480+0000", 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441480+0000", 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441479+0000", 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441488+0000", 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9889999999999998e-06 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441501+0000", 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3811e-05 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.441509+0000", 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9889999999999992e-06 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.448849+0000", 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073399670000000002 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.448876+0000", 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6962e-05 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.464468+0000", 2023-10-09T04:56:55.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015592168 2023-10-09T04:56:55.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.785 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.785 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:47:33.464514+0000", 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5738999999999997e-05 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36225.0:168534 12.8 e324/307 12:15729ae9:::benchmark_data_smithi029_442458_object73284:head v 324'10567, mlcod=324'10565)", 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:48:12.796708+0000", 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 522.97512789899997, 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026130677000000001, 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.786 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796708+0000", 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796708+0000", 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796707+0000", 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796718+0000", 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1420000000000001e-05 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796742+0000", 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3431999999999999e-05 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.796751+0000", 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4760000000000004e-06 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.798093+0000", 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013422600000000001 2023-10-09T04:56:55.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.798126+0000", 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2511000000000001e-05 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.822781+0000", 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024655274000000001 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:48:12.822839+0000", 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7722000000000001e-05 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:77555 14.5 e459/457 14:a29673af:::benchmark_data_smithi029_550670_object77554:head v 459'4859, mlcod=0'0)", 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:11.665942+0000", 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 104.105893854, 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018888186000000001, 2023-10-09T04:56:55.791 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.792 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.665942+0000", 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.665942+0000", 2023-10-09T04:56:55.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.665942+0000", 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666033+0000", 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0927000000000005e-05 2023-10-09T04:56:55.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666050+0000", 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7331999999999999e-05 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666062+0000", 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1446e-05 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666660+0000", 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059858000000000003 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666692+0000", 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2064999999999999e-05 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.684775+0000", 2023-10-09T04:56:55.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018082463 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.684830+0000", 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5773999999999998e-05 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.797 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.5 464/457 [PushOp(14:a2a6fac7:::benchmark_data_smithi029_550670_object9:head, version: 444'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:a2a6fac7:::benchmark_data_smithi029_550670_object9:head@444'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:20.039506+0000", 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 95.732330106999996, 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042652549999999997, 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.039506+0000", 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.039506+0000", 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.039505+0000", 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043032+0000", 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035274920000000001 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043053+0000", 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0726000000000001e-05 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043071+0000", 2023-10-09T04:56:55.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8454e-05 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043161+0000", 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0148999999999994e-05 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043177+0000", 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5755000000000001e-05 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.082159+0000", 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038981364999999997 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.5 464/457 [PushOp(14:a903ccc8:::benchmark_data_smithi029_550670_object252:head, version: 444'20, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:a903ccc8:::benchmark_data_smithi029_550670_object252:head@444'20, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:20.374281+0000", 2023-10-09T04:56:55.802 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 95.397554772000007, 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093335684000000002, 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374281+0000", 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374281+0000", 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374280+0000", 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374326+0000", 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5745999999999999e-05 2023-10-09T04:56:55.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374359+0000", 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3473999999999998e-05 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374382+0000", 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2942000000000002e-05 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374480+0000", 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7909000000000001e-05 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374495+0000", 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4647000000000001e-05 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.467617+0000", 2023-10-09T04:56:55.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093122307000000001 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78017 14.3 e472/464 14:cd0d5495:::benchmark_data_smithi029_550670_object406:head v 472'4892, mlcod=444'27)", 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.099347+0000", 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 63.672489503000001, 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.171165717, 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.807 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099347+0000", 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099347+0000", 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099345+0000", 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099355+0000", 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5130000000000002e-06 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099366+0000", 2023-10-09T04:56:55.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0995999999999999e-05 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099374+0000", 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0940000000000008e-06 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099428+0000", 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3730000000000002e-05 2023-10-09T04:56:55.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099461+0000", 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3688e-05 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270455+0000", 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17099336600000001 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270512+0000", 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.753e-05 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78128 14.0 e472/457 14:04025a37:::benchmark_data_smithi029_550670_object517:head v 472'4898, mlcod=0'0)", 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.271779+0000", 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 63.500057089000002, 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021498257, 2023-10-09T04:56:55.812 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271779+0000", 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271779+0000", 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271779+0000", 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271784+0000", 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7350000000000001e-06 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271792+0000", 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8779999999999998e-06 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271799+0000", 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.204e-06 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271857+0000", 2023-10-09T04:56:55.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7596000000000002e-05 2023-10-09T04:56:55.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271883+0000", 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6097000000000001e-05 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.293228+0000", 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021345151999999999 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.293277+0000", 2023-10-09T04:56:55.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9097000000000001e-05 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78533 14.b e475/464 14:d1503dc4:::benchmark_data_smithi029_550670_object922:head v 475'5041, mlcod=444'60)", 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:58.828771+0000", 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 56.94306504, 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016809307999999999, 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.818 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828771+0000", 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828771+0000", 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828771+0000", 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828775+0000", 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8410000000000002e-06 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828780+0000", 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8300000000000003e-06 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828784+0000", 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9809999999999997e-06 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.838471+0000", 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096873289999999997 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.838501+0000", 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0340000000000001e-05 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.845548+0000", 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007046971 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.845580+0000", 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2248e-05 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.823 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78538 14.b e475/464 14:d6e8583a:::benchmark_data_smithi029_550670_object927:head v 475'5042, mlcod=444'61)", 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:58.869616+0000", 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 56.902219981999998, 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017353989, 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869616+0000", 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869616+0000", 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869616+0000", 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869621+0000", 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8389999999999997e-06 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869627+0000", 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0800000000000002e-06 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869632+0000", 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9599999999999999e-06 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.872224+0000", 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025921640000000001 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.872266+0000", 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2097e-05 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.886914+0000", 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014648458 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.886970+0000", 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5702999999999998e-05 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:79930 14.c e485/483 14:3e975341:::benchmark_data_smithi029_550670_object2319:head v 485'5016, mlcod=0'0)", 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:36.736090+0000", 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 19.035746140000001, 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077403710000000002, 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.829 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736090+0000", 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736090+0000", 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736088+0000", 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736104+0000", 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5610000000000001e-05 2023-10-09T04:56:55.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736129+0000", 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4839000000000001e-05 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736141+0000", 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2211e-05 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736215+0000", 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4031999999999999e-05 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.736251+0000", 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6032000000000003e-05 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.743777+0000", 2023-10-09T04:56:55.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075257240000000001 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:36.743830+0000", 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3680000000000001e-05 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.834 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 487/484 [PushOp(14:9892ddc2:::benchmark_data_smithi029_550670_object3656:head, version: 444'215, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:9892ddc2:::benchmark_data_smithi029_550670_object3656:head@444'215, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:37.950563+0000", 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.821272766, 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084308709999999995, 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.950563+0000", 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.950563+0000", 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.950561+0000", 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.950623+0000", 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1848999999999997e-05 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.950655+0000", 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1605e-05 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.950675+0000", 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0446e-05 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.956298+0000", 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056229569999999996 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.956314+0000", 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5591e-05 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:37.958994+0000", 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026803949999999999 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 487/484 [PushOp(14:9a17a395:::benchmark_data_smithi029_550670_object3667:head, version: 444'218, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:9a17a395:::benchmark_data_smithi029_550670_object3667:head@444'218, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:38.018065+0000", 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.753770693, 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079183580000000003, 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.018065+0000", 2023-10-09T04:56:55.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.018065+0000", 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.018063+0000", 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.018124+0000", 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0214000000000003e-05 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.018159+0000", 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5374999999999999e-05 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.018180+0000", 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1098999999999999e-05 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.023305+0000", 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051253280000000002 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.023323+0000", 2023-10-09T04:56:55.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8043999999999999e-05 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.025984+0000", 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026603500000000001 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 487/484 [PushOp(14:9520ad7c:::benchmark_data_smithi029_550670_object4088:head, version: 444'249, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:9520ad7c:::benchmark_data_smithi029_550670_object4088:head@444'249, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.844 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:38.563136+0000", 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 17.208699842000001, 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.008069962, 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.563136+0000", 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.563136+0000", 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.563134+0000", 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.563198+0000", 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3995e-05 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.563232+0000", 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4258999999999998e-05 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.563257+0000", 2023-10-09T04:56:55.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4766999999999999e-05 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.569097+0000", 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005839637 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.569109+0000", 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2213e-05 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:38.571206+0000", 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020973160000000001 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 488/484 [PushOp(14:921d2d87:::benchmark_data_smithi029_550670_object5190:head, version: 444'317, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:921d2d87:::benchmark_data_smithi029_550670_object5190:head@444'317, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:39.722184+0000", 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 16.049652498, 2023-10-09T04:56:55.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082100509999999995, 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.722184+0000", 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.722184+0000", 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.722181+0000", 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.722241+0000", 2023-10-09T04:56:55.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9938000000000001e-05 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.722274+0000", 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2756000000000003e-05 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.722294+0000", 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9913999999999999e-05 2023-10-09T04:56:55.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.727570+0000", 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052756139999999997 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.727589+0000", 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9165e-05 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:39.730394+0000", 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002804913 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.8 488/464 [PushOp(14:139ead53:::benchmark_data_smithi029_550670_object2324:head, version: 444'155, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:139ead53:::benchmark_data_smithi029_550670_object2324:head@444'155, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.684903+0000", 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 12.086933589999999, 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089983719999999993, 2023-10-09T04:56:55.854 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684903+0000", 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684903+0000", 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684901+0000", 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684938+0000", 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7183e-05 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684962+0000", 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3017e-05 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684980+0000", 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8731000000000001e-05 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.691174+0000", 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061934370000000004 2023-10-09T04:56:55.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.691188+0000", 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3966e-05 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.693901+0000", 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002713268 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.858 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:79935 14.8 e488/464 14:139ead53:::benchmark_data_smithi029_550670_object2324:head v 488'5149, mlcod=444'154)", 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.709849+0000", 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 12.061986896000001, 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015277671, 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709849+0000", 2023-10-09T04:56:55.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709849+0000", 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709847+0000", 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709862+0000", 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4329e-05 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709889+0000", 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7208000000000001e-05 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709899+0000", 2023-10-09T04:56:55.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0189e-05 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.713197+0000", 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032979020000000001 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.713225+0000", 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8089000000000002e-05 2023-10-09T04:56:55.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.725042+0000", 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011816488 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.725127+0000", 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5196000000000002e-05 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.863 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 488/484 [PushOp(14:99167b6d:::benchmark_data_smithi029_550670_object9156:head, version: 444'556, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:99167b6d:::benchmark_data_smithi029_550670_object9156:head@444'556, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.710775+0000", 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 12.061061453000001, 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083697210000000001, 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.710775+0000", 2023-10-09T04:56:55.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.710775+0000", 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.710774+0000", 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.710833+0000", 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9431000000000001e-05 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.710861+0000", 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7463000000000001e-05 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.710877+0000", 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6096999999999999e-05 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.716486+0000", 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056088420000000002 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.716501+0000", 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5451000000000002e-05 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.719144+0000", 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002643276 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 488/484 [PushOp(14:91bba494:::benchmark_data_smithi029_550670_object9316:head, version: 444'565, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:91bba494:::benchmark_data_smithi029_550670_object9316:head@444'565, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.875065+0000", 2023-10-09T04:56:55.868 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 11.896770998999999, 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011887887, 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875065+0000", 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875065+0000", 2023-10-09T04:56:55.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875063+0000", 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875136+0000", 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2479999999999997e-05 2023-10-09T04:56:55.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884072+0000", 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089364830000000003 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884105+0000", 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2656e-05 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884447+0000", 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000341503 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884461+0000", 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4121000000000001e-05 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.886953+0000", 2023-10-09T04:56:55.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024922669999999998 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 490/484 [PushOp(14:9dbe5c56:::benchmark_data_smithi029_550670_object14195:head, version: 444'870, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:9dbe5c56:::benchmark_data_smithi029_550670_object14195:head@444'870, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:48.524751+0000", 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 7.2470855439999999, 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011665902000000001, 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:56:55.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524751+0000", 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524751+0000", 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:56:55.874 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524750+0000", 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524798+0000", 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8872000000000002e-05 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524828+0000", 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9504000000000001e-05 2023-10-09T04:56:55.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524847+0000", 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9279e-05 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524943+0000", 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5675999999999997e-05 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524956+0000", 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3382e-05 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.536416+0000", 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011460187 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:56:55.877 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:56:55.878 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:55.910 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:55.956 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:55.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:56.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:56:56.087 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:56.184 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:56:56.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:55 smithi177 ceph-mon[195797]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:55 smithi177 ceph-mon[195797]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.189 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:55 smithi177 ceph-mon[195797]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:55 smithi177 ceph-mon[195797]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 62353/225957 objects degraded (27.595%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:56.190 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:55 smithi177 ceph-mon[195797]: osdmap e496: 8 total, 7 up, 6 in 2023-10-09T04:56:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:56.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:56:56.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:56:56.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[216903]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[216903]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[216903]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[216903]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 62353/225957 objects degraded (27.595%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[216903]: osdmap e496: 8 total, 7 up, 6 in 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[223858]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[223858]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[223858]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[223858]: failed to encode map e495 with expected crc 2023-10-09T04:56:56.297 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 62353/225957 objects degraded (27.595%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:56:56.297 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:55 smithi029 ceph-mon[223858]: osdmap e496: 8 total, 7 up, 6 in 2023-10-09T04:56:56.325 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:56.670 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:56:56 smithi029 conmon[275254]: 2023-10-09T04:56:56.320+0000 7fee0a60f700 -1 osd.3 496 get_health_metrics reporting 13 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:56:56.814 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: pgmap v1704: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 17 peering, 2 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 62351/225957 objects degraded (27.594%); 9881/225957 objects misplaced (4.373%); 3.4 MiB/s, 56 objects/s recovering 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/631438510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: 13 slow requests (by type [ 'delayed' : 13 ] most affected pool [ 'unique_pool_9' : 13 ]) 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[216903]: osdmap e497: 8 total, 7 up, 6 in 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: pgmap v1704: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 17 peering, 2 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 62351/225957 objects degraded (27.594%); 9881/225957 objects misplaced (4.373%); 3.4 MiB/s, 56 objects/s recovering 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/631438510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: 13 slow requests (by type [ 'delayed' : 13 ] most affected pool [ 'unique_pool_9' : 13 ]) 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:56 smithi029 ceph-mon[223858]: osdmap e497: 8 total, 7 up, 6 in 2023-10-09T04:56:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: pgmap v1704: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 17 peering, 2 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 8 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 62351/225957 objects degraded (27.594%); 9881/225957 objects misplaced (4.373%); 3.4 MiB/s, 56 objects/s recovering 2023-10-09T04:56:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: failed to encode map e496 with expected crc 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/631438510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: 13 slow requests (by type [ 'delayed' : 13 ] most affected pool [ 'unique_pool_9' : 13 ]) 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:56:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:56 smithi177 ceph-mon[195797]: osdmap e497: 8 total, 7 up, 6 in 2023-10-09T04:56:57.920 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:56:57 smithi029 conmon[275254]: 2023-10-09T04:56:57.289+0000 7fee0a60f700 -1 osd.3 497 get_health_metrics reporting 13 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:56:58.058 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:56:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:58.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:56:58.162 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:56:58.196 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:57 smithi177 ceph-mon[195797]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:57 smithi177 ceph-mon[195797]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:57 smithi177 ceph-mon[195797]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:57 smithi177 ceph-mon[195797]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:57 smithi177 ceph-mon[195797]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:57 smithi177 ceph-mon[195797]: 13 slow requests (by type [ 'delayed' : 13 ] most affected pool [ 'unique_pool_9' : 13 ]) 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[216903]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[216903]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[216903]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[216903]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[216903]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[216903]: 13 slow requests (by type [ 'delayed' : 13 ] most affected pool [ 'unique_pool_9' : 13 ]) 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[223858]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[223858]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[223858]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[223858]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[223858]: failed to encode map e497 with expected crc 2023-10-09T04:56:58.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:57 smithi029 ceph-mon[223858]: 13 slow requests (by type [ 'delayed' : 13 ] most affected pool [ 'unique_pool_9' : 13 ]) 2023-10-09T04:56:58.670 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:56:58 smithi029 conmon[275254]: 2023-10-09T04:56:58.326+0000 7fee0a60f700 -1 osd.3 497 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:56:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:58 smithi177 ceph-mon[195797]: pgmap v1707: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 12 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 8 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 62287/225957 objects degraded (27.566%); 9815/225957 objects misplaced (4.344%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T04:56:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:58 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:56:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:59.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[216903]: pgmap v1707: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 12 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 8 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 62287/225957 objects degraded (27.566%); 9815/225957 objects misplaced (4.344%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T04:56:59.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:56:59.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:59.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:59.355 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[223858]: pgmap v1707: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 12 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 8 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 63 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 62287/225957 objects degraded (27.566%); 9815/225957 objects misplaced (4.344%); 1.2 MiB/s, 20 objects/s recovering 2023-10-09T04:56:59.355 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:56:59.355 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:56:59.355 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:56:59.671 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:56:59 smithi029 conmon[275254]: 2023-10-09T04:56:59.353+0000 7fee0a60f700 -1 osd.3 497 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:56:59.698 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T04:56:59.698 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T04:56:59.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:56:59.725 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:56:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:59.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:56:59.829 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:56:59.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:56:59.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:56:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:56:59.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:56:59.985 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:56:59 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:00.007 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:00.051 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:00.065 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 6, 1] out_osds: [4, 0] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T04:57:00.065 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:57:00.065 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [5, 7, 2, 3, 6, 1] 2023-10-09T04:57:00.065 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-09T04:57:00.124 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:00.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:00.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:00.255 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:56:59 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:00.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:00.289 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:00.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:00.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:00.389 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:56:59 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:00.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:00.447 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:00.526 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:00.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:00.631 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:00.657 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:00.660 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:00 smithi029 conmon[275254]: 2023-10-09T04:57:00.388+0000 7fee0a60f700 -1 osd.3 497 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:00.701 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:00.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:00.809 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:00.835 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:00.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:00.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:00.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:00.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:01.011 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:01.058 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:01.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:01.167 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:01.195 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:01.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:01.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:00 smithi177 ceph-mon[195797]: pgmap v1708: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 10 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 6 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 80446/225900 objects degraded (35.611%); 9799/225900 objects misplaced (4.338%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:57:01.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:00 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:01.245 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80446/225900 objects degraded (35.611%), 17 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:01.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:01.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:01.373 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:01.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:01.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:00 smithi029 ceph-mon[216903]: pgmap v1708: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 10 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 6 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 80446/225900 objects degraded (35.611%); 9799/225900 objects misplaced (4.338%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:57:01.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:00 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80446/225900 objects degraded (35.611%), 17 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:00 smithi029 ceph-mon[223858]: pgmap v1708: 99 pgs: 1 active+recovery_wait+degraded+laggy, 1 active+clean+remapped, 10 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 6 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 80446/225900 objects degraded (35.611%); 9799/225900 objects misplaced (4.338%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T04:57:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:00 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80446/225900 objects degraded (35.611%), 17 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:01.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:01.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:01.550 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:01.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:01.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:01.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:01.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:57:01.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:01.736 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:01.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:57:01.882 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:01.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:01.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:01.918 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:01.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:02.046 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:02.190 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:02.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:02.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:02.295 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:02.322 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:02.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:02.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:02.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:02.477 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:02.505 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:02.550 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:02.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:02.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:02.658 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:02.691 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79979 14.8 14.1cc1c4c8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.399715+0000", 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 5.5450245489999999, 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5450554619999997, 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T04:57:02.945 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79979 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399715+0000", 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399715+0000", 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399713+0000", 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:02.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399723+0000", 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0036e-05 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399739+0000", 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5603999999999998e-05 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399747+0000", 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4919999999999993e-06 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:02.948 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:02.949 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:02.949 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:57:02.949 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 1 2023-10-09T04:57:02.949 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:57:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:02 smithi029 ceph-mon[216903]: pgmap v1709: 99 pgs: 1 active+clean+remapped, 9 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 88543/225861 objects degraded (39.202%); 9799/225861 objects misplaced (4.339%); 860 KiB/s, 13 objects/s recovering 2023-10-09T04:57:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:57:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:57:02] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-10-09T04:57:03.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:02 smithi029 ceph-mon[223858]: pgmap v1709: 99 pgs: 1 active+clean+remapped, 9 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 88543/225861 objects degraded (39.202%); 9799/225861 objects misplaced (4.339%); 860 KiB/s, 13 objects/s recovering 2023-10-09T04:57:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:02 smithi177 ceph-mon[195797]: pgmap v1709: 99 pgs: 1 active+clean+remapped, 9 peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 66 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 88543/225861 objects degraded (39.202%); 9799/225861 objects misplaced (4.339%); 860 KiB/s, 13 objects/s recovering 2023-10-09T04:57:03.589 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:03.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:03.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:03.694 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:03.728 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:03.786 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:03.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:03.891 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:03.921 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:04.572 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:04.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:04.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:04.704 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:04.749 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:04.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:04.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T04:57:04.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:04.880 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:04.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:04.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:04.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:05.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:05.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:04 smithi029 ceph-mon[216903]: pgmap v1710: 99 pgs: 1 activating, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 88468/225858 objects degraded (39.170%); 9724/225858 objects misplaced (4.305%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:57:05.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:04 smithi029 ceph-mon[223858]: pgmap v1710: 99 pgs: 1 activating, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 88468/225858 objects degraded (39.170%); 9724/225858 objects misplaced (4.305%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:57:05.081 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:05.081 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:05.084 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:05.179 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:05.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:05.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:04 smithi177 ceph-mon[195797]: pgmap v1710: 99 pgs: 1 activating, 1 active+clean+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 4 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 88468/225858 objects degraded (39.170%); 9724/225858 objects misplaced (4.305%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:57:05.291 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:05.329 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:05.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:05 smithi029 conmon[216878]: 2023-10-09T04:57:05.568+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e497 definitely_dead 0 2023-10-09T04:57:05.837 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:05.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:05.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:05.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:05.977 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:06.056 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.6. 2023-10-09T04:57:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2179128273' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:57:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:05 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:57:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:05 smithi177 ceph-mon[195797]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:57:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 88468/225858 objects degraded (39.170%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:06.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2179128273' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:57:06.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:57:06.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[216903]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:57:06.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 88468/225858 objects degraded (39.170%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:06.322 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2179128273' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:57:06.322 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-09T04:57:06.322 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[223858]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-09T04:57:06.322 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 88468/225858 objects degraded (39.170%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[223858]: pgmap v1711: 99 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+clean+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 97453/225837 objects degraded (43.152%); 9724/225837 objects misplaced (4.306%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[223858]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[223858]: osdmap e498: 8 total, 7 up, 5 in 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[223858]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[223858]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[216903]: pgmap v1711: 99 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+clean+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 97453/225837 objects degraded (43.152%); 9724/225837 objects misplaced (4.306%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:57:07.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[216903]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:57:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[216903]: osdmap e498: 8 total, 7 up, 5 in 2023-10-09T04:57:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[216903]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:07 smithi029 ceph-mon[216903]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:07 smithi177 ceph-mon[195797]: pgmap v1711: 99 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+clean+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 97453/225837 objects degraded (43.152%); 9724/225837 objects misplaced (4.306%); 1.1 MiB/s, 17 objects/s recovering 2023-10-09T04:57:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:07 smithi177 ceph-mon[195797]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:07 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-09T04:57:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:07 smithi177 ceph-mon[195797]: osdmap e498: 8 total, 7 up, 5 in 2023-10-09T04:57:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:07 smithi177 ceph-mon[195797]: failed to encode map e498 with expected crc 2023-10-09T04:57:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:07 smithi177 ceph-mon[195797]: failed to encode map e498 with expected crc 2023-10-09T04:57:08.014 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:07 smithi029 conmon[275254]: 2023-10-09T04:57:07.752+0000 7fee0a60f700 -1 osd.3 499 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:08.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[216903]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[216903]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[216903]: osdmap e499: 8 total, 7 up, 5 in 2023-10-09T04:57:08.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[216903]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[216903]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[216903]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.119 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:08.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:08.238 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:08.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:08 smithi177 ceph-mon[195797]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:08 smithi177 ceph-mon[195797]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:08 smithi177 ceph-mon[195797]: osdmap e499: 8 total, 7 up, 5 in 2023-10-09T04:57:08.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:08 smithi177 ceph-mon[195797]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:08 smithi177 ceph-mon[195797]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:08 smithi177 ceph-mon[195797]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.265 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:08.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:08.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:08.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[223858]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[223858]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[223858]: osdmap e499: 8 total, 7 up, 5 in 2023-10-09T04:57:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[223858]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[223858]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:08 smithi029 ceph-mon[223858]: failed to encode map e499 with expected crc 2023-10-09T04:57:08.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:08.504 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:08.920 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:08 smithi029 conmon[275254]: 2023-10-09T04:57:08.747+0000 7fee0a60f700 -1 osd.3 500 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: pgmap v1714: 99 pgs: 1 activating+degraded, 4 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 2 op/s; 84468/225837 objects degraded (37.402%); 4567/225837 objects misplaced (2.022%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: Health check failed: 7 slow ops, oldest one blocked for 42 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: osdmap e500: 8 total, 7 up, 5 in 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:09 smithi177 ceph-mon[195797]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: pgmap v1714: 99 pgs: 1 activating+degraded, 4 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 2 op/s; 84468/225837 objects degraded (37.402%); 4567/225837 objects misplaced (2.022%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: Health check failed: 7 slow ops, oldest one blocked for 42 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: osdmap e500: 8 total, 7 up, 5 in 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[216903]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: pgmap v1714: 99 pgs: 1 activating+degraded, 4 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 2 op/s; 84468/225837 objects degraded (37.402%); 4567/225837 objects misplaced (2.022%); 1.2 MiB/s, 18 objects/s recovering 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: Health check failed: 7 slow ops, oldest one blocked for 42 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: osdmap e500: 8 total, 7 up, 5 in 2023-10-09T04:57:09.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: failed to encode map e500 with expected crc 2023-10-09T04:57:09.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:09 smithi029 ceph-mon[223858]: failed to encode map e500 with expected crc 2023-10-09T04:57:10.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:10 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:10.171 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:09 smithi029 conmon[275254]: 2023-10-09T04:57:09.777+0000 7fee0a60f700 -1 osd.3 500 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:10.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:10 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:10.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:10 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:10.759 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:10.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:10.856 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:10.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:10.964 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:11.027 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:10 smithi029 conmon[275254]: 2023-10-09T04:57:10.809+0000 7fee0a60f700 -1 osd.3 500 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:11.256 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:11.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:11.284 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:11 smithi029 ceph-mon[216903]: pgmap v1716: 99 pgs: 1 activating+degraded, 6 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 70734/225837 objects degraded (31.321%); 4567/225837 objects misplaced (2.022%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:57:11.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:11 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:11.285 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 70734/225837 objects degraded (31.321%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:11.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:11 smithi029 ceph-mon[223858]: pgmap v1716: 99 pgs: 1 activating+degraded, 6 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 70734/225837 objects degraded (31.321%); 4567/225837 objects misplaced (2.022%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:57:11.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:11 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:11.285 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 70734/225837 objects degraded (31.321%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:11.372 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:11.397 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:11 smithi177 ceph-mon[195797]: pgmap v1716: 99 pgs: 1 activating+degraded, 6 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 70734/225837 objects degraded (31.321%); 4567/225837 objects misplaced (2.022%); 832 KiB/s, 12 objects/s recovering 2023-10-09T04:57:11.397 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:11 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:11.398 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 70734/225837 objects degraded (31.321%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:11.400 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:11.454 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:11.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:11.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:11.640 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:11.899 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:11 smithi029 conmon[275254]: 2023-10-09T04:57:11.768+0000 7fee0a60f700 -1 osd.3 500 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:11.969 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:11.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:11.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:12.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:12.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:12 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:12.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:12.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:12.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:12 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:12.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:12.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:12.197 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:12.425 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:57:12.464 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:12.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:12 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:12.578 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:12.607 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:12.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:12.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:12.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:12.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:12.936 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:12.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:12.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:13.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:13.077 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:57:13.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: pgmap v1717: 99 pgs: 1 activating+degraded, 6 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 68146/225837 objects degraded (30.175%); 6612/225837 objects misplaced (2.928%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T04:57:13.309 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:13.310 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1764238034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:13.310 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/11849620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:13.310 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:13.310 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:13.310 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:13.310 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:57:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:57:12] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-10-09T04:57:13.311 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 04:57:12 smithi029 conmon[275254]: 2023-10-09T04:57:12.809+0000 7fee0a60f700 -1 osd.3 500 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478) 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: pgmap v1717: 99 pgs: 1 activating+degraded, 6 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 68146/225837 objects degraded (30.175%); 6612/225837 objects misplaced (2.928%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1764238034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/11849620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:13.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:13.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:13.312 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:13.313 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:13.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:13.413 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:13.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: pgmap v1717: 99 pgs: 1 activating+degraded, 6 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 68146/225837 objects degraded (30.175%); 6612/225837 objects misplaced (2.928%); 5.6 MiB/s, 90 objects/s recovering 2023-10-09T04:57:13.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:13.443 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1764238034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:13.444 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/11849620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:57:13.444 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:57:13.444 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:57:13.444 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:13.446 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:13.500 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:13.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:13.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:13.653 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:57:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1764238034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: osdmap e501: 8 total, 7 up, 5 in 2023-10-09T04:57:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1764238034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: osdmap e501: 8 total, 7 up, 5 in 2023-10-09T04:57:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:14.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: 7 slow requests (by type [ 'delayed' : 7 ] most affected pool [ 'unique_pool_9' : 7 ]) 2023-10-09T04:57:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:57:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1764238034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: osdmap e501: 8 total, 7 up, 5 in 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: failed to encode map e501 with expected crc 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:15.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:15 smithi029 ceph-mon[216903]: pgmap v1719: 99 pgs: 1 activating+degraded, 4 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80794/225837 objects degraded (35.775%); 6739/225837 objects misplaced (2.984%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:57:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:15 smithi029 ceph-mon[223858]: pgmap v1719: 99 pgs: 1 activating+degraded, 4 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80794/225837 objects degraded (35.775%); 6739/225837 objects misplaced (2.984%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:57:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:15 smithi177 ceph-mon[195797]: pgmap v1719: 99 pgs: 1 activating+degraded, 4 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80794/225837 objects degraded (35.775%); 6739/225837 objects misplaced (2.984%); 6.8 MiB/s, 109 objects/s recovering 2023-10-09T04:57:15.535 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1] out_osds: [4, 0, 6] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T04:57:15.535 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:57:15.536 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-10-09T04:57:15.536 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-10-09T04:57:15.536 DEBUG:teuthology.orchestra.run.smithi177:osd.4> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T04:57:15.538 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl start ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T04:57:15.585 INFO:journalctl@ceph.osd.4.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T04:57:15.801 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:15.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:15.911 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:15.958 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:16.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:15 smithi177 systemd[1]: Starting Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T04:57:16.036 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:16.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:16.060 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80794/225837 objects degraded (35.775%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:16.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:16.187 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:16.262 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:16.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:16.371 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:16.404 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80794/225837 objects degraded (35.775%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:16.409 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:16.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80794/225837 objects degraded (35.775%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:16.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:16.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:16.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:16.628 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:17.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:17 smithi177 ceph-mon[195797]: pgmap v1720: 99 pgs: 1 activating+degraded, 4 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80534/225837 objects degraded (35.660%); 6739/225837 objects misplaced (2.984%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T04:57:17.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:16 smithi177 bash[370950]: --> ceph-volume lvm activate successful for osd ID: 4 2023-10-09T04:57:17.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:17 smithi029 ceph-mon[223858]: pgmap v1720: 99 pgs: 1 activating+degraded, 4 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80534/225837 objects degraded (35.660%); 6739/225837 objects misplaced (2.984%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T04:57:17.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:17 smithi029 ceph-mon[216903]: pgmap v1720: 99 pgs: 1 activating+degraded, 4 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80534/225837 objects degraded (35.660%); 6739/225837 objects misplaced (2.984%); 7.6 MiB/s, 121 objects/s recovering 2023-10-09T04:57:17.502 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:17.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:17.611 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:17.667 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:17.758 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:17.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:17.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:17.869 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:17.903 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:17.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:17.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:17.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:18.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:18.098 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:18.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:18.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:18.256 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:18.300 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:18.886 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:18.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:18.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:18.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:19.045 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:19.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:19 smithi029 ceph-mon[216903]: pgmap v1721: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85422/225837 objects degraded (37.825%); 6835/225837 objects misplaced (3.027%); 8.9 MiB/s, 141 objects/s recovering 2023-10-09T04:57:19.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:19 smithi029 ceph-mon[223858]: pgmap v1721: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85422/225837 objects degraded (37.825%); 6835/225837 objects misplaced (3.027%); 8.9 MiB/s, 141 objects/s recovering 2023-10-09T04:57:19.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:19.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:19.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:19.406 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:19.521 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:19.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:19 smithi177 ceph-mon[195797]: pgmap v1721: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85422/225837 objects degraded (37.825%); 6835/225837 objects misplaced (3.027%); 8.9 MiB/s, 141 objects/s recovering 2023-10-09T04:57:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:19.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:19.628 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:57:19.628 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:57:19.628 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:57:19.628 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:57:19.628 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:57:19.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:19.694 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:19.899 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:19.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:19.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:20.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:20.046 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:20.129 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:20.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:20.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:20.266 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:20.317 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:20.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:20.423 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:20.476 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:20.574 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:20.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:20.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:20.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:20.709 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:20.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:20.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:20.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:20.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:57:20.895 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:20.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:20.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:21.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:21.081 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:21.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:21.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:21.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:21.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:21.268 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:21.319 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:21.326 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:21 smithi177 ceph-mon[195797]: pgmap v1722: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85160/225837 objects degraded (37.709%); 6835/225837 objects misplaced (3.027%); 10 MiB/s, 161 objects/s recovering 2023-10-09T04:57:21.326 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 85160/225837 objects degraded (37.709%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:21.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:21 smithi029 ceph-mon[216903]: pgmap v1722: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85160/225837 objects degraded (37.709%); 6835/225837 objects misplaced (3.027%); 10 MiB/s, 161 objects/s recovering 2023-10-09T04:57:21.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 85160/225837 objects degraded (37.709%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:21 smithi029 ceph-mon[223858]: pgmap v1722: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85160/225837 objects degraded (37.709%); 6835/225837 objects misplaced (3.027%); 10 MiB/s, 161 objects/s recovering 2023-10-09T04:57:21.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 85160/225837 objects degraded (37.709%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:57:21.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:21.459 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:21.509 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:21.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:21.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:21.613 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:21.642 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:21.689 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:21.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:21.798 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:21.825 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:21.875 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:21.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:21.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:22.010 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:22.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:22.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:22.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:22.161 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:22.189 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:22.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:22.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:22.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:22.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:22.377 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:22.804 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:22.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:22.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T04:57:22.910 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:22.973 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:23.004 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:23.085 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:23 smithi029 ceph-mon[216903]: pgmap v1723: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84757/225837 objects degraded (37.530%); 6727/225837 objects misplaced (2.979%); 10 MiB/s, 180 objects/s recovering 2023-10-09T04:57:23.086 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:57:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:57:22] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.43.0" 2023-10-09T04:57:23.090 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:23.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:23.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:23.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:23.225 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:23.273 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:23.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:23.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:23.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:23 smithi029 ceph-mon[223858]: pgmap v1723: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84757/225837 objects degraded (37.530%); 6727/225837 objects misplaced (2.979%); 10 MiB/s, 180 objects/s recovering 2023-10-09T04:57:23.431 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:23.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:23 smithi177 ceph-mon[195797]: pgmap v1723: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 2 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84757/225837 objects degraded (37.530%); 6727/225837 objects misplaced (2.979%); 10 MiB/s, 180 objects/s recovering 2023-10-09T04:57:23.672 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:23.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:23.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:23.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:23.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:23.888 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:23.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:23.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:23.994 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:24.037 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:24.101 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:24.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:24.206 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:24.246 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:24.312 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:24.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:24.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:24.454 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:24.506 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:24.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:24.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:24.656 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:25.003 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:57:25.067 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:25.095 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:25.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:25.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:25.233 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:25 smithi177 ceph-mon[195797]: pgmap v1724: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84811/225837 objects degraded (37.554%); 6600/225837 objects misplaced (2.922%); 9.7 MiB/s, 170 objects/s recovering 2023-10-09T04:57:25.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:24 smithi177 bash[372203]: faaf1f5dae86182e625adaeb73753b0c5c0b3d539c17e98ddf25945033dbbe6f 2023-10-09T04:57:25.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:25 smithi177 systemd[1]: Started Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T04:57:25.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:25 smithi177 ceph-osd[372267]: -- 172.21.15.177:0/1631233076 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55acdbd82b60 con 0x55acdbd49400 2023-10-09T04:57:25.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:25.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:25.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:25.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:25 smithi029 ceph-mon[216903]: pgmap v1724: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84811/225837 objects degraded (37.554%); 6600/225837 objects misplaced (2.922%); 9.7 MiB/s, 170 objects/s recovering 2023-10-09T04:57:25.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:25 smithi029 ceph-mon[223858]: pgmap v1724: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84811/225837 objects degraded (37.554%); 6600/225837 objects misplaced (2.922%); 9.7 MiB/s, 170 objects/s recovering 2023-10-09T04:57:25.395 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:25.426 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:57:25.427 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:25.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:25.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:25.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:25.614 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:25.763 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:25.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:25.818 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:25 smithi177 conmon[372243]: 2023-10-09T04:57:25.566+0000 7f082047b540 -1 Falling back to public interface 2023-10-09T04:57:25.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:25.915 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:26.081 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:26.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84811/225837 objects degraded (37.554%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84811/225837 objects degraded (37.554%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:26.186 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:26.233 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84811/225837 objects degraded (37.554%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:26.297 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:26.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:26.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:26.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:26.444 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:26.801 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:26.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:26.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:26.957 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:27.080 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:27.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:27.136 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:57:27.187 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.222 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:27.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:27 smithi029 ceph-mon[216903]: pgmap v1725: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84472/225837 objects degraded (37.404%); 6600/225837 objects misplaced (2.922%); 10 MiB/s, 177 objects/s recovering 2023-10-09T04:57:27.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3228645437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:27.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:27.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:27 smithi029 ceph-mon[223858]: pgmap v1725: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84472/225837 objects degraded (37.404%); 6600/225837 objects misplaced (2.922%); 10 MiB/s, 177 objects/s recovering 2023-10-09T04:57:27.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3228645437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:27.362 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:27.480 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:27.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:27.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:27.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:27 smithi177 ceph-mon[195797]: pgmap v1725: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84472/225837 objects degraded (37.404%); 6600/225837 objects misplaced (2.922%); 10 MiB/s, 177 objects/s recovering 2023-10-09T04:57:27.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3228645437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:27.485 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:57:27.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.618 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:27.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:27.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:27.844 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:27.890 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:28.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:28.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:28.075 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:57:28.075 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:57:28.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:57:28.075 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:123965 13.10 13:0a5f630d:::benchmark_data_smithi029_495570_object29705:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.919443+0000", 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.15045458399999, 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1698377679999998, 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 123965 2023-10-09T04:57:28.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919443+0000", 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919443+0000", 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919441+0000", 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919448+0000", 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9059999999999996e-06 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919464+0000", 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5885000000000001e-05 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919472+0000", 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919516+0000", 2023-10-09T04:57:28.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4830999999999998e-05 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.919525+0000", 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0880000000000001e-06 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915497+0000", 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99597117800000001 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915511+0000", 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4846e-05 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089166+0000", 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1736544769999999 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089280+0000", 2023-10-09T04:57:28.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011445 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:123983 13.10 13:0fa29277:::benchmark_data_smithi029_495570_object29723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.922458+0000", 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.14743899799998, 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.171047722, 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.082 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 123983 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922458+0000", 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922458+0000", 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922457+0000", 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922463+0000", 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1689999999999997e-06 2023-10-09T04:57:28.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922476+0000", 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2351000000000001e-05 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922485+0000", 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4579999999999999e-06 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922519+0000", 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3288999999999999e-05 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.922525+0000", 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3319999999999997e-06 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915545+0000", 2023-10-09T04:57:28.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.99301996000000003 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915565+0000", 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9817e-05 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089535+0000", 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.173970545 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089674+0000", 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139209 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089925+0000", 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025076100000000001 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093194+0000", 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032688420000000001 2023-10-09T04:57:28.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093256+0000", 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1579000000000001e-05 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093295+0000", 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9700000000000003e-05 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093382+0000", 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6839e-05 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093417+0000", 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5318999999999998e-05 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093506+0000", 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8469999999999998e-05 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.090 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.091 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:123990 13.10 13:0c5d2d0a:::benchmark_data_smithi029_495570_object29730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.929298+0000", 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.14059947599998, 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1687639679999999, 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.093 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 123990 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929298+0000", 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929298+0000", 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929297+0000", 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929302+0000", 2023-10-09T04:57:28.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.668e-06 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929316+0000", 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4443999999999999e-05 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929324+0000", 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4930000000000001e-06 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929380+0000", 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6214000000000001e-05 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.929393+0000", 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3568e-05 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915590+0000", 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.98619684699999999 2023-10-09T04:57:28.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915610+0000", 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9402000000000002e-05 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.090958+0000", 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1753482129999999 2023-10-09T04:57:28.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.091054+0000", 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6655000000000002e-05 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.091229+0000", 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000174396 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093302+0000", 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00207311 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093345+0000", 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3010000000000003e-05 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095725+0000", 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002379595 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.097880+0000", 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021552210000000001 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.097933+0000", 2023-10-09T04:57:28.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3149999999999998e-05 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098062+0000", 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012882599999999999 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124023 13.10 13:0b027d7d:::benchmark_data_smithi029_495570_object29763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.957144+0000", 2023-10-09T04:57:28.102 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.112753236, 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1411683950000002, 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124023 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957144+0000", 2023-10-09T04:57:28.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957144+0000", 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957143+0000", 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957148+0000", 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1379999999999998e-06 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957163+0000", 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4810999999999999e-05 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957171+0000", 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6420000000000007e-06 2023-10-09T04:57:28.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957219+0000", 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.816e-05 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.957226+0000", 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.844e-06 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915638+0000", 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95841202299999995 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915646+0000", 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5019999999999992e-06 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.091976+0000", 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1763292949999999 2023-10-09T04:57:28.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.092125+0000", 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014919600000000001 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.092343+0000", 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000218345 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093306+0000", 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00096309800000000001 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095671+0000", 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023646850000000001 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095776+0000", 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00010521799999999999 2023-10-09T04:57:28.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098140+0000", 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002363876 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098194+0000", 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4024000000000003e-05 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098312+0000", 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118243 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124034 13.10 13:0f418103:::benchmark_data_smithi029_495570_object29774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:31.969963+0000", 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.09993418099998, 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1372428700000001, 2023-10-09T04:57:28.111 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124034 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969963+0000", 2023-10-09T04:57:28.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969963+0000", 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969961+0000", 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.969975+0000", 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4773000000000001e-05 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970035+0000", 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9525999999999999e-05 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970058+0000", 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3067000000000001e-05 2023-10-09T04:57:28.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970195+0000", 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000137358 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:31.970223+0000", 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7881999999999998e-05 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915671+0000", 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94544759700000003 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915678+0000", 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7260000000000002e-06 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093551+0000", 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.177872775 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093682+0000", 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013097200000000001 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.093902+0000", 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000220268 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098082+0000", 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004179332 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099560+0000", 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001477843 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101720+0000", 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021601229999999999 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107021+0000", 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053014940000000003 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107077+0000", 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6048999999999998e-05 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107206+0000", 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000128582 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124111 13.10 13:0cb9c5a2:::benchmark_data_smithi029_495570_object29851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.035582981, 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0730542339999998, 2023-10-09T04:57:28.120 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124111 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034314+0000", 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.122 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034317+0000", 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7620000000000001e-06 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034326+0000", 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8629999999999997e-06 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034331+0000", 2023-10-09T04:57:28.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9269999999999996e-06 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034373+0000", 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1601999999999999e-05 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.034382+0000", 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0690000000000008e-06 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915702+0000", 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.88132001599999998 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915716+0000", 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4725999999999999e-05 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.094617+0000", 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1789001459999999 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.094748+0000", 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131156 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.094974+0000", 2023-10-09T04:57:28.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000226016 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098090+0000", 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003116438 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099588+0000", 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001498227 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101786+0000", 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021974389999999998 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107256+0000", 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054705459999999997 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107291+0000", 2023-10-09T04:57:28.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4193000000000003e-05 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107368+0000", 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7820999999999999e-05 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124120 13.10 13:0906ebd1:::benchmark_data_smithi029_495570_object29860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.129 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.039809+0000", 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.03008832099999, 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.067902353, 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124120 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039809+0000", 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039809+0000", 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039808+0000", 2023-10-09T04:57:28.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039813+0000", 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.13e-06 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039827+0000", 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4053000000000001e-05 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039835+0000", 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5580000000000003e-06 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039895+0000", 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0350999999999999e-05 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.039908+0000", 2023-10-09T04:57:28.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3132000000000001e-05 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915749+0000", 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.87584070400000003 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915758+0000", 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7099999999999996e-06 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095804+0000", 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1800463809999999 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.095929+0000", 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012519199999999999 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.096163+0000", 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023405100000000001 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098433+0000", 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002269573 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099642+0000", 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001209031 2023-10-09T04:57:28.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107450+0000", 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078077040000000004 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107545+0000", 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5366999999999997e-05 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107600+0000", 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5290000000000001e-05 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107711+0000", 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110988 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124129 13.10 13:0b5bf9cf:::benchmark_data_smithi029_495570_object29869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.02089779699998, 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0589564650000001, 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124129 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.048999+0000", 2023-10-09T04:57:28.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049002+0000", 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5130000000000001e-06 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049015+0000", 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.254e-05 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049019+0000", 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4880000000000001e-06 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049059+0000", 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9400999999999998e-05 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.049067+0000", 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9170000000000006e-06 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915784+0000", 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86671685399999998 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915792+0000", 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7450000000000001e-06 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.096835+0000", 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1810427530000001 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.096968+0000", 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013281400000000001 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.097195+0000", 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022751100000000001 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099508+0000", 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023125870000000001 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103860+0000", 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043520130000000001 2023-10-09T04:57:28.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107490+0000", 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003630365 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107778+0000", 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028744000000000003 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107827+0000", 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9236000000000001e-05 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107956+0000", 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012891700000000001 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.147 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124138 13.10 13:0d79deba:::benchmark_data_smithi029_495570_object29878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.01450179699998, 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0585424350000001, 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124138 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055395+0000", 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055399+0000", 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7149999999999997e-06 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055410+0000", 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1024e-05 2023-10-09T04:57:28.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055417+0000", 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9179999999999997e-06 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055458+0000", 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0586000000000001e-05 2023-10-09T04:57:28.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.055464+0000", 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5459999999999997e-06 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915818+0000", 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86035340100000002 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915827+0000", 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7239999999999998e-06 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098460+0000", 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1826330460000001 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098589+0000", 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012895300000000001 2023-10-09T04:57:28.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.098800+0000", 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021192199999999999 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101727+0000", 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029267469999999999 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108023+0000", 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062958520000000002 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108412+0000", 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038892199999999999 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113753+0000", 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053406740000000001 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113810+0000", 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7272000000000003e-05 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113938+0000", 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012794 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.156 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124143 13.10 13:0fbfbcd2:::benchmark_data_smithi029_495570_object29883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.059887+0000", 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.01001054099999, 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0542333460000002, 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.157 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124143 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059887+0000", 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059887+0000", 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059886+0000", 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059891+0000", 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8040000000000001e-06 2023-10-09T04:57:28.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059900+0000", 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4399999999999994e-06 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059907+0000", 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8650000000000003e-06 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059951+0000", 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3905000000000001e-05 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.059961+0000", 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9750000000000002e-06 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915850+0000", 2023-10-09T04:57:28.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85588862200000004 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915859+0000", 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0329999999999997e-06 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099666+0000", 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.183807608 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099748+0000", 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2101999999999994e-05 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.099925+0000", 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000177116 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102757+0000", 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028320070000000001 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108084+0000", 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053261519999999998 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108465+0000", 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00038118400000000002 2023-10-09T04:57:28.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.113987+0000", 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055226390000000002 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114037+0000", 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9552999999999998e-05 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114120+0000", 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3090000000000006e-05 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124173 13.10 13:0c5be9df:::benchmark_data_smithi029_495570_object29913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.085066+0000", 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 355.98483113600003, 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0292819039999999, 2023-10-09T04:57:28.166 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124173 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085066+0000", 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085066+0000", 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085065+0000", 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085069+0000", 2023-10-09T04:57:28.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9770000000000002e-06 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085079+0000", 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7419999999999992e-06 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085084+0000", 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8659999999999997e-06 2023-10-09T04:57:28.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085131+0000", 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6662000000000001e-05 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.085138+0000", 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9120000000000001e-06 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915882+0000", 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83074457800000001 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915893+0000", 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0726e-05 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.100647+0000", 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1847542309999999 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.100789+0000", 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014199300000000001 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101022+0000", 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023304200000000001 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103867+0000", 2023-10-09T04:57:28.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002845237 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108138+0000", 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042705249999999998 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108521+0000", 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000383373 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114177+0000", 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056552 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114229+0000", 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2278000000000003e-05 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114348+0000", 2023-10-09T04:57:28.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00011919 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124226 13.10 13:0ca3c500:::benchmark_data_smithi029_495570_object29966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.131197+0000", 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 355.93870010699999, 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9834688599999999, 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.175 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124226 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131197+0000", 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131197+0000", 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131196+0000", 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131201+0000", 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2409999999999997e-06 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131212+0000", 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1579999999999999e-05 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131221+0000", 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5250000000000005e-06 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131262+0000", 2023-10-09T04:57:28.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1656999999999998e-05 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.131270+0000", 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5569999999999998e-06 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915917+0000", 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78464664799999995 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915925+0000", 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9119999999999998e-06 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101819+0000", 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1858948819999999 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.101906+0000", 2023-10-09T04:57:28.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6304000000000006e-05 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102073+0000", 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016688400000000001 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.106960+0000", 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048878719999999997 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108190+0000", 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001229562 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108573+0000", 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000383011 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114500+0000", 2023-10-09T04:57:28.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059266700000000002 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114551+0000", 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1011999999999999e-05 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114666+0000", 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115226 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.183 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124272 13.10 13:0b31e4b8:::benchmark_data_smithi029_495570_object30012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.185036+0000", 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 355.88486124600001, 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.929835937, 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124272 2023-10-09T04:57:28.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185036+0000", 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185036+0000", 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185035+0000", 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185040+0000", 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6419999999999998e-06 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185052+0000", 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2128e-05 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185058+0000", 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185096+0000", 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8209e-05 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.185104+0000", 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8529999999999992e-06 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915950+0000", 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73084573500000005 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915955+0000", 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0930000000000002e-06 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102798+0000", 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1868433789999999 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.102941+0000", 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014232200000000001 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103162+0000", 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00022179900000000001 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107378+0000", 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042155780000000002 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108258+0000", 2023-10-09T04:57:28.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00087970400000000003 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108607+0000", 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034909900000000001 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114707+0000", 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061005299999999998 2023-10-09T04:57:28.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114757+0000", 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9990000000000001e-05 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114872+0000", 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000114527 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.192 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.193 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.193 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124309 13.10 13:096f70c8:::benchmark_data_smithi029_495570_object30049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.228544+0000", 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 355.84135310900001, 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.886540643, 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.194 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124309 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228544+0000", 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228544+0000", 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228543+0000", 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228548+0000", 2023-10-09T04:57:28.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4449999999999998e-06 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228559+0000", 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1073999999999999e-05 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228564+0000", 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3850000000000001e-06 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228624+0000", 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9373999999999998e-05 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.228637+0000", 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3105000000000001e-05 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915980+0000", 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.68734309599999999 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.915988+0000", 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3829999999999992e-06 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.103909+0000", 2023-10-09T04:57:28.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.187921037 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.104053+0000", 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014338099999999999 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.104268+0000", 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021500299999999999 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107383+0000", 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031151859999999998 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108309+0000", 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00092590800000000003 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108663+0000", 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035403000000000002 2023-10-09T04:57:28.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114912+0000", 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062496920000000003 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114963+0000", 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0989000000000003e-05 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115085+0000", 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000121316 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124317 13.10 13:098473ff:::benchmark_data_smithi029_495570_object30057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e367)", 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:32.240369+0000", 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 355.82952858900001, 2023-10-09T04:57:28.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.874924493, 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124317 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240369+0000", 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240369+0000", 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240368+0000", 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240373+0000", 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1200000000000001e-06 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240385+0000", 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1861e-05 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240391+0000", 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9109999999999998e-06 2023-10-09T04:57:28.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240434+0000", 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3297000000000002e-05 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.240443+0000", 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7900000000000005e-06 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.916024+0000", 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.675581608 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:32.916032+0000", 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9799999999999998e-06 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.104947+0000", 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1889144270000001 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.105088+0000", 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000140989 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.105302+0000", 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021454699999999999 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.107388+0000", 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020854229999999999 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.108363+0000", 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00097550399999999998 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.114429+0000", 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060662110000000002 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115131+0000", 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00070187500000000002 2023-10-09T04:57:28.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115182+0000", 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0986999999999998e-05 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.115293+0000", 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000110894 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124366 13.11 13:8cf34284:::benchmark_data_smithi029_495570_object30106:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:34.089772+0000", 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 353.98012561299998, 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8917853280000001, 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124366 2023-10-09T04:57:28.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089772+0000", 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089772+0000", 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089770+0000", 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089777+0000", 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6850000000000004e-06 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089791+0000", 2023-10-09T04:57:28.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4003e-05 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089801+0000", 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0427e-05 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089853+0000", 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2114999999999998e-05 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.089861+0000", 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6159999999999996e-06 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925469+0000", 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83560847900000002 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925484+0000", 2023-10-09T04:57:28.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4106e-05 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.977705+0000", 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0522217629999999 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.977842+0000", 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013667 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.978091+0000", 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00024873499999999998 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.979457+0000", 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001366183 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.979466+0000", 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1379999999999996e-06 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981363+0000", 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001896589 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981394+0000", 2023-10-09T04:57:28.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1341999999999999e-05 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981431+0000", 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7092999999999999e-05 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.981557+0000", 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012592500000000001 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.221 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124543 13.11 13:8cb72e80:::benchmark_data_smithi029_495570_object30283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:34.255796+0000", 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 353.81410132299999, 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.910879671, 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.222 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124543 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255796+0000", 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255796+0000", 2023-10-09T04:57:28.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255795+0000", 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255800+0000", 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8890000000000001e-06 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255811+0000", 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1471000000000001e-05 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255817+0000", 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4820000000000002e-06 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255865+0000", 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8118999999999998e-05 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.255874+0000", 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0999999999999993e-06 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925795+0000", 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.66992072000000003 2023-10-09T04:57:28.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925803+0000", 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4989999999999994e-06 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.991295+0000", 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0654915330000001 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.991388+0000", 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3405000000000004e-05 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.991539+0000", 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015072100000000001 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.994220+0000", 2023-10-09T04:57:28.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002680956 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.995297+0000", 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010776519999999999 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.999638+0000", 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043407749999999998 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.001196+0000", 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015575929999999999 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.001226+0000", 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9669999999999999e-05 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166676+0000", 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.165450035 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.46418.0:124583 13.11 13:8a76f3cb:::benchmark_data_smithi029_495570_object30323:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 353.77480890200002, 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.871790938, 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.231 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.46418", 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 124583 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295088+0000", 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295092+0000", 2023-10-09T04:57:28.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3189999999999996e-06 2023-10-09T04:57:28.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295102+0000", 2023-10-09T04:57:28.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0542e-05 2023-10-09T04:57:28.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:28.235 INFO:teuthology.orchestra.run.smithi177.stderr:no valid command found; 10 closest matches: 2023-10-09T04:57:28.235 INFO:teuthology.orchestra.run.smithi177.stderr:0 2023-10-09T04:57:28.235 INFO:teuthology.orchestra.run.smithi177.stderr:1 2023-10-09T04:57:28.235 INFO:teuthology.orchestra.run.smithi177.stderr:2 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:abort 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:assert 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:bluefs debug_inject_read_zeros 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:bluefs files list 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:bluefs stats 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:bluestore bluefs device info [] 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:config diff 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi177.stderr:admin_socket: invalid command 2023-10-09T04:57:28.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295107+0000", 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9490000000000004e-06 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295154+0000", 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6842999999999999e-05 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.295162+0000", 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0860000000000002e-06 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925832+0000", 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.63066938500000003 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:34.925840+0000", 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7269999999999996e-06 2023-10-09T04:57:28.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.992437+0000", 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.066596439 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.992547+0000", 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109912 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,7", 2023-10-09T04:57:28.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.992733+0000", 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018629900000000001 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.995172+0000", 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024386080000000001 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.997259+0000", 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020876649999999998 2023-10-09T04:57:28.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:35.999711+0000", 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024519519999999999 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166727+0000", 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16701532499999999 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166770+0000", 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3504000000000002e-05 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:51:36.166879+0000", 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109095 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79957 14.1 14:89e5ec84:::benchmark_data_smithi029_550670_object2346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:55.728853+0000", 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 32.341044447999998, 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.36544397099999998, 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79957 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.243 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.728853+0000", 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.728853+0000", 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.728850+0000", 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.728862+0000", 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.218e-05 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.728888+0000", 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5604000000000001e-05 2023-10-09T04:57:28.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.728899+0000", 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1024e-05 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.731033+0000", 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021342209999999999 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:28.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.753945+0000", 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022911714999999999 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.825723+0000", 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.071777707999999996 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.845188+0000", 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019465567999999999 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.014456+0000", 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16926798300000001 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.026363+0000", 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011906757 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T04:57:28.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.045289+0000", 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018925742999999998 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.073522+0000", 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028233181 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.075547+0000", 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020251190000000001 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.076236+0000", 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00068859100000000001 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.079588+0000", 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033522719999999999 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.094297+0000", 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014708847000000001 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79964 14.f 14:feb7e600:::benchmark_data_smithi029_550670_object2353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.026023+0000", 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 31.043874678000002, 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21200465399999999, 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:28.251 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79964 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.026023+0000", 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.026023+0000", 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.026020+0000", 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.026028+0000", 2023-10-09T04:57:28.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7989999999999994e-06 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.026043+0000", 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4487e-05 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.026054+0000", 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1542999999999999e-05 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.027784+0000", 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001730014 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.037290+0000", 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095060160000000008 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.070504+0000", 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033213452999999997 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.077478+0000", 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006974142 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.094929+0000", 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017450713 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.100161+0000", 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005232357 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,7", 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.110746+0000", 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010584641000000001 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.121110+0000", 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010364485 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.122938+0000", 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00182844 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.123531+0000", 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059237899999999997 2023-10-09T04:57:28.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.126504+0000", 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029728710000000002 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.238027+0000", 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11152347 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.259 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.260 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:28.260 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:28.260 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:57:28.313 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:28.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[216903]: osdmap e502: 8 total, 7 up, 5 in 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[216903]: failed to encode map e502 with expected crc 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[216903]: failed to encode map e502 with expected crc 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[223858]: osdmap e502: 8 total, 7 up, 5 in 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[223858]: failed to encode map e502 with expected crc 2023-10-09T04:57:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:28 smithi029 ceph-mon[223858]: failed to encode map e502 with expected crc 2023-10-09T04:57:28.429 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:28 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:28.430 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:28 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:57:28.430 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:28 smithi177 ceph-mon[195797]: osdmap e502: 8 total, 7 up, 5 in 2023-10-09T04:57:28.430 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:28 smithi177 ceph-mon[195797]: failed to encode map e502 with expected crc 2023-10-09T04:57:28.430 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:28 smithi177 ceph-mon[195797]: failed to encode map e502 with expected crc 2023-10-09T04:57:28.534 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:28.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:28.638 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:28.680 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:28.745 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:28.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:28.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:28.892 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:29.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:29 smithi029 ceph-mon[223858]: pgmap v1727: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85144/225837 objects degraded (37.702%); 5486/225837 objects misplaced (2.429%); 9.3 MiB/s, 165 objects/s recovering 2023-10-09T04:57:29.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:29.106 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:29.231 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:29.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:29.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:29.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T04:57:29.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:29.369 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:29 smithi177 ceph-mon[195797]: pgmap v1727: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85144/225837 objects degraded (37.702%); 5486/225837 objects misplaced (2.429%); 9.3 MiB/s, 165 objects/s recovering 2023-10-09T04:57:29.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:29.370 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:29.371 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:29.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:29 smithi029 ceph-mon[216903]: pgmap v1727: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85144/225837 objects degraded (37.702%); 5486/225837 objects misplaced (2.429%); 9.3 MiB/s, 165 objects/s recovering 2023-10-09T04:57:29.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:29.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:29.436 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:29.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T04:57:29.503 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-10-09T04:57:29.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:29.592 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:29.684 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:29.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:29.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:29.796 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:29.835 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:29.848 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:29.934 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:29.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:30.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:30.083 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:30.167 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:30.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:30.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:30.273 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:57:30.302 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:30.353 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:30.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:30.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:30.502 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:30.690 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:30.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:30.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:30.823 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:30 smithi177 ceph-mon[195797]: pgmap v1728: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85144/225837 objects degraded (37.702%); 5486/225837 objects misplaced (2.429%); 7.7 MiB/s, 138 objects/s recovering 2023-10-09T04:57:30.823 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:30 smithi177 ceph-mon[195797]: osdmap e503: 8 total, 7 up, 5 in 2023-10-09T04:57:30.823 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:30 smithi177 ceph-mon[195797]: failed to encode map e503 with expected crc 2023-10-09T04:57:30.824 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:30 smithi177 ceph-mon[195797]: failed to encode map e503 with expected crc 2023-10-09T04:57:30.824 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:30 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:57:30.826 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:30.884 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:30.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:30.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:30.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[223858]: pgmap v1728: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85144/225837 objects degraded (37.702%); 5486/225837 objects misplaced (2.429%); 7.7 MiB/s, 138 objects/s recovering 2023-10-09T04:57:30.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[223858]: osdmap e503: 8 total, 7 up, 5 in 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[223858]: failed to encode map e503 with expected crc 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[223858]: failed to encode map e503 with expected crc 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[216903]: pgmap v1728: 99 pgs: 1 activating+degraded, 3 remapped+peering, 12 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 85144/225837 objects degraded (37.702%); 5486/225837 objects misplaced (2.429%); 7.7 MiB/s, 138 objects/s recovering 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[216903]: osdmap e503: 8 total, 7 up, 5 in 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[216903]: failed to encode map e503 with expected crc 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[216903]: failed to encode map e503 with expected crc 2023-10-09T04:57:30.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:30 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:57:30.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:31.021 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:31.071 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:31.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:31.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:31.214 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:31.542 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:31.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:31.642 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:31.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:31.678 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:31.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:31.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:31.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:31.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:31 smithi029 ceph-mon[216903]: osdmap e504: 8 total, 7 up, 5 in 2023-10-09T04:57:31.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:31 smithi029 ceph-mon[216903]: failed to encode map e504 with expected crc 2023-10-09T04:57:31.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 85144/225837 objects degraded (37.702%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:31.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:31 smithi029 ceph-mon[223858]: osdmap e504: 8 total, 7 up, 5 in 2023-10-09T04:57:31.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:31 smithi029 ceph-mon[223858]: failed to encode map e504 with expected crc 2023-10-09T04:57:31.871 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 85144/225837 objects degraded (37.702%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:31.875 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:31 smithi177 ceph-mon[195797]: osdmap e504: 8 total, 7 up, 5 in 2023-10-09T04:57:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:31 smithi177 ceph-mon[195797]: failed to encode map e504 with expected crc 2023-10-09T04:57:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 85144/225837 objects degraded (37.702%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:32.131 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:32.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:32.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79909 14.3 14:c89e223c:::benchmark_data_smithi029_550670_object2298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:19.249452+0000", 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 72.953445172000002, 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 72.953471190000002, 2023-10-09T04:57:32.206 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79909 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249452+0000", 2023-10-09T04:57:32.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249452+0000", 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249450+0000", 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249458+0000", 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8399999999999995e-06 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249476+0000", 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7584e-05 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249489+0000", 2023-10-09T04:57:32.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3209e-05 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.274291+0000", 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024802175999999999 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.280783+0000", 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064916069999999999 2023-10-09T04:57:32.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.391863+0000", 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11108014500000001 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.402525+0000", 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010662081 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853378+0000", 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45085308099999999 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853385+0000", 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4819999999999999e-06 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609740+0000", 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75635546200000003 2023-10-09T04:57:32.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609746+0000", 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4909999999999996e-06 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479692+0000", 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869946210999998 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.213 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479700+0000", 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3299999999999999e-06 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810285+0000", 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330585087999999 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810293+0000", 2023-10-09T04:57:32.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4039999999999998e-06 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:16.477201+0000", 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 75.725695838999997, 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 75.725824227000004, 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.215 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79757 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477201+0000", 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477201+0000", 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477199+0000", 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477210+0000", 2023-10-09T04:57:32.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0400999999999999e-05 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477228+0000", 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8615e-05 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477244+0000", 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5529e-05 2023-10-09T04:57:32.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.576458+0000", 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099213778000000002 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.589138+0000", 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012680442 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.686934+0000", 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.097796147 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.690497+0000", 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035627340000000001 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.725047+0000", 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0345502309999999 2023-10-09T04:57:32.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.739987+0000", 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014940142999999999 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.753857+0000", 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013869105 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.221 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.851838+0000", 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.097981357000000005 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.851856+0000", 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7640999999999999e-05 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609540+0000", 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75768437499999997 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609549+0000", 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7979999999999995e-06 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.347755+0000", 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.738205788999998 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for readable", 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.372463+0000", 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024708866999999999 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.457625+0000", 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.085161613999999997 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.457639+0000", 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3849000000000001e-05 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.783127+0000", 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.325488567000001 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.783140+0000", 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2784e-05 2023-10-09T04:57:32.225 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79758 14.3 14:c150ef7d:::benchmark_data_smithi029_550670_object2147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:16.495699+0000", 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 75.707198035999994, 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 75.707423367000004, 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.226 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79758 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495699+0000", 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495699+0000", 2023-10-09T04:57:32.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495698+0000", 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495704+0000", 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4579999999999998e-06 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495717+0000", 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2663000000000001e-05 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495723+0000", 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5640000000000002e-06 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.691098+0000", 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19537406900000001 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.698472+0000", 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073741989999999997 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.760536+0000", 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062063859999999998 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.764106+0000", 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035706599999999998 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853578+0000", 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.089471852 2023-10-09T04:57:32.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853584+0000", 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0599999999999996e-06 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.619983+0000", 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76639892300000001 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.620016+0000", 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2444000000000003e-05 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479878+0000", 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.859862771000003 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479887+0000", 2023-10-09T04:57:32.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0559999999999994e-06 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810492+0000", 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330605093000001 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810503+0000", 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0464e-05 2023-10-09T04:57:32.234 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79820 14.3 14:c9f76994:::benchmark_data_smithi029_550670_object2209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.859490+0000", 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 74.343406971999997, 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 74.343708668000005, 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.235 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79820 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859490+0000", 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859490+0000", 2023-10-09T04:57:32.236 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859489+0000", 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859495+0000", 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8369999999999999e-06 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.237 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859511+0000", 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.579e-05 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859519+0000", 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6799999999999993e-06 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.238 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.062278+0000", 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20275945500000001 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.082170+0000", 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019891170999999999 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.245916+0000", 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.163746419 2023-10-09T04:57:32.239 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.255117+0000", 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092011539999999996 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853194+0000", 2023-10-09T04:57:32.240 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5980772079999999 2023-10-09T04:57:32.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.241 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.241 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.241 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853202+0000", 2023-10-09T04:57:32.241 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.418e-06 2023-10-09T04:57:32.241 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.242 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.242 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609644+0000", 2023-10-09T04:57:32.242 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75644263599999995 2023-10-09T04:57:32.242 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.242 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609650+0000", 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1390000000000003e-06 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.243 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479565+0000", 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869915575999997 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479574+0000", 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4810000000000006e-06 2023-10-09T04:57:32.244 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810173+0000", 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330599312 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.245 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.246 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810180+0000", 2023-10-09T04:57:32.246 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1199999999999996e-06 2023-10-09T04:57:32.246 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.246 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.246 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79804 14.3 14:c249cac1:::benchmark_data_smithi029_550670_object2193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.538618+0000", 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 74.664279160000007, 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 74.664659431999993, 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.247 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79804 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538618+0000", 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.248 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538618+0000", 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538617+0000", 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538622+0000", 2023-10-09T04:57:32.249 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3639999999999998e-06 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538633+0000", 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0539e-05 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538641+0000", 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4940000000000003e-06 2023-10-09T04:57:32.250 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.653399+0000", 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.114757684 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.667887+0000", 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014487718 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.735134+0000", 2023-10-09T04:57:32.251 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067247468000000005 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.742578+0000", 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074440770000000003 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853537+0000", 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.110958981 2023-10-09T04:57:32.252 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853543+0000", 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.302e-06 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609853+0000", 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75630989500000001 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.253 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609862+0000", 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2040000000000003e-06 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479818+0000", 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869955306999998 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.254 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479826+0000", 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4160000000000004e-06 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810426+0000", 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330599409 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810436+0000", 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0859999999999999e-05 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.255 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79892 14.3 14:c950d3cc:::benchmark_data_smithi029_550670_object2281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:18.909858+0000", 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 73.293039295, 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 73.293496223999995, 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.256 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79892 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909858+0000", 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909858+0000", 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.257 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909856+0000", 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909865+0000", 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4330000000000004e-06 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.258 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909884+0000", 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9949000000000002e-05 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909894+0000", 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0299999999999999e-06 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.168818+0000", 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25892472300000002 2023-10-09T04:57:32.259 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.177007+0000", 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081884290000000005 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.261383+0000", 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.084376240000000005 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.260 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.265033+0000", 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003650412 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853319+0000", 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58828595299999997 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853336+0000", 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7039999999999999e-05 2023-10-09T04:57:32.261 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609718+0000", 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.756382153 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609724+0000", 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9900000000000002e-06 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.262 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479650+0000", 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869925195999997 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479658+0000", 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2889999999999998e-06 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810246+0000", 2023-10-09T04:57:32.263 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330588298 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810251+0000", 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3259999999999997e-06 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79904 14.3 14:cad8faf7:::benchmark_data_smithi029_550670_object2293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:19.120909+0000", 2023-10-09T04:57:32.264 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 73.081988503999995, 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 73.082522136999998, 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79904 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.265 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120909+0000", 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120909+0000", 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.266 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120907+0000", 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120914+0000", 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2509999999999997e-06 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120929+0000", 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4507e-05 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.267 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120936+0000", 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4959999999999999e-06 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.265693+0000", 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14475647 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.268 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.272235+0000", 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006542254 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.315620+0000", 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043384616000000001 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.325545+0000", 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099251589999999994 2023-10-09T04:57:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853098+0000", 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.52755283399999997 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853106+0000", 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5140000000000001e-06 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.270 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609604+0000", 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75649816999999997 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609609+0000", 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1209999999999998e-06 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.271 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479481+0000", 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869871203000002 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479489+0000", 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5120000000000008e-06 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810097+0000", 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330607722 2023-10-09T04:57:32.272 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810106+0000", 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6710000000000005e-06 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79976 14.3 14.6cdb3e03 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.318902+0000", 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 34.883994797, 2023-10-09T04:57:32.273 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.884611612, 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79976 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.274 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318902+0000", 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318902+0000", 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318900+0000", 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.275 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318911+0000", 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0587999999999999e-05 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318927+0000", 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5954e-05 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.276 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318946+0000", 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9470000000000002e-05 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810552+0000", 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4916053680000001 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810560+0000", 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4349999999999997e-06 2023-10-09T04:57:32.277 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79978 14.3 14.9588cee3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.381276+0000", 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 34.821621297, 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.822282303000001, 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.278 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79978 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381276+0000", 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.279 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381276+0000", 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381273+0000", 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381284+0000", 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0746e-05 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381298+0000", 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3678e-05 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381308+0000", 2023-10-09T04:57:32.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0115999999999999e-05 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810640+0000", 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4293319370000006 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810645+0000", 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0540000000000002e-06 2023-10-09T04:57:32.282 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79872 14.3 14:ca72c40b:::benchmark_data_smithi029_550670_object2261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:18.684575+0000", 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 73.518321841000002, 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 73.519023235999995, 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.283 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79872 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684575+0000", 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684575+0000", 2023-10-09T04:57:32.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684573+0000", 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684582+0000", 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1499999999999999e-06 2023-10-09T04:57:32.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684599+0000", 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6986999999999998e-05 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684608+0000", 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2199999999999998e-06 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.838960+0000", 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15435184900000001 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.847114+0000", 2023-10-09T04:57:32.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.008154293 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.944551+0000", 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.097437539000000004 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.953685+0000", 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091330890000000005 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853152+0000", 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.89946761799999997 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853160+0000", 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4800000000000004e-06 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609625+0000", 2023-10-09T04:57:32.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75646519700000003 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609630+0000", 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1309999999999997e-06 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479522+0000", 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869892389999997 2023-10-09T04:57:32.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479531+0000", 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7630000000000006e-06 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810134+0000", 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330602589 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810139+0000", 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79787 14.3 14:c432b4dc:::benchmark_data_smithi029_550670_object2176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.261604+0000", 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 74.941293314999996, 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 74.942071213999995, 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.292 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79787 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261604+0000", 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261604+0000", 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261602+0000", 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261609+0000", 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:57:32.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261625+0000", 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5787999999999999e-05 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261634+0000", 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.350021+0000", 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.088387458000000002 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.357600+0000", 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075785660000000001 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.425983+0000", 2023-10-09T04:57:32.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.068383474999999999 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.431794+0000", 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058106700000000004 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853430+0000", 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4216359760000001 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853444+0000", 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3927000000000001e-05 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609761+0000", 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75631734699999997 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609767+0000", 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3410000000000002e-06 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479734+0000", 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869966953999999 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479742+0000", 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2160000000000006e-06 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810324+0000", 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330582115 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810331+0000", 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2300000000000002e-06 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.300 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79873 14.3 14:cb79ed6c:::benchmark_data_smithi029_550670_object2262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:18.688707+0000", 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 73.514190252000006, 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 73.515054620000001, 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.301 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79873 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688707+0000", 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688707+0000", 2023-10-09T04:57:32.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688705+0000", 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688716+0000", 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0555e-05 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688731+0000", 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5116000000000001e-05 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688741+0000", 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7950000000000002e-06 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.848701+0000", 2023-10-09T04:57:32.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.159960773 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.866178+0000", 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017476926 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.010337+0000", 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.144158444 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.021015+0000", 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0106788 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853066+0000", 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83205003899999996 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853076+0000", 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0491e-05 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609585+0000", 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75650865300000003 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609590+0000", 2023-10-09T04:57:32.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1630000000000004e-06 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479437+0000", 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869847352000001 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479446+0000", 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7739999999999993e-06 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810060+0000", 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330614213 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810070+0000", 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6760000000000002e-06 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79774 14.3 14:c3f22083:::benchmark_data_smithi029_550670_object2163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:16.958069+0000", 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 75.244828248000005, 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 75.245768869000003, 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79774 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.310 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958069+0000", 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958069+0000", 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958067+0000", 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958075+0000", 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6380000000000004e-06 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958090+0000", 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4800999999999999e-05 2023-10-09T04:57:32.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958097+0000", 2023-10-09T04:57:32.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2309999999999999e-06 2023-10-09T04:57:32.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.002573+0000", 2023-10-09T04:57:32.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044476104000000002 2023-10-09T04:57:32.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.014293+0000", 2023-10-09T04:57:32.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01172015 2023-10-09T04:57:32.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.139819+0000", 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.125526362 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.143287+0000", 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034675069999999999 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853492+0000", 2023-10-09T04:57:32.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7102050950000001 2023-10-09T04:57:32.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853502+0000", 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0429e-05 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609804+0000", 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75630144899999996 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609816+0000", 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2148e-05 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479775+0000", 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869958529999998 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479783+0000", 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6400000000000003e-06 2023-10-09T04:57:32.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810362+0000", 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330579004000001 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810370+0000", 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5349999999999999e-06 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79817 14.3 14:ce68fda3:::benchmark_data_smithi029_550670_object2206:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.774689+0000", 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 74.428208275000003, 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 74.429225098000003, 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79817 2023-10-09T04:57:32.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774689+0000", 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774689+0000", 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774687+0000", 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774695+0000", 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9279999999999993e-06 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774710+0000", 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4842e-05 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774720+0000", 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0339e-05 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.028560+0000", 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25383992999999999 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.039867+0000", 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011306327 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.124041+0000", 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.084174914000000003 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.129614+0000", 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055721319999999996 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.852984+0000", 2023-10-09T04:57:32.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7233700650000001 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.852998+0000", 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4202999999999999e-05 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609565+0000", 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75656672000000003 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609570+0000", 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4e-06 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479383+0000", 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869812754000002 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479399+0000", 2023-10-09T04:57:32.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5804999999999999e-05 2023-10-09T04:57:32.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810001+0000", 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330602220999999 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810026+0000", 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5031000000000001e-05 2023-10-09T04:57:32.331 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79818 14.3 14:c9aea1d4:::benchmark_data_smithi029_550670_object2207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:32.332 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 74.428159995000001, 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 74.4292528, 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:32.333 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79818 2023-10-09T04:57:32.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.334 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:32.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:32.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:32.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:32.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:32.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:32.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774740+0000", 2023-10-09T04:57:32.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7460000000000001e-06 2023-10-09T04:57:32.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774748+0000", 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7879999999999995e-06 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774754+0000", 2023-10-09T04:57:32.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.913e-06 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.041445+0000", 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26669056299999999 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.060426+0000", 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018981576 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.179168+0000", 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.118741192 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.185905+0000", 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067369719999999999 2023-10-09T04:57:32.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853242+0000", 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6673378809999999 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853261+0000", 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8505999999999999e-05 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609682+0000", 2023-10-09T04:57:32.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75642128600000003 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609696+0000", 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4161999999999999e-05 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479607+0000", 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869910902000001 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479616+0000", 2023-10-09T04:57:32.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3720000000000005e-06 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810210+0000", 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330594581 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:32.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810215+0000", 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4100000000000001e-06 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.347 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:32.348 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:57:32.348 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 15 2023-10-09T04:57:32.348 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:57:32.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:32.390 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:32 smithi029 ceph-mon[216903]: pgmap v1731: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84792/225834 objects degraded (37.546%); 5486/225834 objects misplaced (2.429%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:57:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:32 smithi029 ceph-mon[216903]: Health check update: 7 slow ops, oldest one blocked for 55 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:57:32.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:32 smithi029 ceph-mon[223858]: pgmap v1731: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84792/225834 objects degraded (37.546%); 5486/225834 objects misplaced (2.429%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:57:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:32 smithi029 ceph-mon[223858]: Health check update: 7 slow ops, oldest one blocked for 55 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:57:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:32 smithi177 ceph-mon[195797]: pgmap v1731: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84792/225834 objects degraded (37.546%); 5486/225834 objects misplaced (2.429%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:57:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:32 smithi177 ceph-mon[195797]: Health check update: 7 slow ops, oldest one blocked for 55 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:57:33.098 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:33.100 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:57:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:57:32] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-10-09T04:57:33.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:33.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:33.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:33.231 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:33.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:33.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:33.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:33.421 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:33.524 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:33.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:33.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:33.629 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:33.660 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:33.928 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T04:57:34.104 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:34.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:34.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:34.208 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:34.252 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:34.369 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:34.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:34.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:34.412 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:34.475 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:34.503 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:34.505 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:57:34.574 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:34.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:34.683 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:34.715 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:34.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:34.768 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:34 smithi177 ceph-mon[195797]: pgmap v1732: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84730/225831 objects degraded (37.519%); 5486/225831 objects misplaced (2.429%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:57:34.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:34.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:34.870 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:34.904 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:34.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:34 smithi029 ceph-mon[216903]: pgmap v1732: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84730/225831 objects degraded (37.519%); 5486/225831 objects misplaced (2.429%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:57:34.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:34 smithi029 ceph-mon[223858]: pgmap v1732: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84730/225831 objects degraded (37.519%); 5486/225831 objects misplaced (2.429%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:57:34.967 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:34.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:35.038 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:57:35.074 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:35.115 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:35.225 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:35.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:35.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:35.330 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:35.363 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:35.419 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:35.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:35.523 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:35.559 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:35.592 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4056419816' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:35.592 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:35.598 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:57:35.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:35.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:35.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:35.862 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4056419816' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:35.862 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:35.865 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:35.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4056419816' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:35.962 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:57:36.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:35 smithi177 conmon[372243]: 2023-10-09T04:57:35.959+0000 7f082047b540 -1 osd.4 472 log_to_monitors true 2023-10-09T04:57:36.323 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:36.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:36.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:36.457 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:36.505 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:36.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:36.609 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:36.637 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:36.685 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:36.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: pgmap v1733: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84357/225825 objects degraded (37.355%); 5486/225825 objects misplaced (2.429%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T04:57:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:57:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: failed to encode map e505 with expected crc 2023-10-09T04:57:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: osdmap e505: 8 total, 7 up, 5 in 2023-10-09T04:57:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84357/225825 objects degraded (37.355%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: from='osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:57:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:36 smithi177 ceph-mon[195797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:57:36.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:36.824 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:36.873 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:36.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:36.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: pgmap v1733: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84357/225825 objects degraded (37.355%); 5486/225825 objects misplaced (2.429%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: failed to encode map e505 with expected crc 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: osdmap e505: 8 total, 7 up, 5 in 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84357/225825 objects degraded (37.355%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: from='osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[216903]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:57:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: pgmap v1733: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84357/225825 objects degraded (37.355%); 5486/225825 objects misplaced (2.429%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: failed to encode map e505 with expected crc 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: osdmap e505: 8 total, 7 up, 5 in 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84357/225825 objects degraded (37.355%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: from='osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:57:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:36 smithi029 ceph-mon[223858]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T04:57:36.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:37.017 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:37.089 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:37.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:37.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:37.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:37.232 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:37.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 04:57:36 smithi177 conmon[372243]: 2023-10-09T04:57:36.820+0000 7f081125a700 -1 osd.4 472 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T04:57:37.287 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:37.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:37.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:37.419 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:37.435 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79909 14.3 14:c89e223c:::benchmark_data_smithi029_550670_object2298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:19.249452+0000", 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.184028894999997, 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 78.184050561000006, 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.436 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79909 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249452+0000", 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249452+0000", 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249450+0000", 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249458+0000", 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8399999999999995e-06 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249476+0000", 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7584e-05 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.249489+0000", 2023-10-09T04:57:37.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3209e-05 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.274291+0000", 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024802175999999999 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.280783+0000", 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064916069999999999 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.391863+0000", 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11108014500000001 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.402525+0000", 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010662081 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853378+0000", 2023-10-09T04:57:37.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45085308099999999 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853385+0000", 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4819999999999999e-06 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609740+0000", 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75635546200000003 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609746+0000", 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4909999999999996e-06 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479692+0000", 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869946210999998 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479700+0000", 2023-10-09T04:57:37.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3299999999999999e-06 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810285+0000", 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330585087999999 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810293+0000", 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4039999999999998e-06 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.444 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79757 14.3 14:cd374ce1:::benchmark_data_smithi029_550670_object2146:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:16.477201+0000", 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 80.956279562000006, 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 80.956390135999996, 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.445 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79757 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477201+0000", 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477201+0000", 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477199+0000", 2023-10-09T04:57:37.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477210+0000", 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0400999999999999e-05 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477228+0000", 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8615e-05 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.477244+0000", 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5529e-05 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.576458+0000", 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099213778000000002 2023-10-09T04:57:37.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.589138+0000", 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012680442 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.686934+0000", 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.097796147 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.690497+0000", 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035627340000000001 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.725047+0000", 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0345502309999999 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.739987+0000", 2023-10-09T04:57:37.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014940142999999999 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.753857+0000", 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013869105 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.851838+0000", 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.097981357000000005 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.851856+0000", 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7640999999999999e-05 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609540+0000", 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75768437499999997 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609549+0000", 2023-10-09T04:57:37.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7979999999999995e-06 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.347755+0000", 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.738205788999998 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for readable", 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.372463+0000", 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024708866999999999 2023-10-09T04:57:37.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.457625+0000", 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.085161613999999997 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.457639+0000", 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3849000000000001e-05 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.783127+0000", 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.325488567000001 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.783140+0000", 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2784e-05 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79758 14.3 14:c150ef7d:::benchmark_data_smithi029_550670_object2147:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:16.495699+0000", 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 80.937781759000003, 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 80.937989314999996, 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79758 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495699+0000", 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495699+0000", 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495698+0000", 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495704+0000", 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4579999999999998e-06 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495717+0000", 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2663000000000001e-05 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.495723+0000", 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5640000000000002e-06 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.691098+0000", 2023-10-09T04:57:37.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19537406900000001 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.698472+0000", 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073741989999999997 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.760536+0000", 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062063859999999998 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.764106+0000", 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035706599999999998 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853578+0000", 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.089471852 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853584+0000", 2023-10-09T04:57:37.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0599999999999996e-06 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.619983+0000", 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76639892300000001 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.620016+0000", 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2444000000000003e-05 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479878+0000", 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.859862771000003 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479887+0000", 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0559999999999994e-06 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810492+0000", 2023-10-09T04:57:37.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330605093000001 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810503+0000", 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0464e-05 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.465 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79820 14.3 14:c9f76994:::benchmark_data_smithi029_550670_object2209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.859490+0000", 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 79.573990695000006, 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 79.574283726999994, 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79820 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859490+0000", 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859490+0000", 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859489+0000", 2023-10-09T04:57:37.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859495+0000", 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8369999999999999e-06 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859511+0000", 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.579e-05 2023-10-09T04:57:37.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.859519+0000", 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6799999999999993e-06 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.062278+0000", 2023-10-09T04:57:37.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20275945500000001 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.082170+0000", 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019891170999999999 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.245916+0000", 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.163746419 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.255117+0000", 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092011539999999996 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853194+0000", 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5980772079999999 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853202+0000", 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.418e-06 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609644+0000", 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75644263599999995 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609650+0000", 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1390000000000003e-06 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479565+0000", 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869915575999997 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479574+0000", 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4810000000000006e-06 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810173+0000", 2023-10-09T04:57:37.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330599312 2023-10-09T04:57:37.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810180+0000", 2023-10-09T04:57:37.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1199999999999996e-06 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79804 14.3 14:c249cac1:::benchmark_data_smithi029_550670_object2193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.475 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.538618+0000", 2023-10-09T04:57:37.476 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 79.894862883000002, 2023-10-09T04:57:37.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 79.895235322000005, 2023-10-09T04:57:37.476 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.476 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.476 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79804 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538618+0000", 2023-10-09T04:57:37.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538618+0000", 2023-10-09T04:57:37.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538617+0000", 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538622+0000", 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3639999999999998e-06 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538633+0000", 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0539e-05 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.538641+0000", 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4940000000000003e-06 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.653399+0000", 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.114757684 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.667887+0000", 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014487718 2023-10-09T04:57:37.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.735134+0000", 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.067247468000000005 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.742578+0000", 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074440770000000003 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853537+0000", 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.110958981 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853543+0000", 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.302e-06 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609853+0000", 2023-10-09T04:57:37.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75630989500000001 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609862+0000", 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2040000000000003e-06 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479818+0000", 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869955306999998 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479826+0000", 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4160000000000004e-06 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810426+0000", 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330599409 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810436+0000", 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0859999999999999e-05 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79892 14.3 14:c950d3cc:::benchmark_data_smithi029_550670_object2281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:18.909858+0000", 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.523623017999995, 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 78.524072305000004, 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.486 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79892 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909858+0000", 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909858+0000", 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909856+0000", 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909865+0000", 2023-10-09T04:57:37.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4330000000000004e-06 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909884+0000", 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9949000000000002e-05 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.909894+0000", 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0299999999999999e-06 2023-10-09T04:57:37.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.168818+0000", 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25892472300000002 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.177007+0000", 2023-10-09T04:57:37.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081884290000000005 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.261383+0000", 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.084376240000000005 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.265033+0000", 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003650412 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853319+0000", 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.58828595299999997 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853336+0000", 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7039999999999999e-05 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609718+0000", 2023-10-09T04:57:37.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.756382153 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609724+0000", 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9900000000000002e-06 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479650+0000", 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869925195999997 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479658+0000", 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2889999999999998e-06 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810246+0000", 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330588298 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810251+0000", 2023-10-09T04:57:37.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3259999999999997e-06 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79904 14.3 14:cad8faf7:::benchmark_data_smithi029_550670_object2293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:19.120909+0000", 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.312572227000004, 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 78.313109822000001, 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.495 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79904 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120909+0000", 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.496 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120909+0000", 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120907+0000", 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120914+0000", 2023-10-09T04:57:37.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2509999999999997e-06 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120929+0000", 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4507e-05 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.120936+0000", 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4959999999999999e-06 2023-10-09T04:57:37.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.265693+0000", 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14475647 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.272235+0000", 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006542254 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.315620+0000", 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043384616000000001 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.325545+0000", 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099251589999999994 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853098+0000", 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.52755283399999997 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853106+0000", 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5140000000000001e-06 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609604+0000", 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75649816999999997 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609609+0000", 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1209999999999998e-06 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479481+0000", 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869871203000002 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479489+0000", 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5120000000000008e-06 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810097+0000", 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330607722 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810106+0000", 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6710000000000005e-06 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79976 14.3 14.6cdb3e03 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:57:37.504 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.318902+0000", 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 40.114578520000002, 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 40.115198386000003, 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79976 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318902+0000", 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318902+0000", 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318900+0000", 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318911+0000", 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0587999999999999e-05 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318927+0000", 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5954e-05 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.318946+0000", 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9470000000000002e-05 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810552+0000", 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4916053680000001 2023-10-09T04:57:37.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810560+0000", 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4349999999999997e-06 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79978 14.3 14.9588cee3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.381276+0000", 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 40.052205020000002, 2023-10-09T04:57:37.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 40.052868320000002, 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79978 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.510 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381276+0000", 2023-10-09T04:57:37.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381276+0000", 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381273+0000", 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:57:37.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381284+0000", 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0746e-05 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381298+0000", 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3678e-05 2023-10-09T04:57:37.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.381308+0000", 2023-10-09T04:57:37.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0115999999999999e-05 2023-10-09T04:57:37.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810640+0000", 2023-10-09T04:57:37.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4293319370000006 2023-10-09T04:57:37.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810645+0000", 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0540000000000002e-06 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79872 14.3 14:ca72c40b:::benchmark_data_smithi029_550670_object2261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:18.684575+0000", 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.748905563999998, 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 78.749618373000004, 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79872 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684575+0000", 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684575+0000", 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684573+0000", 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684582+0000", 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1499999999999999e-06 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684599+0000", 2023-10-09T04:57:37.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6986999999999998e-05 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.684608+0000", 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2199999999999998e-06 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.838960+0000", 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15435184900000001 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.847114+0000", 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.008154293 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.944551+0000", 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.097437539000000004 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.953685+0000", 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091330890000000005 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853152+0000", 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.89946761799999997 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853160+0000", 2023-10-09T04:57:37.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4800000000000004e-06 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609625+0000", 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75646519700000003 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609630+0000", 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1309999999999997e-06 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479522+0000", 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869892389999997 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479531+0000", 2023-10-09T04:57:37.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7630000000000006e-06 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810134+0000", 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330602589 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810139+0000", 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6049999999999997e-06 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.528 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79787 14.3 14:c432b4dc:::benchmark_data_smithi029_550670_object2176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.261604+0000", 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 80.171877038000005, 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 80.172674060000006, 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.529 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79787 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261604+0000", 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261604+0000", 2023-10-09T04:57:37.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261602+0000", 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261609+0000", 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3799999999999996e-06 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261625+0000", 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5787999999999999e-05 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.261634+0000", 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6489999999999997e-06 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.350021+0000", 2023-10-09T04:57:37.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.088387458000000002 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.357600+0000", 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075785660000000001 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.425983+0000", 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.068383474999999999 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.431794+0000", 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058106700000000004 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853430+0000", 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4216359760000001 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853444+0000", 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3927000000000001e-05 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609761+0000", 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75631734699999997 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609767+0000", 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3410000000000002e-06 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479734+0000", 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869966953999999 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479742+0000", 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2160000000000006e-06 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810324+0000", 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330582115 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810331+0000", 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2300000000000002e-06 2023-10-09T04:57:37.537 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79873 14.3 14:cb79ed6c:::benchmark_data_smithi029_550670_object2262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:18.688707+0000", 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 78.744773975000001, 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 78.745647453000004, 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.538 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79873 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688707+0000", 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688707+0000", 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688705+0000", 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688716+0000", 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0555e-05 2023-10-09T04:57:37.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688731+0000", 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5116000000000001e-05 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.688741+0000", 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7950000000000002e-06 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.848701+0000", 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.159960773 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.866178+0000", 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017476926 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.010337+0000", 2023-10-09T04:57:37.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.144158444 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.021015+0000", 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0106788 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853066+0000", 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.83205003899999996 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853076+0000", 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0491e-05 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609585+0000", 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75650865300000003 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609590+0000", 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1630000000000004e-06 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479437+0000", 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869847352000001 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479446+0000", 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7739999999999993e-06 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810060+0000", 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330614213 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810070+0000", 2023-10-09T04:57:37.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6760000000000002e-06 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79774 14.3 14:c3f22083:::benchmark_data_smithi029_550670_object2163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:16.958069+0000", 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 80.475411971, 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 80.476361173000001, 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.547 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79774 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958069+0000", 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958069+0000", 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958067+0000", 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958075+0000", 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6380000000000004e-06 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958090+0000", 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4800999999999999e-05 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:16.958097+0000", 2023-10-09T04:57:37.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2309999999999999e-06 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.002573+0000", 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044476104000000002 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.014293+0000", 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01172015 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.139819+0000", 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.125526362 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.143287+0000", 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034675069999999999 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853492+0000", 2023-10-09T04:57:37.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7102050950000001 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853502+0000", 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0429e-05 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609804+0000", 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75630144899999996 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609816+0000", 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2148e-05 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479775+0000", 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869958529999998 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.554 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479783+0000", 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6400000000000003e-06 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810362+0000", 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330579004000001 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.555 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810370+0000", 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5349999999999999e-06 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79817 14.3 14:ce68fda3:::benchmark_data_smithi029_550670_object2206:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.774689+0000", 2023-10-09T04:57:37.556 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 79.658791997999998, 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 79.659817054000001, 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79817 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.557 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774689+0000", 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774689+0000", 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774687+0000", 2023-10-09T04:57:37.558 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774695+0000", 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9279999999999993e-06 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774710+0000", 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4842e-05 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.559 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774720+0000", 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0339e-05 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.560 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.028560+0000", 2023-10-09T04:57:37.561 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25383992999999999 2023-10-09T04:57:37.561 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.561 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.561 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.561 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.039867+0000", 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011306327 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.124041+0000", 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.084174914000000003 2023-10-09T04:57:37.562 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.129614+0000", 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055721319999999996 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.852984+0000", 2023-10-09T04:57:37.563 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7233700650000001 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.852998+0000", 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4202999999999999e-05 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.564 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609565+0000", 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75656672000000003 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.565 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609570+0000", 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4e-06 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479383+0000", 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869812754000002 2023-10-09T04:57:37.566 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479399+0000", 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5804999999999999e-05 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.567 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810001+0000", 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330602220999999 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810026+0000", 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5031000000000001e-05 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.568 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79818 14.3 14:c9aea1d4:::benchmark_data_smithi029_550670_object2207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 79.658743717999997, 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 79.659845013999998, 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:57:37.569 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79818 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:37.570 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774737+0000", 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774740+0000", 2023-10-09T04:57:37.571 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7460000000000001e-06 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774748+0000", 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7879999999999995e-06 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:17.774754+0000", 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.913e-06 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.572 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.041445+0000", 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26669056299999999 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.060426+0000", 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018981576 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.179168+0000", 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.118741192 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:18.185905+0000", 2023-10-09T04:57:37.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067369719999999999 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853242+0000", 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6673378809999999 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:19.853261+0000", 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8505999999999999e-05 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609682+0000", 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75642128600000003 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:20.609696+0000", 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4161999999999999e-05 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479607+0000", 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 34.869910902000001 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.479616+0000", 2023-10-09T04:57:37.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3720000000000005e-06 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810210+0000", 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.330594581 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:06.810215+0000", 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.4100000000000001e-06 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.578 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:57:37.579 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.579 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:57:37.579 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T04:57:37.579 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:57:37.579 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 15 2023-10-09T04:57:37.579 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:57:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:37.580 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:37.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:37.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:37.708 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:37.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:37.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:37.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:37.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:37.943 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:37 smithi177 ceph-mon[195797]: failed to encode map e506 with expected crc 2023-10-09T04:57:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:37 smithi177 ceph-mon[195797]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:57:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:37 smithi177 ceph-mon[195797]: osdmap e506: 8 total, 7 up, 5 in 2023-10-09T04:57:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:37 smithi177 ceph-mon[195797]: from='osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:57:38.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:37 smithi177 ceph-mon[195797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[216903]: failed to encode map e506 with expected crc 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[216903]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[216903]: osdmap e506: 8 total, 7 up, 5 in 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[216903]: from='osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[216903]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[223858]: failed to encode map e506 with expected crc 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[223858]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T04:57:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[223858]: osdmap e506: 8 total, 7 up, 5 in 2023-10-09T04:57:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[223858]: from='osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:57:38.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:37 smithi029 ceph-mon[223858]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T04:57:38.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:38.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:38.318 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:38.352 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:38.403 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:38.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:38.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:38.539 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:38.595 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:38.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:38.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:38.730 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:38.806 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:38 smithi177 ceph-mon[195797]: pgmap v1736: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84292/225825 objects degraded (37.326%); 5487/225825 objects misplaced (2.430%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:57:38.806 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:38 smithi177 ceph-mon[195797]: osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366] boot 2023-10-09T04:57:38.806 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:38 smithi177 ceph-mon[195797]: osdmap e507: 8 total, 8 up, 5 in 2023-10-09T04:57:38.806 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:38 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:57:38.806 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:38 smithi177 ceph-mon[195797]: failed to encode map e507 with expected crc 2023-10-09T04:57:38.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T04:57:38.865 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:38.869 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:57:38.869 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:57:38.869 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:57:38.870 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:57:38.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:38.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:38.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:39.003 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:39.010 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:57:39.049 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[223858]: pgmap v1736: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84292/225825 objects degraded (37.326%); 5487/225825 objects misplaced (2.430%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[223858]: osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366] boot 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[223858]: osdmap e507: 8 total, 8 up, 5 in 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[223858]: failed to encode map e507 with expected crc 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[216903]: pgmap v1736: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 84292/225825 objects degraded (37.326%); 5487/225825 objects misplaced (2.430%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T04:57:39.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[216903]: osd.4 [v2:172.21.15.177:6800/1501862366,v1:172.21.15.177:6801/1501862366] boot 2023-10-09T04:57:39.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[216903]: osdmap e507: 8 total, 8 up, 5 in 2023-10-09T04:57:39.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T04:57:39.051 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:38 smithi029 ceph-mon[216903]: failed to encode map e507 with expected crc 2023-10-09T04:57:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:39.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:39.152 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:39.194 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:39.287 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:39.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:39.368 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:39.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:39.437 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:39.627 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:39.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:39.731 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:39.758 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:39.982 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:40.009 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e473 with expected crc 2023-10-09T04:57:40.009 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e507 with expected crc 2023-10-09T04:57:40.009 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: osdmap e508: 8 total, 8 up, 5 in 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[223858]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e473 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e507 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: osdmap e508: 8 total, 8 up, 5 in 2023-10-09T04:57:40.010 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:39 smithi029 ceph-mon[216903]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e473 with expected crc 2023-10-09T04:57:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e507 with expected crc 2023-10-09T04:57:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: osdmap e508: 8 total, 8 up, 5 in 2023-10-09T04:57:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:39 smithi177 ceph-mon[195797]: failed to encode map e508 with expected crc 2023-10-09T04:57:40.035 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:40.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:40.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:40.139 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:40.211 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:40.385 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:40.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:40.490 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:40.523 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[216903]: pgmap v1739: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84292/225825 objects degraded (37.326%); 5487/225825 objects misplaced (2.430%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[216903]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[216903]: osdmap e509: 8 total, 8 up, 5 in 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[216903]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[216903]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84292/225825 objects degraded (37.326%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[223858]: pgmap v1739: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84292/225825 objects degraded (37.326%); 5487/225825 objects misplaced (2.430%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T04:57:41.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[223858]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[223858]: osdmap e509: 8 total, 8 up, 5 in 2023-10-09T04:57:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[223858]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[223858]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84292/225825 objects degraded (37.326%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:40 smithi177 ceph-mon[195797]: pgmap v1739: 99 pgs: 1 activating+degraded, 3 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 6 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 84292/225825 objects degraded (37.326%); 5487/225825 objects misplaced (2.430%); 1024 KiB/s, 15 objects/s recovering 2023-10-09T04:57:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:40 smithi177 ceph-mon[195797]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:40 smithi177 ceph-mon[195797]: osdmap e509: 8 total, 8 up, 5 in 2023-10-09T04:57:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:40 smithi177 ceph-mon[195797]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:40 smithi177 ceph-mon[195797]: failed to encode map e509 with expected crc 2023-10-09T04:57:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84292/225825 objects degraded (37.326%), 22 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:42.488 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:57:42.517 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:57:42.960 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:42.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:43.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:43.091 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:42 smithi177 ceph-mon[195797]: pgmap v1741: 99 pgs: 1 activating+degraded, 4 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80113/225825 objects degraded (35.476%); 4567/225825 objects misplaced (2.022%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:57:43.094 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:43.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:43.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:43.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:42 smithi029 ceph-mon[216903]: pgmap v1741: 99 pgs: 1 activating+degraded, 4 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80113/225825 objects degraded (35.476%); 4567/225825 objects misplaced (2.022%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:57:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:57:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:57:42] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-10-09T04:57:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:42 smithi029 ceph-mon[223858]: pgmap v1741: 99 pgs: 1 activating+degraded, 4 remapped+peering, 11 peering, 1 activating+undersized+degraded+remapped, 1 activating, 4 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 64 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 80113/225825 objects degraded (35.476%); 4567/225825 objects misplaced (2.022%); 3.0 MiB/s, 47 objects/s recovering 2023-10-09T04:57:43.242 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:43.269 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:43.311 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:43.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:43.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:43.442 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:43.483 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:57:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:43.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:43.557 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:57:43.588 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:57:43.589 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:57:43.614 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:43.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:57:43.655 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:57:43.666 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:57:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:43.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:57:43.689 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.693 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:57:43.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:57:43.822 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:43.831 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:45.098 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1] out_osds: [4, 0, 6] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T04:57:45.098 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:57:45.098 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-09T04:57:45.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-09T04:57:45.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: pgmap v1742: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 63106/225813 objects degraded (27.946%); 5459/225813 objects misplaced (2.417%); 2.7 MiB/s, 63 objects/s recovering 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: osdmap e510: 8 total, 8 up, 5 in 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[223858]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.134 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: pgmap v1742: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 63106/225813 objects degraded (27.946%); 5459/225813 objects misplaced (2.417%); 2.7 MiB/s, 63 objects/s recovering 2023-10-09T04:57:45.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:57:45.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: osdmap e510: 8 total, 8 up, 5 in 2023-10-09T04:57:45.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:44 smithi029 ceph-mon[216903]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: pgmap v1742: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 63106/225813 objects degraded (27.946%); 5459/225813 objects misplaced (2.417%); 2.7 MiB/s, 63 objects/s recovering 2023-10-09T04:57:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-09T04:57:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: osdmap e510: 8 total, 8 up, 5 in 2023-10-09T04:57:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: failed to encode map e510 with expected crc 2023-10-09T04:57:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:44 smithi177 ceph-mon[195797]: failed to encode map e510 with expected crc 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: osdmap e511: 8 total, 8 up, 5 in 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 63106/225813 objects degraded (27.946%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:46.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: osdmap e511: 8 total, 8 up, 5 in 2023-10-09T04:57:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 63106/225813 objects degraded (27.946%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: osdmap e511: 8 total, 8 up, 5 in 2023-10-09T04:57:46.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: failed to encode map e511 with expected crc 2023-10-09T04:57:46.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 63106/225813 objects degraded (27.946%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:46 smithi029 ceph-mon[216903]: pgmap v1745: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 57353/225813 objects degraded (25.398%); 11088/225813 objects misplaced (4.910%); 3.4 MiB/s, 73 objects/s recovering 2023-10-09T04:57:47.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:46 smithi029 ceph-mon[223858]: pgmap v1745: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 57353/225813 objects degraded (25.398%); 11088/225813 objects misplaced (4.910%); 3.4 MiB/s, 73 objects/s recovering 2023-10-09T04:57:47.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:46 smithi177 ceph-mon[195797]: pgmap v1745: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 57353/225813 objects degraded (25.398%); 11088/225813 objects misplaced (4.910%); 3.4 MiB/s, 73 objects/s recovering 2023-10-09T04:57:48.306 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:57:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:48.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:57:48.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:48.444 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:48 smithi029 ceph-mon[216903]: pgmap v1746: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 57350/225807 objects degraded (25.398%); 11088/225807 objects misplaced (4.910%); 0 B/s, 11 objects/s recovering 2023-10-09T04:57:49.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:48 smithi029 ceph-mon[223858]: pgmap v1746: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 57350/225807 objects degraded (25.398%); 11088/225807 objects misplaced (4.910%); 0 B/s, 11 objects/s recovering 2023-10-09T04:57:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:48 smithi177 ceph-mon[195797]: pgmap v1746: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 57350/225807 objects degraded (25.398%); 11088/225807 objects misplaced (4.910%); 0 B/s, 11 objects/s recovering 2023-10-09T04:57:50.723 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:50.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:50.828 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:50.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:50 smithi029 ceph-mon[216903]: pgmap v1747: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 73903/225798 objects degraded (32.730%); 10789/225798 objects misplaced (4.778%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T04:57:50.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 73903/225798 objects degraded (32.730%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:50.855 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:50.906 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:50.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:51.014 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:51.022 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:57:51.022 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:57:51.041 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:51.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:57:51.088 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:50 smithi177 ceph-mon[195797]: pgmap v1747: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 73903/225798 objects degraded (32.730%); 10789/225798 objects misplaced (4.778%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T04:57:51.088 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 73903/225798 objects degraded (32.730%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:51.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:50 smithi029 ceph-mon[223858]: pgmap v1747: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 73903/225798 objects degraded (32.730%); 10789/225798 objects misplaced (4.778%); 3.7 MiB/s, 75 objects/s recovering 2023-10-09T04:57:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 73903/225798 objects degraded (32.730%), 21 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:51.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:51.227 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:52.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:51 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T04:57:52.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:51 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T04:57:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:51 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T04:57:52.926 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:57:53.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:52 smithi029 ceph-mon[216903]: pgmap v1748: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79062/225783 objects degraded (35.017%); 10253/225783 objects misplaced (4.541%); 4.9 MiB/s, 81 objects/s recovering 2023-10-09T04:57:53.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:52 smithi029 ceph-mon[223858]: pgmap v1748: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79062/225783 objects degraded (35.017%); 10253/225783 objects misplaced (4.541%); 4.9 MiB/s, 81 objects/s recovering 2023-10-09T04:57:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:57:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:57:52] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.43.0" 2023-10-09T04:57:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:52 smithi177 ceph-mon[195797]: pgmap v1748: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79062/225783 objects degraded (35.017%); 10253/225783 objects misplaced (4.541%); 4.9 MiB/s, 81 objects/s recovering 2023-10-09T04:57:53.832 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:57:53.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:53.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:57:53.936 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:53.965 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:54.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:54.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:54.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:54.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T04:57:54.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:53 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T04:57:54.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:53 smithi029 ceph-mon[223858]: Health check cleared: SLOW_OPS (was: 7 slow ops, oldest one blocked for 55 sec, osd.3 has slow ops) 2023-10-09T04:57:54.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:54.121 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:53 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T04:57:54.122 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:53 smithi029 ceph-mon[216903]: Health check cleared: SLOW_OPS (was: 7 slow ops, oldest one blocked for 55 sec, osd.3 has slow ops) 2023-10-09T04:57:54.149 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:54.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:54.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:54.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:54.238 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:53 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-10-09T04:57:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:53 smithi177 ceph-mon[195797]: Health check cleared: SLOW_OPS (was: 7 slow ops, oldest one blocked for 55 sec, osd.3 has slow ops) 2023-10-09T04:57:54.303 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:57:54.331 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:54.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:57:54.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:54.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:57:54.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:54.506 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:57:54.516 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:54.525 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:57:54.568 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:57:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:54.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:54.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:54.704 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:54.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:57:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:54.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:54.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:57:54.891 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:54.943 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:57:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:54.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:57:54.997 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:57:55.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:54 smithi029 ceph-mon[216903]: pgmap v1749: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79060/225777 objects degraded (35.017%); 9833/225777 objects misplaced (4.355%); 6.8 MiB/s, 110 objects/s recovering 2023-10-09T04:57:55.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2999320111' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:55.022 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:54 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:55.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:54 smithi029 ceph-mon[223858]: pgmap v1749: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79060/225777 objects degraded (35.017%); 9833/225777 objects misplaced (4.355%); 6.8 MiB/s, 110 objects/s recovering 2023-10-09T04:57:55.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2999320111' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:55.023 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:54 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:55.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:57:55.099 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:54 smithi177 ceph-mon[195797]: pgmap v1749: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79060/225777 objects degraded (35.017%); 9833/225777 objects misplaced (4.355%); 6.8 MiB/s, 110 objects/s recovering 2023-10-09T04:57:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2999320111' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:54 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:57:55.934 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:57:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:55.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:57:56.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:57:56.070 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:56.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:57:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:56.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:57:56.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:55 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:55 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:57:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:55 smithi177 ceph-mon[195797]: osdmap e512: 8 total, 8 up, 5 in 2023-10-09T04:57:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:55 smithi177 ceph-mon[195797]: failed to encode map e512 with expected crc 2023-10-09T04:57:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 79060/225777 objects degraded (35.017%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:56.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:56.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:57:56.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[216903]: osdmap e512: 8 total, 8 up, 5 in 2023-10-09T04:57:56.289 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[216903]: failed to encode map e512 with expected crc 2023-10-09T04:57:56.290 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 79060/225777 objects degraded (35.017%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:56.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:57:56.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:57:56.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[223858]: osdmap e512: 8 total, 8 up, 5 in 2023-10-09T04:57:56.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[223858]: failed to encode map e512 with expected crc 2023-10-09T04:57:56.290 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 79060/225777 objects degraded (35.017%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:57:56.300 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:56 smithi177 ceph-mon[195797]: pgmap v1751: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79062/225759 objects degraded (35.021%); 9293/225759 objects misplaced (4.116%); 7.9 MiB/s, 128 objects/s recovering 2023-10-09T04:57:57.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:56 smithi029 ceph-mon[216903]: pgmap v1751: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79062/225759 objects degraded (35.021%); 9293/225759 objects misplaced (4.116%); 7.9 MiB/s, 128 objects/s recovering 2023-10-09T04:57:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:56 smithi029 ceph-mon[223858]: pgmap v1751: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 79062/225759 objects degraded (35.021%); 9293/225759 objects misplaced (4.116%); 7.9 MiB/s, 128 objects/s recovering 2023-10-09T04:57:57.526 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:57.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:57.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:57:57.664 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:58.423 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:57:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:57:58.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:57:58.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:57:58.569 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:57:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:58 smithi177 ceph-mon[195797]: pgmap v1752: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 6 op/s; 79006/225621 objects degraded (35.017%); 9301/225621 objects misplaced (4.122%); 8.1 MiB/s, 131 objects/s recovering 2023-10-09T04:57:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:57:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:59.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:58 smithi029 ceph-mon[216903]: pgmap v1752: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 6 op/s; 79006/225621 objects degraded (35.017%); 9301/225621 objects misplaced (4.122%); 8.1 MiB/s, 131 objects/s recovering 2023-10-09T04:57:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:57:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:58 smithi029 ceph-mon[223858]: pgmap v1752: 99 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 6 op/s; 79006/225621 objects degraded (35.017%); 9301/225621 objects misplaced (4.122%); 8.1 MiB/s, 131 objects/s recovering 2023-10-09T04:57:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:57:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:57:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:57:59.712 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:57:59.712 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:00.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T04:58:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:00 smithi177 ceph-mon[195797]: pgmap v1753: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 10 op/s; 78964/225489 objects degraded (35.019%); 8949/225489 objects misplaced (3.969%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:58:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 78964/225489 objects degraded (35.019%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:01.382 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:01.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:00 smithi029 ceph-mon[216903]: pgmap v1753: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 10 op/s; 78964/225489 objects degraded (35.019%); 8949/225489 objects misplaced (3.969%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:58:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 78964/225489 objects degraded (35.019%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:00 smithi029 ceph-mon[223858]: pgmap v1753: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 10 op/s; 78964/225489 objects degraded (35.019%); 8949/225489 objects misplaced (3.969%); 8.5 MiB/s, 136 objects/s recovering 2023-10-09T04:58:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 78964/225489 objects degraded (35.019%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:02.575 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:02.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:02.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:02.710 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:02.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:02.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:02.862 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:02.893 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:02.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:02.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:02.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:03.080 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:03.111 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:03.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:02 smithi029 ceph-mon[216903]: pgmap v1754: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 19 op/s; 78958/225210 objects degraded (35.060%); 8372/225210 objects misplaced (3.717%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T04:58:03.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:03 smithi029 ceph-mon[223858]: pgmap v1754: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 19 op/s; 78958/225210 objects degraded (35.060%); 8372/225210 objects misplaced (3.717%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T04:58:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:58:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:58:02] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-10-09T04:58:03.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:03.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:03 smithi177 ceph-mon[195797]: pgmap v1754: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 19 op/s; 78958/225210 objects degraded (35.060%); 8372/225210 objects misplaced (3.717%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T04:58:03.308 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:58:03.309 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:03.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:03.348 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:03.546 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:03.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:03.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:03.699 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:04.228 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:04.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:04.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:04.333 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:04.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T04:58:04.363 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:04.412 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:04.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:04.517 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:04.545 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:04.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:04.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:04.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:04.760 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:04.780 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:04.965 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:04.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:05.025 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:05 smithi029 ceph-mon[216903]: pgmap v1755: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 29 op/s; 78839/224877 objects degraded (35.059%); 8015/224877 objects misplaced (3.564%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:58:05.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:05.103 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:05.150 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:05.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:05.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:05.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:05.268 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:58:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:05 smithi177 ceph-mon[195797]: pgmap v1755: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 29 op/s; 78839/224877 objects degraded (35.059%); 8015/224877 objects misplaced (3.564%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:58:05.320 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:58:05.365 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:05 smithi029 ceph-mon[223858]: pgmap v1755: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 29 op/s; 78839/224877 objects degraded (35.059%); 8015/224877 objects misplaced (3.564%); 8.8 MiB/s, 140 objects/s recovering 2023-10-09T04:58:05.388 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:05.388 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:58:05.421 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:58:05.453 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:58:05.511 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:58:05.564 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:58:05.576 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:05.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:05.618 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:58:05.627 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.628 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:58:05.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:05.759 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:05.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:05 smithi029 conmon[216878]: 2023-10-09T04:58:05.847+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e512 definitely_dead 0 2023-10-09T04:58:05.974 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:05.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:05.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:06.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:06.108 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:06.164 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:58:06.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3082105563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:06.200 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 78839/224877 objects degraded (35.059%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:06.201 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1749703082' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:58:06.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3082105563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:06.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 78839/224877 objects degraded (35.059%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:06.201 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1749703082' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:58:06.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3082105563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 78839/224877 objects degraded (35.059%), 22 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:06.289 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1749703082' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-09T04:58:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:06.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:06.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:06.435 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:06.556 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:06.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:06.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:06.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:06.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:06.889 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.6. 2023-10-09T04:58:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:06.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187694 12.c 12:314f2354:::benchmark_data_smithi029_442458_object92444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.109140+0000", 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.88620702900005, 2023-10-09T04:58:06.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0346555909999999, 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187694 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109140+0000", 2023-10-09T04:58:06.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109140+0000", 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109138+0000", 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109145+0000", 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4239999999999998e-06 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109158+0000", 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3264999999999999e-05 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109163+0000", 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8679999999999998e-06 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109239+0000", 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5788e-05 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.109249+0000", 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0169e-05 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.135498+0000", 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0262491389999999 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.135590+0000", 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2022999999999996e-05 2023-10-09T04:58:07.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.135691+0000", 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010091599999999999 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138806+0000", 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031145320000000002 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143694+0000", 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004888674 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143736+0000", 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1980000000000001e-05 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143769+0000", 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3108000000000002e-05 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143795+0000", 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5899999999999999e-05 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187700 12.c 12:3cbc49fb:::benchmark_data_smithi029_442458_object92450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.122655+0000", 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.87269148200005, 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0213057059999999, 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187700 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122655+0000", 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122655+0000", 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122653+0000", 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122662+0000", 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7339999999999997e-06 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122679+0000", 2023-10-09T04:58:07.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6849000000000001e-05 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122686+0000", 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5719999999999999e-06 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122746+0000", 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0888999999999999e-05 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.122761+0000", 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4823e-05 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138823+0000", 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.016062021 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138896+0000", 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2527000000000005e-05 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.138981+0000", 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5233000000000002e-05 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143831+0000", 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048499040000000004 2023-10-09T04:58:07.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143848+0000", 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6872e-05 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143909+0000", 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1285999999999996e-05 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143935+0000", 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6083000000000001e-05 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.143961+0000", 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5778e-05 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.014 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187715 12.c 12:36b39d4a:::benchmark_data_smithi029_442458_object92465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.135898+0000", 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.85944910000001, 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.014459099, 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:07.015 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187715 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135898+0000", 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135898+0000", 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135896+0000", 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135905+0000", 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7730000000000005e-06 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135921+0000", 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6716000000000001e-05 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135929+0000", 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.362e-06 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135983+0000", 2023-10-09T04:58:07.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4255999999999997e-05 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.135993+0000", 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8509999999999994e-06 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.141569+0000", 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0055756890000001 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.141650+0000", 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1264e-05 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.141717+0000", 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6793999999999998e-05 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144008+0000", 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022912869999999999 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.146873+0000", 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002865126 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150256+0000", 2023-10-09T04:58:07.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033833079999999998 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150312+0000", 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5896000000000002e-05 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150357+0000", 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4393e-05 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.022 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187724 12.c 12:3953830f:::benchmark_data_smithi029_442458_object92474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.152662+0000", 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.84268444700001, 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0087711610000001, 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:07.023 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187724 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152662+0000", 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152662+0000", 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152660+0000", 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152671+0000", 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0864e-05 2023-10-09T04:58:07.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152687+0000", 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6359999999999999e-05 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152694+0000", 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6409999999999996e-06 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152755+0000", 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0846e-05 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.152768+0000", 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3054000000000001e-05 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144036+0000", 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.99126849699999997 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144124+0000", 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7875999999999997e-05 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.144251+0000", 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012712699999999999 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150409+0000", 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061577849999999998 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150416+0000", 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1930000000000004e-06 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161346+0000", 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010929179000000001 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161391+0000", 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5664999999999997e-05 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161433+0000", 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2184000000000001e-05 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187752 12.c 12:33babd0f:::benchmark_data_smithi029_442458_object92502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.196820+0000", 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.798526498, 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.964821496, 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.031 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187752 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196820+0000", 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196820+0000", 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196819+0000", 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196827+0000", 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6640000000000004e-06 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196843+0000", 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5953000000000001e-05 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196851+0000", 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0090000000000007e-06 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196921+0000", 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9896999999999998e-05 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.196936+0000", 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4489e-05 2023-10-09T04:58:07.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.146966+0000", 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95003068899999998 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.147089+0000", 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000122676 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.147221+0000", 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013162199999999999 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150454+0000", 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003233194 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.037 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153417+0000", 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029634829999999998 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161534+0000", 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081165960000000002 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161583+0000", 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9410999999999998e-05 2023-10-09T04:58:07.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161642+0000", 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8391999999999999e-05 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187760 12.c 12:31175804:::benchmark_data_smithi029_442458_object92510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.207903+0000", 2023-10-09T04:58:07.039 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.787444155, 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95389128000000001, 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187760 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207903+0000", 2023-10-09T04:58:07.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207903+0000", 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207901+0000", 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207909+0000", 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9100000000000005e-06 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207930+0000", 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0650999999999999e-05 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.207938+0000", 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4989999999999994e-06 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.208018+0000", 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9079000000000006e-05 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.208030+0000", 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2801e-05 2023-10-09T04:58:07.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150486+0000", 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94245597299999995 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150580+0000", 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3826999999999995e-05 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.150697+0000", 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000116768 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161318+0000", 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010620983000000001 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161443+0000", 2023-10-09T04:58:07.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000125341 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161726+0000", 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028248599999999998 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161771+0000", 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5303999999999999e-05 2023-10-09T04:58:07.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161794+0000", 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2821000000000001e-05 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36225.0:187771 12.c 12:3ad3988e:::benchmark_data_smithi029_442458_object92521:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:48:45.225905+0000", 2023-10-09T04:58:07.047 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 561.76944171100001, 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94938117399999999, 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36225", 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3550493553", 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 187771 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225905+0000", 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225905+0000", 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225904+0000", 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225911+0000", 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.199e-06 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225925+0000", 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.383e-05 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225931+0000", 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5119999999999997e-06 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.225996+0000", 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5029000000000004e-05 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:45.226005+0000", 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4179999999999997e-06 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153442+0000", 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92743711900000003 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153541+0000", 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8963999999999996e-05 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,7", 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.153611+0000", 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0160999999999995e-05 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161449+0000", 2023-10-09T04:58:07.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078379420000000005 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.161495+0000", 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6381999999999997e-05 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175187+0000", 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013692205000000001 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175241+0000", 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3088999999999999e-05 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:48:46.175286+0000", 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5627000000000002e-05 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.055 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:160734 13.9 13:964ae340:::benchmark_data_smithi029_495570_object66474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:27.302515+0000", 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.69283135000001, 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97664355199999997, 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 160734 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.056 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302515+0000", 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302515+0000", 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302514+0000", 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302521+0000", 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9439999999999999e-06 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302531+0000", 2023-10-09T04:58:07.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0336999999999999e-05 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302537+0000", 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7769999999999999e-06 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302584+0000", 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6378000000000002e-05 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.302591+0000", 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2810000000000003e-06 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.272642+0000", 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97005163900000002 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.272763+0000", 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120357 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.273095+0000", 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033170600000000003 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.275096+0000", 2023-10-09T04:58:07.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020016460000000002 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.277012+0000", 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001915513 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279074+0000", 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020624010000000002 2023-10-09T04:58:07.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279113+0000", 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9291e-05 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279159+0000", 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5529999999999999e-05 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.063 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:160763 13.9 13:96d5c004:::benchmark_data_smithi029_495570_object66503:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:27.335356+0000", 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.65999086199997, 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94878702000000004, 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:58:07.064 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 160763 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335356+0000", 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335356+0000", 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335355+0000", 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335360+0000", 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7639999999999996e-06 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335373+0000", 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2924e-05 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335379+0000", 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7400000000000001e-06 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335424+0000", 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5850000000000003e-05 2023-10-09T04:58:07.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.335434+0000", 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6360000000000006e-06 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.275115+0000", 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93968091899999995 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.275193+0000", 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7721999999999999e-05 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.275434+0000", 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024119199999999999 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279270+0000", 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038357069999999998 2023-10-09T04:58:07.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279310+0000", 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0042e-05 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.284059+0000", 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047495660000000002 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.284105+0000", 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5694e-05 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.284143+0000", 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7948000000000003e-05 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.46418.0:160777 13.9 13:90ab2f56:::benchmark_data_smithi029_495570_object66517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e407)", 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:52:27.354149+0000", 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.64119769400003, 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.93013162000000005, 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.46418", 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3903523296", 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 160777 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.072 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354149+0000", 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354149+0000", 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354148+0000", 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354155+0000", 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8970000000000002e-06 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354171+0000", 2023-10-09T04:58:07.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6047000000000001e-05 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354177+0000", 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4330000000000001e-06 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354227+0000", 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0266000000000003e-05 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:27.354236+0000", 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4770000000000003e-06 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.277041+0000", 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92280562899999996 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.277158+0000", 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011612600000000001 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,4", 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.277403+0000", 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024509500000000002 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279300+0000", 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00189697 2023-10-09T04:58:07.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.279318+0000", 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8201000000000001e-05 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.284212+0000", 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0048941219999999999 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.284249+0000", 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6882999999999999e-05 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:52:28.284281+0000", 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1804999999999998e-05 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75806 14.9 14:9060921b:::benchmark_data_smithi029_550670_object75805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:47.525187+0000", 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 199.47015927999999, 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.549752115, 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.080 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75806 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525187+0000", 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525187+0000", 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525186+0000", 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525348+0000", 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016178399999999999 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525362+0000", 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4474000000000001e-05 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525368+0000", 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7139999999999998e-06 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525430+0000", 2023-10-09T04:58:07.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2201999999999998e-05 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525608+0000", 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017740200000000001 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.525812+0000", 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020489999999999999 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.532468+0000", 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006655434 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.532530+0000", 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2014999999999995e-05 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.072548+0000", 2023-10-09T04:58:07.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.540017938 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.072618+0000", 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9914999999999997e-05 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.074940+0000", 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023217929999999999 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.086 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75808 14.9 14:97ba7be1:::benchmark_data_smithi029_550670_object75807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:47.529184+0000", 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 199.466162873, 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.551240014999999, 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.087 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75808 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529184+0000", 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529184+0000", 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529182+0000", 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529227+0000", 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4872e-05 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529246+0000", 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9527000000000001e-05 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529255+0000", 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1359999999999997e-06 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529308+0000", 2023-10-09T04:58:07.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3671999999999997e-05 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529482+0000", 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017409000000000001 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.529611+0000", 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000128262 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.532749+0000", 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031381730000000002 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.532861+0000", 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000112729 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.077704+0000", 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.544842866 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.077744+0000", 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9592000000000003e-05 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.080424+0000", 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026799620000000001 2023-10-09T04:58:07.093 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75811 14.9 14:9fabdfce:::benchmark_data_smithi029_550670_object75810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:47.543944+0000", 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 199.45140281400001, 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.539227425, 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.094 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75811 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.543944+0000", 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.543944+0000", 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.543942+0000", 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.543991+0000", 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8652000000000001e-05 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.544005+0000", 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4100999999999999e-05 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.544011+0000", 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9109999999999998e-06 2023-10-09T04:58:07.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.544064+0000", 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2966000000000001e-05 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.544225+0000", 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016122099999999999 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.544339+0000", 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113575 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.546860+0000", 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025207099999999998 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.547726+0000", 2023-10-09T04:58:07.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00086675000000000001 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.080491+0000", 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.532764674999999 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.080558+0000", 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6835000000000006e-05 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.083171+0000", 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026135339999999998 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75832 14.9 14:9f6ea33a:::benchmark_data_smithi029_550670_object75831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:47.581829+0000", 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 199.41351727599999, 2023-10-09T04:58:07.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.521427095, 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75832 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581829+0000", 2023-10-09T04:58:07.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581829+0000", 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581828+0000", 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581872+0000", 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3609000000000002e-05 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581891+0000", 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9162000000000001e-05 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581899+0000", 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7819999999999994e-06 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.581957+0000", 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8962999999999998e-05 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.582158+0000", 2023-10-09T04:58:07.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020013 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.582364+0000", 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00020602500000000001 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585031+0000", 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026669269999999999 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.589099+0000", 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.004068538 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.100696+0000", 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.511597323 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.100734+0000", 2023-10-09T04:58:07.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7554e-05 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.103257+0000", 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025225669999999999 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75834 14.9 14:9dadb6d4:::benchmark_data_smithi029_550670_object75833:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:58:07.108 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:47.585333+0000", 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 199.41001359000001, 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.631734492, 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75834 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585333+0000", 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585333+0000", 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585332+0000", 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585373+0000", 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1628000000000002e-05 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585389+0000", 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5611e-05 2023-10-09T04:58:07.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585397+0000", 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8960000000000003e-06 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585448+0000", 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.079e-05 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585627+0000", 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017934799999999999 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.585801+0000", 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000174508 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.588659+0000", 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028571989999999999 2023-10-09T04:58:07.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.589129+0000", 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00047010299999999998 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.214308+0000", 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.625179796999999 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.214353+0000", 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4258000000000002e-05 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.217068+0000", 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027148749999999998 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.115 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75840 14.9 14:9bb8bd02:::benchmark_data_smithi029_550670_object75839:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:47.614074+0000", 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 199.38127244, 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.606040184999999, 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75840 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614074+0000", 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614074+0000", 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614073+0000", 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:07.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614124+0000", 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0837000000000001e-05 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614134+0000", 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3030000000000005e-06 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614138+0000", 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0289999999999999e-06 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614179+0000", 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1826999999999998e-05 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614310+0000", 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013080000000000001 2023-10-09T04:58:07.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,4", 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.614434+0000", 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012410399999999999 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.616794+0000", 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023599910000000001 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:47.617071+0000", 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000276383 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.217125+0000", 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 10.600054354999999 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.217373+0000", 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024792100000000001 2023-10-09T04:58:07.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:58.220114+0000", 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027415120000000002 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78157 14.9 14:939d7b7c:::benchmark_data_smithi029_550670_object546:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:53.304866+0000", 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 133.69048056899999, 2023-10-09T04:58:07.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94639081700000005, 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78157 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304866+0000", 2023-10-09T04:58:07.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304866+0000", 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304865+0000", 2023-10-09T04:58:07.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304871+0000", 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6760000000000004e-06 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304884+0000", 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2947e-05 2023-10-09T04:58:07.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304891+0000", 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9070000000000001e-06 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304942+0000", 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1650000000000002e-05 2023-10-09T04:58:07.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.304949+0000", 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0879999999999997e-06 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.234062+0000", 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92911269399999996 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.234211+0000", 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014915800000000001 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,3", 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.234345+0000", 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000134253 2023-10-09T04:58:07.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.248020+0000", 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013674388000000001 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.248078+0000", 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8260000000000001e-05 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.248091+0000", 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.291e-05 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.248158+0000", 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6618000000000005e-05 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.251257+0000", 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030993819999999999 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:79966 14.a 14:527c8ee9:::benchmark_data_smithi029_550670_object2355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:56:57.133767+0000", 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 69.861579622999997, 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.095076214000000006, 2023-10-09T04:58:07.131 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 79966 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133767+0000", 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133767+0000", 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133766+0000", 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133771+0000", 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2189999999999998e-06 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133781+0000", 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0574000000000001e-05 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133786+0000", 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5959999999999997e-06 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.133834+0000", 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8628999999999998e-05 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.134014+0000", 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017935 2023-10-09T04:58:07.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.134235+0000", 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00022105000000000001 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.136694+0000", 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002459658 2023-10-09T04:58:07.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.137138+0000", 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000443421 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.225711+0000", 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.088573467000000003 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.225778+0000", 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6798000000000007e-05 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:57.228843+0000", 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030652449999999999 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:80528 14.9 14:952cdb30:::benchmark_data_smithi029_550670_object2917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:58:06.012747+0000", 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.982600056, 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.23290339099999999, 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 80528 2023-10-09T04:58:07.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012747+0000", 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012747+0000", 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012745+0000", 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012752+0000", 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0049999999999998e-06 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012772+0000", 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9604000000000001e-05 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012780+0000", 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7349999999999996e-06 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012853+0000", 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3802999999999998e-05 2023-10-09T04:58:07.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.012981+0000", 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012738699999999999 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.013101+0000", 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000120191 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.015676+0000", 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025754979999999998 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.015834+0000", 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00015745399999999999 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.242361+0000", 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.226527543 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.242430+0000", 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8786000000000002e-05 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.245650+0000", 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003219824 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:80552 14.9 14:9cc91a08:::benchmark_data_smithi029_550670_object2941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:58:06.760288+0000", 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 0.23505916700000001, 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14362602699999999, 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 80552 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.146 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760288+0000", 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760288+0000", 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760286+0000", 2023-10-09T04:58:07.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760292+0000", 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4670000000000003e-06 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760304+0000", 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1841999999999999e-05 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760311+0000", 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4760000000000001e-06 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760362+0000", 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0912000000000003e-05 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760540+0000", 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000177553 2023-10-09T04:58:07.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.760733+0000", 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00019359299999999999 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.763013+0000", 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022799550000000002 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.763144+0000", 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000130925 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.899828+0000", 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.136684003 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.899899+0000", 2023-10-09T04:58:07.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1310999999999995e-05 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:06.903914+0000", 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0040141500000000002 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:07.152 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:58:07.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:07.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:07 smithi029 ceph-mon[216903]: pgmap v1756: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 33 op/s; 78858/224688 objects degraded (35.097%); 7452/224688 objects misplaced (3.317%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T04:58:07.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1749703082' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:58:07.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:07 smithi029 ceph-mon[216903]: osdmap e513: 8 total, 8 up, 6 in 2023-10-09T04:58:07.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:07 smithi029 ceph-mon[223858]: pgmap v1756: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 33 op/s; 78858/224688 objects degraded (35.097%); 7452/224688 objects misplaced (3.317%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T04:58:07.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1749703082' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:58:07.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:07 smithi029 ceph-mon[223858]: osdmap e513: 8 total, 8 up, 6 in 2023-10-09T04:58:07.193 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:07 smithi177 ceph-mon[195797]: pgmap v1756: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 activating+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 33 op/s; 78858/224688 objects degraded (35.097%); 7452/224688 objects misplaced (3.317%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T04:58:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1749703082' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-09T04:58:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:07 smithi177 ceph-mon[195797]: osdmap e513: 8 total, 8 up, 6 in 2023-10-09T04:58:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[216903]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[216903]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[216903]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[216903]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[216903]: failed to encode map e514 with expected crc 2023-10-09T04:58:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[216903]: osdmap e514: 8 total, 8 up, 6 in 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[223858]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[223858]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[223858]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[223858]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[223858]: failed to encode map e514 with expected crc 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:08 smithi029 ceph-mon[223858]: osdmap e514: 8 total, 8 up, 6 in 2023-10-09T04:58:08.422 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:08 smithi177 ceph-mon[195797]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:08 smithi177 ceph-mon[195797]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:08 smithi177 ceph-mon[195797]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:08 smithi177 ceph-mon[195797]: failed to encode map e513 with expected crc 2023-10-09T04:58:08.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:08 smithi177 ceph-mon[195797]: failed to encode map e514 with expected crc 2023-10-09T04:58:08.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:08 smithi177 ceph-mon[195797]: osdmap e514: 8 total, 8 up, 6 in 2023-10-09T04:58:08.444 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T04:58:08.824 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:09.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: pgmap v1758: 99 pgs: 10 peering, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 78800/224469 objects degraded (35.105%); 7453/224469 objects misplaced (3.320%); 8.8 MiB/s, 139 objects/s recovering 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: osdmap e515: 8 total, 8 up, 6 in 2023-10-09T04:58:09.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[216903]: failed to encode map e515 with expected crc 2023-10-09T04:58:09.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: pgmap v1758: 99 pgs: 10 peering, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 78800/224469 objects degraded (35.105%); 7453/224469 objects misplaced (3.320%); 8.8 MiB/s, 139 objects/s recovering 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: osdmap e515: 8 total, 8 up, 6 in 2023-10-09T04:58:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:09 smithi029 ceph-mon[223858]: failed to encode map e515 with expected crc 2023-10-09T04:58:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: pgmap v1758: 99 pgs: 10 peering, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 5 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 78800/224469 objects degraded (35.105%); 7453/224469 objects misplaced (3.320%); 8.8 MiB/s, 139 objects/s recovering 2023-10-09T04:58:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: failed to encode map e514 with expected crc 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: osdmap e515: 8 total, 8 up, 6 in 2023-10-09T04:58:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:09 smithi177 ceph-mon[195797]: failed to encode map e515 with expected crc 2023-10-09T04:58:09.675 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:09.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:09.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:09.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:09.867 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:09.927 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:09.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:10.032 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:10.060 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:10.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[216903]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[216903]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[216903]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[216903]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[216903]: 1.8 scrub starts 2023-10-09T04:58:10.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[216903]: osdmap e516: 8 total, 8 up, 6 in 2023-10-09T04:58:10.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[223858]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[223858]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[223858]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[223858]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[223858]: 1.8 scrub starts 2023-10-09T04:58:10.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:10 smithi029 ceph-mon[223858]: osdmap e516: 8 total, 8 up, 6 in 2023-10-09T04:58:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:10 smithi177 ceph-mon[195797]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:10 smithi177 ceph-mon[195797]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:10 smithi177 ceph-mon[195797]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:10 smithi177 ceph-mon[195797]: failed to encode map e515 with expected crc 2023-10-09T04:58:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:10 smithi177 ceph-mon[195797]: 1.8 scrub starts 2023-10-09T04:58:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:10 smithi177 ceph-mon[195797]: osdmap e516: 8 total, 8 up, 6 in 2023-10-09T04:58:10.966 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:10.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:11.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:11.101 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:11.105 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-09T04:58:11.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:11.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:11.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:11.164 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:11.251 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: pgmap v1761: 99 pgs: 1 remapped, 14 remapped+peering, 14 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 51772/224181 objects degraded (23.094%); 8815/224181 objects misplaced (3.932%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 51772/224181 objects degraded (23.094%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:11 smithi177 ceph-mon[195797]: osdmap e517: 8 total, 8 up, 6 in 2023-10-09T04:58:11.279 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:11.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: pgmap v1761: 99 pgs: 1 remapped, 14 remapped+peering, 14 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 51772/224181 objects degraded (23.094%); 8815/224181 objects misplaced (3.932%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 51772/224181 objects degraded (23.094%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[216903]: osdmap e517: 8 total, 8 up, 6 in 2023-10-09T04:58:11.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: pgmap v1761: 99 pgs: 1 remapped, 14 remapped+peering, 14 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 51772/224181 objects degraded (23.094%); 8815/224181 objects misplaced (3.932%); 6.7 MiB/s, 106 objects/s recovering 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: failed to encode map e516 with expected crc 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 51772/224181 objects degraded (23.094%), 14 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:11.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:11 smithi029 ceph-mon[223858]: osdmap e517: 8 total, 8 up, 6 in 2023-10-09T04:58:11.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:11.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:11.695 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:11.728 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:11.788 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:11.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:11.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:11.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:11.927 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:11.981 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:11.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:12.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:12.119 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:12.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:12.182 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:12 smithi177 ceph-mon[195797]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.182 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:12 smithi177 ceph-mon[195797]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.182 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:12 smithi177 ceph-mon[195797]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:12 smithi177 ceph-mon[195797]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:12 smithi177 ceph-mon[195797]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:12.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:12.284 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:12.285 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:58:12.285 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:58:12.285 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:58:12.285 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:58:12.313 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:12.361 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:12.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[216903]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[216903]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[216903]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[216903]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.363 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[216903]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.364 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[223858]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.364 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[223858]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.364 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[223858]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.364 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[223858]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.364 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:12 smithi029 ceph-mon[223858]: failed to encode map e517 with expected crc 2023-10-09T04:58:12.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:12.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:12.466 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:12.501 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:12.559 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:12.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:12.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:12.702 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:12.996 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:13.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:13.004 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:58:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:58:12] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-10-09T04:58:13.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:58:13.105 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:13.135 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:13.345 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:13.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:13 smithi177 ceph-mon[195797]: pgmap v1764: 99 pgs: 1 remapped, 14 remapped+peering, 14 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 31 op/s; 48740/224094 objects degraded (21.750%); 11014/224094 objects misplaced (4.915%); 12 MiB/s, 196 objects/s recovering 2023-10-09T04:58:13.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:13 smithi177 ceph-mon[195797]: 1.0 scrub starts 2023-10-09T04:58:13.354 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:13.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:13.422 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T04:58:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:13 smithi029 ceph-mon[223858]: pgmap v1764: 99 pgs: 1 remapped, 14 remapped+peering, 14 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 31 op/s; 48740/224094 objects degraded (21.750%); 11014/224094 objects misplaced (4.915%); 12 MiB/s, 196 objects/s recovering 2023-10-09T04:58:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:13 smithi029 ceph-mon[223858]: 1.0 scrub starts 2023-10-09T04:58:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:13 smithi029 ceph-mon[216903]: pgmap v1764: 99 pgs: 1 remapped, 14 remapped+peering, 14 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 56 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 31 op/s; 48740/224094 objects degraded (21.750%); 11014/224094 objects misplaced (4.915%); 12 MiB/s, 196 objects/s recovering 2023-10-09T04:58:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:13 smithi029 ceph-mon[216903]: 1.0 scrub starts 2023-10-09T04:58:13.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:58:13.456 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:13.498 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:13.569 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:13.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:13.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:13.729 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:13.867 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:13.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:13.891 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:13.974 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:14.008 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:14.027 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:14.166 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:14.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:14.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:14.304 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:14.363 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:14.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:14.502 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:14.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:14 smithi177 ceph-mon[195797]: pgmap v1765: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 13 op/s; 71354/224067 objects degraded (31.845%); 11014/224067 objects misplaced (4.915%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:58:14.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:14.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:14.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1414766440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:14.560 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:14.561 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:14 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-10-09T04:58:14.573 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:58:14.761 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:14.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:14.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:14.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[216903]: pgmap v1765: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 13 op/s; 71354/224067 objects degraded (31.845%); 11014/224067 objects misplaced (4.915%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:58:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1414766440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:14.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-10-09T04:58:14.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[223858]: pgmap v1765: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 13 op/s; 71354/224067 objects degraded (31.845%); 11014/224067 objects misplaced (4.915%); 5.4 MiB/s, 86 objects/s recovering 2023-10-09T04:58:14.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:14.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:14.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1414766440' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:14.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:14.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:14 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-10-09T04:58:14.907 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:15.470 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:15.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:15.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:15.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:15.605 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:15.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[216903]: failed to encode map e518 with expected crc 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[216903]: osdmap e518: 8 total, 8 up, 6 in 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[223858]: failed to encode map e518 with expected crc 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:15 smithi029 ceph-mon[223858]: osdmap e518: 8 total, 8 up, 6 in 2023-10-09T04:58:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:15 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:15 smithi177 ceph-mon[195797]: failed to encode map e518 with expected crc 2023-10-09T04:58:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:15 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:15 smithi177 ceph-mon[195797]: osdmap e518: 8 total, 8 up, 6 in 2023-10-09T04:58:16.106 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6] out_osds: [4, 0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T04:58:16.106 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:58:16.106 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2023-10-09T04:58:16.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:58:16.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:16.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:16.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:16.509 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:16.543 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:16.613 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:16.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:16 smithi029 ceph-mon[216903]: pgmap v1767: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 71354/224061 objects degraded (31.846%); 10557/224061 objects misplaced (4.712%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T04:58:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 71354/224061 objects degraded (31.846%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:16 smithi029 ceph-mon[223858]: pgmap v1767: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 71354/224061 objects degraded (31.846%); 10557/224061 objects misplaced (4.712%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T04:58:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 71354/224061 objects degraded (31.846%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:16.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:16.764 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:16 smithi177 ceph-mon[195797]: pgmap v1767: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 71354/224061 objects degraded (31.846%); 10557/224061 objects misplaced (4.712%); 9.3 MiB/s, 149 objects/s recovering 2023-10-09T04:58:16.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 71354/224061 objects degraded (31.846%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:16.829 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:16.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:16.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:16.976 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:17.001 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:17.068 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:17.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:17.168 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:58:17.174 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:17.190 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:58:17.191 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:58:17.191 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:58:17.191 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:58:17.191 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:58:17.211 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:17.334 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:17.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:17.443 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:17.476 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:17.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2871192944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:17.575 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2871192944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:17.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2871192944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:18.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:58:18.578 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:18.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:18.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:18.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T04:58:18.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:18.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:18.836 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:18 smithi029 ceph-mon[216903]: pgmap v1768: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 71354/224061 objects degraded (31.846%); 10557/224061 objects misplaced (4.712%); 3.8 MiB/s, 59 objects/s recovering 2023-10-09T04:58:18.837 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:18 smithi029 ceph-mon[223858]: pgmap v1768: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 71354/224061 objects degraded (31.846%); 10557/224061 objects misplaced (4.712%); 3.8 MiB/s, 59 objects/s recovering 2023-10-09T04:58:18.861 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:18.867 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:18 smithi177 ceph-mon[195797]: pgmap v1768: 99 pgs: 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 71354/224061 objects degraded (31.846%); 10557/224061 objects misplaced (4.712%); 3.8 MiB/s, 59 objects/s recovering 2023-10-09T04:58:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:18.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:18.969 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:19.007 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:19.059 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:19.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:19.151 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:19.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:19.208 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:19.330 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:19.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:19.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:19.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:19.469 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:19.532 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:19.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:19.537 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:19.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:19.671 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:58:19.739 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:58:19.774 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:19.914 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:58:19.917 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:58:20.189 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:20.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:20.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:20.340 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:20.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:20 smithi029 ceph-mon[223858]: 1.6 deep-scrub starts 2023-10-09T04:58:20.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:20 smithi029 ceph-mon[223858]: pgmap v1769: 99 pgs: 1 activating+undersized+degraded+remapped, 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 71022/224031 objects degraded (31.702%); 13659/224031 objects misplaced (6.097%); 3.4 MiB/s, 52 objects/s recovering 2023-10-09T04:58:20.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:20 smithi029 ceph-mon[223858]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:58:20.612 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:20.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:20.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:20.767 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:20 smithi029 ceph-mon[216903]: 1.6 deep-scrub starts 2023-10-09T04:58:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:20 smithi029 ceph-mon[216903]: pgmap v1769: 99 pgs: 1 activating+undersized+degraded+remapped, 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 71022/224031 objects degraded (31.702%); 13659/224031 objects misplaced (6.097%); 3.4 MiB/s, 52 objects/s recovering 2023-10-09T04:58:20.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:20 smithi029 ceph-mon[216903]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:58:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:20 smithi177 ceph-mon[195797]: 1.6 deep-scrub starts 2023-10-09T04:58:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:20 smithi177 ceph-mon[195797]: pgmap v1769: 99 pgs: 1 activating+undersized+degraded+remapped, 6 remapped+peering, 1 peering, 4 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 71022/224031 objects degraded (31.702%); 13659/224031 objects misplaced (6.097%); 3.4 MiB/s, 52 objects/s recovering 2023-10-09T04:58:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:20 smithi177 ceph-mon[195797]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:58:21.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:21 smithi029 ceph-mon[216903]: 1.3 deep-scrub starts 2023-10-09T04:58:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 71022/224031 objects degraded (31.702%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:21.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:21 smithi029 ceph-mon[223858]: 1.3 deep-scrub starts 2023-10-09T04:58:21.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 71022/224031 objects degraded (31.702%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:22.017 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:21 smithi177 ceph-mon[195797]: 1.3 deep-scrub starts 2023-10-09T04:58:22.017 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 71022/224031 objects degraded (31.702%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:22.088 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:22.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:22.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:22.193 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:22.231 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75763 14.d 14.8965ffd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:44.624146+0000", 2023-10-09T04:58:22.379 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 217.75175465199999, 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96808801799999999, 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75763 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.380 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624146+0000", 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624146+0000", 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624144+0000", 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:22.381 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624280+0000", 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013587199999999999 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624315+0000", 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5182000000000003e-05 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.382 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624323+0000", 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6939999999999995e-06 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624390+0000", 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6968e-05 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624402+0000", 2023-10-09T04:58:22.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2313000000000001e-05 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592224+0000", 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96782124000000003 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592234+0000", 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0292000000000001e-05 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.384 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75766 14.d 14.883bde8d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:44.624346+0000", 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 217.75155430300001, 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.967922485, 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.385 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75766 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624346+0000", 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624346+0000", 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624346+0000", 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624394+0000", 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8247000000000001e-05 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624403+0000", 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0699999999999996e-06 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624410+0000", 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3130000000000001e-06 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624446+0000", 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5889999999999997e-05 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624452+0000", 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9750000000000004e-06 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592260+0000", 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96780793899999995 2023-10-09T04:58:22.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592269+0000", 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5900000000000008e-06 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75774 14.d 14.93849fbd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:58:22.390 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:44.624423+0000", 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 217.75147786100001, 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96787062400000001, 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75774 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624423+0000", 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624423+0000", 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624422+0000", 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624456+0000", 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3958000000000003e-05 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624463+0000", 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9959999999999996e-06 2023-10-09T04:58:22.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624467+0000", 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1609999999999996e-06 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624591+0000", 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000124086 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624597+0000", 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.755e-06 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592286+0000", 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96768886300000001 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592293+0000", 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.255e-06 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:75775 14.d 14.c323126d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:54:44.624500+0000", 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 217.75140067699999, 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.967817606, 2023-10-09T04:58:22.396 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 75775 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624500+0000", 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.397 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624500+0000", 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624499+0000", 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:22.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624538+0000", 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8705000000000003e-05 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624551+0000", 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3219e-05 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624557+0000", 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3640000000000004e-06 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624616+0000", 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8575000000000001e-05 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:44.624620+0000", 2023-10-09T04:58:22.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2540000000000003e-06 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592311+0000", 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96769127899999996 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:54:45.592318+0000", 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0650000000000004e-06 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.401 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78105 14.2 14:4896c452:::benchmark_data_smithi029_550670_object494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:51.829616+0000", 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.546284888, 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1173021729999999, 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.402 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78105 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:51.829616+0000", 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.403 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:51.829616+0000", 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:51.829614+0000", 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:51.829624+0000", 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0346000000000001e-05 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:51.829651+0000", 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6757000000000001e-05 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:51.829660+0000", 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1190000000000003e-06 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.111081+0000", 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.28142043300000003 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.127563+0000", 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01648268 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.214929+0000", 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.087365850999999994 2023-10-09T04:58:22.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.224079+0000", 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0091497269999999999 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.306020+0000", 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081941073000000003 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.310381+0000", 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043613590000000004 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.835901+0000", 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.52552011099999996 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.842202+0000", 2023-10-09T04:58:22.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063003219999999997 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.852871+0000", 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010669494999999999 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.893958+0000", 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.041087225999999998 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.926763+0000", 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.032804316 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.928793+0000", 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020303320000000001 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.932153+0000", 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033603650000000001 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:52.946918+0000", 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014764482000000001 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.411 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78136 14.d 14:b962615d:::benchmark_data_smithi029_550670_object525:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:53.119162+0000", 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 149.25673896800001, 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.807735605, 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78136 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.412 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119162+0000", 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119162+0000", 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119160+0000", 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119167+0000", 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1029999999999998e-06 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119178+0000", 2023-10-09T04:58:22.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1148999999999999e-05 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119184+0000", 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0719999999999996e-06 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119233+0000", 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9398999999999999e-05 2023-10-09T04:58:22.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119241+0000", 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.807e-06 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.132867+0000", 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013626289999999999 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.416 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.132876+0000", 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4959999999999996e-06 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.293474+0000", 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1605982830000001 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.305201+0000", 2023-10-09T04:58:22.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011727044000000001 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.388919+0000", 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.083718230000000005 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.409031+0000", 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020111832 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.722309+0000", 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31327797200000002 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.732693+0000", 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010383580999999999 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:58:22.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.755958+0000", 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023265058000000002 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.892521+0000", 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.13656343500000001 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.894039+0000", 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001517914 2023-10-09T04:58:22.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.896846+0000", 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028064349999999999 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.901496+0000", 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046509139999999999 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.926897+0000", 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025400763999999999 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78166 14.d 14:b7194a4d:::benchmark_data_smithi029_550670_object555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:53.965078+0000", 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 148.410822309, 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.9897513, 2023-10-09T04:58:22.422 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78166 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965078+0000", 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965078+0000", 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965077+0000", 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965083+0000", 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6489999999999996e-06 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965097+0000", 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3827e-05 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965101+0000", 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7990000000000001e-06 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965164+0000", 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2409999999999994e-05 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.965171+0000", 2023-10-09T04:58:22.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.6190000000000002e-06 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.411158+0000", 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44598642900000002 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.427367+0000", 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016209467000000002 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.497293+0000", 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.069926099000000005 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.509373+0000", 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012079418999999999 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.818357+0000", 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30898435099999999 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.835434+0000", 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017076457999999999 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,5", 2023-10-09T04:58:22.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.867150+0000", 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.031716092000000001 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.928265+0000", 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.061114903999999998 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.932494+0000", 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042288650000000001 2023-10-09T04:58:22.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.935477+0000", 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029838109999999998 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.940273+0000", 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047953149999999996 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.954830+0000", 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014556952999999999 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78312 14.2 14:494aaae4:::benchmark_data_smithi029_550670_object701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:55.986327+0000", 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 146.389573324, 2023-10-09T04:58:22.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.88571777799999996, 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78312 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:55.986327+0000", 2023-10-09T04:58:22.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:55.986327+0000", 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:55.986326+0000", 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:55.986335+0000", 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6779999999999995e-06 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:55.986353+0000", 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8043e-05 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:55.986364+0000", 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0209e-05 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.096753+0000", 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.110389111 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.105809+0000", 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0090561099999999992 2023-10-09T04:58:22.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.183342+0000", 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.077532827999999998 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.194183+0000", 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010840944 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.691416+0000", 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.49723355299999999 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.707448+0000", 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016032213999999999 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.732368+0000", 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024919584000000002 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.846891+0000", 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.114523267 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.850497+0000", 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036060530000000001 2023-10-09T04:58:22.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.853840+0000", 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033432470000000001 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.857546+0000", 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037054509999999998 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.872045+0000", 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014499133000000001 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78336 14.2 14:429c852a:::benchmark_data_smithi029_550670_object725:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:56.316978+0000", 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 146.058922405, 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.86977689000000002, 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.441 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78336 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.316978+0000", 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.316978+0000", 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.316976+0000", 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.316987+0000", 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1778e-05 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.317003+0000", 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5591999999999999e-05 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.317012+0000", 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7590000000000003e-06 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.361019+0000", 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.044007304999999997 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.379816+0000", 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018796659 2023-10-09T04:58:22.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.518943+0000", 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.139127474 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:56.527589+0000", 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086460459999999992 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.078893+0000", 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.55130332400000004 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.093209+0000", 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014315972 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.118599+0000", 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025390721000000002 2023-10-09T04:58:22.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.140173+0000", 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021573498999999999 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.146162+0000", 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059896130000000004 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.148192+0000", 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020294200000000001 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.157743+0000", 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095512349999999999 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.186755+0000", 2023-10-09T04:58:22.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029011987999999999 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78448 14.2 14:4e1c5922:::benchmark_data_smithi029_550670_object837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:57.712407+0000", 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 144.663494007, 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.84266759300000005, 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.450 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78448 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.712407+0000", 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.712407+0000", 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.712405+0000", 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.712415+0000", 2023-10-09T04:58:22.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0227999999999999e-05 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.712432+0000", 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6801999999999999e-05 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.712441+0000", 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7660000000000004e-06 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.834281+0000", 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.121840353 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.843798+0000", 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095166769999999994 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.919286+0000", 2023-10-09T04:58:22.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.075488518000000004 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.926106+0000", 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068194019999999996 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.370256+0000", 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44415046499999999 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.379552+0000", 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0092962740000000002 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.395052+0000", 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015499552999999999 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.428303+0000", 2023-10-09T04:58:22.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.033251171000000003 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.522229+0000", 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.093925807 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.525218+0000", 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029893390000000001 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.530844+0000", 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056256910000000004 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.555074+0000", 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024230336000000002 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.458 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78461 14.2 14:43f649b7:::benchmark_data_smithi029_550670_object850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:57.805786+0000", 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 144.57011501700001, 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.78068058100000004, 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.459 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78461 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.805786+0000", 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.805786+0000", 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.805784+0000", 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.805792+0000", 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9910000000000002e-06 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.805805+0000", 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3111e-05 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.805812+0000", 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.08e-06 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.845042+0000", 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.039229343999999999 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.855602+0000", 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010560025000000001 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.973081+0000", 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.117479062 2023-10-09T04:58:22.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.979968+0000", 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068871100000000001 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.480582+0000", 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50061456800000004 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.491154+0000", 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010571548 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.506176+0000", 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015021932 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.555877+0000", 2023-10-09T04:58:22.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.049701644000000003 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.558869+0000", 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002991217 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.561464+0000", 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002595183 2023-10-09T04:58:22.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.566572+0000", 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051077750000000002 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.586466+0000", 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019894543000000001 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.467 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78478 14.2 14:4198c2e2:::benchmark_data_smithi029_550670_object867:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:57.924188+0000", 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 144.45171222100001, 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.85669856600000005, 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.468 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78478 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.924188+0000", 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.924188+0000", 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.924186+0000", 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.924201+0000", 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5257e-05 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.924220+0000", 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8649999999999999e-05 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.924231+0000", 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0953e-05 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.090935+0000", 2023-10-09T04:58:22.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.16670363599999999 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.099339+0000", 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008404178 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.199848+0000", 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.100509697 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.208020+0000", 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0081719830000000007 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.660062+0000", 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.45204149999999998 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.667094+0000", 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0070321309999999996 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.680512+0000", 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01341754 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.706145+0000", 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025633453 2023-10-09T04:58:22.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.763052+0000", 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.056906579999999998 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.764972+0000", 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019207149999999999 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.768152+0000", 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003179711 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.780887+0000", 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01273489 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.476 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78479 14.2 14:49ca40d7:::benchmark_data_smithi029_550670_object868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:57.925692+0000", 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 144.45020887999999, 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89130951300000005, 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78479 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.925692+0000", 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.925692+0000", 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.925690+0000", 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.925697+0000", 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9500000000000004e-06 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.925710+0000", 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2553e-05 2023-10-09T04:58:22.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:57.925716+0000", 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5089999999999999e-06 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.209076+0000", 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.283359056 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.218504+0000", 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0094289730000000002 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.288532+0000", 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.070027919999999994 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.296502+0000", 2023-10-09T04:58:22.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079697919999999998 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.738530+0000", 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44202794899999998 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.744201+0000", 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056713029999999999 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.754301+0000", 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01009935 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.782836+0000", 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028535171000000002 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.784848+0000", 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020122489999999998 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.790468+0000", 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056198760000000002 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.797847+0000", 2023-10-09T04:58:22.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073784189999999998 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:58.817001+0000", 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019154705000000001 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78581 14.2 14:44175baf:::benchmark_data_smithi029_550670_object970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.485 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:59.042351+0000", 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 143.33354916799999, 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.797921027, 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78581 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.042351+0000", 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.042351+0000", 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.042351+0000", 2023-10-09T04:58:22.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.042356+0000", 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9150000000000001e-06 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.042370+0000", 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3738e-05 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.042376+0000", 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1500000000000004e-06 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.072902+0000", 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030525237 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.090518+0000", 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017616612 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.290802+0000", 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.20028427100000001 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.298471+0000", 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.007668729 2023-10-09T04:58:22.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.747373+0000", 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.44890143300000002 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.753998+0000", 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066255919999999996 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.766993+0000", 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012994518 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.808572+0000", 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.041579496000000001 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.811749+0000", 2023-10-09T04:58:22.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031765169999999998 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.814680+0000", 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002930761 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819678+0000", 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049987360000000002 2023-10-09T04:58:22.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.840272+0000", 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020594207999999999 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78590 14.2 14:45804694:::benchmark_data_smithi029_550670_object979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:59.142045+0000", 2023-10-09T04:58:22.494 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 143.23385526300001, 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97660918600000002, 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78590 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.142045+0000", 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.142045+0000", 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.142044+0000", 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T04:58:22.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.142053+0000", 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7299999999999994e-06 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.142066+0000", 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3919e-05 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.142075+0000", 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5310000000000001e-06 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.385720+0000", 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.243645472 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.395017+0000", 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0092967299999999996 2023-10-09T04:58:22.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.452923+0000", 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.057906182 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.457698+0000", 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0047745399999999999 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.960900+0000", 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50320244700000005 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.974222+0000", 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013321859 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.991736+0000", 2023-10-09T04:58:22.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017513672000000001 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.081243+0000", 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.089506746999999998 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.093961+0000", 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012718706999999999 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.096443+0000", 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002481427 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.100947+0000", 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045045090000000003 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.118655+0000", 2023-10-09T04:58:22.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017707211 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78603 14.2 14:4e08a781:::benchmark_data_smithi029_550670_object992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:59.366761+0000", 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 143.00913986500001, 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.968040389, 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.503 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78603 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.366761+0000", 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.366761+0000", 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.366759+0000", 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.366769+0000", 2023-10-09T04:58:22.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8230000000000006e-06 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.366786+0000", 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7507999999999999e-05 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.366794+0000", 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0320000000000003e-06 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.396343+0000", 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029548689999999999 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.407549+0000", 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011206376000000001 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.488775+0000", 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081225321000000003 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.494337+0000", 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055626060000000003 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.253147+0000", 2023-10-09T04:58:22.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.75880921700000004 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.260843+0000", 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076965200000000001 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.276297+0000", 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015453441 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.308846+0000", 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.032549913 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.309674+0000", 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082791400000000002 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.312573+0000", 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028989150000000002 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.317727+0000", 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051542250000000001 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.334801+0000", 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017073629999999999 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78604 14.2 14:48a415ea:::benchmark_data_smithi029_550670_object993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:59.430537+0000", 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 142.945363603, 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.173957513, 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.512 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78604 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.430537+0000", 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.430537+0000", 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.430534+0000", 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999971 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.430544+0000", 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7189999999999996e-06 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.430559+0000", 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4973999999999999e-05 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.430566+0000", 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4959999999999999e-06 2023-10-09T04:58:22.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.495276+0000", 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.064709357999999995 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.503355+0000", 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080788609999999997 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.592357+0000", 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.089002607999999997 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.602104+0000", 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097464079999999998 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.529950+0000", 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92784608300000004 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.536011+0000", 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060613840000000004 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.546916+0000", 2023-10-09T04:58:22.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010905089 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.568337+0000", 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021420805000000001 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.570612+0000", 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022748769999999998 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.585955+0000", 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015342667000000001 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.589751+0000", 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037964159999999999 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.604495+0000", 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014743464 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78623 14.2 14:4e556a5f:::benchmark_data_smithi029_550670_object1012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:59.819398+0000", 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 142.55650254400001, 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92356028300000004, 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.521 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78623 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819398+0000", 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819398+0000", 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819396+0000", 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819404+0000", 2023-10-09T04:58:22.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3780000000000001e-06 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819421+0000", 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.632e-05 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819427+0000", 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5810000000000002e-06 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.865361+0000", 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.045934033999999999 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.880004+0000", 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014642489 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.037506+0000", 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.15750251600000001 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.044309+0000", 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068025550000000001 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.637818+0000", 2023-10-09T04:58:22.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.59350855499999999 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.653283+0000", 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01546515 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.674988+0000", 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021705609000000001 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.702472+0000", 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.027483410999999999 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.704023+0000", 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015509860000000001 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.709431+0000", 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054082160000000004 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.715948+0000", 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065170360000000004 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.742958+0000", 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.02701042 2023-10-09T04:58:22.529 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78624 14.2 14:46530f74:::benchmark_data_smithi029_550670_object1013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e476)", 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:59.819827+0000", 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 142.556073809, 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0981859220000001, 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T04:58:22.530 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78624 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819827+0000", 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819827+0000", 2023-10-09T04:58:22.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819826+0000", 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819830+0000", 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0670000000000002e-06 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819841+0000", 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0263e-05 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.819845+0000", 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.702e-06 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.881008+0000", 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.061162900999999999 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:59.893030+0000", 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012022089 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.084369+0000", 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.19133866299999999 2023-10-09T04:58:22.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.091006+0000", 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066375430000000001 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.824218+0000", 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.73321144599999999 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.838594+0000", 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014375664 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.856596+0000", 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018002695999999999 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.878471+0000", 2023-10-09T04:58:22.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021874327999999998 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.882336+0000", 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038652970000000002 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.883624+0000", 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001288218 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.889727+0000", 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061028050000000002 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:00.918013+0000", 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028285845 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.538 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(14.2 507/499 [PushOp(14:46d6768f:::benchmark_data_smithi029_550670_object12344:head, version: 444'757, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:46d6768f:::benchmark_data_smithi029_550670_object12344:head@444'757, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:57:37.986655+0000", 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 44.389245533, 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5003260059999999, 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986655+0000", 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986655+0000", 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986654+0000", 2023-10-09T04:58:22.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986682+0000", 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8099999999999999e-05 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986698+0000", 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5310000000000001e-05 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986706+0000", 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7530000000000007e-06 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:40.486950+0000", 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5002433499999999 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:40.486981+0000", 2023-10-09T04:58:22.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1350999999999998e-05 2023-10-09T04:58:22.543 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.543 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.543 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.543 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T04:58:22.543 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T04:58:22.543 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:58:22.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:22.544 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:22.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:22.545 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1769,"stamp":"2023-10-09T04:58:19.548078+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4892989368,"num_objects":74677,"num_object_clones":0,"num_object_copies":224031,"num_objects_missing_on_primary":19016,"num_objects_missing":19016,"num_objects_degraded":71022,"num_objects_misplaced":13659,"num_objects_unfound":0,"num_objects_dirty":74677,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":80865,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26378,"num_bytes_recovered":1701676332,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75341,"ondisk_log_size":75341,"up":297,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18875440,"kb_used_data":12317568,"kb_used_omap":14,"kb_used_meta":6557809,"kb_avail":543595472,"statfs":{"total":575970213888,"available":556641763328,"internally_reserved":0,"allocated":12613189632,"data_stored":12603120851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15340,"internal_metadata":6715196436},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,2,0,0,0,4,3,2],"upper_bound":16384},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1376256,"num_objects":-21,"num_object_clones":0,"num_object_copies":-63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22282,"num_objects_misplaced":2645,"num_objects_unfound":0,"num_objects_dirty":-21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":461,"num_bytes_recovered":30676126,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.718127"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":320,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.842747+0000","last_change":"2023-10-09T04:56:53.922296+0000","last_active":"2023-10-09T04:58:06.842747+0000","last_peered":"2023-10-09T04:58:06.842747+0000","last_clean":"2023-10-09T04:58:06.842747+0000","last_became_active":"2023-10-09T04:56:53.921591+0000","last_became_peered":"2023-10-09T04:56:53.921591+0000","last_unstale":"2023-10-09T04:58:06.842747+0000","last_undegraded":"2023-10-09T04:58:06.842747+0000","last_fullsized":"2023-10-09T04:58:06.842747+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:08:43.567917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":107,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.833108+0000","last_change":"2023-10-09T04:57:07.316322+0000","last_active":"2023-10-09T04:57:44.833108+0000","last_peered":"2023-10-09T04:57:44.833108+0000","last_clean":"2023-10-09T04:57:44.833108+0000","last_became_active":"2023-10-09T04:57:07.316054+0000","last_became_peered":"2023-10-09T04:57:07.316054+0000","last_unstale":"2023-10-09T04:57:44.833108+0000","last_undegraded":"2023-10-09T04:57:44.833108+0000","last_fullsized":"2023-10-09T04:57:44.833108+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:20:10.805216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":64,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.889785+0000","last_change":"2023-10-09T04:58:07.889785+0000","last_active":"2023-10-09T04:58:07.889785+0000","last_peered":"2023-10-09T04:58:07.889785+0000","last_clean":"2023-10-09T04:58:07.889785+0000","last_became_active":"2023-10-09T04:58:07.889365+0000","last_became_peered":"2023-10-09T04:58:07.889365+0000","last_unstale":"2023-10-09T04:58:07.889785+0000","last_undegraded":"2023-10-09T04:58:07.889785+0000","last_fullsized":"2023-10-09T04:58:07.889785+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:14.043179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":107,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.892648+0000","last_change":"2023-10-09T04:58:07.892648+0000","last_active":"2023-10-09T04:58:07.892648+0000","last_peered":"2023-10-09T04:58:07.892648+0000","last_clean":"2023-10-09T04:58:07.892648+0000","last_became_active":"2023-10-09T04:58:07.891762+0000","last_became_peered":"2023-10-09T04:58:07.891762+0000","last_unstale":"2023-10-09T04:58:07.892648+0000","last_undegraded":"2023-10-09T04:58:07.892648+0000","last_fullsized":"2023-10-09T04:58:07.892648+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:13:32.446354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.830135+0000","last_change":"2023-10-09T04:56:55.859522+0000","last_active":"2023-10-09T04:57:44.830135+0000","last_peered":"2023-10-09T04:57:44.830135+0000","last_clean":"2023-10-09T04:57:44.830135+0000","last_became_active":"2023-10-09T04:56:55.859254+0000","last_became_peered":"2023-10-09T04:56:55.859254+0000","last_unstale":"2023-10-09T04:57:44.830135+0000","last_undegraded":"2023-10-09T04:57:44.830135+0000","last_fullsized":"2023-10-09T04:57:44.830135+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:30.229851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.840301+0000","last_change":"2023-10-09T04:58:07.895975+0000","last_active":"2023-10-09T04:58:10.840301+0000","last_peered":"2023-10-09T04:58:10.840301+0000","last_clean":"2023-10-09T04:58:10.840301+0000","last_became_active":"2023-10-09T04:58:07.893663+0000","last_became_peered":"2023-10-09T04:58:07.893663+0000","last_unstale":"2023-10-09T04:58:10.840301+0000","last_undegraded":"2023-10-09T04:58:10.840301+0000","last_fullsized":"2023-10-09T04:58:10.840301+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:35.053280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":560,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.851030+0000","last_change":"2023-10-09T04:58:07.879586+0000","last_active":"2023-10-09T04:58:10.851030+0000","last_peered":"2023-10-09T04:58:10.851030+0000","last_clean":"2023-10-09T04:58:10.851030+0000","last_became_active":"2023-10-09T04:58:07.879041+0000","last_became_peered":"2023-10-09T04:58:07.879041+0000","last_unstale":"2023-10-09T04:58:10.851030+0000","last_undegraded":"2023-10-09T04:58:10.851030+0000","last_fullsized":"2023-10-09T04:58:10.851030+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:12.975185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":102,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.830046+0000","last_change":"2023-10-09T04:57:29.319082+0000","last_active":"2023-10-09T04:57:44.830046+0000","last_peered":"2023-10-09T04:57:44.830046+0000","last_clean":"2023-10-09T04:57:44.830046+0000","last_became_active":"2023-10-09T04:57:29.318781+0000","last_became_peered":"2023-10-09T04:57:29.318781+0000","last_unstale":"2023-10-09T04:57:44.830046+0000","last_undegraded":"2023-10-09T04:57:44.830046+0000","last_fullsized":"2023-10-09T04:57:44.830046+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":502,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:47:52.766285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":508,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.845292+0000","last_change":"2023-10-09T04:55:20.004951+0000","last_active":"2023-10-09T04:58:10.845292+0000","last_peered":"2023-10-09T04:58:10.845292+0000","last_clean":"2023-10-09T04:58:10.845292+0000","last_became_active":"2023-10-09T04:55:20.004659+0000","last_became_peered":"2023-10-09T04:55:20.004659+0000","last_unstale":"2023-10-09T04:58:10.845292+0000","last_undegraded":"2023-10-09T04:58:10.845292+0000","last_fullsized":"2023-10-09T04:58:10.845292+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:48.952780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":92,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.831305+0000","last_change":"2023-10-09T04:57:07.270877+0000","last_active":"2023-10-09T04:57:44.831305+0000","last_peered":"2023-10-09T04:57:44.831305+0000","last_clean":"2023-10-09T04:57:44.831305+0000","last_became_active":"2023-10-09T04:57:07.270621+0000","last_became_peered":"2023-10-09T04:57:07.270621+0000","last_unstale":"2023-10-09T04:57:44.831305+0000","last_undegraded":"2023-10-09T04:57:44.831305+0000","last_fullsized":"2023-10-09T04:57:44.831305+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:07:17.571059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":115,"reported_epoch":511,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:57:44.832945+0000","last_change":"2023-10-09T04:57:07.279095+0000","last_active":"2023-10-09T04:57:44.832945+0000","last_peered":"2023-10-09T04:57:44.832945+0000","last_clean":"2023-10-09T04:56:56.814041+0000","last_became_active":"2023-10-09T04:57:07.278825+0000","last_became_peered":"2023-10-09T04:57:07.278825+0000","last_unstale":"2023-10-09T04:57:44.832945+0000","last_undegraded":"2023-10-09T04:57:07.264245+0000","last_fullsized":"2023-10-09T04:57:44.832945+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:57:30.656516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":["3","4","6","7"],"object_location_counts":[{"shards":"3,4,6,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.886648+0000","last_change":"2023-10-09T04:58:07.886648+0000","last_active":"2023-10-09T04:58:07.886648+0000","last_peered":"2023-10-09T04:58:07.886648+0000","last_clean":"2023-10-09T04:58:07.886648+0000","last_became_active":"2023-10-09T04:58:07.886050+0000","last_became_peered":"2023-10-09T04:58:07.886050+0000","last_unstale":"2023-10-09T04:58:07.886648+0000","last_undegraded":"2023-10-09T04:58:07.886648+0000","last_fullsized":"2023-10-09T04:58:07.886648+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:08.968095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":342,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839642+0000","last_change":"2023-10-09T04:58:08.168101+0000","last_active":"2023-10-09T04:58:10.839642+0000","last_peered":"2023-10-09T04:58:10.839642+0000","last_clean":"2023-10-09T04:58:10.839642+0000","last_became_active":"2023-10-09T04:58:07.981931+0000","last_became_peered":"2023-10-09T04:58:07.981931+0000","last_unstale":"2023-10-09T04:58:10.839642+0000","last_undegraded":"2023-10-09T04:58:10.839642+0000","last_fullsized":"2023-10-09T04:58:10.839642+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:20:22.999993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.840082+0000","last_change":"2023-10-09T04:58:07.919705+0000","last_active":"2023-10-09T04:58:10.840082+0000","last_peered":"2023-10-09T04:58:10.840082+0000","last_clean":"2023-10-09T04:58:10.840082+0000","last_became_active":"2023-10-09T04:58:07.919100+0000","last_became_peered":"2023-10-09T04:58:07.919100+0000","last_unstale":"2023-10-09T04:58:10.840082+0000","last_undegraded":"2023-10-09T04:58:10.840082+0000","last_fullsized":"2023-10-09T04:58:10.840082+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:36:21.158489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":507,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:12.682554+0000","last_change":"2023-10-09T04:58:07.893941+0000","last_active":"2023-10-09T04:58:12.682554+0000","last_peered":"2023-10-09T04:58:12.682554+0000","last_clean":"2023-10-09T04:58:12.682554+0000","last_became_active":"2023-10-09T04:58:07.893620+0000","last_became_peered":"2023-10-09T04:58:07.893620+0000","last_unstale":"2023-10-09T04:58:12.682554+0000","last_undegraded":"2023-10-09T04:58:12.682554+0000","last_fullsized":"2023-10-09T04:58:12.682554+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:04:59.822554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":77,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:47.122081+0000","last_change":"2023-10-09T04:55:19.974453+0000","last_active":"2023-10-09T04:57:47.122081+0000","last_peered":"2023-10-09T04:57:47.122081+0000","last_clean":"2023-10-09T04:57:47.122081+0000","last_became_active":"2023-10-09T04:55:19.973871+0000","last_became_peered":"2023-10-09T04:55:19.973871+0000","last_unstale":"2023-10-09T04:57:47.122081+0000","last_undegraded":"2023-10-09T04:57:47.122081+0000","last_fullsized":"2023-10-09T04:57:47.122081+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:47.239847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":54,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.831287+0000","last_change":"2023-10-09T04:56:53.922079+0000","last_active":"2023-10-09T04:58:06.831287+0000","last_peered":"2023-10-09T04:58:06.831287+0000","last_clean":"2023-10-09T04:58:06.831287+0000","last_became_active":"2023-10-09T04:56:53.921311+0000","last_became_peered":"2023-10-09T04:56:53.921311+0000","last_unstale":"2023-10-09T04:58:06.831287+0000","last_undegraded":"2023-10-09T04:58:06.831287+0000","last_fullsized":"2023-10-09T04:58:06.831287+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:20.368978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":517,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.852176+0000","last_change":"2023-10-09T04:54:25.766303+0000","last_active":"2023-10-09T04:58:10.852176+0000","last_peered":"2023-10-09T04:58:10.852176+0000","last_clean":"2023-10-09T04:58:10.852176+0000","last_became_active":"2023-10-09T04:54:25.765766+0000","last_became_peered":"2023-10-09T04:54:25.765766+0000","last_unstale":"2023-10-09T04:58:10.852176+0000","last_undegraded":"2023-10-09T04:58:10.852176+0000","last_fullsized":"2023-10-09T04:58:10.852176+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:48.044596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839080+0000","last_change":"2023-10-09T04:58:07.879173+0000","last_active":"2023-10-09T04:58:10.839080+0000","last_peered":"2023-10-09T04:58:10.839080+0000","last_clean":"2023-10-09T04:58:10.839080+0000","last_became_active":"2023-10-09T04:58:07.877129+0000","last_became_peered":"2023-10-09T04:58:07.877129+0000","last_unstale":"2023-10-09T04:58:10.839080+0000","last_undegraded":"2023-10-09T04:58:10.839080+0000","last_fullsized":"2023-10-09T04:58:10.839080+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:51.344295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":110,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.831904+0000","last_change":"2023-10-09T04:57:07.282058+0000","last_active":"2023-10-09T04:57:44.831904+0000","last_peered":"2023-10-09T04:57:44.831904+0000","last_clean":"2023-10-09T04:57:44.831904+0000","last_became_active":"2023-10-09T04:57:07.281721+0000","last_became_peered":"2023-10-09T04:57:07.281721+0000","last_unstale":"2023-10-09T04:57:44.831904+0000","last_undegraded":"2023-10-09T04:57:44.831904+0000","last_fullsized":"2023-10-09T04:57:44.831904+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:48.745423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":112,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839644+0000","last_change":"2023-10-09T04:58:07.908453+0000","last_active":"2023-10-09T04:58:10.839644+0000","last_peered":"2023-10-09T04:58:10.839644+0000","last_clean":"2023-10-09T04:58:10.839644+0000","last_became_active":"2023-10-09T04:58:07.904674+0000","last_became_peered":"2023-10-09T04:58:07.904674+0000","last_unstale":"2023-10-09T04:58:10.839644+0000","last_undegraded":"2023-10-09T04:58:10.839644+0000","last_fullsized":"2023-10-09T04:58:10.839644+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:18.595337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.930062+0000","last_change":"2023-10-09T04:58:07.930062+0000","last_active":"2023-10-09T04:58:07.930062+0000","last_peered":"2023-10-09T04:58:07.930062+0000","last_clean":"2023-10-09T04:58:07.930062+0000","last_became_active":"2023-10-09T04:58:07.929708+0000","last_became_peered":"2023-10-09T04:58:07.929708+0000","last_unstale":"2023-10-09T04:58:07.930062+0000","last_undegraded":"2023-10-09T04:58:07.930062+0000","last_fullsized":"2023-10-09T04:58:07.930062+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:54.726957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.2","version":"512'4987","reported_seq":6907,"reported_epoch":515,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:18.021315+0000","last_change":"2023-10-09T04:58:18.049312+0000","last_active":"2023-10-09T04:58:06.921101+0000","last_peered":"2023-10-09T04:58:06.620468+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T04:58:01.211386+0000","last_became_peered":"2023-10-09T04:58:01.211386+0000","last_unstale":"2023-10-09T04:58:18.021315+0000","last_undegraded":"2023-10-09T04:58:18.021315+0000","last_fullsized":"2023-10-09T04:58:08.837096+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4987,"log_dups_size":0,"ondisk_log_size":4987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:05:26.826634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304283648,"num_objects":4643,"num_object_clones":0,"num_object_copies":13929,"num_objects_missing_on_primary":958,"num_objects_missing":958,"num_objects_degraded":5090,"num_objects_misplaced":3102,"num_objects_unfound":0,"num_objects_dirty":4643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4987,"num_write_kb":308160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":551,"num_bytes_recovered":36044800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[7,1],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"1,7","objects":583},{"shards":"5","objects":720},{"shards":"5,6","objects":96},{"shards":"5,6,7","objects":1},{"shards":"5,7","objects":3243}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":39,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.831434+0000","last_change":"2023-10-09T04:57:38.881660+0000","last_active":"2023-10-09T04:58:06.831434+0000","last_peered":"2023-10-09T04:58:06.831434+0000","last_clean":"2023-10-09T04:58:06.831434+0000","last_became_active":"2023-10-09T04:56:53.957391+0000","last_became_peered":"2023-10-09T04:56:53.957391+0000","last_unstale":"2023-10-09T04:58:06.831434+0000","last_undegraded":"2023-10-09T04:58:06.831434+0000","last_fullsized":"2023-10-09T04:58:06.831434+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:01:54.139621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":537,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.844068+0000","last_change":"2023-10-09T04:58:07.861064+0000","last_active":"2023-10-09T04:58:10.844068+0000","last_peered":"2023-10-09T04:58:10.844068+0000","last_clean":"2023-10-09T04:58:10.844068+0000","last_became_active":"2023-10-09T04:58:07.860783+0000","last_became_peered":"2023-10-09T04:58:07.860783+0000","last_unstale":"2023-10-09T04:58:10.844068+0000","last_undegraded":"2023-10-09T04:58:10.844068+0000","last_fullsized":"2023-10-09T04:58:10.844068+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:04:35.042137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"512'5057","reported_seq":5874,"reported_epoch":512,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:05.980783+0000","last_change":"2023-10-09T04:57:52.668900+0000","last_active":"2023-10-09T04:58:05.980783+0000","last_peered":"2023-10-09T04:58:05.980783+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:57:52.374702+0000","last_became_peered":"2023-10-09T04:57:52.374702+0000","last_unstale":"2023-10-09T04:58:05.980783+0000","last_undegraded":"2023-10-09T04:57:27.353215+0000","last_fullsized":"2023-10-09T04:57:07.009369+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.130277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306118656,"num_objects":4671,"num_object_clones":0,"num_object_copies":14013,"num_objects_missing_on_primary":3845,"num_objects_missing":3845,"num_objects_degraded":4263,"num_objects_misplaced":822,"num_objects_unfound":0,"num_objects_dirty":4671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5057,"num_write_kb":311296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":183,"num_bytes_recovered":11927552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[3,5],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":16},{"shards":"2,3,5,7","objects":716},{"shards":"2,5,7","objects":3845},{"shards":"3,5","objects":-12},{"shards":"3,5,6","objects":94}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":367,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.835532+0000","last_change":"2023-10-09T04:56:53.917853+0000","last_active":"2023-10-09T04:58:06.835532+0000","last_peered":"2023-10-09T04:58:06.835532+0000","last_clean":"2023-10-09T04:58:06.835532+0000","last_became_active":"2023-10-09T04:56:53.917041+0000","last_became_peered":"2023-10-09T04:56:53.917041+0000","last_unstale":"2023-10-09T04:58:06.835532+0000","last_undegraded":"2023-10-09T04:58:06.835532+0000","last_fullsized":"2023-10-09T04:58:06.835532+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:56.423242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":570,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.900033+0000","last_change":"2023-10-09T04:58:07.889629+0000","last_active":"2023-10-09T04:58:07.900033+0000","last_peered":"2023-10-09T04:58:07.900033+0000","last_clean":"2023-10-09T04:58:07.900033+0000","last_became_active":"2023-10-09T04:58:07.888972+0000","last_became_peered":"2023-10-09T04:58:07.888972+0000","last_unstale":"2023-10-09T04:58:07.900033+0000","last_undegraded":"2023-10-09T04:58:07.900033+0000","last_fullsized":"2023-10-09T04:58:07.900033+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:45.487836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.0","version":"515'5053","reported_seq":5282,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.844703+0000","last_change":"2023-10-09T04:58:08.217251+0000","last_active":"2023-10-09T04:58:10.844703+0000","last_peered":"2023-10-09T04:58:10.844703+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:08.205555+0000","last_became_peered":"2023-10-09T04:58:08.205555+0000","last_unstale":"2023-10-09T04:58:10.844703+0000","last_undegraded":"2023-10-09T04:58:07.865867+0000","last_fullsized":"2023-10-09T04:58:07.837976+0000","mapping_epoch":513,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4515,"log_dups_size":0,"ondisk_log_size":4515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:28:09.965868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306249728,"num_objects":4673,"num_object_clones":0,"num_object_copies":14019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5053,"num_write_kb":311232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":9},{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":4542},{"shards":"2,5","objects":116}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":164,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.852252+0000","last_change":"2023-10-09T04:58:08.028736+0000","last_active":"2023-10-09T04:58:10.852252+0000","last_peered":"2023-10-09T04:58:10.852252+0000","last_clean":"2023-10-09T04:58:10.852252+0000","last_became_active":"2023-10-09T04:58:07.937180+0000","last_became_peered":"2023-10-09T04:58:07.937180+0000","last_unstale":"2023-10-09T04:58:10.852252+0000","last_undegraded":"2023-10-09T04:58:10.852252+0000","last_fullsized":"2023-10-09T04:58:10.852252+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:43:23.362583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.840328+0000","last_change":"2023-10-09T04:58:07.872446+0000","last_active":"2023-10-09T04:58:10.840328+0000","last_peered":"2023-10-09T04:58:10.840328+0000","last_clean":"2023-10-09T04:58:10.840328+0000","last_became_active":"2023-10-09T04:58:07.872159+0000","last_became_peered":"2023-10-09T04:58:07.872159+0000","last_unstale":"2023-10-09T04:58:10.840328+0000","last_undegraded":"2023-10-09T04:58:10.840328+0000","last_fullsized":"2023-10-09T04:58:10.840328+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:25.849479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"512'5046","reported_seq":5668,"reported_epoch":514,"state":"remapped+peering","last_fresh":"2023-10-09T04:58:07.830372+0000","last_change":"2023-10-09T04:58:07.830372+0000","last_active":"2023-10-09T04:58:07.019173+0000","last_peered":"2023-10-09T04:58:06.778029+0000","last_clean":"2023-10-09T04:53:31.951851+0000","last_became_active":"2023-10-09T04:57:42.352928+0000","last_became_peered":"2023-10-09T04:57:42.352928+0000","last_unstale":"2023-10-09T04:58:07.830372+0000","last_undegraded":"2023-10-09T04:58:07.830372+0000","last_fullsized":"2023-10-09T04:58:07.830372+0000","mapping_epoch":514,"log_start":"445'1388","ondisk_log_start":"445'1388","created":440,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3658,"log_dups_size":0,"ondisk_log_size":3658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:59:35.911226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":4705,"num_objects_missing":4705,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5046,"num_write_kb":310976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":187,"num_bytes_recovered":12255232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":578,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.845187+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T04:58:10.845187+0000","last_peered":"2023-10-09T04:58:10.845187+0000","last_clean":"2023-10-09T04:58:10.845187+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T04:58:10.845187+0000","last_undegraded":"2023-10-09T04:58:10.845187+0000","last_fullsized":"2023-10-09T04:58:10.845187+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.837819+0000","last_change":"2023-10-09T04:57:06.810767+0000","last_active":"2023-10-09T04:58:06.837819+0000","last_peered":"2023-10-09T04:58:06.837819+0000","last_clean":"2023-10-09T04:58:06.837819+0000","last_became_active":"2023-10-09T04:57:06.810483+0000","last_became_peered":"2023-10-09T04:57:06.810483+0000","last_unstale":"2023-10-09T04:58:06.837819+0000","last_undegraded":"2023-10-09T04:58:06.837819+0000","last_fullsized":"2023-10-09T04:58:06.837819+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:48.624201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":999,"reported_epoch":512,"state":"remapped+peering","last_fresh":"2023-10-09T04:57:55.010303+0000","last_change":"2023-10-09T04:58:07.828136+0000","last_active":"2023-10-09T04:58:07.828136+0000","last_peered":"2023-10-09T04:57:55.010303+0000","last_clean":"2023-10-09T04:57:55.010303+0000","last_became_active":"2023-10-09T04:57:07.270217+0000","last_became_peered":"2023-10-09T04:57:07.270217+0000","last_unstale":"2023-10-09T04:57:55.010303+0000","last_undegraded":"2023-10-09T04:57:55.010303+0000","last_fullsized":"2023-10-09T04:57:55.010303+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6,7],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"514'4893","reported_seq":5564,"reported_epoch":517,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:10.857142+0000","last_change":"2023-10-09T04:57:45.001341+0000","last_active":"2023-10-09T04:58:10.857142+0000","last_peered":"2023-10-09T04:58:10.857142+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:57:44.954832+0000","last_became_peered":"2023-10-09T04:57:44.954832+0000","last_unstale":"2023-10-09T04:58:10.857142+0000","last_undegraded":"2023-10-09T04:57:44.934565+0000","last_fullsized":"2023-10-09T04:58:10.857142+0000","mapping_epoch":510,"log_start":"444'386","ondisk_log_start":"444'386","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:06:04.945913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4893,"num_write_kb":301824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":11665408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":666},{"shards":"2,3,6","objects":115},{"shards":"2,4,5","objects":3756}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":79,"reported_epoch":511,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:57:52.314234+0000","last_change":"2023-10-09T04:57:06.809884+0000","last_active":"2023-10-09T04:57:52.314234+0000","last_peered":"2023-10-09T04:57:52.314234+0000","last_clean":"2023-10-09T04:55:49.512244+0000","last_became_active":"2023-10-09T04:57:06.809328+0000","last_became_peered":"2023-10-09T04:57:06.809328+0000","last_unstale":"2023-10-09T04:57:52.314234+0000","last_undegraded":"2023-10-09T04:56:55.791687+0000","last_fullsized":"2023-10-09T04:57:52.314234+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":463,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:33.752397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":304,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-09T04:57:55.035978+0000","last_change":"2023-10-09T04:57:07.316142+0000","last_active":"2023-10-09T04:57:55.035978+0000","last_peered":"2023-10-09T04:57:55.035978+0000","last_clean":"2023-10-09T04:57:55.035978+0000","last_became_active":"2023-10-09T04:57:07.315602+0000","last_became_peered":"2023-10-09T04:57:07.315602+0000","last_unstale":"2023-10-09T04:57:55.035978+0000","last_undegraded":"2023-10-09T04:57:55.035978+0000","last_fullsized":"2023-10-09T04:57:55.035978+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:14.906030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1021,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839439+0000","last_change":"2023-10-09T04:58:08.870256+0000","last_active":"2023-10-09T04:58:10.839439+0000","last_peered":"2023-10-09T04:58:10.839439+0000","last_clean":"2023-10-09T04:58:10.839439+0000","last_became_active":"2023-10-09T04:58:07.937068+0000","last_became_peered":"2023-10-09T04:58:07.937068+0000","last_unstale":"2023-10-09T04:58:10.839439+0000","last_undegraded":"2023-10-09T04:58:10.839439+0000","last_fullsized":"2023-10-09T04:58:10.839439+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"515'5071","reported_seq":10219,"reported_epoch":518,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:14.739820+0000","last_change":"2023-10-09T04:58:08.168387+0000","last_active":"2023-10-09T04:58:14.739820+0000","last_peered":"2023-10-09T04:58:14.739820+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:56:55.119428+0000","last_became_peered":"2023-10-09T04:56:55.119428+0000","last_unstale":"2023-10-09T04:58:14.739820+0000","last_undegraded":"2023-10-09T04:56:54.929985+0000","last_fullsized":"2023-10-09T04:56:54.902789+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5071,"log_dups_size":0,"ondisk_log_size":5071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:37:57.470599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308215808,"num_objects":4703,"num_object_clones":0,"num_object_copies":14109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2473,"num_objects_unfound":0,"num_objects_dirty":4703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5071,"num_write_kb":312768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2500,"num_bytes_recovered":161939456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"0,2,4,5","objects":7},{"shards":"2,4,5","objects":2466},{"shards":"2,5","objects":2228}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839508+0000","last_change":"2023-10-09T04:58:07.886012+0000","last_active":"2023-10-09T04:58:10.839508+0000","last_peered":"2023-10-09T04:58:10.839508+0000","last_clean":"2023-10-09T04:58:10.839508+0000","last_became_active":"2023-10-09T04:58:07.879094+0000","last_became_peered":"2023-10-09T04:58:07.879094+0000","last_unstale":"2023-10-09T04:58:10.839508+0000","last_undegraded":"2023-10-09T04:58:10.839508+0000","last_fullsized":"2023-10-09T04:58:10.839508+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:41:22.518511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":323,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.843624+0000","last_change":"2023-10-09T04:58:07.882492+0000","last_active":"2023-10-09T04:58:10.843624+0000","last_peered":"2023-10-09T04:58:10.843624+0000","last_clean":"2023-10-09T04:58:10.843624+0000","last_became_active":"2023-10-09T04:58:07.882161+0000","last_became_peered":"2023-10-09T04:58:07.882161+0000","last_unstale":"2023-10-09T04:58:10.843624+0000","last_undegraded":"2023-10-09T04:58:10.843624+0000","last_fullsized":"2023-10-09T04:58:10.843624+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:06.813717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"514'5064","reported_seq":5791,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.846271+0000","last_change":"2023-10-09T04:58:08.088672+0000","last_active":"2023-10-09T04:58:10.846271+0000","last_peered":"2023-10-09T04:58:10.846271+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:58:08.077966+0000","last_became_peered":"2023-10-09T04:58:08.077966+0000","last_unstale":"2023-10-09T04:58:10.846271+0000","last_undegraded":"2023-10-09T04:58:07.975481+0000","last_fullsized":"2023-10-09T04:58:07.836987+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:39:57.553552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5064,"num_write_kb":311040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":204,"num_bytes_recovered":13369344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[7,3],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":713},{"shards":"2,4,7","objects":3832},{"shards":"3,7","objects":111}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.842395+0000","last_change":"2023-10-09T04:56:53.449447+0000","last_active":"2023-10-09T04:58:10.842395+0000","last_peered":"2023-10-09T04:58:10.842395+0000","last_clean":"2023-10-09T04:58:10.842395+0000","last_became_active":"2023-10-09T04:56:53.449076+0000","last_became_peered":"2023-10-09T04:56:53.449076+0000","last_unstale":"2023-10-09T04:58:10.842395+0000","last_undegraded":"2023-10-09T04:58:10.842395+0000","last_fullsized":"2023-10-09T04:58:10.842395+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:59.853051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.840037+0000","last_change":"2023-10-09T04:58:07.885456+0000","last_active":"2023-10-09T04:58:10.840037+0000","last_peered":"2023-10-09T04:58:10.840037+0000","last_clean":"2023-10-09T04:58:10.840037+0000","last_became_active":"2023-10-09T04:58:07.885146+0000","last_became_peered":"2023-10-09T04:58:07.885146+0000","last_unstale":"2023-10-09T04:58:10.840037+0000","last_undegraded":"2023-10-09T04:58:10.840037+0000","last_fullsized":"2023-10-09T04:58:10.840037+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:56:04.393780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1004,"reported_epoch":512,"state":"remapped+peering","last_fresh":"2023-10-09T04:57:54.976548+0000","last_change":"2023-10-09T04:58:07.829807+0000","last_active":"2023-10-09T04:58:07.829807+0000","last_peered":"2023-10-09T04:57:54.976548+0000","last_clean":"2023-10-09T04:57:54.976548+0000","last_became_active":"2023-10-09T04:57:07.267695+0000","last_became_peered":"2023-10-09T04:57:07.267695+0000","last_unstale":"2023-10-09T04:57:54.976548+0000","last_undegraded":"2023-10-09T04:57:54.976548+0000","last_fullsized":"2023-10-09T04:57:54.976548+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6,7],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.5","version":"515'5062","reported_seq":12180,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.846758+0000","last_change":"2023-10-09T04:57:21.869577+0000","last_active":"2023-10-09T04:58:10.846758+0000","last_peered":"2023-10-09T04:58:10.846758+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T04:56:56.964854+0000","last_became_peered":"2023-10-09T04:56:56.964854+0000","last_unstale":"2023-10-09T04:58:10.846758+0000","last_undegraded":"2023-10-09T04:56:56.899925+0000","last_fullsized":"2023-10-09T04:56:56.806827+0000","mapping_epoch":496,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4816,"log_dups_size":0,"ondisk_log_size":4816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:09.640416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305397760,"num_objects":4660,"num_object_clones":0,"num_object_copies":13980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3681,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5062,"num_write_kb":311104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3475,"num_bytes_recovered":217448448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[7,5],"avail_no_missing":["7","4","5"],"object_location_counts":[{"shards":"0,4,5,7","objects":1836},{"shards":"4,5,7","objects":1842},{"shards":"5,7","objects":979}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":78,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.030923+0000","last_change":"2023-10-09T04:56:35.819568+0000","last_active":"2023-10-09T04:58:07.030923+0000","last_peered":"2023-10-09T04:58:07.030923+0000","last_clean":"2023-10-09T04:58:07.030923+0000","last_became_active":"2023-10-09T04:56:35.819270+0000","last_became_peered":"2023-10-09T04:56:35.819270+0000","last_unstale":"2023-10-09T04:58:07.030923+0000","last_undegraded":"2023-10-09T04:58:07.030923+0000","last_fullsized":"2023-10-09T04:58:07.030923+0000","mapping_epoch":484,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":485,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:10:47.939607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":70,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.839272+0000","last_change":"2023-10-09T04:56:43.191752+0000","last_active":"2023-10-09T04:58:06.839272+0000","last_peered":"2023-10-09T04:58:06.839272+0000","last_clean":"2023-10-09T04:58:06.839272+0000","last_became_active":"2023-10-09T04:56:43.191401+0000","last_became_peered":"2023-10-09T04:56:43.191401+0000","last_unstale":"2023-10-09T04:58:06.839272+0000","last_undegraded":"2023-10-09T04:58:06.839272+0000","last_fullsized":"2023-10-09T04:58:06.839272+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:59:27.441237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1021,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.885744+0000","last_change":"2023-10-09T04:58:10.885483+0000","last_active":"2023-10-09T04:58:10.885744+0000","last_peered":"2023-10-09T04:58:10.885744+0000","last_clean":"2023-10-09T04:58:10.885744+0000","last_became_active":"2023-10-09T04:58:10.885055+0000","last_became_peered":"2023-10-09T04:58:10.885055+0000","last_unstale":"2023-10-09T04:58:10.885744+0000","last_undegraded":"2023-10-09T04:58:10.885744+0000","last_fullsized":"2023-10-09T04:58:10.885744+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"513'5198","reported_seq":5931,"reported_epoch":513,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:07.495118+0000","last_change":"2023-10-09T04:57:29.321923+0000","last_active":"2023-10-09T04:58:07.495118+0000","last_peered":"2023-10-09T04:58:07.495118+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:57:28.961713+0000","last_became_peered":"2023-10-09T04:57:28.961713+0000","last_unstale":"2023-10-09T04:58:07.495118+0000","last_undegraded":"2023-10-09T04:57:03.533931+0000","last_fullsized":"2023-10-09T04:56:56.795217+0000","mapping_epoch":496,"log_start":"445'1536","ondisk_log_start":"445'1536","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3662,"log_dups_size":0,"ondisk_log_size":3662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:27:26.179389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313917440,"num_objects":4790,"num_object_clones":0,"num_object_copies":14370,"num_objects_missing_on_primary":4790,"num_objects_missing":4790,"num_objects_degraded":8899,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5198,"num_write_kb":319616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":13631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":7},{"shards":"2,3,5","objects":828},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":3953}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.835758+0000","last_change":"2023-10-09T04:56:53.920990+0000","last_active":"2023-10-09T04:58:06.835758+0000","last_peered":"2023-10-09T04:58:06.835758+0000","last_clean":"2023-10-09T04:58:06.835758+0000","last_became_active":"2023-10-09T04:56:53.920700+0000","last_became_peered":"2023-10-09T04:56:53.920700+0000","last_unstale":"2023-10-09T04:58:06.835758+0000","last_undegraded":"2023-10-09T04:58:06.835758+0000","last_fullsized":"2023-10-09T04:58:06.835758+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:06.039757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":562,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.852110+0000","last_change":"2023-10-09T04:54:25.766092+0000","last_active":"2023-10-09T04:58:10.852110+0000","last_peered":"2023-10-09T04:58:10.852110+0000","last_clean":"2023-10-09T04:58:10.852110+0000","last_became_active":"2023-10-09T04:54:25.761061+0000","last_became_peered":"2023-10-09T04:54:25.761061+0000","last_unstale":"2023-10-09T04:58:10.852110+0000","last_undegraded":"2023-10-09T04:58:10.852110+0000","last_fullsized":"2023-10-09T04:58:10.852110+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:39.310415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.842825+0000","last_change":"2023-10-09T04:56:53.922494+0000","last_active":"2023-10-09T04:58:06.842825+0000","last_peered":"2023-10-09T04:58:06.842825+0000","last_clean":"2023-10-09T04:58:06.842825+0000","last_became_active":"2023-10-09T04:56:53.921862+0000","last_became_peered":"2023-10-09T04:56:53.921862+0000","last_unstale":"2023-10-09T04:58:06.842825+0000","last_undegraded":"2023-10-09T04:58:06.842825+0000","last_fullsized":"2023-10-09T04:58:06.842825+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:28.414327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":997,"reported_epoch":512,"state":"remapped+peering","last_fresh":"2023-10-09T04:57:55.010383+0000","last_change":"2023-10-09T04:58:07.831283+0000","last_active":"2023-10-09T04:58:07.831283+0000","last_peered":"2023-10-09T04:57:55.010383+0000","last_clean":"2023-10-09T04:57:55.010383+0000","last_became_active":"2023-10-09T04:57:07.315309+0000","last_became_peered":"2023-10-09T04:57:07.315309+0000","last_unstale":"2023-10-09T04:57:55.010383+0000","last_undegraded":"2023-10-09T04:57:55.010383+0000","last_fullsized":"2023-10-09T04:57:55.010383+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6,7],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"515'4965","reported_seq":14793,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.846939+0000","last_change":"2023-10-09T04:58:08.980751+0000","last_active":"2023-10-09T04:58:10.846939+0000","last_peered":"2023-10-09T04:58:10.846939+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T04:58:08.968351+0000","last_became_peered":"2023-10-09T04:58:08.968351+0000","last_unstale":"2023-10-09T04:58:10.846939+0000","last_undegraded":"2023-10-09T04:58:08.946779+0000","last_fullsized":"2023-10-09T04:58:08.836235+0000","mapping_epoch":514,"log_start":"444'442","ondisk_log_start":"444'442","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4523,"log_dups_size":0,"ondisk_log_size":4523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:34.530238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302710784,"num_objects":4619,"num_object_clones":0,"num_object_copies":13857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4965,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4464,"num_bytes_recovered":292552704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,1],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"1,2","objects":705},{"shards":"2,3,4","objects":170},{"shards":"2,4,6","objects":111},{"shards":"2,4,6,7","objects":3633}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":572,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.846104+0000","last_change":"2023-10-09T04:58:07.914635+0000","last_active":"2023-10-09T04:58:10.846104+0000","last_peered":"2023-10-09T04:58:10.846104+0000","last_clean":"2023-10-09T04:58:10.846104+0000","last_became_active":"2023-10-09T04:58:07.913894+0000","last_became_peered":"2023-10-09T04:58:07.913894+0000","last_unstale":"2023-10-09T04:58:10.846104+0000","last_undegraded":"2023-10-09T04:58:10.846104+0000","last_fullsized":"2023-10-09T04:58:10.846104+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:32.545025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839878+0000","last_change":"2023-10-09T04:58:07.874107+0000","last_active":"2023-10-09T04:58:10.839878+0000","last_peered":"2023-10-09T04:58:10.839878+0000","last_clean":"2023-10-09T04:58:10.839878+0000","last_became_active":"2023-10-09T04:58:07.873816+0000","last_became_peered":"2023-10-09T04:58:07.873816+0000","last_unstale":"2023-10-09T04:58:10.839878+0000","last_undegraded":"2023-10-09T04:58:10.839878+0000","last_fullsized":"2023-10-09T04:58:10.839878+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:11:41.486018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":89,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.888094+0000","last_change":"2023-10-09T04:58:07.888094+0000","last_active":"2023-10-09T04:58:07.888094+0000","last_peered":"2023-10-09T04:58:07.888094+0000","last_clean":"2023-10-09T04:58:07.888094+0000","last_became_active":"2023-10-09T04:58:07.887823+0000","last_became_peered":"2023-10-09T04:58:07.887823+0000","last_unstale":"2023-10-09T04:58:07.888094+0000","last_undegraded":"2023-10-09T04:58:07.888094+0000","last_fullsized":"2023-10-09T04:58:07.888094+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:51.526193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1004,"reported_epoch":512,"state":"remapped+peering","last_fresh":"2023-10-09T04:57:54.976636+0000","last_change":"2023-10-09T04:58:07.830982+0000","last_active":"2023-10-09T04:58:07.830982+0000","last_peered":"2023-10-09T04:57:54.976636+0000","last_clean":"2023-10-09T04:57:54.976636+0000","last_became_active":"2023-10-09T04:57:07.268860+0000","last_became_peered":"2023-10-09T04:57:07.268860+0000","last_unstale":"2023-10-09T04:57:54.976636+0000","last_undegraded":"2023-10-09T04:57:54.976636+0000","last_fullsized":"2023-10-09T04:57:54.976636+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6,7],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"512'4983","reported_seq":5356,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.848154+0000","last_change":"2023-10-09T04:58:08.159256+0000","last_active":"2023-10-09T04:58:10.848154+0000","last_peered":"2023-10-09T04:58:10.848154+0000","last_clean":"2023-10-09T04:54:44.521216+0000","last_became_active":"2023-10-09T04:58:08.008690+0000","last_became_peered":"2023-10-09T04:58:08.008690+0000","last_unstale":"2023-10-09T04:58:10.848154+0000","last_undegraded":"2023-10-09T04:58:08.155169+0000","last_fullsized":"2023-10-09T04:58:07.833710+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4983,"log_dups_size":0,"ondisk_log_size":4983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:06:08.836303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038488,"num_objects":4625,"num_object_clones":0,"num_object_copies":13875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4557,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4625,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4983,"num_write_kb":307393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":8192000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[5,2],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"2,4,5","objects":4497},{"shards":"2,5","objects":128}],"blocked_by":[],"up_primary":6,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":206,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.842514+0000","last_change":"2023-10-09T04:58:07.893285+0000","last_active":"2023-10-09T04:58:10.842514+0000","last_peered":"2023-10-09T04:58:10.842514+0000","last_clean":"2023-10-09T04:58:10.842514+0000","last_became_active":"2023-10-09T04:58:07.892815+0000","last_became_peered":"2023-10-09T04:58:07.892815+0000","last_unstale":"2023-10-09T04:58:10.842514+0000","last_undegraded":"2023-10-09T04:58:10.842514+0000","last_fullsized":"2023-10-09T04:58:10.842514+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:21.327306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.915179+0000","last_change":"2023-10-09T04:58:07.915179+0000","last_active":"2023-10-09T04:58:07.915179+0000","last_peered":"2023-10-09T04:58:07.915179+0000","last_clean":"2023-10-09T04:58:07.915179+0000","last_became_active":"2023-10-09T04:58:07.914564+0000","last_became_peered":"2023-10-09T04:58:07.914564+0000","last_unstale":"2023-10-09T04:58:07.915179+0000","last_undegraded":"2023-10-09T04:58:07.915179+0000","last_fullsized":"2023-10-09T04:58:07.915179+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:41.098943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":64,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.839740+0000","last_change":"2023-10-09T04:56:53.963152+0000","last_active":"2023-10-09T04:58:06.839740+0000","last_peered":"2023-10-09T04:58:06.839740+0000","last_clean":"2023-10-09T04:58:06.839740+0000","last_became_active":"2023-10-09T04:56:53.957811+0000","last_became_peered":"2023-10-09T04:56:53.957811+0000","last_unstale":"2023-10-09T04:58:06.839740+0000","last_undegraded":"2023-10-09T04:58:06.839740+0000","last_fullsized":"2023-10-09T04:58:06.839740+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:17.669923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"514'4967","reported_seq":6761,"reported_epoch":514,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:08.316155+0000","last_change":"2023-10-09T04:58:08.192608+0000","last_active":"2023-10-09T04:58:08.316155+0000","last_peered":"2023-10-09T04:58:08.316155+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T04:56:43.024070+0000","last_became_peered":"2023-10-09T04:56:43.024070+0000","last_unstale":"2023-10-09T04:58:08.316155+0000","last_undegraded":"2023-10-09T04:56:29.350566+0000","last_fullsized":"2023-10-09T04:56:21.601498+0000","mapping_epoch":480,"log_start":"444'1316","ondisk_log_start":"444'1316","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3651,"log_dups_size":0,"ondisk_log_size":3651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:37.940860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300875776,"num_objects":4591,"num_object_clones":0,"num_object_copies":13773,"num_objects_missing_on_primary":4718,"num_objects_missing":4718,"num_objects_degraded":1,"num_objects_misplaced":4572,"num_objects_unfound":0,"num_objects_dirty":4591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4967,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":846,"num_bytes_recovered":55443456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,3,5,7","objects":1},{"shards":"1,5","objects":17},{"shards":"2,3,5,7","objects":580},{"shards":"2,5,7","objects":3216},{"shards":"3,5,7","objects":776},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":519,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.844572+0000","last_change":"2023-10-09T04:52:10.727903+0000","last_active":"2023-10-09T04:58:10.844572+0000","last_peered":"2023-10-09T04:58:10.844572+0000","last_clean":"2023-10-09T04:58:10.844572+0000","last_became_active":"2023-10-09T04:52:10.727522+0000","last_became_peered":"2023-10-09T04:52:10.727522+0000","last_unstale":"2023-10-09T04:58:10.844572+0000","last_undegraded":"2023-10-09T04:58:10.844572+0000","last_fullsized":"2023-10-09T04:58:10.844572+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:22:45.141607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1021,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:11.885541+0000","last_change":"2023-10-09T04:58:11.885502+0000","last_active":"2023-10-09T04:58:11.885541+0000","last_peered":"2023-10-09T04:58:11.885541+0000","last_clean":"2023-10-09T04:58:11.885541+0000","last_became_active":"2023-10-09T04:58:10.854933+0000","last_became_peered":"2023-10-09T04:58:10.854933+0000","last_unstale":"2023-10-09T04:58:11.885541+0000","last_undegraded":"2023-10-09T04:58:11.885541+0000","last_fullsized":"2023-10-09T04:58:11.885541+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":471,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.846187+0000","last_change":"2023-10-09T04:58:07.914800+0000","last_active":"2023-10-09T04:58:10.846187+0000","last_peered":"2023-10-09T04:58:10.846187+0000","last_clean":"2023-10-09T04:58:10.846187+0000","last_became_active":"2023-10-09T04:58:07.914304+0000","last_became_peered":"2023-10-09T04:58:07.914304+0000","last_unstale":"2023-10-09T04:58:10.846187+0000","last_undegraded":"2023-10-09T04:58:10.846187+0000","last_fullsized":"2023-10-09T04:58:10.846187+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:18:43.951251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.839377+0000","last_change":"2023-10-09T04:58:07.874980+0000","last_active":"2023-10-09T04:58:10.839377+0000","last_peered":"2023-10-09T04:58:10.839377+0000","last_clean":"2023-10-09T04:58:10.839377+0000","last_became_active":"2023-10-09T04:58:07.874669+0000","last_became_peered":"2023-10-09T04:58:07.874669+0000","last_unstale":"2023-10-09T04:58:10.839377+0000","last_undegraded":"2023-10-09T04:58:10.839377+0000","last_fullsized":"2023-10-09T04:58:10.839377+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:09:33.146344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1021,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.857999+0000","last_change":"2023-10-09T04:58:10.856173+0000","last_active":"2023-10-09T04:58:10.857999+0000","last_peered":"2023-10-09T04:58:10.857999+0000","last_clean":"2023-10-09T04:58:10.857999+0000","last_became_active":"2023-10-09T04:58:10.855794+0000","last_became_peered":"2023-10-09T04:58:10.855794+0000","last_unstale":"2023-10-09T04:58:10.857999+0000","last_undegraded":"2023-10-09T04:58:10.857999+0000","last_fullsized":"2023-10-09T04:58:10.857999+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"514'5049","reported_seq":9633,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.846747+0000","last_change":"2023-10-09T04:58:08.096284+0000","last_active":"2023-10-09T04:58:10.846747+0000","last_peered":"2023-10-09T04:58:10.846747+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T04:58:07.977339+0000","last_became_peered":"2023-10-09T04:58:07.977339+0000","last_unstale":"2023-10-09T04:58:10.846747+0000","last_undegraded":"2023-10-09T04:58:07.859262+0000","last_fullsized":"2023-10-09T04:58:07.831166+0000","mapping_epoch":513,"log_start":"444'541","ondisk_log_start":"444'541","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4508,"log_dups_size":0,"ondisk_log_size":4508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:44.269547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4670,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5049,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2043,"num_bytes_recovered":124583936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":2},{"shards":"1,2,4,5","objects":1663},{"shards":"2,4,5","objects":2943},{"shards":"2,5","objects":92}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":67,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.032481+0000","last_change":"2023-10-09T04:56:55.417824+0000","last_active":"2023-10-09T04:58:07.032481+0000","last_peered":"2023-10-09T04:58:07.032481+0000","last_clean":"2023-10-09T04:58:07.032481+0000","last_became_active":"2023-10-09T04:56:55.417430+0000","last_became_peered":"2023-10-09T04:56:55.417430+0000","last_unstale":"2023-10-09T04:58:07.032481+0000","last_undegraded":"2023-10-09T04:58:07.032481+0000","last_fullsized":"2023-10-09T04:58:07.032481+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:22.097654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":149,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.850837+0000","last_change":"2023-10-09T04:56:53.470340+0000","last_active":"2023-10-09T04:58:10.850837+0000","last_peered":"2023-10-09T04:58:10.850837+0000","last_clean":"2023-10-09T04:58:10.850837+0000","last_became_active":"2023-10-09T04:56:53.469975+0000","last_became_peered":"2023-10-09T04:56:53.469975+0000","last_unstale":"2023-10-09T04:58:10.850837+0000","last_undegraded":"2023-10-09T04:58:10.850837+0000","last_fullsized":"2023-10-09T04:58:10.850837+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:21.768457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":50,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.834951+0000","last_change":"2023-10-09T04:56:53.920993+0000","last_active":"2023-10-09T04:58:06.834951+0000","last_peered":"2023-10-09T04:58:06.834951+0000","last_clean":"2023-10-09T04:58:06.834951+0000","last_became_active":"2023-10-09T04:56:53.920701+0000","last_became_peered":"2023-10-09T04:56:53.920701+0000","last_unstale":"2023-10-09T04:58:06.834951+0000","last_undegraded":"2023-10-09T04:58:06.834951+0000","last_fullsized":"2023-10-09T04:58:06.834951+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:04:49.918589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1021,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.854997+0000","last_change":"2023-10-09T04:58:10.854010+0000","last_active":"2023-10-09T04:58:10.854997+0000","last_peered":"2023-10-09T04:58:10.854997+0000","last_clean":"2023-10-09T04:58:10.854997+0000","last_became_active":"2023-10-09T04:58:10.853647+0000","last_became_peered":"2023-10-09T04:58:10.853647+0000","last_unstale":"2023-10-09T04:58:10.854997+0000","last_undegraded":"2023-10-09T04:58:10.854997+0000","last_fullsized":"2023-10-09T04:58:10.854997+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"515'5063","reported_seq":16681,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.848870+0000","last_change":"2023-10-09T04:56:55.563288+0000","last_active":"2023-10-09T04:58:10.848870+0000","last_peered":"2023-10-09T04:58:10.848870+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:56:55.550670+0000","last_became_peered":"2023-10-09T04:56:55.550670+0000","last_unstale":"2023-10-09T04:58:10.848870+0000","last_undegraded":"2023-10-09T04:56:55.385420+0000","last_fullsized":"2023-10-09T04:56:54.897927+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:19.611795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309002240,"num_objects":4715,"num_object_clones":0,"num_object_copies":14145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5063,"num_write_kb":312896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5744,"num_bytes_recovered":374013952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","2","3","4"],"object_location_counts":[{"shards":"0,2,3,4,5","objects":783},{"shards":"2,3,4,5","objects":3932},{"shards":"3,5","objects":-35}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":68,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.884732+0000","last_change":"2023-10-09T04:58:07.884732+0000","last_active":"2023-10-09T04:58:07.884732+0000","last_peered":"2023-10-09T04:58:07.884732+0000","last_clean":"2023-10-09T04:58:07.884732+0000","last_became_active":"2023-10-09T04:58:07.884296+0000","last_became_peered":"2023-10-09T04:58:07.884296+0000","last_unstale":"2023-10-09T04:58:07.884732+0000","last_undegraded":"2023-10-09T04:58:07.884732+0000","last_fullsized":"2023-10-09T04:58:07.884732+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:19.231150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":62,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.834434+0000","last_change":"2023-10-09T04:55:19.977673+0000","last_active":"2023-10-09T04:58:06.834434+0000","last_peered":"2023-10-09T04:58:06.834434+0000","last_clean":"2023-10-09T04:58:06.834434+0000","last_became_active":"2023-10-09T04:55:19.977334+0000","last_became_peered":"2023-10-09T04:55:19.977334+0000","last_unstale":"2023-10-09T04:58:06.834434+0000","last_undegraded":"2023-10-09T04:58:06.834434+0000","last_fullsized":"2023-10-09T04:58:06.834434+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:09.956791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":412,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.844433+0000","last_change":"2023-10-09T04:56:53.302772+0000","last_active":"2023-10-09T04:58:10.844433+0000","last_peered":"2023-10-09T04:58:10.844433+0000","last_clean":"2023-10-09T04:58:10.844433+0000","last_became_active":"2023-10-09T04:56:53.302425+0000","last_became_peered":"2023-10-09T04:56:53.302425+0000","last_unstale":"2023-10-09T04:58:10.844433+0000","last_undegraded":"2023-10-09T04:58:10.844433+0000","last_fullsized":"2023-10-09T04:58:10.844433+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:00.445110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1021,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.855577+0000","last_change":"2023-10-09T04:58:10.854839+0000","last_active":"2023-10-09T04:58:10.855577+0000","last_peered":"2023-10-09T04:58:10.855577+0000","last_clean":"2023-10-09T04:58:10.855577+0000","last_became_active":"2023-10-09T04:58:10.854463+0000","last_became_peered":"2023-10-09T04:58:10.854463+0000","last_unstale":"2023-10-09T04:58:10.855577+0000","last_undegraded":"2023-10-09T04:58:10.855577+0000","last_fullsized":"2023-10-09T04:58:10.855577+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"515'4943","reported_seq":15482,"reported_epoch":518,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:14.762298+0000","last_change":"2023-10-09T04:57:43.528290+0000","last_active":"2023-10-09T04:58:14.762298+0000","last_peered":"2023-10-09T04:58:14.762298+0000","last_clean":"2023-10-09T04:54:25.113773+0000","last_became_active":"2023-10-09T04:55:21.623063+0000","last_became_peered":"2023-10-09T04:55:21.623063+0000","last_unstale":"2023-10-09T04:58:14.762298+0000","last_undegraded":"2023-10-09T04:55:21.089953+0000","last_fullsized":"2023-10-09T04:55:20.961092+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:40:11.340362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301400064,"num_objects":4599,"num_object_clones":0,"num_object_copies":13797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":166,"num_objects_misplaced":2690,"num_objects_unfound":0,"num_objects_dirty":4599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4943,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4602,"num_bytes_recovered":301596672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3","4","7"],"object_location_counts":[{"shards":"3,4,5","objects":865},{"shards":"3,4,5,7","objects":1825},{"shards":"3,5","objects":1743}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":96,"reported_epoch":511,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:57:44.831518+0000","last_change":"2023-10-09T04:56:55.831188+0000","last_active":"2023-10-09T04:57:44.831518+0000","last_peered":"2023-10-09T04:57:44.831518+0000","last_clean":"2023-10-09T04:56:55.360208+0000","last_became_active":"2023-10-09T04:56:55.830839+0000","last_became_peered":"2023-10-09T04:56:55.830839+0000","last_unstale":"2023-10-09T04:57:44.831518+0000","last_undegraded":"2023-10-09T04:56:55.790046+0000","last_fullsized":"2023-10-09T04:57:44.831518+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:54.131102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":94,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.892816+0000","last_change":"2023-10-09T04:58:07.892816+0000","last_active":"2023-10-09T04:58:07.892816+0000","last_peered":"2023-10-09T04:58:07.892816+0000","last_clean":"2023-10-09T04:58:07.892816+0000","last_became_active":"2023-10-09T04:58:07.892203+0000","last_became_peered":"2023-10-09T04:58:07.892203+0000","last_unstale":"2023-10-09T04:58:07.892816+0000","last_undegraded":"2023-10-09T04:58:07.892816+0000","last_fullsized":"2023-10-09T04:58:07.892816+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:24:24.377113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-09T04:58:07.914991+0000","last_change":"2023-10-09T04:58:07.914991+0000","last_active":"2023-10-09T04:58:07.914991+0000","last_peered":"2023-10-09T04:58:07.914991+0000","last_clean":"2023-10-09T04:58:07.914991+0000","last_became_active":"2023-10-09T04:58:07.914214+0000","last_became_peered":"2023-10-09T04:58:07.914214+0000","last_unstale":"2023-10-09T04:58:07.914991+0000","last_undegraded":"2023-10-09T04:58:07.914991+0000","last_fullsized":"2023-10-09T04:58:07.914991+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:59:07.039940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1009,"reported_epoch":512,"state":"remapped+peering","last_fresh":"2023-10-09T04:57:54.977593+0000","last_change":"2023-10-09T04:58:07.828765+0000","last_active":"2023-10-09T04:58:07.828765+0000","last_peered":"2023-10-09T04:57:54.977593+0000","last_clean":"2023-10-09T04:57:54.977593+0000","last_became_active":"2023-10-09T04:57:08.419054+0000","last_became_peered":"2023-10-09T04:57:08.419054+0000","last_unstale":"2023-10-09T04:57:54.977593+0000","last_undegraded":"2023-10-09T04:57:54.977593+0000","last_fullsized":"2023-10-09T04:57:54.977593+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":810709,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6,7],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.b","version":"515'5164","reported_seq":7235,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:10.848698+0000","last_change":"2023-10-09T04:58:09.220159+0000","last_active":"2023-10-09T04:58:10.848698+0000","last_peered":"2023-10-09T04:58:10.848698+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T04:58:09.204385+0000","last_became_peered":"2023-10-09T04:58:09.204385+0000","last_unstale":"2023-10-09T04:58:10.848698+0000","last_undegraded":"2023-10-09T04:58:08.984698+0000","last_fullsized":"2023-10-09T04:58:08.834068+0000","mapping_epoch":514,"log_start":"444'657","ondisk_log_start":"444'657","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:04.803237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314310656,"num_objects":4796,"num_object_clones":0,"num_object_copies":14388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8947,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5164,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":883,"num_bytes_recovered":57868288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"1,2","objects":-2},{"shards":"1,2,4,7","objects":698},{"shards":"2,4,7","objects":3972},{"shards":"2,6,7","objects":125}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.840160+0000","last_change":"2023-10-09T04:58:07.881659+0000","last_active":"2023-10-09T04:58:10.840160+0000","last_peered":"2023-10-09T04:58:10.840160+0000","last_clean":"2023-10-09T04:58:10.840160+0000","last_became_active":"2023-10-09T04:58:07.881399+0000","last_became_peered":"2023-10-09T04:58:07.881399+0000","last_unstale":"2023-10-09T04:58:10.840160+0000","last_undegraded":"2023-10-09T04:58:10.840160+0000","last_fullsized":"2023-10-09T04:58:10.840160+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:31.736513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":112,"reported_epoch":514,"state":"peering","last_fresh":"2023-10-09T04:58:07.834708+0000","last_change":"2023-10-09T04:58:07.834869+0000","last_active":"2023-10-09T04:58:06.841070+0000","last_peered":"2023-10-09T04:57:54.967035+0000","last_clean":"2023-10-09T04:57:54.967035+0000","last_became_active":"2023-10-09T04:57:08.213590+0000","last_became_peered":"2023-10-09T04:57:08.213590+0000","last_unstale":"2023-10-09T04:58:07.834708+0000","last_undegraded":"2023-10-09T04:58:07.834708+0000","last_fullsized":"2023-10-09T04:58:07.834708+0000","mapping_epoch":514,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:32:17.060854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":316,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.850749+0000","last_change":"2023-10-09T04:58:07.878605+0000","last_active":"2023-10-09T04:58:10.850749+0000","last_peered":"2023-10-09T04:58:10.850749+0000","last_clean":"2023-10-09T04:58:10.850749+0000","last_became_active":"2023-10-09T04:58:07.878282+0000","last_became_peered":"2023-10-09T04:58:07.878282+0000","last_unstale":"2023-10-09T04:58:10.850749+0000","last_undegraded":"2023-10-09T04:58:10.850749+0000","last_fullsized":"2023-10-09T04:58:10.850749+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:00:51.900516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":49,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-09T04:58:06.836594+0000","last_change":"2023-10-09T04:56:55.500748+0000","last_active":"2023-10-09T04:58:06.836594+0000","last_peered":"2023-10-09T04:58:06.836594+0000","last_clean":"2023-10-09T04:58:06.836594+0000","last_became_active":"2023-10-09T04:56:55.500429+0000","last_became_peered":"2023-10-09T04:56:55.500429+0000","last_unstale":"2023-10-09T04:58:06.836594+0000","last_undegraded":"2023-10-09T04:58:06.836594+0000","last_fullsized":"2023-10-09T04:58:06.836594+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:27.304569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.851021+0000","last_change":"2023-10-09T04:55:19.967907+0000","last_active":"2023-10-09T04:58:10.851021+0000","last_peered":"2023-10-09T04:58:10.851021+0000","last_clean":"2023-10-09T04:58:10.851021+0000","last_became_active":"2023-10-09T04:55:19.967695+0000","last_became_peered":"2023-10-09T04:55:19.967695+0000","last_unstale":"2023-10-09T04:58:10.851021+0000","last_undegraded":"2023-10-09T04:58:10.851021+0000","last_fullsized":"2023-10-09T04:58:10.851021+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:46:03.915896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":599,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.845285+0000","last_change":"2023-10-09T04:58:07.991736+0000","last_active":"2023-10-09T04:58:10.845285+0000","last_peered":"2023-10-09T04:58:10.845285+0000","last_clean":"2023-10-09T04:58:10.845285+0000","last_became_active":"2023-10-09T04:58:07.890883+0000","last_became_peered":"2023-10-09T04:58:07.890883+0000","last_unstale":"2023-10-09T04:58:10.845285+0000","last_undegraded":"2023-10-09T04:58:10.845285+0000","last_fullsized":"2023-10-09T04:58:10.845285+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:15:54.738471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":300,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:52.313815+0000","last_change":"2023-10-09T04:56:55.480416+0000","last_active":"2023-10-09T04:57:52.313815+0000","last_peered":"2023-10-09T04:57:52.313815+0000","last_clean":"2023-10-09T04:57:52.313815+0000","last_became_active":"2023-10-09T04:56:55.480090+0000","last_became_peered":"2023-10-09T04:56:55.480090+0000","last_unstale":"2023-10-09T04:57:52.313815+0000","last_undegraded":"2023-10-09T04:57:52.313815+0000","last_fullsized":"2023-10-09T04:57:52.313815+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:18.359338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":568,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.852038+0000","last_change":"2023-10-09T04:58:08.037002+0000","last_active":"2023-10-09T04:58:10.852038+0000","last_peered":"2023-10-09T04:58:10.852038+0000","last_clean":"2023-10-09T04:58:10.852038+0000","last_became_active":"2023-10-09T04:58:07.937676+0000","last_became_peered":"2023-10-09T04:58:07.937676+0000","last_unstale":"2023-10-09T04:58:10.852038+0000","last_undegraded":"2023-10-09T04:58:10.852038+0000","last_fullsized":"2023-10-09T04:58:10.852038+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:50:40.208097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.830835+0000","last_change":"2023-10-09T04:57:07.279151+0000","last_active":"2023-10-09T04:57:44.830835+0000","last_peered":"2023-10-09T04:57:44.830835+0000","last_clean":"2023-10-09T04:57:44.830835+0000","last_became_active":"2023-10-09T04:57:07.278810+0000","last_became_peered":"2023-10-09T04:57:07.278810+0000","last_unstale":"2023-10-09T04:57:44.830835+0000","last_undegraded":"2023-10-09T04:57:44.830835+0000","last_fullsized":"2023-10-09T04:57:44.830835+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:55:59.075114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":92,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.831819+0000","last_change":"2023-10-09T04:57:07.324006+0000","last_active":"2023-10-09T04:57:44.831819+0000","last_peered":"2023-10-09T04:57:44.831819+0000","last_clean":"2023-10-09T04:57:44.831819+0000","last_became_active":"2023-10-09T04:57:07.323660+0000","last_became_peered":"2023-10-09T04:57:07.323660+0000","last_unstale":"2023-10-09T04:57:44.831819+0000","last_undegraded":"2023-10-09T04:57:44.831819+0000","last_fullsized":"2023-10-09T04:57:44.831819+0000","mapping_epoch":498,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:53:53.994925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-09T04:57:44.831382+0000","last_change":"2023-10-09T04:56:55.857967+0000","last_active":"2023-10-09T04:57:44.831382+0000","last_peered":"2023-10-09T04:57:44.831382+0000","last_clean":"2023-10-09T04:57:44.831382+0000","last_became_active":"2023-10-09T04:56:55.857506+0000","last_became_peered":"2023-10-09T04:56:55.857506+0000","last_unstale":"2023-10-09T04:57:44.831382+0000","last_undegraded":"2023-10-09T04:57:44.831382+0000","last_fullsized":"2023-10-09T04:57:44.831382+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:57.553326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":552,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.851216+0000","last_change":"2023-10-09T04:58:07.965270+0000","last_active":"2023-10-09T04:58:10.851216+0000","last_peered":"2023-10-09T04:58:10.851216+0000","last_clean":"2023-10-09T04:58:10.851216+0000","last_became_active":"2023-10-09T04:58:07.879397+0000","last_became_peered":"2023-10-09T04:58:07.879397+0000","last_unstale":"2023-10-09T04:58:10.851216+0000","last_undegraded":"2023-10-09T04:58:10.851216+0000","last_fullsized":"2023-10-09T04:58:10.851216+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:13:51.386740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-09T04:58:10.840953+0000","last_change":"2023-10-09T04:58:07.938201+0000","last_active":"2023-10-09T04:58:10.840953+0000","last_peered":"2023-10-09T04:58:10.840953+0000","last_clean":"2023-10-09T04:58:10.840953+0000","last_became_active":"2023-10-09T04:58:07.937536+0000","last_became_peered":"2023-10-09T04:58:07.937536+0000","last_unstale":"2023-10-09T04:58:10.840953+0000","last_undegraded":"2023-10-09T04:58:10.840953+0000","last_fullsized":"2023-10-09T04:58:10.840953+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:47:03.717056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":4892393496,"num_objects":74653,"num_object_clones":0,"num_object_copies":223959,"num_objects_missing_on_primary":19016,"num_objects_missing":19016,"num_objects_degraded":71017,"num_objects_misplaced":13659,"num_objects_unfound":0,"num_objects_dirty":74653,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":80565,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26243,"num_bytes_recovered":1695809536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16502046720,"data_stored":16502030432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73515,"ondisk_log_size":73515,"up":48,"acting":33,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":16786,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31051,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5850010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2580480,"data_stored":2559168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":28,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234613,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4041732,"kb_used_data":2536572,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":89703420,"statfs":{"total":95995035648,"available":91856302080,"internally_reserved":0,"allocated":2597449728,"data_stored":2596301569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.53900000000000003,"15min":0.97899999999999998},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.21099999999999999},"max":{"1min":0.57499999999999996,"5min":3.4420000000000002,"15min":14.970000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.77300000000000002,"15min":1.4319999999999999},"min":{"1min":0.315,"5min":0.26700000000000002,"15min":0.249},"max":{"1min":2.2519999999999998,"5min":6.657,"15min":21.216000000000001},"last":0.42799999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.76700000000000002,"15min":3.0649999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.246},"max":{"1min":2.9980000000000002,"5min":5.0990000000000002,"15min":128.601},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.0680000000000001,"5min":0.84399999999999997,"15min":3.0089999999999999},"min":{"1min":0.36299999999999999,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":8.1920000000000002,"5min":8.1920000000000002,"15min":128.381},"last":0.40699999999999997}]},{"osd":2,"last update":"Mon Oct 9 04:57:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":3.4689999999999999,"15min":4.6479999999999997},"min":{"1min":0.34100000000000003,"5min":0.30299999999999999,"15min":0.27100000000000002},"max":{"1min":11.731,"5min":133.601,"15min":207.96600000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.661,"5min":3.609,"15min":4.8959999999999999},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":11.781000000000001,"5min":133.68799999999999,"15min":207.97900000000001},"last":0.51800000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:57:53 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":1.7130000000000001,"15min":1.6879999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29499999999999998},"max":{"1min":3.4809999999999999,"5min":74.259,"15min":74.259},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.742,"15min":2.008},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.2440000000000002,"5min":74.100999999999999,"15min":74.100999999999999},"last":0.57299999999999995}]},{"osd":4,"last update":"Mon Oct 9 04:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":2.0139999999999998,"15min":1.897},"min":{"1min":0.29499999999999998,"5min":0.28199999999999997,"15min":0.22600000000000001},"max":{"1min":0.99299999999999999,"5min":21.355,"15min":21.355},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.85199999999999998,"5min":2.2599999999999998,"15min":1.5860000000000001},"min":{"1min":0.312,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":4.5860000000000003,"5min":21.489999999999998,"15min":21.489999999999998},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:57:35 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":1.952,"15min":1.7549999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.503,"5min":131.84899999999999,"15min":131.84899999999999},"last":0.441},{"interface":"front","average":{"1min":0.42299999999999999,"5min":2.2879999999999998,"15min":2.2759999999999998},"min":{"1min":0.29099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.86699999999999999,"5min":135.74199999999999,"15min":135.74199999999999},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:57:35 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.82799999999999996,"15min":1.1539999999999999},"min":{"1min":0.32400000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57799999999999996,"5min":7.1920000000000002,"15min":10.047000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.88700000000000001,"15min":1.5129999999999999},"min":{"1min":0.30599999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.51900000000000002,"5min":8.7650000000000006,"15min":21.321999999999999},"last":2.0299999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496246,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1306640,"kb_used_data":400264,"kb_used_omap":0,"kb_used_meta":906368,"kb_avail":92438512,"statfs":{"total":95995035648,"available":94657036288,"internally_reserved":0,"allocated":409870336,"data_stored":408718631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928120832},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.57799999999999996,"15min":0.89300000000000002},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.20499999999999999},"max":{"1min":1.004,"5min":4.6239999999999997,"15min":9.4979999999999993},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.70799999999999996,"15min":1.1399999999999999},"min":{"1min":0.32100000000000001,"5min":0.28000000000000003,"15min":0.26700000000000002},"max":{"1min":1.351,"5min":7.3129999999999997,"15min":16.574000000000002},"last":0.33200000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.98999999999999999,"15min":5.2729999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.0730000000000004,"5min":9.7379999999999995,"15min":260.738},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.83799999999999997,"15min":5.0140000000000002},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.26300000000000001},"max":{"1min":6.7069999999999999,"5min":7.1890000000000001,"15min":261.26100000000002},"last":0.56299999999999994}]},{"osd":2,"last update":"Mon Oct 9 04:57:27 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":3.044,"15min":4.3440000000000003},"min":{"1min":0.373,"5min":0.28199999999999997,"15min":0.27800000000000002},"max":{"1min":7.8630000000000004,"5min":133.535,"15min":133.535},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.4610000000000001,"5min":3.1179999999999999,"15min":4.1390000000000002},"min":{"1min":0.39600000000000002,"5min":0.31,"15min":0.31},"max":{"1min":7.8890000000000002,"5min":133.51900000000001,"15min":133.51900000000001},"last":0.48199999999999998}]},{"osd":3,"last update":"Mon Oct 9 04:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":3.4660000000000002,"15min":3.1499999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":7.0190000000000001,"5min":147.43600000000001,"15min":147.43600000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":3.484,"15min":2.9009999999999998},"min":{"1min":0.35499999999999998,"5min":0.34799999999999998,"15min":0.29999999999999999},"max":{"1min":11.420999999999999,"5min":147.41999999999999,"15min":147.41999999999999},"last":0.57799999999999996}]},{"osd":4,"last update":"Mon Oct 9 04:55:50 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":1.4039999999999999,"15min":2.1059999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":3.222,"5min":10.609,"15min":112.691},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.5389999999999999,"15min":2.0329999999999999},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.26500000000000001},"max":{"1min":1.8899999999999999,"5min":21.664000000000001,"15min":109.024},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Oct 9 04:57:56 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.80100000000000005,"15min":1.9670000000000001},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":0.60499999999999998,"5min":4.5839999999999996,"15min":83.168999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.0089999999999999,"15min":2.0110000000000001},"min":{"1min":0.33900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.66300000000000003,"5min":13.241,"15min":83.126000000000005},"last":0.61899999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:57:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.0129999999999999,"15min":1.4530000000000001},"min":{"1min":0.32900000000000001,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":0.56699999999999995,"5min":5.5049999999999999,"15min":36.204999999999998},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.437,"5min":1.0249999999999999,"15min":1.4299999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71499999999999997,"5min":6.1790000000000003,"15min":10.661},"last":0.51800000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141325,"num_pgs":83,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1577508,"kb_used_data":468380,"kb_used_omap":4,"kb_used_meta":1109115,"kb_avail":92167644,"statfs":{"total":95995035648,"available":94379667456,"internally_reserved":0,"allocated":479621120,"data_stored":477124844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1135734670},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,0,2,0,0,0,4,3],"upper_bound":8192},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:57:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":0.72899999999999998,"15min":1.1859999999999999},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":11.763999999999999,"5min":11.763999999999999,"15min":21.021999999999998},"last":9.6999999999999993},{"interface":"front","average":{"1min":1.2450000000000001,"5min":0.84799999999999998,"15min":1.3839999999999999},"min":{"1min":0.24199999999999999,"5min":0.214,"15min":0.20499999999999999},"max":{"1min":6.2130000000000001,"5min":6.2130000000000001,"15min":21.280999999999999},"last":0.5}]},{"osd":2,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":4.3949999999999996,"15min":4.6360000000000001},"min":{"1min":0.29399999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":5.274,"5min":263.089,"15min":263.60500000000002},"last":10.167999999999999},{"interface":"front","average":{"1min":1.204,"5min":4.3860000000000001,"15min":4.7229999999999999},"min":{"1min":0.28899999999999998,"5min":0.222,"15min":0.222},"max":{"1min":6.2759999999999998,"5min":254.50899999999999,"15min":263.65199999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.841,"15min":2.5409999999999999},"min":{"1min":0.307,"5min":0.28000000000000003,"15min":0.23000000000000001},"max":{"1min":11.670999999999999,"5min":77.819999999999993,"15min":379.80900000000003},"last":39.289000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.857,"15min":2.6720000000000002},"min":{"1min":0.307,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":5.8109999999999999,"5min":77.796000000000006,"15min":379.95800000000003},"last":14.212}]},{"osd":4,"last update":"Mon Oct 9 04:55:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5149999999999999,"5min":4.2229999999999999,"15min":2.9249999999999998},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":7.8040000000000003,"5min":163.94200000000001,"15min":163.94200000000001},"last":19.634},{"interface":"front","average":{"1min":1.498,"5min":4.3029999999999999,"15min":2.9609999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":9.6899999999999995,"5min":164.00700000000001,"15min":164.00700000000001},"last":13.045}]},{"osd":5,"last update":"Mon Oct 9 04:57:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2729999999999999,"15min":1.675},"min":{"1min":0.39800000000000002,"5min":0.35199999999999998,"15min":0.33000000000000002},"max":{"1min":5.6559999999999997,"5min":7.4980000000000002,"15min":9.782},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":1.0940000000000001,"15min":1.6950000000000001},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.7629999999999999,"5min":7.633,"15min":20.021000000000001},"last":19.597999999999999}]},{"osd":6,"last update":"Mon Oct 9 04:57:20 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.151,"15min":1.972},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":6.1619999999999999,"5min":6.2069999999999999,"15min":132.36099999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.0569999999999999,"15min":2.1000000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.32400000000000001},"max":{"1min":2.847,"5min":7.734,"15min":132.17699999999999},"last":19.548999999999999}]},{"osd":7,"last update":"Mon Oct 9 04:57:20 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.1439999999999999,"15min":2.6459999999999999},"min":{"1min":0.41999999999999998,"5min":0.38800000000000001,"15min":0.35899999999999999},"max":{"1min":2.8159999999999998,"5min":7.7000000000000002,"15min":132.136},"last":19.664999999999999},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.2110000000000001,"15min":2.3540000000000001},"min":{"1min":0.39300000000000002,"5min":0.378,"15min":0.378},"max":{"1min":4.194,"5min":10.167999999999999,"15min":132.12},"last":9.8710000000000004}]}]},{"osd":0,"up_from":0,"seq":390842024397,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772673,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4630104,"kb_used_data":3866896,"kb_used_omap":7,"kb_used_meta":763192,"kb_avail":89115048,"statfs":{"total":95995035648,"available":91253809152,"internally_reserved":0,"allocated":3959701504,"data_stored":3957551113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":781508689},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.91300000000000003,"15min":1.452},"min":{"1min":0.23699999999999999,"5min":0.192,"15min":0.183},"max":{"1min":2.1829999999999998,"5min":13.042999999999999,"15min":25.899999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.78500000000000003,"15min":1.171},"min":{"1min":0.26100000000000001,"5min":0.25,"15min":0.19500000000000001},"max":{"1min":1.0940000000000001,"5min":12.161,"15min":16.550000000000001},"last":1.0600000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.1299999999999999,"15min":3.8359999999999999},"min":{"1min":0.28999999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.076000000000001,"5min":24.765999999999998,"15min":170.172},"last":1.196},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.2050000000000001,"15min":3.9510000000000001},"min":{"1min":0.36599999999999999,"5min":0.219,"15min":0.219},"max":{"1min":10.346,"5min":29.244,"15min":170.511},"last":3.661}]},{"osd":3,"last update":"Mon Oct 9 04:57:28 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.6819999999999999,"15min":3.8769999999999998},"min":{"1min":0.35499999999999998,"5min":0.248,"15min":0.21199999999999999},"max":{"1min":10.393000000000001,"5min":45.664000000000001,"15min":225.85499999999999},"last":0.78600000000000003},{"interface":"front","average":{"1min":2.0089999999999999,"5min":1.857,"15min":3.984},"min":{"1min":0.32800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":12.196999999999999,"5min":45.686999999999998,"15min":233.53299999999999},"last":0.71499999999999997}]},{"osd":4,"last update":"Mon Oct 9 04:55:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.1040000000000001,"15min":2.6859999999999999},"min":{"1min":0.31,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.2599999999999998,"5min":6.4699999999999998,"15min":211.642},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.157,"5min":1.1799999999999999,"15min":2.673},"min":{"1min":0.375,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":4.7199999999999998,"5min":11.984,"15min":211.614},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":2.6000000000000001,"15min":3.698},"min":{"1min":0.42199999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66000000000000003,"5min":175.01599999999999,"15min":272.04700000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.8919999999999999,"15min":3.7709999999999999},"min":{"1min":0.41199999999999998,"5min":0.33500000000000002,"15min":0.32200000000000001},"max":{"1min":10.427,"5min":174.02799999999999,"15min":271.89699999999999},"last":0.76000000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.98199999999999998,"15min":1.504},"min":{"1min":0.39500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.534,"5min":6.8730000000000002,"15min":16.425999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.96899999999999997,"15min":1.355},"min":{"1min":0.38800000000000001,"5min":0.38700000000000001,"15min":0.34699999999999998},"max":{"1min":0.66300000000000003,"5min":7.6120000000000001,"15min":10.977},"last":0.69199999999999995}]},{"osd":7,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.2809999999999999,"15min":1.8380000000000001},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.34300000000000003},"max":{"1min":6.1710000000000003,"5min":6.1710000000000003,"15min":16.462},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.1519999999999999,"15min":1.6950000000000001},"min":{"1min":0.44500000000000001,"5min":0.40799999999999997,"15min":0.373},"max":{"1min":0.77600000000000002,"5min":5.5199999999999996,"15min":11.669},"last":0.63}]}]},{"osd":3,"up_from":142,"seq":609885356374,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1890604,"kb_used_data":1118116,"kb_used_omap":2,"kb_used_meta":772477,"kb_avail":91854548,"statfs":{"total":95995035648,"available":94059057152,"internally_reserved":0,"allocated":1144950784,"data_stored":1142965133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":791016501},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.57599999999999996,"15min":1.1819999999999999},"min":{"1min":0.30299999999999999,"5min":0.20699999999999999,"15min":0.19},"max":{"1min":0.92500000000000004,"5min":5.2850000000000001,"15min":75.132999999999996},"last":0.499},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.72599999999999998,"15min":1.4239999999999999},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.22},"max":{"1min":1.149,"5min":6.5229999999999997,"15min":76.897999999999996},"last":4.9580000000000002}]},{"osd":1,"last update":"Mon Oct 9 04:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.80100000000000005,"15min":3.2639999999999998},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":1.484,"5min":6.9710000000000001,"15min":125.233},"last":8.5199999999999996},{"interface":"front","average":{"1min":1.3260000000000001,"5min":0.84699999999999998,"15min":3.165},"min":{"1min":0.32600000000000001,"5min":0.25,"15min":0.21299999999999999},"max":{"1min":7.9359999999999999,"5min":7.9359999999999999,"15min":126.102},"last":0.65700000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":2.6619999999999999,"15min":3.3210000000000002},"min":{"1min":0.33000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.9729999999999999,"5min":203.36000000000001,"15min":203.36000000000001},"last":6.7610000000000001},{"interface":"front","average":{"1min":1.446,"5min":2.7200000000000002,"15min":3.5329999999999999},"min":{"1min":0.33900000000000002,"5min":0.27500000000000002,"15min":0.20799999999999999},"max":{"1min":6.5739999999999998,"5min":194.125,"15min":194.125},"last":4.984}]},{"osd":4,"last update":"Mon Oct 9 04:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.323,"15min":2.7320000000000002},"min":{"1min":0.32900000000000001,"5min":0.312,"15min":0.312},"max":{"1min":4.8959999999999999,"5min":12.493,"15min":254.072},"last":4.9109999999999996},{"interface":"front","average":{"1min":0.76000000000000001,"5min":1.377,"15min":2.5720000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.8260000000000001,"5min":12.542,"15min":253.49700000000001},"last":8.5779999999999994}]},{"osd":5,"last update":"Mon Oct 9 04:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.95199999999999996,"15min":1.579},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.30599999999999999},"max":{"1min":0.89800000000000002,"5min":6.7640000000000002,"15min":28.302},"last":0.439},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.2170000000000001,"15min":1.7190000000000001},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":24.986999999999998,"15min":28.280000000000001},"last":4.8630000000000004}]},{"osd":6,"last update":"Mon Oct 9 04:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.89500000000000002,"15min":1.329},"min":{"1min":0.39600000000000002,"5min":0.372,"15min":0.32900000000000001},"max":{"1min":1.216,"5min":5.1589999999999998,"15min":16.439},"last":4.9359999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.306,"15min":1.6739999999999999},"min":{"1min":0.40600000000000003,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":5.4390000000000001,"5min":24.916,"15min":24.916},"last":8.6069999999999993}]},{"osd":7,"last update":"Mon Oct 9 04:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.179,"15min":1.8879999999999999},"min":{"1min":0.437,"5min":0.40200000000000002,"15min":0.375},"max":{"1min":1.516,"5min":25.004000000000001,"15min":25.004000000000001},"last":8.6370000000000005},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.81799999999999995,"15min":1.448},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.41599999999999998},"max":{"1min":1.0549999999999999,"5min":4.6680000000000001,"15min":12.486000000000001},"last":0.46999999999999997}]}]},{"osd":4,"up_from":0,"seq":2177548419081,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757888,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5428852,"kb_used_data":3927340,"kb_used_omap":0,"kb_used_meta":1501504,"kb_avail":88316300,"statfs":{"total":95995035648,"available":90435891200,"internally_reserved":0,"allocated":4021596160,"data_stored":4020459561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1537540096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,2],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.9079999999999999,"15min":2.3399999999999999},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":1.107,"5min":106.465,"15min":163.70699999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.3939999999999999,"5min":2.2269999999999999,"15min":2.371},"min":{"1min":0.28199999999999997,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":11.052,"5min":106.54600000000001,"15min":163.756},"last":6.4249999999999998}]},{"osd":1,"last update":"Mon Oct 9 04:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.82299999999999995,"15min":4.9119999999999999},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.4129999999999998,"5min":5,"15min":211.85400000000001},"last":3.4129999999999998},{"interface":"front","average":{"1min":1.038,"5min":0.91700000000000004,"15min":5.0960000000000001},"min":{"1min":0.30599999999999999,"5min":0.30099999999999999,"15min":0.255},"max":{"1min":11.249000000000001,"5min":11.249000000000001,"15min":211.435},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:57:26 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":1.5,"15min":3.4660000000000002},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.28999999999999998},"max":{"1min":2.496,"5min":66.579999999999998,"15min":86.488},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.875,"15min":3.6339999999999999},"min":{"1min":0.39800000000000002,"5min":0.34799999999999998,"15min":0.29699999999999999},"max":{"1min":8.4239999999999995,"5min":66.557000000000002,"15min":86.510999999999996},"last":3.9260000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:57:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":2.4260000000000002,"15min":2.427},"min":{"1min":0.34799999999999998,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":9.2609999999999992,"5min":105.879,"15min":105.879},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.71499999999999997,"5min":2.2749999999999999,"15min":2.1030000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.29799999999999999},"max":{"1min":2.5659999999999998,"5min":105.625,"15min":105.625},"last":4.9489999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:55:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.3859999999999999,"15min":1.526},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":8.0180000000000007,"5min":10.33,"15min":116.054},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":2.0209999999999999,"15min":2.073},"min":{"1min":0.309,"5min":0.30199999999999999,"15min":0.253},"max":{"1min":6.9000000000000004,"5min":11.888999999999999,"15min":118.47},"last":0.45700000000000002}]},{"osd":6,"last update":"Mon Oct 9 04:57:13 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.89600000000000002,"15min":1.038},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.45600000000000002,"5min":9.1780000000000008,"15min":13.162000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.39000000000000001,"5min":1.1970000000000001,"15min":1.169},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.54800000000000004,"5min":13.083,"15min":13.083},"last":0.503}]},{"osd":7,"last update":"Mon Oct 9 04:57:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.0129999999999999,"15min":2.3690000000000002},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.77,"5min":8.7720000000000002,"15min":110.521},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.89200000000000002,"15min":2.4369999999999998},"min":{"1min":0.34599999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.72599999999999998,"5min":5.0259999999999998,"15min":110.746},"last":0.97999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":197722112,"data_stored":197722112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":479723520,"data_stored":479723520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3952873472,"data_stored":3952869400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1138622464,"data_stored":1138622464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3720351744,"data_stored":3720347672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4016508928,"data_stored":4016504856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":404492288,"data_stored":404488216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2591752192,"data_stored":2591752192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:58:22.552 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T04:58:22.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pg_num 26 2023-10-09T04:58:22.646 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:22.680 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:22.731 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:22.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:22.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[216903]: pgmap v1770: 99 pgs: 2 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 80478/224028 objects degraded (35.923%); 12890/224028 objects misplaced (5.754%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[223858]: pgmap v1770: 99 pgs: 2 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 80478/224028 objects degraded (35.923%); 12890/224028 objects misplaced (5.754%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.783 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.784 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:22 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:22.834 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:22.862 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:22.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:22.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:23.019 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:22 smithi177 ceph-mon[195797]: pgmap v1770: 99 pgs: 2 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 80478/224028 objects degraded (35.923%); 12890/224028 objects misplaced (5.754%); 7.7 MiB/s, 122 objects/s recovering 2023-10-09T04:58:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:22 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.087 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:58:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:58:22] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.43.0" 2023-10-09T04:58:23.096 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:23.244 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:23.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:23.297 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:23.351 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:23.379 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:23.426 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:23.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:23.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:23.530 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:23.558 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:23.764 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:23.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:23.816 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:58:23.824 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T04:58:23.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2732993684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:23.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:23.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.858 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2732993684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:23.858 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:23.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:23.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:23.922 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2732993684' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:23 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:58:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:24.244 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:24.442 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:24.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:24.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:24.582 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:25.160 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: pgmap v1771: 99 pgs: 2 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 80475/224019 objects degraded (35.923%); 12890/224019 objects misplaced (5.754%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: osdmap e519: 8 total, 8 up, 6 in 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: pgmap v1771: 99 pgs: 2 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 80475/224019 objects degraded (35.923%); 12890/224019 objects misplaced (5.754%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: osdmap e519: 8 total, 8 up, 6 in 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:24 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.185 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:58:25.207 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:58:25.236 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:58:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: pgmap v1771: 99 pgs: 2 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 80475/224019 objects degraded (35.923%); 12890/224019 objects misplaced (5.754%); 7.7 MiB/s, 123 objects/s recovering 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: osdmap e519: 8 total, 8 up, 6 in 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:24 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:25.285 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:25.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:25.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:25.428 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:25.715 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:58:25.718 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:58:26.166 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:26.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:26.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:26.270 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:26.303 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:26.722 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:58:26.723 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:26.947 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:26.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:26 smithi177 ceph-mon[195797]: pgmap v1773: 99 pgs: 2 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84876/224001 objects degraded (37.891%); 12296/224001 objects misplaced (5.489%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:58:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:26 smithi177 ceph-mon[195797]: failed to encode map e519 with expected crc 2023-10-09T04:58:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:26 smithi177 ceph-mon[195797]: failed to encode map e520 with expected crc 2023-10-09T04:58:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:26 smithi177 ceph-mon[195797]: osdmap e520: 8 total, 8 up, 6 in 2023-10-09T04:58:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84876/224001 objects degraded (37.891%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:27.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:27.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[216903]: pgmap v1773: 99 pgs: 2 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84876/224001 objects degraded (37.891%); 12296/224001 objects misplaced (5.489%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:58:27.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[216903]: failed to encode map e519 with expected crc 2023-10-09T04:58:27.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[216903]: failed to encode map e520 with expected crc 2023-10-09T04:58:27.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[216903]: osdmap e520: 8 total, 8 up, 6 in 2023-10-09T04:58:27.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84876/224001 objects degraded (37.891%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:27.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[223858]: pgmap v1773: 99 pgs: 2 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84876/224001 objects degraded (37.891%); 12296/224001 objects misplaced (5.489%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:58:27.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[223858]: failed to encode map e519 with expected crc 2023-10-09T04:58:27.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[223858]: failed to encode map e520 with expected crc 2023-10-09T04:58:27.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[223858]: osdmap e520: 8 total, 8 up, 6 in 2023-10-09T04:58:27.085 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84876/224001 objects degraded (37.891%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:27.088 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:27.151 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:27.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:27.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:27.258 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:27.292 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:27.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:27.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:27.469 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:27.500 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:27.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:27.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:27.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:27.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:27.691 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:27.749 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:27.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:27.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:27 smithi177 ceph-mon[195797]: failed to encode map e521 with expected crc 2023-10-09T04:58:27.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:27 smithi177 ceph-mon[195797]: osdmap e521: 8 total, 8 up, 6 in 2023-10-09T04:58:27.812 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:27 smithi177 ceph-mon[195797]: failed to encode map e521 with expected crc 2023-10-09T04:58:27.813 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1782495261' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:58:27.821 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 14 pg_num to 26 2023-10-09T04:58:27.855 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:27.882 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:28.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:58:28.138 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[216903]: failed to encode map e521 with expected crc 2023-10-09T04:58:28.138 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[216903]: osdmap e521: 8 total, 8 up, 6 in 2023-10-09T04:58:28.138 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[216903]: failed to encode map e521 with expected crc 2023-10-09T04:58:28.138 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1782495261' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:58:28.138 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[223858]: failed to encode map e521 with expected crc 2023-10-09T04:58:28.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[223858]: osdmap e521: 8 total, 8 up, 6 in 2023-10-09T04:58:28.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[223858]: failed to encode map e521 with expected crc 2023-10-09T04:58:28.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1782495261' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-10-09T04:58:28.163 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:28.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:28.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:28.223 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T04:58:28.269 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:28.298 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:28.355 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:28.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:28.463 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:28.496 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:28.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:28.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:28.657 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:28.661 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:28.689 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: pgmap v1776: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84874/223995 objects degraded (37.891%); 12296/223995 objects misplaced (5.489%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1782495261' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: osdmap e522: 8 total, 8 up, 6 in 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: failed to encode map e522 with expected crc 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: failed to encode map e522 with expected crc 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: pgmap v1776: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84874/223995 objects degraded (37.891%); 12296/223995 objects misplaced (5.489%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1782495261' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: osdmap e522: 8 total, 8 up, 6 in 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: failed to encode map e522 with expected crc 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: failed to encode map e522 with expected crc 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:29.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:58:29.145 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: pgmap v1776: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84874/223995 objects degraded (37.891%); 12296/223995 objects misplaced (5.489%); 7.3 MiB/s, 117 objects/s recovering 2023-10-09T04:58:29.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1782495261' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-10-09T04:58:29.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: osdmap e522: 8 total, 8 up, 6 in 2023-10-09T04:58:29.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: failed to encode map e522 with expected crc 2023-10-09T04:58:29.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: failed to encode map e522 with expected crc 2023-10-09T04:58:29.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:29.146 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:29.362 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:29.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:29.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:29.468 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:29.487 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:29.520 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:29.524 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:29.594 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:29.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:29.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:29.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:29.729 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:29.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:29 smithi029 ceph-mon[216903]: failed to encode map e522 with expected crc 2023-10-09T04:58:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:29 smithi029 ceph-mon[223858]: failed to encode map e522 with expected crc 2023-10-09T04:58:30.038 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:30.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:30.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:30.146 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:30.176 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:29 smithi177 ceph-mon[195797]: failed to encode map e522 with expected crc 2023-10-09T04:58:30.180 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:30.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:30.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:30.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:30.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:30.374 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:30.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:30 smithi177 ceph-mon[195797]: pgmap v1778: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84874/223995 objects degraded (37.891%); 12296/223995 objects misplaced (5.489%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T04:58:30.868 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84874/223995 objects degraded (37.891%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:31.098 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:31.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:30 smithi029 ceph-mon[216903]: pgmap v1778: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84874/223995 objects degraded (37.891%); 12296/223995 objects misplaced (5.489%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T04:58:31.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84874/223995 objects degraded (37.891%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:31.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:30 smithi029 ceph-mon[223858]: pgmap v1778: 99 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 1 op/s; 84874/223995 objects degraded (37.891%); 12296/223995 objects misplaced (5.489%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T04:58:31.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84874/223995 objects degraded (37.891%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:31.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:31.155 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:58:31.155 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:31.203 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:31.233 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:32.088 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:32.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:32.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:32.229 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:32.282 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:32.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:32.386 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:32.426 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:32.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T04:58:32.763 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:32.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:32.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:32.920 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:33.017 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:58:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:58:32] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-10-09T04:58:33.018 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:32 smithi029 ceph-mon[223858]: pgmap v1779: 99 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 79784/223995 objects degraded (35.619%); 16147/223995 objects misplaced (7.209%); 11 MiB/s, 169 objects/s recovering 2023-10-09T04:58:33.018 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:32 smithi029 ceph-mon[216903]: pgmap v1779: 99 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 79784/223995 objects degraded (35.619%); 16147/223995 objects misplaced (7.209%); 11 MiB/s, 169 objects/s recovering 2023-10-09T04:58:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:32 smithi177 ceph-mon[195797]: pgmap v1779: 99 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 79784/223995 objects degraded (35.619%); 16147/223995 objects misplaced (7.209%); 11 MiB/s, 169 objects/s recovering 2023-10-09T04:58:33.087 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:58:33.701 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:33.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:33.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:33.850 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:34.161 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:34.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:34.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:34.298 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:34.347 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:34.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:34.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:34.380 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6] out_osds: [4, 0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T04:58:34.380 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:58:34.381 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.6824777217101583 2023-10-09T04:58:34.381 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 0.6824777217101583 2023-10-09T04:58:34.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:34.498 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:34.566 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:58:34.580 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:34.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:34.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:34.684 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:34.721 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:34.837 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:58:34.839 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:34.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:34.877 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:34.954 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:34.985 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:35.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:35.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:35.076 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: pgmap v1780: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79784/223995 objects degraded (35.619%); 16147/223995 objects misplaced (7.209%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3769188486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.077 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/881823937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: pgmap v1780: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79784/223995 objects degraded (35.619%); 16147/223995 objects misplaced (7.209%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3769188486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:35.078 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.079 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/881823937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:35.150 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:35.184 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:35.220 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: pgmap v1780: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79784/223995 objects degraded (35.619%); 16147/223995 objects misplaced (7.209%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3769188486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:35.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/881823937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:35.238 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:35.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:35.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:35.378 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:35.434 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:35.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:35.499 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:58:35.499 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:35.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:35.573 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:35.629 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:35.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:35.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:35.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:35.767 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:35.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:35.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:35.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:35.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[216903]: failed to encode map e523 with expected crc 2023-10-09T04:58:35.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:35.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[216903]: osdmap e523: 8 total, 8 up, 6 in 2023-10-09T04:58:35.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 79784/223995 objects degraded (35.619%), 16 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:35.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:35.951 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:35.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:36.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:36.103 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:36.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:36.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[223858]: failed to encode map e523 with expected crc 2023-10-09T04:58:36.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:36.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[223858]: osdmap e523: 8 total, 8 up, 6 in 2023-10-09T04:58:36.134 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 79784/223995 objects degraded (35.619%), 16 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:36.137 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:36.152 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:58:36.187 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:58:36.258 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:36.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:35 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:35 smithi177 ceph-mon[195797]: failed to encode map e523 with expected crc 2023-10-09T04:58:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:35 smithi177 ceph-mon[195797]: osdmap e523: 8 total, 8 up, 6 in 2023-10-09T04:58:36.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 79784/223995 objects degraded (35.619%), 16 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:36.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:36.398 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:36.616 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:36.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:36.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:36.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:36.773 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:36.815 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:58:36.851 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:58:36.879 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:58:36.917 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:36.922 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:58:36.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:36.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:36.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T04:58:37.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:37.052 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:37.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:37.100 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:36 smithi177 ceph-mon[195797]: pgmap v1782: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79784/223995 objects degraded (35.619%); 14884/223995 objects misplaced (6.645%); 18 MiB/s, 285 objects/s recovering 2023-10-09T04:58:37.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:37.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:37.121 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.122 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.123 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:58:37.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:36 smithi029 ceph-mon[216903]: pgmap v1782: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79784/223995 objects degraded (35.619%); 14884/223995 objects misplaced (6.645%); 18 MiB/s, 285 objects/s recovering 2023-10-09T04:58:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:36 smithi029 ceph-mon[223858]: pgmap v1782: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 79784/223995 objects degraded (35.619%); 14884/223995 objects misplaced (6.645%); 18 MiB/s, 285 objects/s recovering 2023-10-09T04:58:37.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:37.231 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:37.280 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:37.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:37.387 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:37.414 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:37.459 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:37.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:37.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:37.564 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:37.591 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:37.641 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:37.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:37.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:37.785 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:38.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[216903]: osdmap e524: 8 total, 8 up, 6 in 2023-10-09T04:58:38.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[216903]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[216903]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[216903]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[216903]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[223858]: osdmap e524: 8 total, 8 up, 6 in 2023-10-09T04:58:38.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[223858]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[223858]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[223858]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:37 smithi029 ceph-mon[223858]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:58:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:37 smithi177 ceph-mon[195797]: osdmap e524: 8 total, 8 up, 6 in 2023-10-09T04:58:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:37 smithi177 ceph-mon[195797]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:37 smithi177 ceph-mon[195797]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:37 smithi177 ceph-mon[195797]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:37 smithi177 ceph-mon[195797]: failed to encode map e524 with expected crc 2023-10-09T04:58:38.366 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:38.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:38.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:38.585 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:38.670 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:38.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:38.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:38.815 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:38.863 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:38.898 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:38.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:38.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[216903]: pgmap v1784: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 84387/223989 objects degraded (37.675%); 10365/223989 objects misplaced (4.627%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:58:38.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[216903]: failed to encode map e525 with expected crc 2023-10-09T04:58:38.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[216903]: osdmap e525: 8 total, 8 up, 6 in 2023-10-09T04:58:38.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[216903]: failed to encode map e525 with expected crc 2023-10-09T04:58:38.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[216903]: failed to encode map e525 with expected crc 2023-10-09T04:58:38.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[223858]: pgmap v1784: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 84387/223989 objects degraded (37.675%); 10365/223989 objects misplaced (4.627%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:58:38.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[223858]: failed to encode map e525 with expected crc 2023-10-09T04:58:38.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[223858]: osdmap e525: 8 total, 8 up, 6 in 2023-10-09T04:58:38.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[223858]: failed to encode map e525 with expected crc 2023-10-09T04:58:38.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:38 smithi029 ceph-mon[223858]: failed to encode map e525 with expected crc 2023-10-09T04:58:38.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:39.010 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.038 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:39.076 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:39.086 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:39.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:39.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:39.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:39.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:38 smithi177 ceph-mon[195797]: pgmap v1784: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 84387/223989 objects degraded (37.675%); 10365/223989 objects misplaced (4.627%); 19 MiB/s, 303 objects/s recovering 2023-10-09T04:58:39.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:38 smithi177 ceph-mon[195797]: failed to encode map e525 with expected crc 2023-10-09T04:58:39.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:38 smithi177 ceph-mon[195797]: osdmap e525: 8 total, 8 up, 6 in 2023-10-09T04:58:39.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:38 smithi177 ceph-mon[195797]: failed to encode map e525 with expected crc 2023-10-09T04:58:39.218 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:38 smithi177 ceph-mon[195797]: failed to encode map e525 with expected crc 2023-10-09T04:58:39.220 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:39.270 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:39.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:39.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:39.405 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:39.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:39.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:39.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:39.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:39.589 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:39.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:39.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:39.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:39.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:39.778 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:39.908 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 0.682478 (aeb6) 2023-10-09T04:58:39.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[216903]: osdmap e526: 8 total, 8 up, 6 in 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[216903]: failed to encode map e526 with expected crc 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[216903]: failed to encode map e526 with expected crc 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[216903]: failed to encode map e526 with expected crc 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1266746720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]: dispatch 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]: dispatch 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[223858]: osdmap e526: 8 total, 8 up, 6 in 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[223858]: failed to encode map e526 with expected crc 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[223858]: failed to encode map e526 with expected crc 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[223858]: failed to encode map e526 with expected crc 2023-10-09T04:58:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1266746720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]: dispatch 2023-10-09T04:58:39.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:39 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]: dispatch 2023-10-09T04:58:40.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:39 smithi177 ceph-mon[195797]: osdmap e526: 8 total, 8 up, 6 in 2023-10-09T04:58:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:39 smithi177 ceph-mon[195797]: failed to encode map e526 with expected crc 2023-10-09T04:58:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:39 smithi177 ceph-mon[195797]: failed to encode map e526 with expected crc 2023-10-09T04:58:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:39 smithi177 ceph-mon[195797]: failed to encode map e526 with expected crc 2023-10-09T04:58:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1266746720' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]: dispatch 2023-10-09T04:58:40.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:39 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]: dispatch 2023-10-09T04:58:40.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:58:40.785 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:40.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:40.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:40.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:40.923 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: pgmap v1787: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 84380/223944 objects degraded (37.679%); 10365/223944 objects misplaced (4.628%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]': finished 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: osdmap e527: 8 total, 8 up, 6 in 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84380/223944 objects degraded (37.679%), 16 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[216903]: osdmap e528: 8 total, 8 up, 6 in 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: pgmap v1787: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 84380/223944 objects degraded (37.679%); 10365/223944 objects misplaced (4.628%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]': finished 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: osdmap e527: 8 total, 8 up, 6 in 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84380/223944 objects degraded (37.679%), 16 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:41.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:40 smithi029 ceph-mon[223858]: osdmap e528: 8 total, 8 up, 6 in 2023-10-09T04:58:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: pgmap v1787: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 84380/223944 objects degraded (37.679%); 10365/223944 objects misplaced (4.628%); 2.1 MiB/s, 33 objects/s recovering 2023-10-09T04:58:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.6824777217101583}]': finished 2023-10-09T04:58:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: osdmap e527: 8 total, 8 up, 6 in 2023-10-09T04:58:41.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: failed to encode map e527 with expected crc 2023-10-09T04:58:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84380/223944 objects degraded (37.679%), 16 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:41.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:40 smithi177 ceph-mon[195797]: osdmap e528: 8 total, 8 up, 6 in 2023-10-09T04:58:41.431 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:41.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:41.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:41.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:41.563 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:41.609 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:41.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:41.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:41.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:41.741 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:41.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:41.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:41.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:41.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:41.927 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:42.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[216903]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[216903]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[216903]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[216903]: osdmap e529: 8 total, 8 up, 6 in 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[223858]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[223858]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[223858]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:41 smithi029 ceph-mon[223858]: osdmap e529: 8 total, 8 up, 6 in 2023-10-09T04:58:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:41 smithi177 ceph-mon[195797]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:41 smithi177 ceph-mon[195797]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:41 smithi177 ceph-mon[195797]: failed to encode map e528 with expected crc 2023-10-09T04:58:42.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:41 smithi177 ceph-mon[195797]: osdmap e529: 8 total, 8 up, 6 in 2023-10-09T04:58:42.680 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:42.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:42.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:42.804 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:42.828 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:42.878 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:58:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:58:42] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-10-09T04:58:42.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[223858]: pgmap v1790: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 3 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 85019/223644 objects degraded (38.015%); 8621/223644 objects misplaced (3.855%); 15 MiB/s, 288 objects/s recovering 2023-10-09T04:58:42.879 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:58:42.879 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:58:42.879 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T04:58:42.879 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:58:42.896 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:42.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:43.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:43.036 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:43.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[223858]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[223858]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[223858]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[223858]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[216903]: pgmap v1790: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 3 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 85019/223644 objects degraded (38.015%); 8621/223644 objects misplaced (3.855%); 15 MiB/s, 288 objects/s recovering 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[216903]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[216903]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[216903]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:42 smithi029 ceph-mon[216903]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:42 smithi177 ceph-mon[195797]: pgmap v1790: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 3 peering, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 85019/223644 objects degraded (38.015%); 8621/223644 objects misplaced (3.855%); 15 MiB/s, 288 objects/s recovering 2023-10-09T04:58:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:42 smithi177 ceph-mon[195797]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:42 smithi177 ceph-mon[195797]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:42 smithi177 ceph-mon[195797]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:42 smithi177 ceph-mon[195797]: failed to encode map e529 with expected crc 2023-10-09T04:58:43.901 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:58:43.999 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:44.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:44.104 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:44.133 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:44.133 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2936717474' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:58:44.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:44.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2936717474' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:58:44.141 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:44.193 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:44.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2936717474' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:58:44.297 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:44.325 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:44.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: pgmap v1792: 99 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 27 op/s; 84988/223554 objects degraded (38.017%); 8621/223554 objects misplaced (3.856%); 13 MiB/s, 249 objects/s recovering 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2936717474' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: osdmap e530: 8 total, 8 up, 6 in 2023-10-09T04:58:45.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[216903]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: pgmap v1792: 99 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 27 op/s; 84988/223554 objects degraded (38.017%); 8621/223554 objects misplaced (3.856%); 13 MiB/s, 249 objects/s recovering 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2936717474' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: osdmap e530: 8 total, 8 up, 6 in 2023-10-09T04:58:45.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:44 smithi029 ceph-mon[223858]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.128 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:45.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:45.232 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: pgmap v1792: 99 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 27 op/s; 84988/223554 objects degraded (38.017%); 8621/223554 objects misplaced (3.856%); 13 MiB/s, 249 objects/s recovering 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2936717474' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: osdmap e530: 8 total, 8 up, 6 in 2023-10-09T04:58:45.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:44 smithi177 ceph-mon[195797]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:45.304 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:58:45.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:45.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:58:45.411 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:45.438 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:45.482 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:45.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:45.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:45.589 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:45.622 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:45.854 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:45.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:45.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:45 smithi029 ceph-mon[216903]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:45 smithi029 ceph-mon[216903]: failed to encode map e530 with expected crc 2023-10-09T04:58:45.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:45 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 84988/223554 objects degraded (38.017%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:45.959 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:45.987 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:46.011 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:46.031 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:46.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:46.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:46.160 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:45 smithi177 ceph-mon[195797]: failed to encode map e530 with expected crc 2023-10-09T04:58:46.160 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:45 smithi177 ceph-mon[195797]: failed to encode map e530 with expected crc 2023-10-09T04:58:46.160 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:45 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 84988/223554 objects degraded (38.017%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:46.162 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:45 smithi029 ceph-mon[223858]: failed to encode map e530 with expected crc 2023-10-09T04:58:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:45 smithi029 ceph-mon[223858]: failed to encode map e530 with expected crc 2023-10-09T04:58:46.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:45 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 84988/223554 objects degraded (38.017%), 15 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:46.212 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:46.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:46.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:46.319 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:46.345 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:46.392 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:46.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:46.502 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:46.515 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:58:46.543 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:58:46.543 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:46.564 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6] out_osds: [4, 0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T04:58:46.564 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:58:46.564 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-09T04:58:46.564 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-09T04:58:46.572 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:58:46.605 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:58:46.630 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:46.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:46.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:46.635 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:58:46.659 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:58:46.681 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:58:46.701 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:58:46.703 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.704 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:58:46.734 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:46.760 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:46.808 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:46.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:46.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: pgmap v1794: 99 pgs: 3 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 89853/223488 objects degraded (40.205%); 8621/223488 objects misplaced (3.857%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: osdmap e531: 8 total, 8 up, 6 in 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: failed to encode map e531 with expected crc 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: failed to encode map e531 with expected crc 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: failed to encode map e531 with expected crc 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: failed to encode map e531 with expected crc 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: failed to encode map e531 with expected crc 2023-10-09T04:58:46.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/670512359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:46.903 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:58:46.913 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:46.982 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: pgmap v1794: 99 pgs: 3 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 89853/223488 objects degraded (40.205%); 8621/223488 objects misplaced (3.857%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:58:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: osdmap e531: 8 total, 8 up, 6 in 2023-10-09T04:58:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/670512359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:47.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: pgmap v1794: 99 pgs: 3 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 22 op/s; 89853/223488 objects degraded (40.205%); 8621/223488 objects misplaced (3.857%); 1.3 MiB/s, 20 objects/s recovering 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: osdmap e531: 8 total, 8 up, 6 in 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: failed to encode map e531 with expected crc 2023-10-09T04:58:47.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/670512359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:58:48.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:58:48.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:58:48.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-09T04:58:48.537 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:48.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:48.601 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:48.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:48.656 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:48.693 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:48.767 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:48.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:48.871 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:48.897 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:49.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:49.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:49.111 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:49.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: pgmap v1796: 99 pgs: 1 active+undersized+degraded+remapped+laggy, 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 9 op/s; 89851/223479 objects degraded (40.206%); 8486/223479 objects misplaced (3.797%); 938 KiB/s, 14 objects/s recovering 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: Health check failed: 0 slow ops, oldest one blocked for 35 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: osdmap e532: 8 total, 8 up, 6 in 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[223858]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: pgmap v1796: 99 pgs: 1 active+undersized+degraded+remapped+laggy, 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 9 op/s; 89851/223479 objects degraded (40.206%); 8486/223479 objects misplaced (3.797%); 938 KiB/s, 14 objects/s recovering 2023-10-09T04:58:49.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: Health check failed: 0 slow ops, oldest one blocked for 35 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:58:49.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:58:49.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: osdmap e532: 8 total, 8 up, 6 in 2023-10-09T04:58:49.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:48 smithi029 ceph-mon[216903]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.201 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: pgmap v1796: 99 pgs: 1 active+undersized+degraded+remapped+laggy, 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 9 op/s; 89851/223479 objects degraded (40.206%); 8486/223479 objects misplaced (3.797%); 938 KiB/s, 14 objects/s recovering 2023-10-09T04:58:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: Health check failed: 0 slow ops, oldest one blocked for 35 sec, osd.3 has slow ops (SLOW_OPS) 2023-10-09T04:58:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-09T04:58:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: osdmap e532: 8 total, 8 up, 6 in 2023-10-09T04:58:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:48 smithi177 ceph-mon[195797]: failed to encode map e532 with expected crc 2023-10-09T04:58:49.616 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:58:49.616 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:58:49.703 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:49.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:49.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:58:49.866 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[216903]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[216903]: osdmap e533: 8 total, 8 up, 6 in 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[216903]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[216903]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[216903]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[223858]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[223858]: osdmap e533: 8 total, 8 up, 6 in 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[223858]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[223858]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:49 smithi029 ceph-mon[223858]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:49 smithi177 ceph-mon[195797]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:49 smithi177 ceph-mon[195797]: osdmap e533: 8 total, 8 up, 6 in 2023-10-09T04:58:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:49 smithi177 ceph-mon[195797]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:49 smithi177 ceph-mon[195797]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:49 smithi177 ceph-mon[195797]: failed to encode map e533 with expected crc 2023-10-09T04:58:50.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: pgmap v1799: 109 pgs: 10 unknown, 1 active+undersized+degraded+remapped+laggy, 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 89836/223425 objects degraded (40.209%); 8486/223425 objects misplaced (3.798%); 891 KiB/s, 13 objects/s recovering 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: osdmap e534: 8 total, 8 up, 6 in 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: failed to encode map e534 with expected crc 2023-10-09T04:58:50.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 89836/223425 objects degraded (40.209%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:50.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[216903]: osdmap e535: 8 total, 8 up, 6 in 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: pgmap v1799: 109 pgs: 10 unknown, 1 active+undersized+degraded+remapped+laggy, 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 89836/223425 objects degraded (40.209%); 8486/223425 objects misplaced (3.798%); 891 KiB/s, 13 objects/s recovering 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: osdmap e534: 8 total, 8 up, 6 in 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 89836/223425 objects degraded (40.209%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:50 smithi029 ceph-mon[223858]: osdmap e535: 8 total, 8 up, 6 in 2023-10-09T04:58:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: pgmap v1799: 109 pgs: 10 unknown, 1 active+undersized+degraded+remapped+laggy, 1 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 12 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 89836/223425 objects degraded (40.209%); 8486/223425 objects misplaced (3.798%); 891 KiB/s, 13 objects/s recovering 2023-10-09T04:58:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: osdmap e534: 8 total, 8 up, 6 in 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: failed to encode map e534 with expected crc 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 89836/223425 objects degraded (40.209%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:50 smithi177 ceph-mon[195797]: osdmap e535: 8 total, 8 up, 6 in 2023-10-09T04:58:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:51 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:51 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:51 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:51 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:51 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:51 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:51 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:52.975 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:58:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:52.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:58:53.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:53.107 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:52 smithi029 ceph-mon[216903]: failed to encode map e535 with expected crc 2023-10-09T04:58:53.107 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:52 smithi029 ceph-mon[216903]: pgmap v1802: 109 pgs: 7 unknown, 1 active+undersized+degraded+remapped+laggy, 4 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 84504/223383 objects degraded (37.829%); 8486/223383 objects misplaced (3.799%); 176 KiB/s, 3 objects/s recovering 2023-10-09T04:58:53.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:52 smithi029 ceph-mon[223858]: failed to encode map e535 with expected crc 2023-10-09T04:58:53.108 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:52 smithi029 ceph-mon[223858]: pgmap v1802: 109 pgs: 7 unknown, 1 active+undersized+degraded+remapped+laggy, 4 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 84504/223383 objects degraded (37.829%); 8486/223383 objects misplaced (3.799%); 176 KiB/s, 3 objects/s recovering 2023-10-09T04:58:53.108 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:58:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:58:52] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.43.0" 2023-10-09T04:58:53.113 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:52 smithi177 ceph-mon[195797]: failed to encode map e535 with expected crc 2023-10-09T04:58:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:52 smithi177 ceph-mon[195797]: pgmap v1802: 109 pgs: 7 unknown, 1 active+undersized+degraded+remapped+laggy, 4 peering, 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 84504/223383 objects degraded (37.829%); 8486/223383 objects misplaced (3.799%); 176 KiB/s, 3 objects/s recovering 2023-10-09T04:58:54.012 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:54.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:54.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T04:58:54.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:54.146 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:54.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[216903]: osdmap e536: 8 total, 8 up, 6 in 2023-10-09T04:58:54.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[216903]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[216903]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[216903]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[216903]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[216903]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[223858]: osdmap e536: 8 total, 8 up, 6 in 2023-10-09T04:58:54.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[223858]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[223858]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[223858]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[223858]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:53 smithi029 ceph-mon[223858]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.195 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:54.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:53 smithi177 ceph-mon[195797]: osdmap e536: 8 total, 8 up, 6 in 2023-10-09T04:58:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:53 smithi177 ceph-mon[195797]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:53 smithi177 ceph-mon[195797]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:53 smithi177 ceph-mon[195797]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:53 smithi177 ceph-mon[195797]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:53 smithi177 ceph-mon[195797]: failed to encode map e536 with expected crc 2023-10-09T04:58:54.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:54.348 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:54.454 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:54.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:54.559 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:54.603 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:54.661 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:54.799 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:54.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:54.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:55.008 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:54 smithi029 ceph-mon[223858]: pgmap v1804: 109 pgs: 2 remapped+peering, 1 activating+degraded, 2 activating+undersized+degraded+remapped, 1 activating, 2 unknown, 3 peering, 1 active+clean+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 creating+activating+degraded, 82 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 77721/223335 objects degraded (34.800%); 6579/223335 objects misplaced (2.946%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:58:55.009 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:54 smithi029 ceph-mon[223858]: failed to encode map e536 with expected crc 2023-10-09T04:58:55.009 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:54 smithi029 ceph-mon[223858]: Health check cleared: SLOW_OPS (was: 0 slow ops, oldest one blocked for 35 sec, osd.3 has slow ops) 2023-10-09T04:58:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:54 smithi029 ceph-mon[216903]: pgmap v1804: 109 pgs: 2 remapped+peering, 1 activating+degraded, 2 activating+undersized+degraded+remapped, 1 activating, 2 unknown, 3 peering, 1 active+clean+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 creating+activating+degraded, 82 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 77721/223335 objects degraded (34.800%); 6579/223335 objects misplaced (2.946%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:58:55.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:54 smithi029 ceph-mon[216903]: failed to encode map e536 with expected crc 2023-10-09T04:58:55.020 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:54 smithi029 ceph-mon[216903]: Health check cleared: SLOW_OPS (was: 0 slow ops, oldest one blocked for 35 sec, osd.3 has slow ops) 2023-10-09T04:58:55.020 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:55.261 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:55.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:55.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:54 smithi177 ceph-mon[195797]: pgmap v1804: 109 pgs: 2 remapped+peering, 1 activating+degraded, 2 activating+undersized+degraded+remapped, 1 activating, 2 unknown, 3 peering, 1 active+clean+remapped, 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 creating+activating+degraded, 82 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 77721/223335 objects degraded (34.800%); 6579/223335 objects misplaced (2.946%); 1.6 MiB/s, 24 objects/s recovering 2023-10-09T04:58:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:54 smithi177 ceph-mon[195797]: failed to encode map e536 with expected crc 2023-10-09T04:58:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:54 smithi177 ceph-mon[195797]: Health check cleared: SLOW_OPS (was: 0 slow ops, oldest one blocked for 35 sec, osd.3 has slow ops) 2023-10-09T04:58:55.367 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:55.384 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:58:55.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:55.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:55.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:55.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:55.587 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:55.649 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:55.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:55.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:55.781 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:55 smithi029 conmon[216878]: 2023-10-09T04:58:55.526+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e536 definitely_dead 0 2023-10-09T04:58:55.785 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:55.842 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:55.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:55.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:55.946 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:55.968 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:58:55.968 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.4. 2023-10-09T04:58:55.974 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:56.028 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:56.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:56.134 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:56.161 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:56.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1683227960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:56.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:56.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2842346067' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:58:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 77721/223335 objects degraded (34.800%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1683227960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2842346067' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:58:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 77721/223335 objects degraded (34.800%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:56.204 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:56.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1683227960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:56.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:55 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:58:56.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2842346067' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-09T04:58:56.208 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 77721/223335 objects degraded (34.800%), 18 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:58:56.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:56.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:56.310 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:56.335 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:56.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:56.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:56.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:56.511 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:56.554 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:56.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:56.660 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:58:56.686 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:56.737 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:58:56.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:56.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:58:56.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:58:56.880 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: pgmap v1805: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating+degraded, 3 activating+undersized+degraded+remapped, 1 activating, 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 creating+activating+degraded, 82 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 75797/223305 objects degraded (33.943%); 9253/223305 objects misplaced (4.144%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2842346067' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[216903]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: pgmap v1805: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating+degraded, 3 activating+undersized+degraded+remapped, 1 activating, 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 creating+activating+degraded, 82 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 75797/223305 objects degraded (33.943%); 9253/223305 objects misplaced (4.144%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2842346067' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:58:57.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T04:58:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:56 smithi029 ceph-mon[223858]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: pgmap v1805: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating+degraded, 3 activating+undersized+degraded+remapped, 1 activating, 10 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 creating+activating+degraded, 82 active+clean; 4.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 75797/223305 objects degraded (33.943%); 9253/223305 objects misplaced (4.144%); 1.7 MiB/s, 26 objects/s recovering 2023-10-09T04:58:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:58:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:58:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2842346067' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-09T04:58:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: osdmap e537: 8 total, 8 up, 7 in 2023-10-09T04:58:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: failed to encode map e537 with expected crc 2023-10-09T04:58:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:56 smithi177 ceph-mon[195797]: failed to encode map e537 with expected crc 2023-10-09T04:58:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e537 with expected crc 2023-10-09T04:58:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:57 smithi177 ceph-mon[195797]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e537 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[216903]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e537 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: osdmap e538: 8 total, 8 up, 7 in 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:57 smithi029 ceph-mon[223858]: failed to encode map e538 with expected crc 2023-10-09T04:58:58.511 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-09T04:58:58.690 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:58:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:58.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:58:58.795 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:58:58.821 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:58.866 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:58:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:58.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:58:58.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:59.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: pgmap v1808: 109 pgs: 13 peering, 1 active+clean+remapped, 3 remapped+peering, 2 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65702/223827 objects degraded (29.354%); 14902/223827 objects misplaced (6.658%) 2023-10-09T04:58:59.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:59.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: pgmap v1808: 109 pgs: 13 peering, 1 active+clean+remapped, 3 remapped+peering, 2 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65702/223827 objects degraded (29.354%); 14902/223827 objects misplaced (6.658%) 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.059 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.059 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:59.059 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:59.060 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:59.129 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:58:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:59.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:58:59.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:59.264 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: pgmap v1808: 109 pgs: 13 peering, 1 active+clean+remapped, 3 remapped+peering, 2 activating+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65702/223827 objects degraded (29.354%); 14902/223827 objects misplaced (6.658%) 2023-10-09T04:58:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:58:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: osdmap e539: 8 total, 8 up, 7 in 2023-10-09T04:58:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: failed to encode map e539 with expected crc 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:58:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:58:59.316 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:59.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:59.420 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:58:59.448 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:59.498 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:58:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:59.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:58:59.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:58:59.637 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:58:59.693 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:58:59.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:58:59.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:58:59.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:58:59.832 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:00.188 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:00.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[216903]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[216903]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[216903]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[216903]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[223858]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[223858]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[223858]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.249 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:58:59 smithi029 ceph-mon[223858]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:59 smithi177 ceph-mon[195797]: osdmap e540: 8 total, 8 up, 7 in 2023-10-09T04:59:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:59 smithi177 ceph-mon[195797]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:59 smithi177 ceph-mon[195797]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:58:59 smithi177 ceph-mon[195797]: failed to encode map e540 with expected crc 2023-10-09T04:59:00.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:00.328 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:00.385 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:00.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:00.490 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:00.497 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:59:00.516 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:00.517 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:59:00.538 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:59:00.561 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:59:00.562 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:00.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:00.591 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:59:00.610 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:59:00.634 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:59:00.656 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:59:00.658 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.658 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.658 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.658 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.658 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.659 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.659 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.659 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:59:00.669 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:00.706 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:00.765 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T04:59:00.926 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:59:00.926 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75767 14.6 14:67eeec07:::benchmark_data_smithi029_550670_object75766:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587164+0000", 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.335566223, 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8369943580000001, 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:59:00.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75767 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587164+0000", 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587164+0000", 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587162+0000", 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:59:00.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587206+0000", 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3853000000000002e-05 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587228+0000", 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1642999999999999e-05 2023-10-09T04:59:00.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587239+0000", 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1773999999999999e-05 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587432+0000", 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000193165 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587621+0000", 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000188137 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:59:00.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587810+0000", 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000189638 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.595548+0000", 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077380579999999999 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.595558+0000", 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3070000000000008e-06 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.626520+0000", 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030962798 2023-10-09T04:59:00.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.626536+0000", 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5619000000000001e-05 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578599+0000", 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95206309499999997 2023-10-09T04:59:00.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578612+0000", 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2500000000000001e-05 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.424024+0000", 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.845412107 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.424159+0000", 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013491100000000001 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75768 14.b 14:df4bb7ab:::benchmark_data_smithi029_550670_object75767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587258+0000", 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.33547220499997, 2023-10-09T04:59:00.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9327158849999999, 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75768 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587258+0000", 2023-10-09T04:59:00.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587258+0000", 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587258+0000", 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587297+0000", 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8970000000000001e-05 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587305+0000", 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5720000000000003e-06 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587311+0000", 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8479999999999997e-06 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.591805+0000", 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044936780000000001 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.591955+0000", 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000150617 2023-10-09T04:59:00.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.592115+0000", 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000159679 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619278+0000", 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027162781 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619295+0000", 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6892999999999999e-05 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577449+0000", 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95815421199999995 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.944 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577461+0000", 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2148e-05 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.519849+0000", 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.942388059 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.519974+0000", 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012499899999999999 2023-10-09T04:59:00.945 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75769 14.6 14:640c7ebb:::benchmark_data_smithi029_550670_object75768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587318+0000", 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.33541216399999, 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8637364299999999, 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.946 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75769 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587318+0000", 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587318+0000", 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587318+0000", 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587337+0000", 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9017000000000001e-05 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587344+0000", 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6760000000000001e-06 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587349+0000", 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9899999999999997e-06 2023-10-09T04:59:00.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.590673+0000", 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003324151 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.590837+0000", 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164186 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.591004+0000", 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166897 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.627208+0000", 2023-10-09T04:59:00.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036204436 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.627221+0000", 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2875e-05 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578647+0000", 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95142564900000004 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578657+0000", 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0594999999999999e-05 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.426758+0000", 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8481005879999999 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:00.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.427461+0000", 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00070262299999999998 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.450926+0000", 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023465465000000001 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.451055+0000", 2023-10-09T04:59:00.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012875300000000001 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75770 14.b 14:dc4bda7c:::benchmark_data_smithi029_550670_object75769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587353+0000", 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.33537720200002, 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.961170659, 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.955 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75770 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587353+0000", 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587353+0000", 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587353+0000", 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587375+0000", 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1727000000000001e-05 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587381+0000", 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6769999999999998e-06 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587386+0000", 2023-10-09T04:59:00.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9930000000000003e-06 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.600078+0000", 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012692050999999999 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.600255+0000", 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000176885 2023-10-09T04:59:00.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.600426+0000", 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017075099999999999 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619779+0000", 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019352582 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619791+0000", 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2045000000000001e-05 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577498+0000", 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95770707099999997 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577508+0000", 2023-10-09T04:59:00.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8859999999999999e-06 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523040+0000", 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9455329180000001 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523516+0000", 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00047530000000000001 2023-10-09T04:59:00.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548407+0000", 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024890834000000001 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548524+0000", 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117376 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75771 14.0 14:090f761e:::benchmark_data_smithi029_550670_object75770:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.599830+0000", 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.32290081799999, 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.927342222, 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75771 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.964 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599830+0000", 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599830+0000", 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599827+0000", 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599868+0000", 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0200999999999997e-05 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599883+0000", 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5826000000000001e-05 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599894+0000", 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0865e-05 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.623358+0000", 2023-10-09T04:59:00.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023463588 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.623366+0000", 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9079999999999995e-06 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576263+0000", 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95289678799999999 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576275+0000", 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2527999999999999e-05 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.510946+0000", 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9346704300000002 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.511127+0000", 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018167300000000001 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.511327+0000", 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000199676 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523885+0000", 2023-10-09T04:59:00.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012557972000000001 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.524081+0000", 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000196119 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.524092+0000", 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1073999999999999e-05 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.524147+0000", 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4941000000000001e-05 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.527172+0000", 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030249029999999998 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75778 14.b 14:db8e7e52:::benchmark_data_smithi029_550670_object75777:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.632090+0000", 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.29064086800003, 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.920090541, 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75778 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.973 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632090+0000", 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632090+0000", 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632088+0000", 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632115+0000", 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6480000000000001e-05 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632133+0000", 2023-10-09T04:59:00.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8165999999999999e-05 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632141+0000", 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8639999999999996e-06 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632222+0000", 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1005000000000001e-05 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632232+0000", 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0789e-05 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581266+0000", 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94903347299999996 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581273+0000", 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2880000000000004e-06 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523554+0000", 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9422803749999999 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523699+0000", 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014507999999999999 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523877+0000", 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017794499999999999 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.538285+0000", 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014408287000000001 2023-10-09T04:59:00.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.541528+0000", 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032427770000000001 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548696+0000", 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007167919 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548754+0000", 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8739999999999997e-05 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.552180+0000", 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034258280000000001 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.981 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75780 14.0 14:0dbd6af5:::benchmark_data_smithi029_550670_object75779:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.637479+0000", 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.28525147599998, 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9000195930000001, 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.982 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75780 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637479+0000", 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637479+0000", 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637479+0000", 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637500+0000", 2023-10-09T04:59:00.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1426000000000001e-05 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637511+0000", 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0839000000000001e-05 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637517+0000", 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4470000000000003e-06 2023-10-09T04:59:00.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.639412+0000", 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018943899999999999 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.639418+0000", 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7129999999999999e-06 2023-10-09T04:59:00.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576299+0000", 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93688061199999995 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576307+0000", 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5490000000000001e-06 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520128+0000", 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.943821799 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520331+0000", 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020220200000000001 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520519+0000", 2023-10-09T04:59:00.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000188445 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.527279+0000", 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067602900000000004 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.527328+0000", 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8647999999999999e-05 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.534681+0000", 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073529279999999999 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.534756+0000", 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4950000000000006e-05 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.990 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.537499+0000", 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027427860000000001 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75782 14.6 14:66f59f82:::benchmark_data_smithi029_550670_object75781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.638790+0000", 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 256.28394045099998, 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8162426200000001, 2023-10-09T04:59:00.991 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75782 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638790+0000", 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638790+0000", 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638789+0000", 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:00.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638819+0000", 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0349e-05 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638837+0000", 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8090000000000001e-05 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638847+0000", 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9690000000000006e-06 2023-10-09T04:59:00.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638900+0000", 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3232000000000001e-05 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638910+0000", 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9480000000000003e-06 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578683+0000", 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93977262699999997 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578692+0000", 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8759999999999994e-06 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.427504+0000", 2023-10-09T04:59:00.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.848811832 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.427820+0000", 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031670199999999998 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.428026+0000", 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000205756 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.430801+0000", 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027751910000000002 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.439921+0000", 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091198389999999994 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:00.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.451125+0000", 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01120434 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.451194+0000", 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8336000000000004e-05 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.455033+0000", 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003838957 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:00.999 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75763 14.d 14:bffa6910:::benchmark_data_smithi029_550670_object75762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581881+0000", 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 255.340849792, 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.884113382, 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.000 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75763 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581881+0000", 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581881+0000", 2023-10-09T04:59:01.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581879+0000", 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581922+0000", 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2283999999999997e-05 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581937+0000", 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.552e-05 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581947+0000", 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0441999999999999e-05 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582166+0000", 2023-10-09T04:59:01.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021813 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582175+0000", 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4239999999999999e-06 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.464358+0000", 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.882182969 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.465994+0000", 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001636071 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75766 14.d 14:b17bdc11:::benchmark_data_smithi029_550670_object75765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581967+0000", 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 255.340763573, 2023-10-09T04:59:01.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8861905270000001, 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75766 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581967+0000", 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581967+0000", 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581966+0000", 2023-10-09T04:59:01.007 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582002+0000", 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5385000000000001e-05 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582031+0000", 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9578999999999999e-05 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582039+0000", 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4399999999999999e-06 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582199+0000", 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016029400000000001 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582207+0000", 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7670000000000003e-06 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.468032+0000", 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.885825222 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.468157+0000", 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012543 2023-10-09T04:59:01.010 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75774 14.d 14:bdf921c9:::benchmark_data_smithi029_550670_object75773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.582046+0000", 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 255.34068474899999, 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9239243049999999, 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.011 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75774 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582046+0000", 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582046+0000", 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582045+0000", 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582067+0000", 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2084999999999999e-05 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582073+0000", 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582078+0000", 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1139999999999999e-06 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582232+0000", 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015455800000000001 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582239+0000", 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9959999999999996e-06 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.470621+0000", 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.888381734 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.471199+0000", 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057843300000000005 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T04:59:01.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.471326+0000", 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012646000000000001 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.476776+0000", 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005449756 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.478331+0000", 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001555802 2023-10-09T04:59:01.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.478980+0000", 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064857800000000002 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.503195+0000", 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024215161999999998 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.503259+0000", 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3878000000000001e-05 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.505970+0000", 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027110989999999998 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75775 14.d 14:b648c4c3:::benchmark_data_smithi029_550670_object75774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.582081+0000", 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 255.34064920899999, 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9269593029999998, 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75775 2023-10-09T04:59:01.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582081+0000", 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582081+0000", 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582081+0000", 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582104+0000", 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2773999999999999e-05 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582113+0000", 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0110000000000006e-06 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582119+0000", 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7860000000000002e-06 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582262+0000", 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014312 2023-10-09T04:59:01.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582268+0000", 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8600000000000004e-06 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.473499+0000", 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.891230465 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.473690+0000", 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019085700000000001 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.473900+0000", 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000209731 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.476972+0000", 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030722330000000002 2023-10-09T04:59:01.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.478460+0000", 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001488367 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.479284+0000", 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00082400300000000002 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.506072+0000", 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026788055000000002 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.506156+0000", 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3523000000000007e-05 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.509041+0000", 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002884869 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:78125 14.c 14:3c481121:::benchmark_data_smithi029_550670_object514:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.270549+0000", 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.65218159400001, 2023-10-09T04:59:01.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.060580334, 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 78125 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270549+0000", 2023-10-09T04:59:01.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270549+0000", 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270548+0000", 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270553+0000", 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7250000000000002e-06 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270563+0000", 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5680000000000005e-06 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270569+0000", 2023-10-09T04:59:01.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9610000000000002e-06 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.316048+0000", 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.045478969000000001 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.316056+0000", 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7819999999999994e-06 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138551+0000", 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.82249487200000004 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138560+0000", 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4860000000000003e-06 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.299702+0000", 2023-10-09T04:59:01.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1611424079999999 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.300295+0000", 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059211799999999998 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.311832+0000", 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011537016000000001 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.313194+0000", 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013622269999999999 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.313294+0000", 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100395 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327670+0000", 2023-10-09T04:59:01.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014375489 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327710+0000", 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0241999999999999e-05 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327745+0000", 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5070000000000001e-05 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327792+0000", 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7407999999999997e-05 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.331129+0000", 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033368450000000002 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.041 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:78130 14.c 14:3226741c:::benchmark_data_smithi029_550670_object519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.321301+0000", 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.60142992900001, 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0139341900000001, 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 78130 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321301+0000", 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321301+0000", 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321298+0000", 2023-10-09T04:59:01.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321306+0000", 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0960000000000001e-06 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321324+0000", 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7490999999999999e-05 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321334+0000", 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8709999999999992e-06 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.324175+0000", 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028410649999999998 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.324186+0000", 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0808999999999999e-05 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138579+0000", 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81439388400000001 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138585+0000", 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6629999999999998e-06 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.303198+0000", 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1646125330000001 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.303820+0000", 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00062219 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.324676+0000", 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020855728 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.324801+0000", 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012591999999999999 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.324956+0000", 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154742 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332201+0000", 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072445599999999997 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332255+0000", 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4011999999999997e-05 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332328+0000", 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3379000000000003e-05 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332363+0000", 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5015999999999998e-05 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.335235+0000", 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028716240000000001 2023-10-09T04:59:01.050 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:78138 14.c 14:36c9f53e:::benchmark_data_smithi029_550670_object527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.497048+0000", 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 188.42568202699999, 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.88800638, 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.051 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 78138 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497048+0000", 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497048+0000", 2023-10-09T04:59:01.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497047+0000", 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497057+0000", 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0536e-05 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497073+0000", 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5494000000000001e-05 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497085+0000", 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2255999999999999e-05 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497132+0000", 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7321000000000003e-05 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497141+0000", 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3399999999999998e-06 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138603+0000", 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.64146234800000002 2023-10-09T04:59:01.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138609+0000", 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8409999999999996e-06 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.331198+0000", 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1925896629999999 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.331703+0000", 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00050408300000000001 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.352160+0000", 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020457389999999999 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.352290+0000", 2023-10-09T04:59:01.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130576 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.352446+0000", 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015550699999999999 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.368379+0000", 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015933460999999999 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.378110+0000", 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097306489999999992 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.381781+0000", 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036712279999999999 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.381822+0000", 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0844999999999999e-05 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.385055+0000", 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032326949999999998 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.060 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79944 14.b 14:dc480776:::benchmark_data_smithi029_550670_object2333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:53.873492+0000", 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 127.049238972, 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.313680212, 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79944 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.061 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873492+0000", 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873492+0000", 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873490+0000", 2023-10-09T04:59:01.062 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873500+0000", 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0532e-05 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873520+0000", 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0228e-05 2023-10-09T04:59:01.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873529+0000", 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9409999999999996e-06 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873596+0000", 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6738999999999999e-05 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873608+0000", 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.184e-05 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.899229+0000", 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025621357000000001 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.899240+0000", 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0434999999999999e-05 2023-10-09T04:59:01.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.068466+0000", 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1692258600000001 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.069075+0000", 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00060947900000000001 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.176690+0000", 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10761480599999999 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.176829+0000", 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139042 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.177029+0000", 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020046300000000001 2023-10-09T04:59:01.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.181269+0000", 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042392059999999997 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.183001+0000", 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017322769999999999 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.184173+0000", 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001172551 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.184246+0000", 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2984999999999993e-05 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.187172+0000", 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029253909999999998 2023-10-09T04:59:01.069 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79950 14.6 14:6b7b279e:::benchmark_data_smithi029_550670_object2339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:55.235020+0000", 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 125.687710313, 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0513258090000002, 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.070 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79950 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235020+0000", 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.071 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235020+0000", 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235019+0000", 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235025+0000", 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4529999999999999e-06 2023-10-09T04:59:01.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235038+0000", 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2843000000000001e-05 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235045+0000", 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7139999999999996e-06 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235087+0000", 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1946000000000001e-05 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235097+0000", 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0424e-05 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.806392+0000", 2023-10-09T04:59:01.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57129448100000002 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.806403+0000", 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1620000000000001e-05 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.939987+0000", 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1335838650000001 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.940515+0000", 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052770499999999995 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.276476+0000", 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33596088499999999 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.276610+0000", 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013452700000000001 2023-10-09T04:59:01.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.276798+0000", 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000187529 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283471+0000", 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066732329999999998 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283506+0000", 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5008000000000001e-05 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283526+0000", 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9721999999999999e-05 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283592+0000", 2023-10-09T04:59:01.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5778999999999995e-05 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.286346+0000", 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002754458 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80555 14.b 14:db16aa4d:::benchmark_data_smithi029_550670_object2944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T04:59:01.079 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.903074+0000", 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 54.019656343000001, 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3293849390000001, 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80555 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903074+0000", 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903074+0000", 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903072+0000", 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:59:01.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903084+0000", 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1606999999999999e-05 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903111+0000", 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7042999999999998e-05 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903124+0000", 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3463e-05 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.905222+0000", 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020976089999999998 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.905239+0000", 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6869999999999999e-05 2023-10-09T04:59:01.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846128+0000", 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94088980499999997 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846141+0000", 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2454e-05 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.216737+0000", 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.370596132 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.216952+0000", 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021524599999999999 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.217187+0000", 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023498700000000001 2023-10-09T04:59:01.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228059+0000", 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010872122 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228118+0000", 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9052000000000003e-05 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228177+0000", 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8903999999999997e-05 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228253+0000", 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5881000000000001e-05 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.232459+0000", 2023-10-09T04:59:01.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042059209999999996 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80560 14.b 14:d972c932:::benchmark_data_smithi029_550670_object2949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:07.374946+0000", 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 53.547784256, 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8664385380000001, 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80560 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374946+0000", 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374946+0000", 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374944+0000", 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374955+0000", 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0813e-05 2023-10-09T04:59:01.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374974+0000", 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9607999999999999e-05 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374988+0000", 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3927999999999999e-05 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.375053+0000", 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4863000000000006e-05 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.375065+0000", 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2286000000000001e-05 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846161+0000", 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47109602 2023-10-09T04:59:01.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846168+0000", 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1029999999999996e-06 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224329+0000", 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.378160863 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224524+0000", 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019509599999999999 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224719+0000", 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019503399999999999 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.232728+0000", 2023-10-09T04:59:01.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080089949999999997 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.232876+0000", 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014811100000000001 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.237189+0000", 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043121560000000001 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.237263+0000", 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4667999999999997e-05 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.241385+0000", 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041214670000000002 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80562 14.e 14:732ec9d2:::benchmark_data_smithi029_550670_object2951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:07.495374+0000", 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 53.427356989000003, 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6830923520000001, 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80562 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.097 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495374+0000", 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495374+0000", 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495372+0000", 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495382+0000", 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0210000000000001e-05 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495408+0000", 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5658999999999999e-05 2023-10-09T04:59:01.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495420+0000", 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2395000000000001e-05 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495481+0000", 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1062000000000006e-05 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495490+0000", 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3810000000000005e-06 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.842050+0000", 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34655991600000002 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.842066+0000", 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6103000000000002e-05 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:08.980071+0000", 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1380047360000001 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:08.980571+0000", 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049947900000000005 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T04:59:01.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.116782+0000", 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13621124900000001 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.116901+0000", 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000119215 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.117081+0000", 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018009100000000001 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174495+0000", 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057413829 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174546+0000", 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0671999999999999e-05 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174592+0000", 2023-10-09T04:59:01.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6103000000000002e-05 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174660+0000", 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8343999999999995e-05 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.178466+0000", 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003805775 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.105 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.106 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.106 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T04:59:01.106 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T04:59:01.106 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:59:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:00 smithi177 ceph-mon[195797]: failed to encode map e540 with expected crc 2023-10-09T04:59:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:00 smithi177 ceph-mon[195797]: pgmap v1811: 109 pgs: 14 peering, 1 active+clean+remapped, 4 remapped+peering, 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65837/223827 objects degraded (29.414%); 10835/223827 objects misplaced (4.841%) 2023-10-09T04:59:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3328715267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 65837/223827 objects degraded (29.414%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:01.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[223858]: failed to encode map e540 with expected crc 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[223858]: pgmap v1811: 109 pgs: 14 peering, 1 active+clean+remapped, 4 remapped+peering, 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65837/223827 objects degraded (29.414%); 10835/223827 objects misplaced (4.841%) 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3328715267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 65837/223827 objects degraded (29.414%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[216903]: failed to encode map e540 with expected crc 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[216903]: pgmap v1811: 109 pgs: 14 peering, 1 active+clean+remapped, 4 remapped+peering, 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 4.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65837/223827 objects degraded (29.414%); 10835/223827 objects misplaced (4.841%) 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3328715267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 65837/223827 objects degraded (29.414%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:01.540 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:01.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:01.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:01.644 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:01.675 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:01.727 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:01.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:01.831 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:01.858 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:01.911 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:01.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:02.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:02.047 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:02.099 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:02.105 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:02 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:59:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:02.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:02.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:02 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:59:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:02 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-10-09T04:59:02.207 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:02.235 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:02.803 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T04:59:02.841 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:02.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:02.908 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:02.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:02.980 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:03.035 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:03 smithi029 ceph-mon[216903]: pgmap v1812: 109 pgs: 12 peering, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 15 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 61227/223293 objects degraded (27.420%); 14463/223293 objects misplaced (6.477%); 651 KiB/s, 7 objects/s recovering 2023-10-09T04:59:03.035 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:59:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:59:02] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.43.0" 2023-10-09T04:59:03.036 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:03 smithi029 ceph-mon[223858]: pgmap v1812: 109 pgs: 12 peering, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 15 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 61227/223293 objects degraded (27.420%); 14463/223293 objects misplaced (6.477%); 651 KiB/s, 7 objects/s recovering 2023-10-09T04:59:03.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:03.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:03.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:03.182 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:03 smithi177 ceph-mon[195797]: pgmap v1812: 109 pgs: 12 peering, 3 remapped+peering, 1 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 15 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 61227/223293 objects degraded (27.420%); 14463/223293 objects misplaced (6.477%); 651 KiB/s, 7 objects/s recovering 2023-10-09T04:59:03.183 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:03.244 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:03.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:03.286 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:03.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:03.408 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:03.512 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6, 4] out_osds: [0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T04:59:03.512 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:59:03.512 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_9 2023-10-09T04:59:03.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T04:59:03.554 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:03.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:03.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:03.690 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:03.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:03.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:03.947 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:03.994 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:04.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:04.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:04.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:04.191 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:04.473 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:04.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:04.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:04.597 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:04.609 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:04.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:04.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:04.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:04.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:04.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:04.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:59:04.877 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:04.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:04.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:04.982 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:05.020 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:05 smithi029 ceph-mon[216903]: pgmap v1813: 109 pgs: 1 peering, 1 remapped+peering, 2 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 17 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 1.1 KiB/s wr, 28 op/s; 66362/223305 objects degraded (29.718%); 14463/223305 objects misplaced (6.477%); 11 MiB/s, 6 keys/s, 208 objects/s recovering 2023-10-09T04:59:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:05 smithi029 ceph-mon[223858]: pgmap v1813: 109 pgs: 1 peering, 1 remapped+peering, 2 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 17 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 1.1 KiB/s wr, 28 op/s; 66362/223305 objects degraded (29.718%); 14463/223305 objects misplaced (6.477%); 11 MiB/s, 6 keys/s, 208 objects/s recovering 2023-10-09T04:59:05.239 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:05.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:05.348 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:05.380 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:05 smithi177 ceph-mon[195797]: pgmap v1813: 109 pgs: 1 peering, 1 remapped+peering, 2 activating, 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 17 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 1.1 KiB/s wr, 28 op/s; 66362/223305 objects degraded (29.718%); 14463/223305 objects misplaced (6.477%); 11 MiB/s, 6 keys/s, 208 objects/s recovering 2023-10-09T04:59:05.594 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:05.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:05.698 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:05.727 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:05.777 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:05.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:05.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:05.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:05.909 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:05.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:05.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:05.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:06.059 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:06.069 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T04:59:06.086 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:06.258 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:06.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:06 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1519722331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:59:06.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:06 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:59:06.260 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 66362/223305 objects degraded (29.718%), 32 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:06.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:06.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:06.362 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:06.394 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:06.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:06 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1519722331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:59:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:06 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:59:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 66362/223305 objects degraded (29.718%), 32 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:06 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1519722331' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:59:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:06 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T04:59:06.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 66362/223305 objects degraded (29.718%), 32 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:06.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:06.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:06.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:06.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:06.608 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:06.715 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:06.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:06.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:06.849 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:06.936 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:59:06.936 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:59:06.936 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T04:59:06.936 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:59:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: pgmap v1814: 109 pgs: 1 remapped+peering, 1 activating, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 895 B/s wr, 40 op/s; 66175/222888 objects degraded (29.690%); 14334/222888 objects misplaced (6.431%); 11 MiB/s, 4 keys/s, 191 objects/s recovering 2023-10-09T04:59:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:59:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T04:59:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:07 smithi177 ceph-mon[195797]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.383 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:07.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: pgmap v1814: 109 pgs: 1 remapped+peering, 1 activating, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 895 B/s wr, 40 op/s; 66175/222888 objects degraded (29.690%); 14334/222888 objects misplaced (6.431%); 11 MiB/s, 4 keys/s, 191 objects/s recovering 2023-10-09T04:59:07.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:07.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:59:07.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T04:59:07.385 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[216903]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: pgmap v1814: 109 pgs: 1 remapped+peering, 1 activating, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 895 B/s wr, 40 op/s; 66175/222888 objects degraded (29.690%); 14334/222888 objects misplaced (6.431%); 11 MiB/s, 4 keys/s, 191 objects/s recovering 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: osdmap e541: 8 total, 8 up, 7 in 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.386 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.387 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.387 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:07 smithi029 ceph-mon[223858]: failed to encode map e541 with expected crc 2023-10-09T04:59:07.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:07.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:07.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:07.522 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:07.573 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:07.686 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:07.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:07.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:07.794 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:07.829 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:07.845 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T04:59:07.881 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:07.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:07.986 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:08.024 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:08.077 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:08.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:08.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:08.201 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:59:08.204 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T04:59:08.218 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:08.265 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:09.122 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:09.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:09.227 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:09.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:09 smithi177 ceph-mon[195797]: pgmap v1816: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 834 B/s wr, 39 op/s; 70883/222855 objects degraded (31.807%); 13863/222855 objects misplaced (6.221%); 12 MiB/s, 4 keys/s, 206 objects/s recovering 2023-10-09T04:59:09.256 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:09 smithi177 ceph-mon[195797]: from='client.37257 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:59:09.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:09.382 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:09.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:09.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:09.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:09 smithi029 ceph-mon[216903]: pgmap v1816: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 834 B/s wr, 39 op/s; 70883/222855 objects degraded (31.807%); 13863/222855 objects misplaced (6.221%); 12 MiB/s, 4 keys/s, 206 objects/s recovering 2023-10-09T04:59:09.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:09 smithi029 ceph-mon[216903]: from='client.37257 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:59:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:09 smithi029 ceph-mon[223858]: pgmap v1816: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 834 B/s wr, 39 op/s; 70883/222855 objects degraded (31.807%); 13863/222855 objects misplaced (6.221%); 12 MiB/s, 4 keys/s, 206 objects/s recovering 2023-10-09T04:59:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:09 smithi029 ceph-mon[223858]: from='client.37257 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T04:59:09.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:09.519 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:09.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T04:59:09.601 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:09.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:09.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:09.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:09.740 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:09.848 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1816,"stamp":"2023-10-09T04:59:07.566592+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4867299256,"num_objects":74285,"num_object_clones":0,"num_object_copies":222855,"num_objects_missing_on_primary":27093,"num_objects_missing":27093,"num_objects_degraded":70883,"num_objects_misplaced":13863,"num_objects_unfound":0,"num_objects_dirty":74285,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":81257,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32162,"num_bytes_recovered":2069146706,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122676,"ondisk_log_size":122676,"up":327,"acting":306,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":448,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23296688,"kb_used_data":16280440,"kb_used_omap":26,"kb_used_meta":7016165,"kb_avail":632919376,"statfs":{"total":671965249536,"available":648109441024,"internally_reserved":0,"allocated":16671170560,"data_stored":16659105648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26724,"internal_metadata":7184553884},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,0,0,0,1,3,2,2,12,2],"upper_bound":16384},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-21495752,"num_objects":-324,"num_object_clones":0,"num_object_copies":-972,"num_objects_missing_on_primary":4138,"num_objects_missing":4138,"num_objects_degraded":9434,"num_objects_misplaced":3480,"num_objects_unfound":0,"num_objects_dirty":-324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":335,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1772,"num_bytes_recovered":104211932,"num_keys_recovered":40,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.588137"},"pg_stats":[{"pgid":"14.17","version":"540'5091","reported_seq":13744,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.509261+0000","last_change":"2023-10-09T04:58:58.118094+0000","last_active":"2023-10-09T04:59:04.509261+0000","last_peered":"2023-10-09T04:59:04.509261+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:58:58.106351+0000","last_became_peered":"2023-10-09T04:58:58.106351+0000","last_unstale":"2023-10-09T04:59:04.509261+0000","last_undegraded":"2023-10-09T04:58:58.051523+0000","last_fullsized":"2023-10-09T04:58:58.016329+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153190400,"num_objects":2338,"num_object_clones":0,"num_object_copies":7014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":524,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2550,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2116,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,4,5","objects":6},{"shards":"2,4,5","objects":375},{"shards":"4,5","objects":1957}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.985655+0000","last_change":"2023-10-09T04:58:57.067059+0000","last_active":"2023-10-09T04:58:58.985655+0000","last_peered":"2023-10-09T04:58:58.985655+0000","last_clean":"2023-10-09T04:58:58.985655+0000","last_became_active":"2023-10-09T04:58:57.066168+0000","last_became_peered":"2023-10-09T04:58:57.066168+0000","last_unstale":"2023-10-09T04:58:58.985655+0000","last_undegraded":"2023-10-09T04:58:58.985655+0000","last_fullsized":"2023-10-09T04:58:58.985655+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:23.404016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":153,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.992942+0000","last_change":"2023-10-09T04:58:57.034175+0000","last_active":"2023-10-09T04:58:57.992942+0000","last_peered":"2023-10-09T04:58:57.992942+0000","last_clean":"2023-10-09T04:58:57.992942+0000","last_became_active":"2023-10-09T04:58:57.033913+0000","last_became_peered":"2023-10-09T04:58:57.033913+0000","last_unstale":"2023-10-09T04:58:57.992942+0000","last_undegraded":"2023-10-09T04:58:57.992942+0000","last_fullsized":"2023-10-09T04:58:57.992942+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:50.157531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"540'5083","reported_seq":6608,"reported_epoch":541,"state":"active+recovering","last_fresh":"2023-10-09T04:59:07.267914+0000","last_change":"2023-10-09T04:58:57.362380+0000","last_active":"2023-10-09T04:59:07.267914+0000","last_peered":"2023-10-09T04:59:07.267914+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:58:54.005733+0000","last_became_peered":"2023-10-09T04:58:54.005733+0000","last_unstale":"2023-10-09T04:59:07.267914+0000","last_undegraded":"2023-10-09T04:59:07.267914+0000","last_fullsized":"2023-10-09T04:59:07.267914+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151814144,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":2011,"num_objects_missing":2011,"num_objects_degraded":0,"num_objects_misplaced":3647,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2544,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":351,"num_bytes_recovered":22970368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4,7","objects":349},{"shards":"2,4,7","objects":1649},{"shards":"3,6,7","objects":319}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.995758+0000","last_change":"2023-10-09T04:58:07.889785+0000","last_active":"2023-10-09T04:58:57.995758+0000","last_peered":"2023-10-09T04:58:57.995758+0000","last_clean":"2023-10-09T04:58:57.995758+0000","last_became_active":"2023-10-09T04:58:07.889365+0000","last_became_peered":"2023-10-09T04:58:07.889365+0000","last_unstale":"2023-10-09T04:58:57.995758+0000","last_undegraded":"2023-10-09T04:58:57.995758+0000","last_fullsized":"2023-10-09T04:58:57.995758+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:14.043179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":143,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.978917+0000","last_change":"2023-10-09T04:58:57.071131+0000","last_active":"2023-10-09T04:58:57.978917+0000","last_peered":"2023-10-09T04:58:57.978917+0000","last_clean":"2023-10-09T04:58:57.978917+0000","last_became_active":"2023-10-09T04:58:57.070781+0000","last_became_peered":"2023-10-09T04:58:57.070781+0000","last_unstale":"2023-10-09T04:58:57.978917+0000","last_undegraded":"2023-10-09T04:58:57.978917+0000","last_fullsized":"2023-10-09T04:58:57.978917+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:36:26.261662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.15","version":"540'5087","reported_seq":12230,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.649269+0000","last_change":"2023-10-09T04:58:58.192776+0000","last_active":"2023-10-09T04:59:04.649269+0000","last_peered":"2023-10-09T04:59:04.649269+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T04:58:58.162421+0000","last_became_peered":"2023-10-09T04:58:58.162421+0000","last_unstale":"2023-10-09T04:59:04.649269+0000","last_undegraded":"2023-10-09T04:58:58.104923+0000","last_fullsized":"2023-10-09T04:58:57.987919+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151650304,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":93,"num_objects_misplaced":1830,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2547,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1738,"num_bytes_recovered":108724224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":899},{"shards":"4,5,7","objects":929},{"shards":"5,7","objects":486}],"blocked_by":[],"up_primary":4,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":130,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.990408+0000","last_change":"2023-10-09T04:56:55.859522+0000","last_active":"2023-10-09T04:58:57.990408+0000","last_peered":"2023-10-09T04:58:57.990408+0000","last_clean":"2023-10-09T04:58:57.990408+0000","last_became_active":"2023-10-09T04:56:55.859254+0000","last_became_peered":"2023-10-09T04:56:55.859254+0000","last_unstale":"2023-10-09T04:58:57.990408+0000","last_undegraded":"2023-10-09T04:58:57.990408+0000","last_fullsized":"2023-10-09T04:58:57.990408+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:53:30.229851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":37,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.988478+0000","last_change":"2023-10-09T04:58:07.895975+0000","last_active":"2023-10-09T04:58:58.988478+0000","last_peered":"2023-10-09T04:58:58.988478+0000","last_clean":"2023-10-09T04:58:58.988478+0000","last_became_active":"2023-10-09T04:58:07.893663+0000","last_became_peered":"2023-10-09T04:58:07.893663+0000","last_unstale":"2023-10-09T04:58:58.988478+0000","last_undegraded":"2023-10-09T04:58:58.988478+0000","last_fullsized":"2023-10-09T04:58:58.988478+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:35.053280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":582,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.052571+0000","last_change":"2023-10-09T04:58:07.879586+0000","last_active":"2023-10-09T04:58:58.052571+0000","last_peered":"2023-10-09T04:58:58.052571+0000","last_clean":"2023-10-09T04:58:58.052571+0000","last_became_active":"2023-10-09T04:58:07.879041+0000","last_became_peered":"2023-10-09T04:58:07.879041+0000","last_unstale":"2023-10-09T04:58:58.052571+0000","last_undegraded":"2023-10-09T04:58:58.052571+0000","last_fullsized":"2023-10-09T04:58:58.052571+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:12.975185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":140,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.987743+0000","last_change":"2023-10-09T04:58:07.920414+0000","last_active":"2023-10-09T04:58:58.987743+0000","last_peered":"2023-10-09T04:58:58.987743+0000","last_clean":"2023-10-09T04:58:58.987743+0000","last_became_active":"2023-10-09T04:58:07.920149+0000","last_became_peered":"2023-10-09T04:58:07.920149+0000","last_unstale":"2023-10-09T04:58:58.987743+0000","last_undegraded":"2023-10-09T04:58:58.987743+0000","last_fullsized":"2023-10-09T04:58:58.987743+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:34.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":530,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.988031+0000","last_change":"2023-10-09T04:55:20.004951+0000","last_active":"2023-10-09T04:58:57.988031+0000","last_peered":"2023-10-09T04:58:57.988031+0000","last_clean":"2023-10-09T04:58:57.988031+0000","last_became_active":"2023-10-09T04:55:20.004659+0000","last_became_peered":"2023-10-09T04:55:20.004659+0000","last_unstale":"2023-10-09T04:58:57.988031+0000","last_undegraded":"2023-10-09T04:58:57.988031+0000","last_fullsized":"2023-10-09T04:58:57.988031+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:19:48.952780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":130,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986595+0000","last_change":"2023-10-09T04:58:07.895159+0000","last_active":"2023-10-09T04:58:58.986595+0000","last_peered":"2023-10-09T04:58:58.986595+0000","last_clean":"2023-10-09T04:58:58.986595+0000","last_became_active":"2023-10-09T04:58:07.892354+0000","last_became_peered":"2023-10-09T04:58:07.892354+0000","last_unstale":"2023-10-09T04:58:58.986595+0000","last_undegraded":"2023-10-09T04:58:58.986595+0000","last_fullsized":"2023-10-09T04:58:58.986595+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:02.609962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"540'5224","reported_seq":6070,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.582561+0000","last_change":"2023-10-09T04:58:59.822211+0000","last_active":"2023-10-09T04:59:04.582561+0000","last_peered":"2023-10-09T04:59:04.582561+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:58:59.753906+0000","last_became_peered":"2023-10-09T04:58:59.753906+0000","last_unstale":"2023-10-09T04:59:04.582561+0000","last_undegraded":"2023-10-09T04:58:56.542702+0000","last_fullsized":"2023-10-09T04:58:54.779150+0000","mapping_epoch":535,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3688,"log_dups_size":0,"ondisk_log_size":3688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155910144,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":2353,"num_objects_missing":2353,"num_objects_degraded":4359,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2615,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":7864320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2","objects":26},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1922}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":159,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:57.992447+0000","last_change":"2023-10-09T04:58:56.994920+0000","last_active":"2023-10-09T04:58:57.992447+0000","last_peered":"2023-10-09T04:58:57.992447+0000","last_clean":"2023-10-09T04:58:52.940465+0000","last_became_active":"2023-10-09T04:58:56.994731+0000","last_became_peered":"2023-10-09T04:58:56.994731+0000","last_unstale":"2023-10-09T04:58:57.992447+0000","last_undegraded":"2023-10-09T04:58:56.969401+0000","last_fullsized":"2023-10-09T04:58:57.992447+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:35:05.501001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":720,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":106,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.972365+0000","last_change":"2023-10-09T04:58:07.886648+0000","last_active":"2023-10-09T04:58:57.972365+0000","last_peered":"2023-10-09T04:58:57.972365+0000","last_clean":"2023-10-09T04:58:57.972365+0000","last_became_active":"2023-10-09T04:58:07.886050+0000","last_became_peered":"2023-10-09T04:58:07.886050+0000","last_unstale":"2023-10-09T04:58:57.972365+0000","last_undegraded":"2023-10-09T04:58:57.972365+0000","last_fullsized":"2023-10-09T04:58:57.972365+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:08.968095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"540'5084","reported_seq":16745,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.243326+0000","last_change":"2023-10-09T04:58:58.103577+0000","last_active":"2023-10-09T04:59:04.243326+0000","last_peered":"2023-10-09T04:59:04.243326+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:58:58.100530+0000","last_became_peered":"2023-10-09T04:58:58.100530+0000","last_unstale":"2023-10-09T04:59:04.243326+0000","last_undegraded":"2023-10-09T04:58:57.996576+0000","last_fullsized":"2023-10-09T04:58:57.986120+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5084,"log_dups_size":0,"ondisk_log_size":5084,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153583616,"num_objects":2344,"num_object_clones":0,"num_object_copies":7032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":78,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2546,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2872,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":2344}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":11,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.986583+0000","last_change":"2023-10-09T04:58:57.210260+0000","last_active":"2023-10-09T04:58:58.986583+0000","last_peered":"2023-10-09T04:58:58.986583+0000","last_clean":"2023-10-09T04:54:31.740314+0000","last_became_active":"2023-10-09T04:58:57.210019+0000","last_became_peered":"2023-10-09T04:58:57.210019+0000","last_unstale":"2023-10-09T04:58:58.986583+0000","last_undegraded":"2023-10-09T04:58:57.193095+0000","last_fullsized":"2023-10-09T04:58:58.986583+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:53.045092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.984898+0000","last_change":"2023-10-09T04:58:57.063972+0000","last_active":"2023-10-09T04:58:58.984898+0000","last_peered":"2023-10-09T04:58:58.984898+0000","last_clean":"2023-10-09T04:58:58.984898+0000","last_became_active":"2023-10-09T04:58:57.063636+0000","last_became_peered":"2023-10-09T04:58:57.063636+0000","last_unstale":"2023-10-09T04:58:58.984898+0000","last_undegraded":"2023-10-09T04:58:58.984898+0000","last_fullsized":"2023-10-09T04:58:58.984898+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:19.332300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":11,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.985367+0000","last_change":"2023-10-09T04:58:57.076114+0000","last_active":"2023-10-09T04:58:58.985367+0000","last_peered":"2023-10-09T04:58:58.985367+0000","last_clean":"2023-10-09T04:58:52.925346+0000","last_became_active":"2023-10-09T04:58:57.075811+0000","last_became_peered":"2023-10-09T04:58:57.075811+0000","last_unstale":"2023-10-09T04:58:58.985367+0000","last_undegraded":"2023-10-09T04:58:56.973358+0000","last_fullsized":"2023-10-09T04:58:58.985367+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:45.388764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":2,"num_objects_missing":2,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":105,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.992305+0000","last_change":"2023-10-09T04:55:19.974453+0000","last_active":"2023-10-09T04:58:57.992305+0000","last_peered":"2023-10-09T04:58:57.992305+0000","last_clean":"2023-10-09T04:58:57.992305+0000","last_became_active":"2023-10-09T04:55:19.973871+0000","last_became_peered":"2023-10-09T04:55:19.973871+0000","last_unstale":"2023-10-09T04:58:57.992305+0000","last_undegraded":"2023-10-09T04:58:57.992305+0000","last_fullsized":"2023-10-09T04:58:57.992305+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:47.239847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":81,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.971969+0000","last_change":"2023-10-09T04:56:53.922079+0000","last_active":"2023-10-09T04:58:57.971969+0000","last_peered":"2023-10-09T04:58:57.971969+0000","last_clean":"2023-10-09T04:58:57.971969+0000","last_became_active":"2023-10-09T04:56:53.921311+0000","last_became_peered":"2023-10-09T04:56:53.921311+0000","last_unstale":"2023-10-09T04:58:57.971969+0000","last_undegraded":"2023-10-09T04:58:57.971969+0000","last_fullsized":"2023-10-09T04:58:57.971969+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:20.368978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":539,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.117394+0000","last_change":"2023-10-09T04:54:25.766303+0000","last_active":"2023-10-09T04:58:58.117394+0000","last_peered":"2023-10-09T04:58:58.117394+0000","last_clean":"2023-10-09T04:58:58.117394+0000","last_became_active":"2023-10-09T04:54:25.765766+0000","last_became_peered":"2023-10-09T04:54:25.765766+0000","last_unstale":"2023-10-09T04:58:58.117394+0000","last_undegraded":"2023-10-09T04:58:58.117394+0000","last_fullsized":"2023-10-09T04:58:58.117394+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:25:48.044596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.985030+0000","last_change":"2023-10-09T04:58:56.993529+0000","last_active":"2023-10-09T04:58:58.985030+0000","last_peered":"2023-10-09T04:58:58.985030+0000","last_clean":"2023-10-09T04:58:58.985030+0000","last_became_active":"2023-10-09T04:58:56.993185+0000","last_became_peered":"2023-10-09T04:58:56.993185+0000","last_unstale":"2023-10-09T04:58:58.985030+0000","last_undegraded":"2023-10-09T04:58:58.985030+0000","last_fullsized":"2023-10-09T04:58:58.985030+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:36.864637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.991556+0000","last_change":"2023-10-09T04:58:07.882093+0000","last_active":"2023-10-09T04:58:57.991556+0000","last_peered":"2023-10-09T04:58:57.991556+0000","last_clean":"2023-10-09T04:58:57.991556+0000","last_became_active":"2023-10-09T04:58:07.881272+0000","last_became_peered":"2023-10-09T04:58:07.881272+0000","last_unstale":"2023-10-09T04:58:57.991556+0000","last_undegraded":"2023-10-09T04:58:57.991556+0000","last_fullsized":"2023-10-09T04:58:57.991556+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:09.422315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":135,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986357+0000","last_change":"2023-10-09T04:58:07.908453+0000","last_active":"2023-10-09T04:58:58.986357+0000","last_peered":"2023-10-09T04:58:58.986357+0000","last_clean":"2023-10-09T04:58:58.986357+0000","last_became_active":"2023-10-09T04:58:07.904674+0000","last_became_peered":"2023-10-09T04:58:07.904674+0000","last_unstale":"2023-10-09T04:58:58.986357+0000","last_undegraded":"2023-10-09T04:58:58.986357+0000","last_fullsized":"2023-10-09T04:58:58.986357+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:18.595337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.976269+0000","last_change":"2023-10-09T04:58:07.930062+0000","last_active":"2023-10-09T04:58:57.976269+0000","last_peered":"2023-10-09T04:58:57.976269+0000","last_clean":"2023-10-09T04:58:57.976269+0000","last_became_active":"2023-10-09T04:58:07.929708+0000","last_became_peered":"2023-10-09T04:58:07.929708+0000","last_unstale":"2023-10-09T04:58:57.976269+0000","last_undegraded":"2023-10-09T04:58:57.976269+0000","last_fullsized":"2023-10-09T04:58:57.976269+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:54.726957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.2","version":"540'4999","reported_seq":8005,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:59.028952+0000","last_change":"2023-10-09T04:58:53.640706+0000","last_active":"2023-10-09T04:58:59.028952+0000","last_peered":"2023-10-09T04:58:59.028952+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T04:58:52.534783+0000","last_became_peered":"2023-10-09T04:58:52.534783+0000","last_unstale":"2023-10-09T04:58:59.028952+0000","last_undegraded":"2023-10-09T04:58:51.928205+0000","last_fullsized":"2023-10-09T04:58:49.920402+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151715840,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":338,"num_objects_missing":338,"num_objects_degraded":2281,"num_objects_misplaced":1572,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2500,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":406,"num_bytes_recovered":26574848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[7,5],"avail_no_missing":["4"],"object_location_counts":[{"shards":"3,4,5","objects":224},{"shards":"3,4,5,6","objects":46},{"shards":"3,4,5,6,7","objects":1},{"shards":"4,5,6,7","objects":1},{"shards":"4,5,7","objects":1638},{"shards":"5,7","objects":405}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":11,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.986071+0000","last_change":"2023-10-09T04:58:57.069590+0000","last_active":"2023-10-09T04:58:58.986071+0000","last_peered":"2023-10-09T04:58:58.986071+0000","last_clean":"2023-10-09T04:58:52.920896+0000","last_became_active":"2023-10-09T04:58:57.069252+0000","last_became_peered":"2023-10-09T04:58:57.069252+0000","last_unstale":"2023-10-09T04:58:58.986071+0000","last_undegraded":"2023-10-09T04:58:56.975925+0000","last_fullsized":"2023-10-09T04:58:58.986071+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:17.585270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":569,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.130509+0000","last_change":"2023-10-09T04:58:56.991571+0000","last_active":"2023-10-09T04:58:58.130509+0000","last_peered":"2023-10-09T04:58:58.130509+0000","last_clean":"2023-10-09T04:58:58.130509+0000","last_became_active":"2023-10-09T04:58:56.990879+0000","last_became_peered":"2023-10-09T04:58:56.990879+0000","last_unstale":"2023-10-09T04:58:58.130509+0000","last_undegraded":"2023-10-09T04:58:58.130509+0000","last_fullsized":"2023-10-09T04:58:58.130509+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:38.496827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"540'5073","reported_seq":5998,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:03.979043+0000","last_change":"2023-10-09T04:58:59.237960+0000","last_active":"2023-10-09T04:59:03.979043+0000","last_peered":"2023-10-09T04:59:03.979043+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:58:56.607803+0000","last_became_peered":"2023-10-09T04:58:56.607803+0000","last_unstale":"2023-10-09T04:59:03.979043+0000","last_undegraded":"2023-10-09T04:58:52.524319+0000","last_fullsized":"2023-10-09T04:58:49.922200+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5073,"log_dups_size":0,"ondisk_log_size":5073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":1875,"num_objects_missing":1875,"num_objects_degraded":4252,"num_objects_misplaced":479,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":6750208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[3,6],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":7},{"shards":"2,3,5,7","objects":368},{"shards":"2,5,7","objects":1875},{"shards":"3,5,6","objects":45},{"shards":"3,6","objects":28}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":11,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.985513+0000","last_change":"2023-10-09T04:58:57.073209+0000","last_active":"2023-10-09T04:58:58.985513+0000","last_peered":"2023-10-09T04:58:58.985513+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T04:58:57.071526+0000","last_became_peered":"2023-10-09T04:58:57.071526+0000","last_unstale":"2023-10-09T04:58:58.985513+0000","last_undegraded":"2023-10-09T04:58:56.975190+0000","last_fullsized":"2023-10-09T04:58:58.985513+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:53:23.533644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.984922+0000","last_change":"2023-10-09T04:58:57.563521+0000","last_active":"2023-10-09T04:58:58.984922+0000","last_peered":"2023-10-09T04:58:58.984922+0000","last_clean":"2023-10-09T04:58:58.984922+0000","last_became_active":"2023-10-09T04:58:57.563202+0000","last_became_peered":"2023-10-09T04:58:57.563202+0000","last_unstale":"2023-10-09T04:58:58.984922+0000","last_undegraded":"2023-10-09T04:58:58.984922+0000","last_fullsized":"2023-10-09T04:58:58.984922+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:47:49.413824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"540'5065","reported_seq":5348,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.465627+0000","last_change":"2023-10-09T04:58:58.143125+0000","last_active":"2023-10-09T04:59:04.465627+0000","last_peered":"2023-10-09T04:59:04.465627+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:58.136774+0000","last_became_peered":"2023-10-09T04:58:58.136774+0000","last_unstale":"2023-10-09T04:59:04.465627+0000","last_undegraded":"2023-10-09T04:58:58.128621+0000","last_fullsized":"2023-10-09T04:58:57.986571+0000","mapping_epoch":538,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4527,"log_dups_size":0,"ondisk_log_size":4527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152666112,"num_objects":2329,"num_object_clones":0,"num_object_copies":6987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2529,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2533,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":1671168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":2264},{"shards":"2,5,6","objects":56},{"shards":"4,5","objects":7}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":186,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.117490+0000","last_change":"2023-10-09T04:58:08.028736+0000","last_active":"2023-10-09T04:58:58.117490+0000","last_peered":"2023-10-09T04:58:58.117490+0000","last_clean":"2023-10-09T04:58:58.117490+0000","last_became_active":"2023-10-09T04:58:07.937180+0000","last_became_peered":"2023-10-09T04:58:07.937180+0000","last_unstale":"2023-10-09T04:58:58.117490+0000","last_undegraded":"2023-10-09T04:58:58.117490+0000","last_fullsized":"2023-10-09T04:58:58.117490+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:43:23.362583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":37,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986716+0000","last_change":"2023-10-09T04:58:07.872446+0000","last_active":"2023-10-09T04:58:58.986716+0000","last_peered":"2023-10-09T04:58:58.986716+0000","last_clean":"2023-10-09T04:58:58.986716+0000","last_became_active":"2023-10-09T04:58:07.872159+0000","last_became_peered":"2023-10-09T04:58:07.872159+0000","last_unstale":"2023-10-09T04:58:58.986716+0000","last_undegraded":"2023-10-09T04:58:58.986716+0000","last_fullsized":"2023-10-09T04:58:58.986716+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:21:25.849479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"540'5081","reported_seq":6405,"reported_epoch":540,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:05.144793+0000","last_change":"2023-10-09T04:58:53.420726+0000","last_active":"2023-10-09T04:59:05.144793+0000","last_peered":"2023-10-09T04:59:05.144793+0000","last_clean":"2023-10-09T04:53:31.951851+0000","last_became_active":"2023-10-09T04:58:53.363446+0000","last_became_peered":"2023-10-09T04:58:53.363446+0000","last_unstale":"2023-10-09T04:59:05.144793+0000","last_undegraded":"2023-10-09T04:58:50.593229+0000","last_fullsized":"2023-10-09T04:58:48.897208+0000","mapping_epoch":514,"log_start":"445'1388","ondisk_log_start":"445'1388","created":440,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3693,"log_dups_size":0,"ondisk_log_size":3693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584768,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":2167,"num_objects_missing":2167,"num_objects_degraded":1733,"num_objects_misplaced":2507,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2546,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":304,"num_bytes_recovered":19890176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,2","objects":168},{"shards":"1,2,5","objects":1},{"shards":"2,3,5","objects":362},{"shards":"2,3,6","objects":49},{"shards":"2,5","objects":1733}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":600,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.986225+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T04:58:57.986225+0000","last_peered":"2023-10-09T04:58:57.986225+0000","last_clean":"2023-10-09T04:58:57.986225+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T04:58:57.986225+0000","last_undegraded":"2023-10-09T04:58:57.986225+0000","last_fullsized":"2023-10-09T04:58:57.986225+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":118,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:59:02.687690+0000","last_change":"2023-10-09T04:59:02.687690+0000","last_active":"2023-10-09T04:59:02.687690+0000","last_peered":"2023-10-09T04:59:02.687690+0000","last_clean":"2023-10-09T04:59:02.687690+0000","last_became_active":"2023-10-09T04:59:02.687418+0000","last_became_peered":"2023-10-09T04:59:02.687418+0000","last_unstale":"2023-10-09T04:59:02.687690+0000","last_undegraded":"2023-10-09T04:59:02.687690+0000","last_fullsized":"2023-10-09T04:59:02.687690+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:21.317269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1029,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986528+0000","last_change":"2023-10-09T04:58:38.073050+0000","last_active":"2023-10-09T04:58:58.986528+0000","last_peered":"2023-10-09T04:58:58.986528+0000","last_clean":"2023-10-09T04:58:58.986528+0000","last_became_active":"2023-10-09T04:58:38.072478+0000","last_became_peered":"2023-10-09T04:58:38.072478+0000","last_unstale":"2023-10-09T04:58:58.986528+0000","last_undegraded":"2023-10-09T04:58:58.986528+0000","last_fullsized":"2023-10-09T04:58:58.986528+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"531'4913","reported_seq":5674,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:57.994360+0000","last_change":"2023-10-09T04:58:49.063597+0000","last_active":"2023-10-09T04:58:57.994360+0000","last_peered":"2023-10-09T04:58:57.994360+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:58:49.038106+0000","last_became_peered":"2023-10-09T04:58:49.038106+0000","last_unstale":"2023-10-09T04:58:57.994360+0000","last_undegraded":"2023-10-09T04:58:48.983642+0000","last_fullsized":"2023-10-09T04:58:57.994360+0000","mapping_epoch":510,"log_start":"444'386","ondisk_log_start":"444'386","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4527,"log_dups_size":0,"ondisk_log_size":4527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148078592,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4130,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2456,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":6488064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"1,2,3","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":1827}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":105,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:59:02.687453+0000","last_change":"2023-10-09T04:59:02.687658+0000","last_active":"2023-10-09T04:59:02.687453+0000","last_peered":"2023-10-09T04:59:02.687453+0000","last_clean":"2023-10-09T04:58:52.572314+0000","last_became_active":"2023-10-09T04:59:02.687355+0000","last_became_peered":"2023-10-09T04:59:02.687355+0000","last_unstale":"2023-10-09T04:59:02.687453+0000","last_undegraded":"2023-10-09T04:59:01.713929+0000","last_fullsized":"2023-10-09T04:59:02.687453+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:03.680594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":["3","1","7"],"object_location_counts":[{"shards":"1,3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986738+0000","last_change":"2023-10-09T04:58:57.210297+0000","last_active":"2023-10-09T04:58:58.986738+0000","last_peered":"2023-10-09T04:58:58.986738+0000","last_clean":"2023-10-09T04:58:58.986738+0000","last_became_active":"2023-10-09T04:58:57.209978+0000","last_became_peered":"2023-10-09T04:58:57.209978+0000","last_unstale":"2023-10-09T04:58:58.986738+0000","last_undegraded":"2023-10-09T04:58:58.986738+0000","last_fullsized":"2023-10-09T04:58:58.986738+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:58.875418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1044,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986300+0000","last_change":"2023-10-09T04:58:08.870256+0000","last_active":"2023-10-09T04:58:58.986300+0000","last_peered":"2023-10-09T04:58:58.986300+0000","last_clean":"2023-10-09T04:58:58.986300+0000","last_became_active":"2023-10-09T04:58:07.937068+0000","last_became_peered":"2023-10-09T04:58:07.937068+0000","last_unstale":"2023-10-09T04:58:58.986300+0000","last_undegraded":"2023-10-09T04:58:58.986300+0000","last_fullsized":"2023-10-09T04:58:58.986300+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"540'5087","reported_seq":13743,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.475651+0000","last_change":"2023-10-09T04:58:58.148616+0000","last_active":"2023-10-09T04:59:04.475651+0000","last_peered":"2023-10-09T04:59:04.475651+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:58:58.108418+0000","last_became_peered":"2023-10-09T04:58:58.108418+0000","last_unstale":"2023-10-09T04:59:04.475651+0000","last_undegraded":"2023-10-09T04:58:58.015775+0000","last_fullsized":"2023-10-09T04:58:57.983109+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153452544,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":532,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2545,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2115,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,4,5","objects":1},{"shards":"2,4,5","objects":360},{"shards":"4,5","objects":1980}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":47,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986698+0000","last_change":"2023-10-09T04:58:57.060419+0000","last_active":"2023-10-09T04:58:58.986698+0000","last_peered":"2023-10-09T04:58:58.986698+0000","last_clean":"2023-10-09T04:58:58.986698+0000","last_became_active":"2023-10-09T04:58:57.059935+0000","last_became_peered":"2023-10-09T04:58:57.059935+0000","last_unstale":"2023-10-09T04:58:58.986698+0000","last_undegraded":"2023-10-09T04:58:58.986698+0000","last_fullsized":"2023-10-09T04:58:58.986698+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:44.313186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":345,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.979868+0000","last_change":"2023-10-09T04:58:07.882492+0000","last_active":"2023-10-09T04:58:57.979868+0000","last_peered":"2023-10-09T04:58:57.979868+0000","last_clean":"2023-10-09T04:58:57.979868+0000","last_became_active":"2023-10-09T04:58:07.882161+0000","last_became_peered":"2023-10-09T04:58:07.882161+0000","last_unstale":"2023-10-09T04:58:57.979868+0000","last_undegraded":"2023-10-09T04:58:57.979868+0000","last_fullsized":"2023-10-09T04:58:57.979868+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:58:06.813717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"540'5086","reported_seq":5909,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:59:04.980482+0000","last_change":"2023-10-09T04:58:57.200214+0000","last_active":"2023-10-09T04:59:04.980482+0000","last_peered":"2023-10-09T04:59:04.980482+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:58:55.515597+0000","last_became_peered":"2023-10-09T04:58:55.515597+0000","last_unstale":"2023-10-09T04:59:04.980482+0000","last_undegraded":"2023-10-09T04:58:51.011222+0000","last_fullsized":"2023-10-09T04:59:04.980482+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151617536,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":2306,"num_objects_missing":2306,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2546,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":7634944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,4,7","objects":1920},{"shards":"3,6,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.984749+0000","last_change":"2023-10-09T04:58:57.557023+0000","last_active":"2023-10-09T04:58:58.984749+0000","last_peered":"2023-10-09T04:58:58.984749+0000","last_clean":"2023-10-09T04:58:58.984749+0000","last_became_active":"2023-10-09T04:58:57.556703+0000","last_became_peered":"2023-10-09T04:58:57.556703+0000","last_unstale":"2023-10-09T04:58:58.984749+0000","last_undegraded":"2023-10-09T04:58:58.984749+0000","last_fullsized":"2023-10-09T04:58:58.984749+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:44.046576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.985038+0000","last_change":"2023-10-09T04:58:57.071995+0000","last_active":"2023-10-09T04:58:58.985038+0000","last_peered":"2023-10-09T04:58:58.985038+0000","last_clean":"2023-10-09T04:58:58.985038+0000","last_became_active":"2023-10-09T04:58:57.070855+0000","last_became_peered":"2023-10-09T04:58:57.070855+0000","last_unstale":"2023-10-09T04:58:58.985038+0000","last_undegraded":"2023-10-09T04:58:58.985038+0000","last_fullsized":"2023-10-09T04:58:58.985038+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:29.469962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1038,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.988825+0000","last_change":"2023-10-09T04:58:26.884306+0000","last_active":"2023-10-09T04:58:58.988825+0000","last_peered":"2023-10-09T04:58:58.988825+0000","last_clean":"2023-10-09T04:58:58.988825+0000","last_became_active":"2023-10-09T04:58:26.883876+0000","last_became_peered":"2023-10-09T04:58:26.883876+0000","last_unstale":"2023-10-09T04:58:58.988825+0000","last_undegraded":"2023-10-09T04:58:58.988825+0000","last_fullsized":"2023-10-09T04:58:58.988825+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"540'5087","reported_seq":12230,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.390957+0000","last_change":"2023-10-09T04:58:58.197235+0000","last_active":"2023-10-09T04:59:04.390957+0000","last_peered":"2023-10-09T04:59:04.390957+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T04:58:58.192854+0000","last_became_peered":"2023-10-09T04:58:58.192854+0000","last_unstale":"2023-10-09T04:59:04.390957+0000","last_undegraded":"2023-10-09T04:58:58.114536+0000","last_fullsized":"2023-10-09T04:58:57.983803+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151650304,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":85,"num_objects_misplaced":1851,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2547,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1737,"num_bytes_recovered":108724224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":937},{"shards":"4,5,7","objects":913},{"shards":"5,7","objects":464}],"blocked_by":[],"up_primary":4,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":113,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:59.821378+0000","last_change":"2023-10-09T04:58:59.821378+0000","last_active":"2023-10-09T04:58:59.821378+0000","last_peered":"2023-10-09T04:58:59.821378+0000","last_clean":"2023-10-09T04:58:59.821378+0000","last_became_active":"2023-10-09T04:58:59.820863+0000","last_became_peered":"2023-10-09T04:58:59.820863+0000","last_unstale":"2023-10-09T04:58:59.821378+0000","last_undegraded":"2023-10-09T04:58:59.821378+0000","last_fullsized":"2023-10-09T04:58:59.821378+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:03.664434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.976400+0000","last_change":"2023-10-09T04:58:57.066851+0000","last_active":"2023-10-09T04:58:57.976400+0000","last_peered":"2023-10-09T04:58:57.976400+0000","last_clean":"2023-10-09T04:58:57.976400+0000","last_became_active":"2023-10-09T04:58:57.066522+0000","last_became_peered":"2023-10-09T04:58:57.066522+0000","last_unstale":"2023-10-09T04:58:57.976400+0000","last_undegraded":"2023-10-09T04:58:57.976400+0000","last_fullsized":"2023-10-09T04:58:57.976400+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:48.448579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1044,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.988549+0000","last_change":"2023-10-09T04:58:10.885483+0000","last_active":"2023-10-09T04:58:58.988549+0000","last_peered":"2023-10-09T04:58:58.988549+0000","last_clean":"2023-10-09T04:58:58.988549+0000","last_became_active":"2023-10-09T04:58:10.885055+0000","last_became_peered":"2023-10-09T04:58:10.885055+0000","last_unstale":"2023-10-09T04:58:58.988549+0000","last_undegraded":"2023-10-09T04:58:58.988549+0000","last_fullsized":"2023-10-09T04:58:58.988549+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"540'5224","reported_seq":6067,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:05.100374+0000","last_change":"2023-10-09T04:58:54.736728+0000","last_active":"2023-10-09T04:59:05.100374+0000","last_peered":"2023-10-09T04:59:05.100374+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:58:52.441046+0000","last_became_peered":"2023-10-09T04:58:52.441046+0000","last_unstale":"2023-10-09T04:59:05.100374+0000","last_undegraded":"2023-10-09T04:58:50.668211+0000","last_fullsized":"2023-10-09T04:58:48.896671+0000","mapping_epoch":496,"log_start":"445'1536","ondisk_log_start":"445'1536","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3688,"log_dups_size":0,"ondisk_log_size":3688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155910144,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":2405,"num_objects_missing":2405,"num_objects_degraded":4476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2615,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":7864320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"1,2","objects":-26},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":1999}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":98,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.974244+0000","last_change":"2023-10-09T04:56:53.920990+0000","last_active":"2023-10-09T04:58:57.974244+0000","last_peered":"2023-10-09T04:58:57.974244+0000","last_clean":"2023-10-09T04:58:57.974244+0000","last_became_active":"2023-10-09T04:56:53.920700+0000","last_became_peered":"2023-10-09T04:56:53.920700+0000","last_unstale":"2023-10-09T04:58:57.974244+0000","last_undegraded":"2023-10-09T04:58:57.974244+0000","last_fullsized":"2023-10-09T04:58:57.974244+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:06.039757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":584,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.117318+0000","last_change":"2023-10-09T04:54:25.766092+0000","last_active":"2023-10-09T04:58:58.117318+0000","last_peered":"2023-10-09T04:58:58.117318+0000","last_clean":"2023-10-09T04:58:58.117318+0000","last_became_active":"2023-10-09T04:54:25.761061+0000","last_became_peered":"2023-10-09T04:54:25.761061+0000","last_unstale":"2023-10-09T04:58:58.117318+0000","last_undegraded":"2023-10-09T04:58:58.117318+0000","last_fullsized":"2023-10-09T04:58:58.117318+0000","mapping_epoch":448,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:33:39.310415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.972291+0000","last_change":"2023-10-09T04:58:57.074281+0000","last_active":"2023-10-09T04:58:57.972291+0000","last_peered":"2023-10-09T04:58:57.972291+0000","last_clean":"2023-10-09T04:58:57.972291+0000","last_became_active":"2023-10-09T04:58:57.073900+0000","last_became_peered":"2023-10-09T04:58:57.073900+0000","last_unstale":"2023-10-09T04:58:57.972291+0000","last_undegraded":"2023-10-09T04:58:57.972291+0000","last_fullsized":"2023-10-09T04:58:57.972291+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:13.677579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1027,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986430+0000","last_change":"2023-10-09T04:58:38.072798+0000","last_active":"2023-10-09T04:58:58.986430+0000","last_peered":"2023-10-09T04:58:58.986430+0000","last_clean":"2023-10-09T04:58:58.986430+0000","last_became_active":"2023-10-09T04:58:38.072032+0000","last_became_peered":"2023-10-09T04:58:38.072032+0000","last_unstale":"2023-10-09T04:58:58.986430+0000","last_undegraded":"2023-10-09T04:58:58.986430+0000","last_fullsized":"2023-10-09T04:58:58.986430+0000","mapping_epoch":524,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":525,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"531'4981","reported_seq":14872,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:58.102906+0000","last_change":"2023-10-09T04:58:58.102906+0000","last_active":"2023-10-09T04:58:58.102906+0000","last_peered":"2023-10-09T04:58:58.102906+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T04:58:58.066199+0000","last_became_peered":"2023-10-09T04:58:58.066199+0000","last_unstale":"2023-10-09T04:58:58.102906+0000","last_undegraded":"2023-10-09T04:58:58.018506+0000","last_fullsized":"2023-10-09T04:58:57.983219+0000","mapping_epoch":538,"log_start":"444'442","ondisk_log_start":"444'442","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4539,"log_dups_size":0,"ondisk_log_size":4539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:46.262165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301662208,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":375,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4981,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4480,"num_bytes_recovered":293601280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[2,6],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,3,4","objects":170},{"shards":"2,6","objects":4433}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":594,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.990818+0000","last_change":"2023-10-09T04:58:07.914635+0000","last_active":"2023-10-09T04:58:57.990818+0000","last_peered":"2023-10-09T04:58:57.990818+0000","last_clean":"2023-10-09T04:58:57.990818+0000","last_became_active":"2023-10-09T04:58:07.913894+0000","last_became_peered":"2023-10-09T04:58:07.913894+0000","last_unstale":"2023-10-09T04:58:57.990818+0000","last_undegraded":"2023-10-09T04:58:57.990818+0000","last_fullsized":"2023-10-09T04:58:57.990818+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:32.545025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.984837+0000","last_change":"2023-10-09T04:58:57.063810+0000","last_active":"2023-10-09T04:58:58.984837+0000","last_peered":"2023-10-09T04:58:58.984837+0000","last_clean":"2023-10-09T04:58:58.984837+0000","last_became_active":"2023-10-09T04:58:57.063532+0000","last_became_peered":"2023-10-09T04:58:57.063532+0000","last_unstale":"2023-10-09T04:58:58.984837+0000","last_undegraded":"2023-10-09T04:58:58.984837+0000","last_fullsized":"2023-10-09T04:58:58.984837+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:01.777681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":115,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.974166+0000","last_change":"2023-10-09T04:58:07.888094+0000","last_active":"2023-10-09T04:58:57.974166+0000","last_peered":"2023-10-09T04:58:57.974166+0000","last_clean":"2023-10-09T04:58:57.974166+0000","last_became_active":"2023-10-09T04:58:07.887823+0000","last_became_peered":"2023-10-09T04:58:07.887823+0000","last_unstale":"2023-10-09T04:58:57.974166+0000","last_undegraded":"2023-10-09T04:58:57.974166+0000","last_fullsized":"2023-10-09T04:58:57.974166+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:43:51.526193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1038,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986781+0000","last_change":"2023-10-09T04:58:26.862301+0000","last_active":"2023-10-09T04:58:58.986781+0000","last_peered":"2023-10-09T04:58:58.986781+0000","last_clean":"2023-10-09T04:58:58.986781+0000","last_became_active":"2023-10-09T04:58:26.862036+0000","last_became_peered":"2023-10-09T04:58:26.862036+0000","last_unstale":"2023-10-09T04:58:58.986781+0000","last_undegraded":"2023-10-09T04:58:58.986781+0000","last_fullsized":"2023-10-09T04:58:58.986781+0000","mapping_epoch":520,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":521,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"531'5005","reported_seq":5397,"reported_epoch":539,"state":"activating+undersized+remapped","last_fresh":"2023-10-09T04:58:57.981736+0000","last_change":"2023-10-09T04:59:07.269218+0000","last_active":"2023-10-09T04:58:57.091314+0000","last_peered":"2023-10-09T04:58:47.141141+0000","last_clean":"2023-10-09T04:54:44.521216+0000","last_became_active":"2023-10-09T04:58:08.008690+0000","last_became_peered":"2023-10-09T04:58:08.008690+0000","last_unstale":"2023-10-09T04:58:57.981736+0000","last_undegraded":"2023-10-09T04:58:57.981736+0000","last_fullsized":"2023-10-09T04:58:57.981736+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:35.050351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301596696,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":4557,"num_objects_missing":4557,"num_objects_degraded":4722,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5005,"num_write_kb":307393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":8192000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"2,4,5","objects":4483},{"shards":"5,6","objects":120}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":229,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.981088+0000","last_change":"2023-10-09T04:58:07.893285+0000","last_active":"2023-10-09T04:58:57.981088+0000","last_peered":"2023-10-09T04:58:57.981088+0000","last_clean":"2023-10-09T04:58:57.981088+0000","last_became_active":"2023-10-09T04:58:07.892815+0000","last_became_peered":"2023-10-09T04:58:07.892815+0000","last_unstale":"2023-10-09T04:58:57.981088+0000","last_undegraded":"2023-10-09T04:58:57.981088+0000","last_fullsized":"2023-10-09T04:58:57.981088+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:21.327306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":98,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T04:58:56.565448+0000","last_change":"2023-10-09T04:58:07.915179+0000","last_active":"2023-10-09T04:58:56.565448+0000","last_peered":"2023-10-09T04:58:56.565448+0000","last_clean":"2023-10-09T04:58:56.565448+0000","last_became_active":"2023-10-09T04:58:07.914564+0000","last_became_peered":"2023-10-09T04:58:07.914564+0000","last_unstale":"2023-10-09T04:58:56.565448+0000","last_undegraded":"2023-10-09T04:58:56.565448+0000","last_fullsized":"2023-10-09T04:58:56.565448+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:41.098943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.976474+0000","last_change":"2023-10-09T04:56:53.963152+0000","last_active":"2023-10-09T04:58:57.976474+0000","last_peered":"2023-10-09T04:58:57.976474+0000","last_clean":"2023-10-09T04:58:57.976474+0000","last_became_active":"2023-10-09T04:56:53.957811+0000","last_became_peered":"2023-10-09T04:56:53.957811+0000","last_unstale":"2023-10-09T04:58:57.976474+0000","last_undegraded":"2023-10-09T04:58:57.976474+0000","last_fullsized":"2023-10-09T04:58:57.976474+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:49:17.669923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"540'4996","reported_seq":7331,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.780945+0000","last_change":"2023-10-09T04:58:25.738930+0000","last_active":"2023-10-09T04:59:04.780945+0000","last_peered":"2023-10-09T04:59:04.780945+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T04:56:43.024070+0000","last_became_peered":"2023-10-09T04:56:43.024070+0000","last_unstale":"2023-10-09T04:59:04.780945+0000","last_undegraded":"2023-10-09T04:56:29.350566+0000","last_fullsized":"2023-10-09T04:56:21.601498+0000","mapping_epoch":480,"log_start":"445'1416","ondisk_log_start":"445'1416","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3580,"log_dups_size":0,"ondisk_log_size":3580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:37.940860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298975232,"num_objects":4562,"num_object_clones":0,"num_object_copies":13686,"num_objects_missing_on_primary":4463,"num_objects_missing":4463,"num_objects_degraded":4608,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4996,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1102,"num_bytes_recovered":72220672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,5","objects":244},{"shards":"2,3,5,7","objects":325},{"shards":"2,5,7","objects":3216},{"shards":"3,5,7","objects":776},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":541,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.985735+0000","last_change":"2023-10-09T04:52:10.727903+0000","last_active":"2023-10-09T04:58:57.985735+0000","last_peered":"2023-10-09T04:58:57.985735+0000","last_clean":"2023-10-09T04:58:57.985735+0000","last_became_active":"2023-10-09T04:52:10.727522+0000","last_became_peered":"2023-10-09T04:52:10.727522+0000","last_unstale":"2023-10-09T04:58:57.985735+0000","last_undegraded":"2023-10-09T04:58:57.985735+0000","last_fullsized":"2023-10-09T04:58:57.985735+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:22:45.141607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1044,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986229+0000","last_change":"2023-10-09T04:58:11.885502+0000","last_active":"2023-10-09T04:58:58.986229+0000","last_peered":"2023-10-09T04:58:58.986229+0000","last_clean":"2023-10-09T04:58:58.986229+0000","last_became_active":"2023-10-09T04:58:10.854933+0000","last_became_peered":"2023-10-09T04:58:10.854933+0000","last_unstale":"2023-10-09T04:58:58.986229+0000","last_undegraded":"2023-10-09T04:58:58.986229+0000","last_fullsized":"2023-10-09T04:58:58.986229+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":503,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.990932+0000","last_change":"2023-10-09T04:58:56.995693+0000","last_active":"2023-10-09T04:58:57.990932+0000","last_peered":"2023-10-09T04:58:57.990932+0000","last_clean":"2023-10-09T04:58:57.990932+0000","last_became_active":"2023-10-09T04:58:56.995379+0000","last_became_peered":"2023-10-09T04:58:56.995379+0000","last_unstale":"2023-10-09T04:58:57.990932+0000","last_undegraded":"2023-10-09T04:58:57.990932+0000","last_fullsized":"2023-10-09T04:58:57.990932+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:42.883549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-09T04:58:56.999352+0000","last_change":"2023-10-09T04:58:56.999352+0000","last_active":"2023-10-09T04:58:56.999352+0000","last_peered":"2023-10-09T04:58:56.999352+0000","last_clean":"2023-10-09T04:58:56.999352+0000","last_became_active":"2023-10-09T04:58:56.999048+0000","last_became_peered":"2023-10-09T04:58:56.999048+0000","last_unstale":"2023-10-09T04:58:56.999352+0000","last_undegraded":"2023-10-09T04:58:56.999352+0000","last_fullsized":"2023-10-09T04:58:56.999352+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:53.056597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1047,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.986705+0000","last_change":"2023-10-09T04:58:19.734140+0000","last_active":"2023-10-09T04:58:58.986705+0000","last_peered":"2023-10-09T04:58:58.986705+0000","last_clean":"2023-10-09T04:58:58.986705+0000","last_became_active":"2023-10-09T04:58:10.855794+0000","last_became_peered":"2023-10-09T04:58:10.855794+0000","last_unstale":"2023-10-09T04:58:58.986705+0000","last_undegraded":"2023-10-09T04:58:58.986705+0000","last_fullsized":"2023-10-09T04:58:58.986705+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"531'5063","reported_seq":9831,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:59.461438+0000","last_change":"2023-10-09T04:58:59.464907+0000","last_active":"2023-10-09T04:58:59.461438+0000","last_peered":"2023-10-09T04:58:59.461438+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T04:58:59.452514+0000","last_became_peered":"2023-10-09T04:58:59.452514+0000","last_unstale":"2023-10-09T04:58:59.461438+0000","last_undegraded":"2023-10-09T04:58:59.441276+0000","last_fullsized":"2023-10-09T04:58:59.025474+0000","mapping_epoch":539,"log_start":"444'541","ondisk_log_start":"444'541","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4522,"log_dups_size":0,"ondisk_log_size":4522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:08.028846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307167232,"num_objects":4687,"num_object_clones":0,"num_object_copies":14061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4752,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5063,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2118,"num_bytes_recovered":129499136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[2,4],"avail_no_missing":["2","0","1","5"],"object_location_counts":[{"shards":"2,4","objects":168}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":91,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T04:58:56.565369+0000","last_change":"2023-10-09T04:56:55.417824+0000","last_active":"2023-10-09T04:58:56.565369+0000","last_peered":"2023-10-09T04:58:56.565369+0000","last_clean":"2023-10-09T04:58:56.565369+0000","last_became_active":"2023-10-09T04:56:55.417430+0000","last_became_peered":"2023-10-09T04:56:55.417430+0000","last_unstale":"2023-10-09T04:58:56.565369+0000","last_undegraded":"2023-10-09T04:58:56.565369+0000","last_fullsized":"2023-10-09T04:58:56.565369+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:22.097654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":171,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.003915+0000","last_change":"2023-10-09T04:56:53.470340+0000","last_active":"2023-10-09T04:58:58.003915+0000","last_peered":"2023-10-09T04:58:58.003915+0000","last_clean":"2023-10-09T04:58:58.003915+0000","last_became_active":"2023-10-09T04:56:53.469975+0000","last_became_peered":"2023-10-09T04:56:53.469975+0000","last_unstale":"2023-10-09T04:58:58.003915+0000","last_undegraded":"2023-10-09T04:58:58.003915+0000","last_fullsized":"2023-10-09T04:58:58.003915+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:36:21.768457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":94,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.978997+0000","last_change":"2023-10-09T04:58:57.069761+0000","last_active":"2023-10-09T04:58:57.978997+0000","last_peered":"2023-10-09T04:58:57.978997+0000","last_clean":"2023-10-09T04:58:57.978997+0000","last_became_active":"2023-10-09T04:58:57.069209+0000","last_became_peered":"2023-10-09T04:58:57.069209+0000","last_unstale":"2023-10-09T04:58:57.978997+0000","last_undegraded":"2023-10-09T04:58:57.978997+0000","last_fullsized":"2023-10-09T04:58:57.978997+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:54.132927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1047,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.988516+0000","last_change":"2023-10-09T04:58:18.749540+0000","last_active":"2023-10-09T04:58:58.988516+0000","last_peered":"2023-10-09T04:58:58.988516+0000","last_clean":"2023-10-09T04:58:58.988516+0000","last_became_active":"2023-10-09T04:58:10.853647+0000","last_became_peered":"2023-10-09T04:58:10.853647+0000","last_unstale":"2023-10-09T04:58:58.988516+0000","last_undegraded":"2023-10-09T04:58:58.988516+0000","last_fullsized":"2023-10-09T04:58:58.988516+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"540'5086","reported_seq":16750,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.633088+0000","last_change":"2023-10-09T04:58:59.000602+0000","last_active":"2023-10-09T04:59:04.633088+0000","last_peered":"2023-10-09T04:59:04.633088+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:58:58.998253+0000","last_became_peered":"2023-10-09T04:58:58.998253+0000","last_unstale":"2023-10-09T04:59:04.633088+0000","last_undegraded":"2023-10-09T04:58:58.992982+0000","last_fullsized":"2023-10-09T04:58:58.986976+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153452544,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":73,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2547,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2872,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":2341}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":14,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.985004+0000","last_change":"2023-10-09T04:58:57.034371+0000","last_active":"2023-10-09T04:58:58.985004+0000","last_peered":"2023-10-09T04:58:58.985004+0000","last_clean":"2023-10-09T04:58:58.985004+0000","last_became_active":"2023-10-09T04:58:57.034021+0000","last_became_peered":"2023-10-09T04:58:57.034021+0000","last_unstale":"2023-10-09T04:58:58.985004+0000","last_undegraded":"2023-10-09T04:58:58.985004+0000","last_fullsized":"2023-10-09T04:58:58.985004+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:22.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":97,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.974174+0000","last_change":"2023-10-09T04:58:40.831293+0000","last_active":"2023-10-09T04:58:57.974174+0000","last_peered":"2023-10-09T04:58:57.974174+0000","last_clean":"2023-10-09T04:58:57.974174+0000","last_became_active":"2023-10-09T04:58:40.830973+0000","last_became_peered":"2023-10-09T04:58:40.830973+0000","last_unstale":"2023-10-09T04:58:57.974174+0000","last_undegraded":"2023-10-09T04:58:57.974174+0000","last_fullsized":"2023-10-09T04:58:57.974174+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:03:20.204250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":434,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.985583+0000","last_change":"2023-10-09T04:56:53.302772+0000","last_active":"2023-10-09T04:58:57.985583+0000","last_peered":"2023-10-09T04:58:57.985583+0000","last_clean":"2023-10-09T04:58:57.985583+0000","last_became_active":"2023-10-09T04:56:53.302425+0000","last_became_peered":"2023-10-09T04:56:53.302425+0000","last_unstale":"2023-10-09T04:58:57.985583+0000","last_undegraded":"2023-10-09T04:58:57.985583+0000","last_fullsized":"2023-10-09T04:58:57.985583+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":493,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:52:00.445110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1041,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T04:58:56.060998+0000","last_change":"2023-10-09T04:58:10.854839+0000","last_active":"2023-10-09T04:58:56.060998+0000","last_peered":"2023-10-09T04:58:56.060998+0000","last_clean":"2023-10-09T04:58:56.060998+0000","last_became_active":"2023-10-09T04:58:10.854463+0000","last_became_peered":"2023-10-09T04:58:10.854463+0000","last_unstale":"2023-10-09T04:58:56.060998+0000","last_undegraded":"2023-10-09T04:58:56.060998+0000","last_fullsized":"2023-10-09T04:58:56.060998+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"540'4965","reported_seq":21249,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:59:04.469676+0000","last_change":"2023-10-09T04:58:39.912315+0000","last_active":"2023-10-09T04:59:04.469676+0000","last_peered":"2023-10-09T04:59:04.469676+0000","last_clean":"2023-10-09T04:59:04.469676+0000","last_became_active":"2023-10-09T04:58:39.903645+0000","last_became_peered":"2023-10-09T04:58:39.903645+0000","last_unstale":"2023-10-09T04:59:04.469676+0000","last_undegraded":"2023-10-09T04:59:04.469676+0000","last_fullsized":"2023-10-09T04:59:04.469676+0000","mapping_epoch":526,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4965,"log_dups_size":0,"ondisk_log_size":4965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:27.838809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299958272,"num_objects":4577,"num_object_clones":0,"num_object_copies":13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4965,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7458,"num_bytes_recovered":477888512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":131,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.987759+0000","last_change":"2023-10-09T04:58:06.874649+0000","last_active":"2023-10-09T04:58:58.987759+0000","last_peered":"2023-10-09T04:58:58.987759+0000","last_clean":"2023-10-09T04:58:58.987759+0000","last_became_active":"2023-10-09T04:56:55.830839+0000","last_became_peered":"2023-10-09T04:56:55.830839+0000","last_unstale":"2023-10-09T04:58:58.987759+0000","last_undegraded":"2023-10-09T04:58:58.987759+0000","last_fullsized":"2023-10-09T04:58:58.987759+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:54.131102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":129,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.979262+0000","last_change":"2023-10-09T04:58:57.063270+0000","last_active":"2023-10-09T04:58:57.979262+0000","last_peered":"2023-10-09T04:58:57.979262+0000","last_clean":"2023-10-09T04:58:57.979262+0000","last_became_active":"2023-10-09T04:58:57.062873+0000","last_became_peered":"2023-10-09T04:58:57.062873+0000","last_unstale":"2023-10-09T04:58:57.979262+0000","last_undegraded":"2023-10-09T04:58:57.979262+0000","last_fullsized":"2023-10-09T04:58:57.979262+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:58.612148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":137,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-09T04:58:56.563696+0000","last_change":"2023-10-09T04:58:07.914991+0000","last_active":"2023-10-09T04:58:56.563696+0000","last_peered":"2023-10-09T04:58:56.563696+0000","last_clean":"2023-10-09T04:58:56.563696+0000","last_became_active":"2023-10-09T04:58:07.914214+0000","last_became_peered":"2023-10-09T04:58:07.914214+0000","last_unstale":"2023-10-09T04:58:56.563696+0000","last_undegraded":"2023-10-09T04:58:56.563696+0000","last_fullsized":"2023-10-09T04:58:56.563696+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:59:07.039940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1028,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:59.003574+0000","last_change":"2023-10-09T04:58:57.150218+0000","last_active":"2023-10-09T04:58:59.003574+0000","last_peered":"2023-10-09T04:58:59.003574+0000","last_clean":"2023-10-09T04:58:59.003574+0000","last_became_active":"2023-10-09T04:58:52.539899+0000","last_became_peered":"2023-10-09T04:58:52.539899+0000","last_unstale":"2023-10-09T04:58:59.003574+0000","last_undegraded":"2023-10-09T04:58:59.003574+0000","last_fullsized":"2023-10-09T04:58:59.003574+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1138389,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"531'5175","reported_seq":7268,"reported_epoch":539,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:58:57.998642+0000","last_change":"2023-10-09T04:58:09.220159+0000","last_active":"2023-10-09T04:58:57.998642+0000","last_peered":"2023-10-09T04:58:57.998642+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T04:58:09.204385+0000","last_became_peered":"2023-10-09T04:58:09.204385+0000","last_unstale":"2023-10-09T04:58:57.998642+0000","last_undegraded":"2023-10-09T04:58:08.984698+0000","last_fullsized":"2023-10-09T04:58:08.834068+0000","mapping_epoch":514,"log_start":"444'657","ondisk_log_start":"444'657","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4518,"log_dups_size":0,"ondisk_log_size":4518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:30:04.803237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313589760,"num_objects":4785,"num_object_clones":0,"num_object_copies":14355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8947,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5175,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":883,"num_bytes_recovered":57868288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"1,2","objects":-13},{"shards":"1,2,4,7","objects":687},{"shards":"2,4,7","objects":3972},{"shards":"2,6,7","objects":125}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":37,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.988359+0000","last_change":"2023-10-09T04:58:07.881659+0000","last_active":"2023-10-09T04:58:58.988359+0000","last_peered":"2023-10-09T04:58:58.988359+0000","last_clean":"2023-10-09T04:58:58.988359+0000","last_became_active":"2023-10-09T04:58:07.881399+0000","last_became_peered":"2023-10-09T04:58:07.881399+0000","last_unstale":"2023-10-09T04:58:58.988359+0000","last_undegraded":"2023-10-09T04:58:58.988359+0000","last_fullsized":"2023-10-09T04:58:58.988359+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:31.736513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":164,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.015847+0000","last_change":"2023-10-09T04:58:58.015847+0000","last_active":"2023-10-09T04:58:58.015847+0000","last_peered":"2023-10-09T04:58:58.015847+0000","last_clean":"2023-10-09T04:58:52.922210+0000","last_became_active":"2023-10-09T04:58:58.015415+0000","last_became_peered":"2023-10-09T04:58:58.015415+0000","last_unstale":"2023-10-09T04:58:58.015847+0000","last_undegraded":"2023-10-09T04:58:57.979615+0000","last_fullsized":"2023-10-09T04:58:58.015847+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:12.808436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:58.984837+0000","last_change":"2023-10-09T04:58:56.998007+0000","last_active":"2023-10-09T04:58:58.984837+0000","last_peered":"2023-10-09T04:58:58.984837+0000","last_clean":"2023-10-09T04:58:58.984837+0000","last_became_active":"2023-10-09T04:58:56.997662+0000","last_became_peered":"2023-10-09T04:58:56.997662+0000","last_unstale":"2023-10-09T04:58:58.984837+0000","last_undegraded":"2023-10-09T04:58:58.984837+0000","last_fullsized":"2023-10-09T04:58:58.984837+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:28:15.981713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"531'4913","reported_seq":5674,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:57.983141+0000","last_change":"2023-10-09T04:58:49.315510+0000","last_active":"2023-10-09T04:58:57.983141+0000","last_peered":"2023-10-09T04:58:57.983141+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:58:49.309184+0000","last_became_peered":"2023-10-09T04:58:49.309184+0000","last_unstale":"2023-10-09T04:58:57.983141+0000","last_undegraded":"2023-10-09T04:58:49.134600+0000","last_fullsized":"2023-10-09T04:58:57.983141+0000","mapping_epoch":532,"log_start":"444'386","ondisk_log_start":"444'386","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4527,"log_dups_size":0,"ondisk_log_size":4527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148078592,"num_objects":2260,"num_object_clones":0,"num_object_copies":6780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4271,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2457,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":6488064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"1,2,3","objects":-36},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":1909}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:59.186040+0000","last_change":"2023-10-09T04:58:59.185788+0000","last_active":"2023-10-09T04:58:59.186040+0000","last_peered":"2023-10-09T04:58:59.186040+0000","last_clean":"2023-10-09T04:58:59.186040+0000","last_became_active":"2023-10-09T04:58:59.185338+0000","last_became_peered":"2023-10-09T04:58:59.185338+0000","last_unstale":"2023-10-09T04:58:59.186040+0000","last_undegraded":"2023-10-09T04:58:59.186040+0000","last_fullsized":"2023-10-09T04:58:59.186040+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:19.825485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":88,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.974087+0000","last_change":"2023-10-09T04:58:57.064053+0000","last_active":"2023-10-09T04:58:57.974087+0000","last_peered":"2023-10-09T04:58:57.974087+0000","last_clean":"2023-10-09T04:58:57.974087+0000","last_became_active":"2023-10-09T04:58:57.063696+0000","last_became_peered":"2023-10-09T04:58:57.063696+0000","last_unstale":"2023-10-09T04:58:57.974087+0000","last_undegraded":"2023-10-09T04:58:57.974087+0000","last_fullsized":"2023-10-09T04:58:57.974087+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:30.110906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"540'5081","reported_seq":5837,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.484190+0000","last_change":"2023-10-09T04:58:57.558096+0000","last_active":"2023-10-09T04:59:04.484190+0000","last_peered":"2023-10-09T04:59:04.484190+0000","last_clean":"2023-10-09T04:53:31.951851+0000","last_became_active":"2023-10-09T04:58:57.516641+0000","last_became_peered":"2023-10-09T04:58:57.516641+0000","last_unstale":"2023-10-09T04:59:04.484190+0000","last_undegraded":"2023-10-09T04:58:55.090806+0000","last_fullsized":"2023-10-09T04:58:53.405485+0000","mapping_epoch":535,"log_start":"445'1388","ondisk_log_start":"445'1388","created":532,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3693,"log_dups_size":0,"ondisk_log_size":3693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584768,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":2306,"num_objects_missing":2306,"num_objects_degraded":4627,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2546,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":7503872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,2","objects":21},{"shards":"2,3,5","objects":363},{"shards":"2,3,6","objects":57},{"shards":"2,5","objects":1872}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":629,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:57.986335+0000","last_change":"2023-10-09T04:58:57.167182+0000","last_active":"2023-10-09T04:58:57.986335+0000","last_peered":"2023-10-09T04:58:57.986335+0000","last_clean":"2023-10-09T04:58:52.959143+0000","last_became_active":"2023-10-09T04:58:57.166878+0000","last_became_peered":"2023-10-09T04:58:57.166878+0000","last_unstale":"2023-10-09T04:58:57.986335+0000","last_undegraded":"2023-10-09T04:58:57.165658+0000","last_fullsized":"2023-10-09T04:58:57.986335+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.097934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:59:02.688039+0000","last_change":"2023-10-09T04:59:02.687722+0000","last_active":"2023-10-09T04:59:02.688039+0000","last_peered":"2023-10-09T04:59:02.688039+0000","last_clean":"2023-10-09T04:59:02.688039+0000","last_became_active":"2023-10-09T04:59:02.687389+0000","last_became_peered":"2023-10-09T04:59:02.687389+0000","last_unstale":"2023-10-09T04:59:02.688039+0000","last_undegraded":"2023-10-09T04:59:02.688039+0000","last_fullsized":"2023-10-09T04:59:02.688039+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:36:14.807001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"540'5076","reported_seq":5358,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.360229+0000","last_change":"2023-10-09T04:58:58.141338+0000","last_active":"2023-10-09T04:59:04.360229+0000","last_peered":"2023-10-09T04:59:04.360229+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:58.134873+0000","last_became_peered":"2023-10-09T04:58:58.134873+0000","last_unstale":"2023-10-09T04:59:04.360229+0000","last_undegraded":"2023-10-09T04:58:58.123193+0000","last_fullsized":"2023-10-09T04:58:58.004066+0000","mapping_epoch":538,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4538,"log_dups_size":0,"ondisk_log_size":4538,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151945216,"num_objects":2319,"num_object_clones":0,"num_object_copies":6957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2537,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2545,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1671168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":2262},{"shards":"2,5,6","objects":62},{"shards":"4,5","objects":-22}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":598,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.117162+0000","last_change":"2023-10-09T04:58:56.997234+0000","last_active":"2023-10-09T04:58:58.117162+0000","last_peered":"2023-10-09T04:58:58.117162+0000","last_clean":"2023-10-09T04:58:52.967489+0000","last_became_active":"2023-10-09T04:58:56.996966+0000","last_became_peered":"2023-10-09T04:58:56.996966+0000","last_unstale":"2023-10-09T04:58:58.117162+0000","last_undegraded":"2023-10-09T04:58:56.980013+0000","last_fullsized":"2023-10-09T04:58:58.117162+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:43.116063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":148,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.991899+0000","last_change":"2023-10-09T04:58:07.903822+0000","last_active":"2023-10-09T04:58:57.991899+0000","last_peered":"2023-10-09T04:58:57.991899+0000","last_clean":"2023-10-09T04:58:57.991899+0000","last_became_active":"2023-10-09T04:58:07.903432+0000","last_became_peered":"2023-10-09T04:58:07.903432+0000","last_unstale":"2023-10-09T04:58:57.991899+0000","last_undegraded":"2023-10-09T04:58:57.991899+0000","last_fullsized":"2023-10-09T04:58:57.991899+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:12:23.655112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"540'5074","reported_seq":5998,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:04.179433+0000","last_change":"2023-10-09T04:59:02.679087+0000","last_active":"2023-10-09T04:59:04.179433+0000","last_peered":"2023-10-09T04:59:04.179433+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:59:01.721052+0000","last_became_peered":"2023-10-09T04:59:01.721052+0000","last_unstale":"2023-10-09T04:59:04.179433+0000","last_undegraded":"2023-10-09T04:58:59.175344+0000","last_fullsized":"2023-10-09T04:58:56.583281+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5074,"log_dups_size":0,"ondisk_log_size":5074,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152174592,"num_objects":2322,"num_object_clones":0,"num_object_copies":6966,"num_objects_missing_on_primary":1946,"num_objects_missing":1946,"num_objects_degraded":4372,"num_objects_misplaced":446,"num_objects_unfound":0,"num_objects_dirty":2322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2542,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":106,"num_bytes_recovered":6881280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[3,6],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":9},{"shards":"2,3,5,7","objects":346},{"shards":"2,5,7","objects":1946},{"shards":"3,5,6","objects":49},{"shards":"3,6","objects":-28}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":130,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.991470+0000","last_change":"2023-10-09T04:58:07.882299+0000","last_active":"2023-10-09T04:58:57.991470+0000","last_peered":"2023-10-09T04:58:57.991470+0000","last_clean":"2023-10-09T04:58:57.991470+0000","last_became_active":"2023-10-09T04:58:07.881746+0000","last_became_peered":"2023-10-09T04:58:07.881746+0000","last_unstale":"2023-10-09T04:58:57.991470+0000","last_undegraded":"2023-10-09T04:58:57.991470+0000","last_fullsized":"2023-10-09T04:58:57.991470+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:14.326353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-09T04:58:57.991674+0000","last_change":"2023-10-09T04:56:55.857967+0000","last_active":"2023-10-09T04:58:57.991674+0000","last_peered":"2023-10-09T04:58:57.991674+0000","last_clean":"2023-10-09T04:58:57.991674+0000","last_became_active":"2023-10-09T04:56:55.857506+0000","last_became_peered":"2023-10-09T04:56:55.857506+0000","last_unstale":"2023-10-09T04:58:57.991674+0000","last_undegraded":"2023-10-09T04:58:57.991674+0000","last_fullsized":"2023-10-09T04:58:57.991674+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:44:57.553326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"540'5003","reported_seq":8007,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T04:59:03.635091+0000","last_change":"2023-10-09T04:58:56.027311+0000","last_active":"2023-10-09T04:59:03.635091+0000","last_peered":"2023-10-09T04:59:03.635091+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T04:58:55.010554+0000","last_became_peered":"2023-10-09T04:58:55.010554+0000","last_unstale":"2023-10-09T04:59:03.635091+0000","last_undegraded":"2023-10-09T04:58:54.438514+0000","last_fullsized":"2023-10-09T04:58:51.946173+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":358,"num_objects_missing":358,"num_objects_degraded":2285,"num_objects_misplaced":1531,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2504,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":406,"num_bytes_recovered":26574848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[7,5],"avail_no_missing":["4"],"object_location_counts":[{"shards":"3,4,5","objects":234},{"shards":"3,4,5,6","objects":50},{"shards":"4,5,7","objects":1605},{"shards":"5,7","objects":422}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":598,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T04:58:58.052806+0000","last_change":"2023-10-09T04:58:56.991316+0000","last_active":"2023-10-09T04:58:58.052806+0000","last_peered":"2023-10-09T04:58:58.052806+0000","last_clean":"2023-10-09T04:58:52.961267+0000","last_became_active":"2023-10-09T04:58:56.991046+0000","last_became_peered":"2023-10-09T04:58:56.991046+0000","last_unstale":"2023-10-09T04:58:58.052806+0000","last_undegraded":"2023-10-09T04:58:56.974156+0000","last_fullsized":"2023-10-09T04:58:58.052806+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:54:11.712349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":["5","2","6","7"],"object_location_counts":[{"shards":"2,5,6,7","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-09T04:58:59.822045+0000","last_change":"2023-10-09T04:58:59.821723+0000","last_active":"2023-10-09T04:58:59.822045+0000","last_peered":"2023-10-09T04:58:59.822045+0000","last_clean":"2023-10-09T04:58:59.822045+0000","last_became_active":"2023-10-09T04:58:59.821428+0000","last_became_peered":"2023-10-09T04:58:59.821428+0000","last_unstale":"2023-10-09T04:58:59.822045+0000","last_undegraded":"2023-10-09T04:58:59.822045+0000","last_fullsized":"2023-10-09T04:58:59.822045+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:15.048879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":4866703384,"num_objects":74261,"num_object_clones":0,"num_object_copies":222783,"num_objects_missing_on_primary":27085,"num_objects_missing":27085,"num_objects_degraded":70867,"num_objects_misplaced":13863,"num_objects_unfound":0,"num_objects_dirty":74261,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":80957,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32014,"num_bytes_recovered":2062942208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16827629568,"data_stored":16827613280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120850,"ondisk_log_size":120850,"up":78,"acting":57,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":8,"num_objects_missing":8,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":26808,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31051,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":6177690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2908160,"data_stored":2886848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234622,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4034460,"kb_used_data":2529300,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":89710692,"statfs":{"total":95995035648,"available":91863748608,"internally_reserved":0,"allocated":2590003200,"data_stored":2588762165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.53900000000000003,"15min":0.97899999999999998},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.21099999999999999},"max":{"1min":0.57499999999999996,"5min":3.4420000000000002,"15min":14.970000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.77300000000000002,"15min":1.4319999999999999},"min":{"1min":0.315,"5min":0.26700000000000002,"15min":0.249},"max":{"1min":2.2519999999999998,"5min":6.657,"15min":21.216000000000001},"last":0.314}]},{"osd":1,"last update":"Mon Oct 9 04:58:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.76700000000000002,"15min":3.0649999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.246},"max":{"1min":2.9980000000000002,"5min":5.0990000000000002,"15min":128.601},"last":0.373},{"interface":"front","average":{"1min":1.0680000000000001,"5min":0.84399999999999997,"15min":3.0089999999999999},"min":{"1min":0.36299999999999999,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":8.1920000000000002,"5min":8.1920000000000002,"15min":128.381},"last":0.52700000000000002}]},{"osd":2,"last update":"Mon Oct 9 04:58:38 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":3.48,"15min":4.5999999999999996},"min":{"1min":0.36799999999999999,"5min":0.30299999999999999,"15min":0.27100000000000002},"max":{"1min":2.6320000000000001,"5min":133.601,"15min":207.96600000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.6000000000000001,"15min":4.7919999999999998},"min":{"1min":0.41899999999999998,"5min":0.32500000000000001,"15min":0.29699999999999999},"max":{"1min":2.6419999999999999,"5min":133.68799999999999,"15min":207.97900000000001},"last":0.35999999999999999}]},{"osd":3,"last update":"Mon Oct 9 04:58:53 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.7170000000000001,"15min":1.681},"min":{"1min":0.36699999999999999,"5min":0.29999999999999999,"15min":0.29499999999999998},"max":{"1min":4.8129999999999997,"5min":74.259,"15min":74.259},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.01,"5min":1.7370000000000001,"15min":1.9930000000000001},"min":{"1min":0.35399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.843,"5min":74.100999999999999,"15min":74.100999999999999},"last":0.47599999999999998}]},{"osd":4,"last update":"Mon Oct 9 04:58:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.986,"5min":1.986,"15min":1.986},"last":0.39800000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:58:38 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.57199999999999995,"15min":1.7050000000000001},"min":{"1min":0.27800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.82199999999999995,"5min":3.6789999999999998,"15min":131.84899999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.84799999999999998,"15min":2.1560000000000001},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.266,"5min":6.6299999999999999,"15min":135.74199999999999},"last":0.44}]},{"osd":6,"last update":"Mon Oct 9 04:58:38 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.59599999999999997,"15min":1.127},"min":{"1min":0.36899999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.76800000000000002,"5min":3.7200000000000002,"15min":10.047000000000001},"last":0.495},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.752,"15min":1.4179999999999999},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.1629999999999998,"5min":6.6950000000000003,"15min":21.321999999999999},"last":0.505}]}]},{"osd":6,"up_from":46,"seq":197568496255,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333016,"kb_used_data":438928,"kb_used_omap":0,"kb_used_meta":894080,"kb_avail":92412136,"statfs":{"total":95995035648,"available":94630027264,"internally_reserved":0,"allocated":449462272,"data_stored":448225012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915537920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,0,0,0,0,0,2,12,2],"upper_bound":16384},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.52700000000000002,"15min":0.89600000000000002},"min":{"1min":0.309,"5min":0.24399999999999999,"15min":0.20499999999999999},"max":{"1min":0.93899999999999995,"5min":3.2109999999999999,"15min":9.4979999999999993},"last":0.504},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.73999999999999999,"15min":1.1399999999999999},"min":{"1min":0.32500000000000001,"5min":0.28000000000000003,"15min":0.26700000000000002},"max":{"1min":6.1980000000000004,"5min":7.3129999999999997,"15min":16.574000000000002},"last":0.40899999999999997}]},{"osd":1,"last update":"Mon Oct 9 04:58:57 2023","interfaces":[{"interface":"back","average":{"1min":5.4610000000000003,"5min":1.9570000000000001,"15min":5.3869999999999996},"min":{"1min":0.312,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":90.278999999999996,"5min":90.278999999999996,"15min":260.738},"last":0.33500000000000002},{"interface":"front","average":{"1min":5.1699999999999999,"5min":1.7190000000000001,"15min":5.0780000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26300000000000001},"max":{"1min":90.414000000000001,"5min":90.414000000000001,"15min":261.26100000000002},"last":0.53600000000000003}]},{"osd":2,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.4910000000000001,"15min":4.2720000000000002},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.27800000000000002},"max":{"1min":9.2759999999999998,"5min":133.535,"15min":133.535},"last":0.44},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.5379999999999998,"15min":4.1020000000000003},"min":{"1min":0.40200000000000002,"5min":0.38400000000000001,"15min":0.31},"max":{"1min":9.2639999999999993,"5min":133.51900000000001,"15min":133.51900000000001},"last":0.39700000000000002}]},{"osd":3,"last update":"Mon Oct 9 04:58:57 2023","interfaces":[{"interface":"back","average":{"1min":5.8049999999999997,"5min":4.4660000000000002,"15min":3.5},"min":{"1min":0.42599999999999999,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":91.260999999999996,"5min":147.43600000000001,"15min":147.43600000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":6.3200000000000003,"5min":4.593,"15min":3.2480000000000002},"min":{"1min":0.41499999999999998,"5min":0.34799999999999998,"15min":0.29999999999999999},"max":{"1min":92.087000000000003,"5min":147.41999999999999,"15min":147.41999999999999},"last":0.49199999999999999}]},{"osd":4,"last update":"Mon Oct 9 04:58:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.378},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.46999999999999997}]},{"osd":5,"last update":"Mon Oct 9 04:58:57 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.79100000000000004,"15min":1.9670000000000001},"min":{"1min":0.30599999999999999,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":3.4180000000000001,"5min":4.5839999999999996,"15min":83.168999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.90900000000000003,"15min":2.02},"min":{"1min":0.34000000000000002,"5min":0.254,"15min":0.254},"max":{"1min":2.3879999999999999,"5min":13.241,"15min":83.126000000000005},"last":0.58499999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.877,"15min":1.4199999999999999},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":0.70899999999999996,"5min":5.5049999999999999,"15min":36.204999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.498,"5min":0.85899999999999999,"15min":1.371},"min":{"1min":0.32600000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.63400000000000001,"5min":4.5359999999999996,"15min":10.661},"last":0.57199999999999995}]}]},{"osd":1,"up_from":118,"seq":506806141332,"num_pgs":84,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1994824,"kb_used_data":835648,"kb_used_omap":4,"kb_used_meta":1159163,"kb_avail":91750328,"statfs":{"total":95995035648,"available":93952335872,"internally_reserved":0,"allocated":855703552,"data_stored":853014445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1186983822},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":0.88600000000000001,"15min":1.1240000000000001},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":9.6999999999999993,"5min":11.763999999999999,"15min":21.021999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.302,"5min":0.91200000000000003,"15min":1.1799999999999999},"min":{"1min":0.254,"5min":0.214,"15min":0.214},"max":{"1min":5.7510000000000003,"5min":6.2130000000000001,"15min":21.280999999999999},"last":1.895}]},{"osd":2,"last update":"Mon Oct 9 04:58:28 2023","interfaces":[{"interface":"back","average":{"1min":4.9420000000000002,"5min":5.2439999999999998,"15min":4.1379999999999999},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":63.143999999999998,"5min":263.089,"15min":263.089},"last":1.4159999999999999},{"interface":"front","average":{"1min":4.3079999999999998,"5min":5.1470000000000002,"15min":4.0810000000000004},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.222},"max":{"1min":66.198999999999998,"5min":254.50899999999999,"15min":254.50899999999999},"last":1.9370000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:59:02 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.4870000000000001,"15min":1.752},"min":{"1min":0.32200000000000001,"5min":0.28699999999999998,"15min":0.23000000000000001},"max":{"1min":39.289000000000001,"5min":77.819999999999993,"15min":77.819999999999993},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.2240000000000002,"15min":1.77},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":14.212,"5min":77.796000000000006,"15min":77.796000000000006},"last":1.9990000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:58:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.8720000000000001,"15min":1.8720000000000001},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":19.634,"5min":19.634,"15min":19.634},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.319,"5min":1.319,"15min":1.319},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":13.045,"5min":13.045,"15min":13.045},"last":0.77700000000000002}]},{"osd":5,"last update":"Mon Oct 9 04:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.212,"15min":1.522},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.33000000000000002},"max":{"1min":5.774,"5min":7.4980000000000002,"15min":9.782},"last":1.964},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.1779999999999999,"15min":1.6439999999999999},"min":{"1min":0.40699999999999997,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":19.597999999999999,"5min":19.597999999999999,"15min":20.021000000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Mon Oct 9 04:58:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":1.1539999999999999,"15min":1.4319999999999999},"min":{"1min":0.43099999999999999,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":5.7930000000000001,"5min":6.2069999999999999,"15min":9.7840000000000007},"last":1.982},{"interface":"front","average":{"1min":2.218,"5min":1.2829999999999999,"15min":1.712},"min":{"1min":0.50700000000000001,"5min":0.38,"15min":0.32400000000000001},"max":{"1min":19.548999999999999,"5min":19.548999999999999,"15min":20.164999999999999},"last":0.52100000000000002}]},{"osd":7,"last update":"Mon Oct 9 04:58:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":1.3009999999999999,"15min":2.2309999999999999},"min":{"1min":0.42699999999999999,"5min":0.38800000000000001,"15min":0.35899999999999999},"max":{"1min":19.664999999999999,"5min":19.664999999999999,"15min":59.972000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.6419999999999999,"5min":1.1759999999999999,"15min":1.9390000000000001},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.378},"max":{"1min":9.8710000000000004,"5min":10.167999999999999,"15min":60.011000000000003},"last":0.68100000000000005}]}]},{"osd":0,"up_from":0,"seq":390842024407,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772682,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4638936,"kb_used_data":3857680,"kb_used_omap":7,"kb_used_meta":781240,"kb_avail":89106216,"statfs":{"total":95995035648,"available":91244765184,"internally_reserved":0,"allocated":3950264320,"data_stored":3947976770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":799989841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.96399999999999997,"15min":1.335},"min":{"1min":0.215,"5min":0.192,"15min":0.183},"max":{"1min":6.0490000000000004,"5min":13.042999999999999,"15min":25.899999999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.79500000000000004,"15min":1.0960000000000001},"min":{"1min":0.255,"5min":0.25,"15min":0.19500000000000001},"max":{"1min":1.145,"5min":12.161,"15min":16.550000000000001},"last":0.85899999999999999}]},{"osd":1,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.272,"15min":3.1629999999999998},"min":{"1min":0.27700000000000002,"5min":0.23300000000000001,"15min":0.20499999999999999},"max":{"1min":19.331,"5min":24.765999999999998,"15min":170.172},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.0049999999999999,"5min":1.4450000000000001,"15min":3.3650000000000002},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":19.283999999999999,"5min":29.244,"15min":170.511},"last":0.45400000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.716,"15min":2.8599999999999999},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.248},"max":{"1min":3.298,"5min":45.664000000000001,"15min":161.12799999999999},"last":0.501},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.9339999999999999,"15min":2.919},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":6.5869999999999997,"5min":45.686999999999998,"15min":161.386},"last":0.63100000000000001}]},{"osd":4,"last update":"Mon Oct 9 04:58:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":1.1040000000000001}]},{"osd":5,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.77500000000000002,"15min":3.649},"min":{"1min":0.374,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.72199999999999998,"5min":4.4450000000000003,"15min":272.04700000000003},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.69899999999999995,"5min":1.1240000000000001,"15min":3.7040000000000002},"min":{"1min":0.375,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":3.343,"5min":10.427,"15min":271.89699999999999},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.94999999999999996,"15min":1.4219999999999999},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.5549999999999997,"5min":6.8730000000000002,"15min":16.425999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.86799999999999999,"15min":1.2949999999999999},"min":{"1min":0.40000000000000002,"5min":0.38700000000000001,"15min":0.34699999999999998},"max":{"1min":0.83399999999999996,"5min":7.6120000000000001,"15min":10.977},"last":0.752}]},{"osd":7,"last update":"Mon Oct 9 04:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.155,"15min":1.7529999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.34300000000000003},"max":{"1min":6.4989999999999997,"5min":6.4989999999999997,"15min":16.462},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.93899999999999995,"15min":1.621},"min":{"1min":0.46500000000000002,"5min":0.40799999999999997,"15min":0.373},"max":{"1min":2.9550000000000001,"5min":4.9880000000000004,"15min":10.935},"last":0.79100000000000004}]}]},{"osd":3,"up_from":142,"seq":609885356380,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917836,"kb_used_data":1123204,"kb_used_omap":2,"kb_used_meta":794621,"kb_avail":91827316,"statfs":{"total":95995035648,"available":94031171584,"internally_reserved":0,"allocated":1150160896,"data_stored":1147994910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":813691957},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,3,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.65400000000000003,"15min":1.1100000000000001},"min":{"1min":0.29799999999999999,"5min":0.20699999999999999,"15min":0.19},"max":{"1min":3.0219999999999998,"5min":5.2850000000000001,"15min":75.132999999999996},"last":1.556},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.86799999999999999,"15min":1.353},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.22},"max":{"1min":4.9580000000000002,"5min":6.5229999999999997,"15min":76.897999999999996},"last":1.988}]},{"osd":1,"last update":"Mon Oct 9 04:58:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4319999999999999,"5min":1.018,"15min":3.2250000000000001},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.21299999999999999},"max":{"1min":8.5199999999999996,"5min":8.5199999999999996,"15min":125.233},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.034,"15min":3.1349999999999998},"min":{"1min":0.255,"5min":0.25,"15min":0.21299999999999999},"max":{"1min":11.44,"5min":11.44,"15min":126.102},"last":1.8109999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:58:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":2.8340000000000001,"15min":3.0059999999999998},"min":{"1min":0.34399999999999997,"5min":0.27200000000000002,"15min":0.23499999999999999},"max":{"1min":6.7610000000000001,"5min":203.36000000000001,"15min":203.36000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.083,"5min":2.851,"15min":3.1030000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.20799999999999999},"max":{"1min":4.984,"5min":194.125,"15min":194.125},"last":2.0510000000000002}]},{"osd":4,"last update":"Mon Oct 9 04:58:43 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":4.9109999999999996,"5min":4.9109999999999996,"15min":4.9109999999999996},"last":1.7090000000000001},{"interface":"front","average":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":8.5779999999999994,"5min":8.5779999999999994,"15min":8.5779999999999994},"last":0.65600000000000003}]},{"osd":5,"last update":"Mon Oct 9 04:58:59 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":0.98099999999999998,"15min":1.5389999999999999},"min":{"1min":0.38700000000000001,"5min":0.35299999999999998,"15min":0.30599999999999999},"max":{"1min":11.368,"5min":11.368,"15min":28.302},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.218,"15min":1.6799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.30199999999999999},"max":{"1min":4.8630000000000004,"5min":24.986999999999998,"15min":28.280000000000001},"last":2.085}]},{"osd":6,"last update":"Mon Oct 9 04:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.91000000000000003,"15min":1.2989999999999999},"min":{"1min":0.41799999999999998,"5min":0.372,"15min":0.32900000000000001},"max":{"1min":4.9359999999999999,"5min":5.1589999999999998,"15min":16.439},"last":2.2269999999999999},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.4339999999999999,"15min":1.6479999999999999},"min":{"1min":0.48299999999999998,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":8.6069999999999993,"5min":24.916,"15min":24.916},"last":0.871}]},{"osd":7,"last update":"Mon Oct 9 04:58:59 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.2869999999999999,"15min":1.863},"min":{"1min":0.46000000000000002,"5min":0.40200000000000002,"15min":0.375},"max":{"1min":8.6370000000000005,"5min":25.004000000000001,"15min":25.004000000000001},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.77400000000000002,"15min":1.3899999999999999},"min":{"1min":0.45700000000000002,"5min":0.42799999999999999,"15min":0.41599999999999998},"max":{"1min":2.2869999999999999,"5min":3.875,"15min":12.486000000000001},"last":0.76100000000000001}]}]},{"osd":4,"up_from":507,"seq":2177548419091,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3977416,"kb_used_data":3580608,"kb_used_omap":11,"kb_used_meta":396788,"kb_avail":89767736,"statfs":{"total":95995035648,"available":91922161664,"internally_reserved":0,"allocated":3666542592,"data_stored":3665324551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":406311816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.4790000000000001,"5min":5.4790000000000001,"15min":5.4790000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.4770000000000003,"5min":5.4770000000000003,"15min":5.4770000000000003},"last":0.45300000000000001}]},{"osd":1,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":8.4550000000000001,"5min":8.4550000000000001,"15min":8.4550000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":147.49600000000001,"5min":147.49600000000001,"15min":147.49600000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":8.7200000000000006,"5min":8.7200000000000006,"15min":8.7200000000000006},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":147.51300000000001,"5min":147.51300000000001,"15min":147.51300000000001},"last":0.54100000000000004}]},{"osd":2,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":11.055999999999999,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.377}]},{"osd":3,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.5679999999999996,"5min":4.5679999999999996,"15min":4.5679999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.4540000000000002,"5min":2.4540000000000002,"15min":2.4540000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":20.388000000000002,"5min":20.388000000000002,"15min":20.388000000000002},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5199999999999996,"5min":4.5199999999999996,"15min":4.5199999999999996},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.508,"5min":1.508,"15min":1.508},"last":0.55400000000000005}]},{"osd":7,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.5949999999999998,"5min":4.5949999999999998,"15min":4.5949999999999998},"last":0.56999999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757898,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5400200,"kb_used_data":3915072,"kb_used_omap":0,"kb_used_meta":1485120,"kb_avail":88344952,"statfs":{"total":95995035648,"available":90465230848,"internally_reserved":0,"allocated":4009033728,"data_stored":4007807795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1520762880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 04:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":1.9079999999999999,"15min":2.3399999999999999},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":1.107,"5min":106.465,"15min":163.70699999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.3939999999999999,"5min":2.2269999999999999,"15min":2.371},"min":{"1min":0.28199999999999997,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":11.052,"5min":106.54600000000001,"15min":163.756},"last":0.34300000000000003}]},{"osd":1,"last update":"Mon Oct 9 04:58:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.82299999999999995,"15min":4.9119999999999999},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.4129999999999998,"5min":5,"15min":211.85400000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.038,"5min":0.91700000000000004,"15min":5.0960000000000001},"min":{"1min":0.30599999999999999,"5min":0.30099999999999999,"15min":0.255},"max":{"1min":11.249000000000001,"5min":11.249000000000001,"15min":211.435},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 04:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.85799999999999998,"15min":3.1030000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.2559999999999998,"5min":4.8689999999999998,"15min":86.488},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1970000000000001,"15min":3.3580000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":5.9729999999999999,"5min":8.4239999999999995,"15min":86.510999999999996},"last":0.52900000000000003}]},{"osd":3,"last update":"Mon Oct 9 04:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.2149999999999999,"15min":2.1269999999999998},"min":{"1min":0.36499999999999999,"5min":0.34399999999999997,"15min":0.30599999999999999},"max":{"1min":1.9339999999999999,"5min":105.879,"15min":105.879},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.4450000000000001,"5min":2.3210000000000002,"15min":1.837},"min":{"1min":0.375,"5min":0.316,"15min":0.29799999999999999},"max":{"1min":8.1910000000000007,"5min":105.625,"15min":105.625},"last":0.746}]},{"osd":4,"last update":"Mon Oct 9 04:58:39 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.2360000000000002,"5min":2.2360000000000002,"15min":2.2360000000000002},"last":0.434},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.40400000000000003}]},{"osd":6,"last update":"Mon Oct 9 04:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.61899999999999999,"15min":1.028},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.1680000000000001,"5min":4.9420000000000002,"15min":13.162000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.754,"15min":1.127},"min":{"1min":0.317,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.66200000000000003,"5min":5.04,"15min":13.083},"last":0.57599999999999996}]},{"osd":7,"last update":"Mon Oct 9 04:58:45 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.72499999999999998,"15min":2.3639999999999999},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78600000000000003,"5min":5.0839999999999996,"15min":110.521},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.73699999999999999,"15min":2.4260000000000002},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.266,"5min":5.0259999999999998,"15min":110.746},"last":1.306}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":197722112,"data_stored":197722112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":846856192,"data_stored":846856192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3943108608,"data_stored":3943104536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1143144448,"data_stored":1143144448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3661172736,"data_stored":3661168664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4000583680,"data_stored":4000579608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":453578752,"data_stored":453574680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2581463040,"data_stored":2581463040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T04:59:09.856 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pgp_num 26 2023-10-09T04:59:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:09.858 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:09.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:09.955 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T04:59:09.955 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T04:59:09.955 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T04:59:09.955 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T04:59:09.955 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T04:59:09.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:09.997 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:10.066 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:10.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:10.170 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:10.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:10.214 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:10.291 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:59:10.311 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:59:10.335 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:59:10.357 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:59:10.376 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:59:10.383 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:10.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:10.395 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:59:10.414 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:59:10.435 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:59:10.488 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:10.520 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:10.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:10.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T04:59:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:10.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:10.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:10.890 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:10.945 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:10.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:11.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:11.078 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:11.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:11 smithi029 ceph-mon[223858]: pgmap v1817: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 716 B/s wr, 40 op/s; 70839/222636 objects degraded (31.818%); 13863/222636 objects misplaced (6.227%); 10 MiB/s, 3 keys/s, 179 objects/s recovering 2023-10-09T04:59:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3973321155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:11.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 70839/222636 objects degraded (31.818%), 33 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:11 smithi029 ceph-mon[216903]: pgmap v1817: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 716 B/s wr, 40 op/s; 70839/222636 objects degraded (31.818%); 13863/222636 objects misplaced (6.227%); 10 MiB/s, 3 keys/s, 179 objects/s recovering 2023-10-09T04:59:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3973321155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 70839/222636 objects degraded (31.818%), 33 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:11.186 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T04:59:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:11 smithi177 ceph-mon[195797]: pgmap v1817: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 716 B/s wr, 40 op/s; 70839/222636 objects degraded (31.818%); 13863/222636 objects misplaced (6.227%); 10 MiB/s, 3 keys/s, 179 objects/s recovering 2023-10-09T04:59:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3973321155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 70839/222636 objects degraded (31.818%), 33 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:11.466 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:11.480 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:11.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:11.585 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:11.621 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:12.307 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:12.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:12.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:12.443 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:12.748 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:12.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:12.852 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:12.884 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:12.884 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:12.890 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:12.943 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:12.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:12.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:13.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:13.087 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:13.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:13 smithi029 ceph-mon[216903]: pgmap v1818: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 716 B/s wr, 23 op/s; 70681/222621 objects degraded (31.749%); 13705/222621 objects misplaced (6.156%); 3.6 MiB/s, 3 keys/s, 60 objects/s recovering 2023-10-09T04:59:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:59:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:59:12] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.43.0" 2023-10-09T04:59:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:13 smithi029 ceph-mon[223858]: pgmap v1818: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 716 B/s wr, 23 op/s; 70681/222621 objects degraded (31.749%); 13705/222621 objects misplaced (6.156%); 3.6 MiB/s, 3 keys/s, 60 objects/s recovering 2023-10-09T04:59:13.194 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:13.198 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:13 smithi177 ceph-mon[195797]: pgmap v1818: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 716 B/s wr, 23 op/s; 70681/222621 objects degraded (31.749%); 13705/222621 objects misplaced (6.156%); 3.6 MiB/s, 3 keys/s, 60 objects/s recovering 2023-10-09T04:59:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:13.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:13.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:13.330 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:13.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T04:59:13.735 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:13.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:13.839 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:13.875 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:14.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:59:14.035 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:14.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:14.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:14.173 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:14.223 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:14.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:14.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:14.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:14.362 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:14.412 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:14.434 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:14.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:14.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:14.542 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:14.614 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:14.615 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T04:59:14.641 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:14.641 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[216903]: pgmap v1819: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 70681/222621 objects degraded (31.749%); 13705/222621 objects misplaced (6.156%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2565387763' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[223858]: pgmap v1819: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 70681/222621 objects degraded (31.749%); 13705/222621 objects misplaced (6.156%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2565387763' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:14.642 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:14 smithi177 ceph-mon[195797]: pgmap v1819: 109 pgs: 1 activating+undersized+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 22 op/s; 70681/222621 objects degraded (31.749%); 13705/222621 objects misplaced (6.156%); 3.6 MiB/s, 58 objects/s recovering 2023-10-09T04:59:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2565387763' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:15.603 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 14 pgp_num to 26 2023-10-09T04:59:15.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2223239043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[223858]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2223239043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:59:15.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:15 smithi029 ceph-mon[216903]: failed to encode map e542 with expected crc 2023-10-09T04:59:15.778 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:15.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:15.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:15.926 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: failed to encode map e542 with expected crc 2023-10-09T04:59:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: failed to encode map e542 with expected crc 2023-10-09T04:59:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: osdmap e542: 8 total, 8 up, 7 in 2023-10-09T04:59:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: failed to encode map e542 with expected crc 2023-10-09T04:59:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2223239043' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:59:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T04:59:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:15 smithi177 ceph-mon[195797]: failed to encode map e542 with expected crc 2023-10-09T04:59:16.365 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:16.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:16.469 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:16.498 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:16.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: pgmap v1821: 109 pgs: 1 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 70516/222603 objects degraded (31.678%); 13153/222603 objects misplaced (5.909%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 70681/222621 objects degraded (31.749%), 33 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: pgmap v1821: 109 pgs: 1 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 70516/222603 objects degraded (31.678%); 13153/222603 objects misplaced (5.909%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: failed to encode map e543 with expected crc 2023-10-09T04:59:16.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 70681/222621 objects degraded (31.749%), 33 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:16.951 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:16.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:16.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: pgmap v1821: 109 pgs: 1 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 70516/222603 objects degraded (31.678%); 13153/222603 objects misplaced (5.909%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T04:59:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T04:59:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: osdmap e543: 8 total, 8 up, 7 in 2023-10-09T04:59:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: failed to encode map e543 with expected crc 2023-10-09T04:59:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: failed to encode map e543 with expected crc 2023-10-09T04:59:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: failed to encode map e543 with expected crc 2023-10-09T04:59:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: failed to encode map e543 with expected crc 2023-10-09T04:59:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: failed to encode map e543 with expected crc 2023-10-09T04:59:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 70681/222621 objects degraded (31.749%), 33 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:17.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:17.095 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:17.207 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:17.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:17.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:17.346 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:17.398 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:17.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:17.503 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:17.539 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:18.116 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:18.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:18.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:18.221 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:18.263 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:18.314 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:18.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:18.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:18.414 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T04:59:18.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:18.513 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:18.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:18 smithi029 ceph-mon[216903]: pgmap v1823: 109 pgs: 1 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 70516/222603 objects degraded (31.678%); 13153/222603 objects misplaced (5.909%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:59:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:18 smithi029 ceph-mon[223858]: pgmap v1823: 109 pgs: 1 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 70516/222603 objects degraded (31.678%); 13153/222603 objects misplaced (5.909%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:59:18.932 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:18.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:18 smithi177 ceph-mon[195797]: pgmap v1823: 109 pgs: 1 active+undersized+degraded+remapped, 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 70516/222603 objects degraded (31.678%); 13153/222603 objects misplaced (5.909%); 4.0 MiB/s, 64 objects/s recovering 2023-10-09T04:59:19.038 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:19.135 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:19.211 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:19.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:19.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:19.362 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:19.437 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:19.437 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:19.511 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:19.644 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:19.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:19.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:19.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:19.792 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:20.570 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:59:20.596 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:59:20.616 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:59:20.637 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:59:20.660 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:59:20.681 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:59:20.703 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:59:20.730 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.732 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T04:59:20.735 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:20.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:20.840 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:20 smithi029 ceph-mon[223858]: pgmap v1824: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 70334/221964 objects degraded (31.687%); 13070/221964 objects misplaced (5.888%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:59:20.869 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3725574029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:20.869 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:20 smithi029 ceph-mon[216903]: pgmap v1824: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 70334/221964 objects degraded (31.687%); 13070/221964 objects misplaced (5.888%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:59:20.869 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3725574029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:20.872 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:20 smithi177 ceph-mon[195797]: pgmap v1824: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 70334/221964 objects degraded (31.687%); 13070/221964 objects misplaced (5.888%); 4.0 MiB/s, 63 objects/s recovering 2023-10-09T04:59:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3725574029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:21.108 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T04:59:21.128 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:21.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:21.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:21.264 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:21.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 70334/221964 objects degraded (31.687%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:21.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 70334/221964 objects degraded (31.687%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:21.926 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:21.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:21.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 70334/221964 objects degraded (31.687%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:22.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:22.044 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6, 4] out_osds: [0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T04:59:22.045 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T04:59:22.045 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T04:59:22.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T04:59:22.064 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:22.115 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:22.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:22.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:22.220 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:22.250 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:22.306 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:22.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:22.416 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:22.444 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:22.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:22.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:22.599 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:22.627 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:22.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:22.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:22.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:22.822 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:22.875 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:22.879 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:22 smithi177 ceph-mon[195797]: pgmap v1825: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 63 op/s; 70120/221088 objects degraded (31.716%); 12938/221088 objects misplaced (5.852%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:59:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:22.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:22.981 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:23.028 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:23.080 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:59:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:59:22] "GET /metrics HTTP/1.1" 200 36698 "" "Prometheus/2.43.0" 2023-10-09T04:59:23.080 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:22 smithi029 ceph-mon[223858]: pgmap v1825: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 63 op/s; 70120/221088 objects degraded (31.716%); 12938/221088 objects misplaced (5.852%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:59:23.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:22 smithi029 ceph-mon[216903]: pgmap v1825: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 63 op/s; 70120/221088 objects degraded (31.716%); 12938/221088 objects misplaced (5.852%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T04:59:23.091 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:23.215 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:23.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:23.279 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:23.320 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:23.349 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:23.408 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:23.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:23.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:23.520 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:23.561 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:23.823 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:23.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:23.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T04:59:23.933 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:23.960 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:24.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:24.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:24.126 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:24.220 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:24 smithi029 ceph-mon[216903]: pgmap v1826: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 70064/220692 objects degraded (31.747%); 12938/220692 objects misplaced (5.862%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:59:25.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:24 smithi029 ceph-mon[223858]: pgmap v1826: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 70064/220692 objects degraded (31.747%); 12938/220692 objects misplaced (5.862%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:59:25.194 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:25.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:25.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:24 smithi177 ceph-mon[195797]: pgmap v1826: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 70064/220692 objects degraded (31.747%); 12938/220692 objects misplaced (5.862%); 3.6 MiB/s, 57 objects/s recovering 2023-10-09T04:59:25.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:25.363 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:25.644 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:25.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:25.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:25.749 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:25.760 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T04:59:25.760 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T04:59:25.760 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T04:59:25.761 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T04:59:25.761 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T04:59:25.799 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:25.907 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T04:59:26.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:25 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2118255842' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:26.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 70064/220692 objects degraded (31.747%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:26.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:25 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2118255842' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:26.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:26.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 70064/220692 objects degraded (31.747%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:25 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2118255842' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T04:59:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 70064/220692 objects degraded (31.747%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:26.796 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:26.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:26.901 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:26.947 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:26.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:27.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:27.103 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:27.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: pgmap v1827: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 69687/219432 objects degraded (31.758%); 12754/219432 objects misplaced (5.812%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:59:27.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:27.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:27.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T04:59:27.129 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.130 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:26 smithi177 ceph-mon[195797]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.132 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:27.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: pgmap v1827: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 69687/219432 objects degraded (31.758%); 12754/219432 objects misplaced (5.812%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[223858]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: pgmap v1827: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 69687/219432 objects degraded (31.758%); 12754/219432 objects misplaced (5.812%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:59:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T04:59:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T04:59:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: osdmap e544: 8 total, 8 up, 7 in 2023-10-09T04:59:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:26 smithi029 ceph-mon[216903]: failed to encode map e544 with expected crc 2023-10-09T04:59:27.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:27.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:27.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:27.335 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:28.058 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:28.100 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:28.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:28.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:28.234 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:28.278 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:28.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:28.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:28.413 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:28.471 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:28.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:28.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:28.577 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T04:59:28.607 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:28.655 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T04:59:28.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:28.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T04:59:28.760 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:28.849 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:29.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T04:59:29.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:28 smithi029 ceph-mon[216903]: pgmap v1829: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 69687/219432 objects degraded (31.758%); 12754/219432 objects misplaced (5.812%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:59:29.101 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:29.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:28 smithi029 ceph-mon[223858]: pgmap v1829: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 69687/219432 objects degraded (31.758%); 12754/219432 objects misplaced (5.812%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:59:29.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:29.120 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:29.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:29.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:28 smithi177 ceph-mon[195797]: pgmap v1829: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 69687/219432 objects degraded (31.758%); 12754/219432 objects misplaced (5.812%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T04:59:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:29.344 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:29.510 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:29.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:29.634 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:29.662 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:29.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:29.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:29.817 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:29.847 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:29.883 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'32136','3':'65536','5':'34833','7':'45899'}"}]: dispatch 2023-10-09T04:59:29.887 INFO:teuthology.orchestra.run.smithi029.stdout:moved 49 / 327 (14.9847%) 2023-10-09T04:59:29.887 INFO:teuthology.orchestra.run.smithi029.stdout:avg 46.7143 2023-10-09T04:59:29.887 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 14.5574 -> 20.6655 (expected baseline 6.32778) 2023-10-09T04:59:29.887 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 29 -> 19 pgs (0.620795 -> 0.406728 * mean) 2023-10-09T04:59:29.887 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 72 -> 74 pgs (1.54128 -> 1.5841 * mean) 2023-10-09T04:59:29.887 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0353 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0388 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 0.8615 -> 0.5315 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 0.8390 -> 0.7004 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 weight 0.6358 -> 1.0000 2023-10-09T04:59:29.888 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.6825 -> 0.4904 2023-10-09T04:59:29.894 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:29.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:29.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'32136','3':'65536','5':'34833','7':'45899'}"}]: dispatch 2023-10-09T04:59:30.031 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:30.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T04:59:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'32136','3':'65536','5':'34833','7':'45899'}"}]: dispatch 2023-10-09T04:59:30.908 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:30.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:31.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:31.050 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:31.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:31.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: pgmap v1830: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 117 op/s; 69456/218424 objects degraded (31.799%); 12673/218424 objects misplaced (5.802%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: from='client.56797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'32136','3':'65536','5':'34833','7':'45899'}"}]': finished 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 69456/218424 objects degraded (31.799%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[216903]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: pgmap v1830: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 117 op/s; 69456/218424 objects degraded (31.799%); 12673/218424 objects misplaced (5.802%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: from='client.56797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:59:31.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'32136','3':'65536','5':'34833','7':'45899'}"}]': finished 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 69456/218424 objects degraded (31.799%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:31.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:30 smithi029 ceph-mon[223858]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T04:59:31.200 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:31.232 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: pgmap v1830: 109 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 20 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 117 op/s; 69456/218424 objects degraded (31.799%); 12673/218424 objects misplaced (5.802%); 4.3 MiB/s, 68 objects/s recovering 2023-10-09T04:59:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: from='client.56797 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T04:59:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'32136','3':'65536','5':'34833','7':'45899'}"}]': finished 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: osdmap e545: 8 total, 8 up, 7 in 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: failed to encode map e545 with expected crc 2023-10-09T04:59:31.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 69456/218424 objects degraded (31.799%), 34 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:31.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:30 smithi177 ceph-mon[195797]: osdmap e546: 8 total, 8 up, 7 in 2023-10-09T04:59:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[216903]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T04:59:32.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:31 smithi029 ceph-mon[223858]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T04:59:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: failed to encode map e546 with expected crc 2023-10-09T04:59:32.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:31 smithi177 ceph-mon[195797]: osdmap e547: 8 total, 8 up, 7 in 2023-10-09T04:59:33.023 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:33.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:33.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:33.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: pgmap v1833: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 6 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 46505/217560 objects degraded (21.376%); 15029/217560 objects misplaced (6.908%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:59:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:59:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[216903]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T04:59:33.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:59:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:59:32] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-10-09T04:59:33.172 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:33.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: pgmap v1833: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 6 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 46505/217560 objects degraded (21.376%); 15029/217560 objects misplaced (6.908%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:59:33.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:59:33.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:32 smithi029 ceph-mon[223858]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T04:59:33.236 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:33.239 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-09T04:59:33.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-09T04:59:33.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: pgmap v1833: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 6 peering, 10 active+recovery_wait+degraded, 1 active+recovering, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 46505/217560 objects degraded (21.376%); 15029/217560 objects misplaced (6.908%); 3.8 MiB/s, 60 objects/s recovering 2023-10-09T04:59:33.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: failed to encode map e547 with expected crc 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-09T04:59:33.241 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:32 smithi177 ceph-mon[195797]: osdmap e548: 8 total, 8 up, 7 in 2023-10-09T04:59:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:33.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:33.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:33.509 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:33.766 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:33.770 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:33.873 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:33.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:33.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:33.941 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T04:59:33.978 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T04:59:33.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:34.027 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T04:59:34.088 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:34.089 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T04:59:34.141 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T04:59:34.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T04:59:34.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2863826935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T04:59:34.165 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2863826935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:34.173 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T04:59:34.230 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T04:59:34.264 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T04:59:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: failed to encode map e548 with expected crc 2023-10-09T04:59:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: osdmap e549: 8 total, 8 up, 7 in 2023-10-09T04:59:34.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2863826935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T04:59:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.296 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.297 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T04:59:34.297 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T04:59:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: pgmap v1836: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 6 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 16 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 49805/217434 objects degraded (22.906%); 21357/217434 objects misplaced (9.822%); 5.9 MiB/s, 2 keys/s, 152 objects/s recovering 2023-10-09T04:59:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: pgmap v1836: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 6 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 16 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 49805/217434 objects degraded (22.906%); 21357/217434 objects misplaced (9.822%); 5.9 MiB/s, 2 keys/s, 152 objects/s recovering 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: pgmap v1836: 109 pgs: 1 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 6 peering, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 16 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 49805/217434 objects degraded (22.906%); 21357/217434 objects misplaced (9.822%); 5.9 MiB/s, 2 keys/s, 152 objects/s recovering 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: failed to encode map e549 with expected crc 2023-10-09T04:59:35.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T04:59:36.233 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:36.242 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 49805/217434 objects degraded (22.906%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:36.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 49805/217434 objects degraded (22.906%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:36.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 49805/217434 objects degraded (22.906%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:36.344 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:36.383 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:36.496 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:36.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:36.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:36.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:36.651 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:37.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:36 smithi177 ceph-mon[195797]: pgmap v1838: 109 pgs: 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 51889/217320 objects degraded (23.877%); 27530/217320 objects misplaced (12.668%); 3.3 MiB/s, 2 keys/s, 105 objects/s recovering 2023-10-09T04:59:37.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:36 smithi029 ceph-mon[216903]: pgmap v1838: 109 pgs: 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 51889/217320 objects degraded (23.877%); 27530/217320 objects misplaced (12.668%); 3.3 MiB/s, 2 keys/s, 105 objects/s recovering 2023-10-09T04:59:37.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:36 smithi029 ceph-mon[223858]: pgmap v1838: 109 pgs: 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 51889/217320 objects degraded (23.877%); 27530/217320 objects misplaced (12.668%); 3.3 MiB/s, 2 keys/s, 105 objects/s recovering 2023-10-09T04:59:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:39 smithi177 ceph-mon[195797]: pgmap v1839: 109 pgs: 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 51889/217320 objects degraded (23.877%); 27530/217320 objects misplaced (12.668%); 2.6 MiB/s, 1 keys/s, 83 objects/s recovering 2023-10-09T04:59:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:39.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:39 smithi029 ceph-mon[216903]: pgmap v1839: 109 pgs: 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 51889/217320 objects degraded (23.877%); 27530/217320 objects misplaced (12.668%); 2.6 MiB/s, 1 keys/s, 83 objects/s recovering 2023-10-09T04:59:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:39.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:39.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:39 smithi029 ceph-mon[223858]: pgmap v1839: 109 pgs: 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 3 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 69 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 51889/217320 objects degraded (23.877%); 27530/217320 objects misplaced (12.668%); 2.6 MiB/s, 1 keys/s, 83 objects/s recovering 2023-10-09T04:59:39.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:39.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:40.589 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:40 smithi029 ceph-mon[223858]: Health check failed: 1902/72434 objects unfound (2.626%) (OBJECT_UNFOUND) 2023-10-09T04:59:40.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:40 smithi029 ceph-mon[216903]: Health check failed: 1902/72434 objects unfound (2.626%) (OBJECT_UNFOUND) 2023-10-09T04:59:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:40 smithi177 ceph-mon[195797]: Health check failed: 1902/72434 objects unfound (2.626%) (OBJECT_UNFOUND) 2023-10-09T04:59:41.071 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:41.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:41.175 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:41.205 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:41.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:41.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:41.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:41.362 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:41.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:41 smithi029 ceph-mon[216903]: pgmap v1840: 109 pgs: 3 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 62059/217302 objects degraded (28.559%); 26029/217302 objects misplaced (11.978%); 1902/72434 objects unfound (2.626%); 4.7 MiB/s, 1 keys/s, 107 objects/s recovering 2023-10-09T04:59:41.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 62059/217302 objects degraded (28.559%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:41.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:41 smithi029 ceph-mon[216903]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T04:59:41.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:41 smithi029 ceph-mon[223858]: pgmap v1840: 109 pgs: 3 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 62059/217302 objects degraded (28.559%); 26029/217302 objects misplaced (11.978%); 1902/72434 objects unfound (2.626%); 4.7 MiB/s, 1 keys/s, 107 objects/s recovering 2023-10-09T04:59:41.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 62059/217302 objects degraded (28.559%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:41.414 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:41 smithi029 ceph-mon[223858]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T04:59:41.418 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:41 smithi177 ceph-mon[195797]: pgmap v1840: 109 pgs: 3 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 11 op/s; 62059/217302 objects degraded (28.559%); 26029/217302 objects misplaced (11.978%); 1902/72434 objects unfound (2.626%); 4.7 MiB/s, 1 keys/s, 107 objects/s recovering 2023-10-09T04:59:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 62059/217302 objects degraded (28.559%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:41 smithi177 ceph-mon[195797]: osdmap e550: 8 total, 8 up, 7 in 2023-10-09T04:59:42.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:42 smithi029 ceph-mon[216903]: failed to encode map e550 with expected crc 2023-10-09T04:59:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:42 smithi029 ceph-mon[216903]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T04:59:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:42 smithi029 ceph-mon[223858]: failed to encode map e550 with expected crc 2023-10-09T04:59:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:42 smithi029 ceph-mon[223858]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T04:59:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:42 smithi177 ceph-mon[195797]: failed to encode map e550 with expected crc 2023-10-09T04:59:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:42 smithi177 ceph-mon[195797]: osdmap e551: 8 total, 8 up, 7 in 2023-10-09T04:59:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:59:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:59:42] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[216903]: pgmap v1842: 109 pgs: 3 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 62059/217302 objects degraded (28.559%); 26029/217302 objects misplaced (11.978%); 1902/72434 objects unfound (2.626%); 3.0 MiB/s, 50 objects/s recovering 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[216903]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[216903]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[216903]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[216903]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[223858]: pgmap v1842: 109 pgs: 3 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 62059/217302 objects degraded (28.559%); 26029/217302 objects misplaced (11.978%); 1902/72434 objects unfound (2.626%); 3.0 MiB/s, 50 objects/s recovering 2023-10-09T04:59:43.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[223858]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[223858]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[223858]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:43 smithi029 ceph-mon[223858]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:43 smithi177 ceph-mon[195797]: pgmap v1842: 109 pgs: 3 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 62059/217302 objects degraded (28.559%); 26029/217302 objects misplaced (11.978%); 1902/72434 objects unfound (2.626%); 3.0 MiB/s, 50 objects/s recovering 2023-10-09T04:59:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:43 smithi177 ceph-mon[195797]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:43 smithi177 ceph-mon[195797]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:43 smithi177 ceph-mon[195797]: failed to encode map e551 with expected crc 2023-10-09T04:59:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:43 smithi177 ceph-mon[195797]: failed to encode map e551 with expected crc 2023-10-09T04:59:44.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:44 smithi029 ceph-mon[216903]: pgmap v1844: 109 pgs: 4 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 61762/217290 objects degraded (28.424%); 25435/217290 objects misplaced (11.706%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:59:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:44.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:44 smithi029 ceph-mon[223858]: pgmap v1844: 109 pgs: 4 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 61762/217290 objects degraded (28.424%); 25435/217290 objects misplaced (11.706%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:59:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:44 smithi177 ceph-mon[195797]: pgmap v1844: 109 pgs: 4 activating+undersized+degraded+remapped, 3 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 61762/217290 objects degraded (28.424%); 25435/217290 objects misplaced (11.706%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T04:59:45.128 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:45.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:45.232 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:45.269 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:45.327 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:45.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:45.433 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:45.460 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:45.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:45.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:45.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:45.615 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:45.641 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:45.690 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:45.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:45 smithi177 ceph-mon[195797]: Health check cleared: OBJECT_UNFOUND (was: 1902/72434 objects unfound (2.626%)) 2023-10-09T04:59:45.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:45.822 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:45.869 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:45.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:45.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:45 smithi029 ceph-mon[223858]: Health check cleared: OBJECT_UNFOUND (was: 1902/72434 objects unfound (2.626%)) 2023-10-09T04:59:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:45 smithi029 ceph-mon[216903]: Health check cleared: OBJECT_UNFOUND (was: 1902/72434 objects unfound (2.626%)) 2023-10-09T04:59:45.978 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:46.006 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:46.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:46.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:46.157 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:46.187 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:46.233 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:46.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:46.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:46.369 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:46.423 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:46.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:46.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:46.559 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:46.608 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:46.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:46.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:46.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:46 smithi029 ceph-mon[223858]: pgmap v1845: 109 pgs: 4 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 61762/217287 objects degraded (28.424%); 25319/217287 objects misplaced (11.652%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:59:46.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 61762/217290 objects degraded (28.424%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:46.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:46 smithi029 ceph-mon[216903]: pgmap v1845: 109 pgs: 4 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 61762/217287 objects degraded (28.424%); 25319/217287 objects misplaced (11.652%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:59:46.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 61762/217290 objects degraded (28.424%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:46.717 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:46.744 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:46.794 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:46.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:46.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:46.902 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:46.945 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:47.004 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:47.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:47.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:46 smithi177 ceph-mon[195797]: pgmap v1845: 109 pgs: 4 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 61762/217287 objects degraded (28.424%); 25319/217287 objects misplaced (11.652%); 5.8 MiB/s, 93 objects/s recovering 2023-10-09T04:59:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 61762/217290 objects degraded (28.424%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T04:59:47.107 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T04:59:47.137 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:47.191 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T04:59:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:47.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T04:59:47.295 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:47.327 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:47.377 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:47.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:47.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:47.507 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:47.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:47.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:47.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:47.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:47.688 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:47.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:47.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:47.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:47.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T04:59:47.873 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:47.920 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T04:59:47.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:47.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T04:59:48.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T04:59:48.058 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:48.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:47 smithi029 ceph-mon[223858]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T04:59:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:47 smithi029 ceph-mon[223858]: failed to encode map e552 with expected crc 2023-10-09T04:59:48.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:47 smithi029 ceph-mon[223858]: failed to encode map e552 with expected crc 2023-10-09T04:59:48.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:47 smithi029 ceph-mon[216903]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T04:59:48.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:47 smithi029 ceph-mon[216903]: failed to encode map e552 with expected crc 2023-10-09T04:59:48.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:47 smithi029 ceph-mon[216903]: failed to encode map e552 with expected crc 2023-10-09T04:59:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:47 smithi177 ceph-mon[195797]: osdmap e552: 8 total, 8 up, 7 in 2023-10-09T04:59:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:47 smithi177 ceph-mon[195797]: failed to encode map e552 with expected crc 2023-10-09T04:59:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:47 smithi177 ceph-mon[195797]: failed to encode map e552 with expected crc 2023-10-09T04:59:48.415 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T04:59:48.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:48.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T04:59:48.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:48.546 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:48.594 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:48.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:48.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:48.741 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:48.872 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:48.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:48.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T04:59:49.006 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:49.051 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T04:59:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:49.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T04:59:49.131 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[216903]: pgmap v1847: 109 pgs: 4 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 61762/217287 objects degraded (28.424%); 25319/217287 objects misplaced (11.652%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[216903]: failed to encode map e553 with expected crc 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[216903]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[216903]: failed to encode map e553 with expected crc 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[223858]: pgmap v1847: 109 pgs: 4 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 61762/217287 objects degraded (28.424%); 25319/217287 objects misplaced (11.652%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[223858]: failed to encode map e553 with expected crc 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[223858]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T04:59:49.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:48 smithi029 ceph-mon[223858]: failed to encode map e553 with expected crc 2023-10-09T04:59:49.153 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T04:59:49.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T04:59:49.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:48 smithi177 ceph-mon[195797]: pgmap v1847: 109 pgs: 4 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 61762/217287 objects degraded (28.424%); 25319/217287 objects misplaced (11.652%); 3.8 MiB/s, 61 objects/s recovering 2023-10-09T04:59:49.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:48 smithi177 ceph-mon[195797]: failed to encode map e553 with expected crc 2023-10-09T04:59:49.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:48 smithi177 ceph-mon[195797]: osdmap e553: 8 total, 8 up, 7 in 2023-10-09T04:59:49.183 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:48 smithi177 ceph-mon[195797]: failed to encode map e553 with expected crc 2023-10-09T04:59:49.185 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:49.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T04:59:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:49.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T04:59:49.337 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:49.397 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:49.473 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:49.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:49.527 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T04:59:49.527 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T04:59:49.579 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T04:59:49.633 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:49.859 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T04:59:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:49 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/325147127' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:49 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:49 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/325147127' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:49 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:49 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/325147127' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:49 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T04:59:50.383 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T04:59:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[216903]: pgmap v1849: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 61301/217206 objects degraded (28.223%); 24285/217206 objects misplaced (11.181%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T04:59:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:59:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[216903]: failed to encode map e554 with expected crc 2023-10-09T04:59:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[216903]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T04:59:51.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 61301/217206 objects degraded (28.223%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[223858]: pgmap v1849: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 61301/217206 objects degraded (28.223%); 24285/217206 objects misplaced (11.181%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T04:59:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:59:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[223858]: failed to encode map e554 with expected crc 2023-10-09T04:59:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[223858]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T04:59:51.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:50 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 61301/217206 objects degraded (28.223%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:50 smithi177 ceph-mon[195797]: pgmap v1849: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 61301/217206 objects degraded (28.223%); 24285/217206 objects misplaced (11.181%); 8.0 MiB/s, 128 objects/s recovering 2023-10-09T04:59:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:50 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T04:59:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:50 smithi177 ceph-mon[195797]: failed to encode map e554 with expected crc 2023-10-09T04:59:51.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:50 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T04:59:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:50 smithi177 ceph-mon[195797]: osdmap e554: 8 total, 8 up, 7 in 2023-10-09T04:59:51.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:50 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 61301/217206 objects degraded (28.223%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:51.848 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T04:59:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:51.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T04:59:51.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T04:59:51.988 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T04:59:53.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:52 smithi029 ceph-mon[216903]: pgmap v1851: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 61301/217158 objects degraded (28.229%); 24285/217158 objects misplaced (11.183%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:59:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 04:59:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:04:59:52] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2023-10-09T04:59:53.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:52 smithi029 ceph-mon[223858]: pgmap v1851: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 61301/217158 objects degraded (28.229%); 24285/217158 objects misplaced (11.183%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:59:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:52 smithi177 ceph-mon[195797]: pgmap v1851: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 61301/217158 objects degraded (28.229%); 24285/217158 objects misplaced (11.183%); 6.0 MiB/s, 96 objects/s recovering 2023-10-09T04:59:55.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:54 smithi029 ceph-mon[216903]: pgmap v1852: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 60571/217158 objects degraded (27.893%); 22825/217158 objects misplaced (10.511%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:59:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:54 smithi029 ceph-mon[223858]: pgmap v1852: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 60571/217158 objects degraded (27.893%); 22825/217158 objects misplaced (10.511%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:59:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:54 smithi177 ceph-mon[195797]: pgmap v1852: 109 pgs: 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 2 unknown, 1 remapped+peering, 7 peering, 1 active+recovering+undersized+degraded+remapped, 19 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 60571/217158 objects degraded (27.893%); 22825/217158 objects misplaced (10.511%); 12 MiB/s, 193 objects/s recovering 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 60571/217158 objects degraded (27.893%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[216903]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 60571/217158 objects degraded (27.893%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:55 smithi029 ceph-mon[223858]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T04:59:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: osdmap e555: 8 total, 8 up, 7 in 2023-10-09T04:59:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: failed to encode map e555 with expected crc 2023-10-09T04:59:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 60571/217158 objects degraded (27.893%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T04:59:56.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:55 smithi177 ceph-mon[195797]: osdmap e556: 8 total, 8 up, 7 in 2023-10-09T04:59:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: pgmap v1854: 109 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 85776/217035 objects degraded (39.522%); 18406/217035 objects misplaced (8.481%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:59:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:59:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[216903]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T04:59:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: pgmap v1854: 109 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 85776/217035 objects degraded (39.522%); 18406/217035 objects misplaced (8.481%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:59:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:59:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:56 smithi029 ceph-mon[223858]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T04:59:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: pgmap v1854: 109 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 85776/217035 objects degraded (39.522%); 18406/217035 objects misplaced (8.481%); 8.1 MiB/s, 129 objects/s recovering 2023-10-09T04:59:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-09T04:59:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: failed to encode map e556 with expected crc 2023-10-09T04:59:57.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:56 smithi177 ceph-mon[195797]: osdmap e557: 8 total, 8 up, 7 in 2023-10-09T04:59:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[216903]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[216903]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[216903]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[216903]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[216903]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[223858]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[223858]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[223858]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[223858]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:57 smithi029 ceph-mon[223858]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:57 smithi177 ceph-mon[195797]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:57 smithi177 ceph-mon[195797]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:57 smithi177 ceph-mon[195797]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:57 smithi177 ceph-mon[195797]: failed to encode map e557 with expected crc 2023-10-09T04:59:58.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:57 smithi177 ceph-mon[195797]: failed to encode map e557 with expected crc 2023-10-09T04:59:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:58 smithi177 ceph-mon[195797]: pgmap v1857: 109 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 85776/217035 objects degraded (39.522%); 18406/217035 objects misplaced (8.481%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:59:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 04:59:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:58 smithi029 ceph-mon[216903]: pgmap v1857: 109 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 85776/217035 objects degraded (39.522%); 18406/217035 objects misplaced (8.481%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:59:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:59.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 04:59:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:59.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:58 smithi029 ceph-mon[223858]: pgmap v1857: 109 pgs: 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 6 op/s; 85776/217035 objects degraded (39.522%); 18406/217035 objects misplaced (8.481%); 9.2 MiB/s, 147 objects/s recovering 2023-10-09T04:59:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T04:59:59.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 04:59:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T04:59:59.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T04:59:59.733 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T04:59:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T04:59:59.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T04:59:59.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:00.014 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:00.046 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:00.126 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:00.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:00.231 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:00.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:00.442 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:00.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:00.550 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:00.591 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:00.697 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:00.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:00.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:00.861 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pgmap v1858: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 38 op/s; 81521/216468 objects degraded (37.660%); 21252/216468 objects misplaced (9.818%) 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: Health detail: HEALTH_WARN Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: [WRN] PG_DEGRADED: Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 1.4 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.0 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-10-09T05:00:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.1 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.2 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.4 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.5 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.7 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.8 is stuck undersized for 70s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-10-09T05:00:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.9 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.b is activating+undersized+degraded+remapped, acting [1,6] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.c is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.d is stuck undersized for 61s, current state active+recovering+undersized+remapped, last acting [6,5] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.e is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,6] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.f is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.10 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.12 is active+recovering+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.13 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.14 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.15 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-10-09T05:00:01.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.16 is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.17 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-10-09T05:00:01.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.18 is stuck undersized for 64s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-10-09T05:00:01.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: pg 14.19 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:01.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:01.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pgmap v1858: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 38 op/s; 81521/216468 objects degraded (37.660%); 21252/216468 objects misplaced (9.818%) 2023-10-09T05:00:01.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: Health detail: HEALTH_WARN Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized 2023-10-09T05:00:01.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: [WRN] PG_DEGRADED: Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized 2023-10-09T05:00:01.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 1.4 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2023-10-09T05:00:01.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.0 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.1 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.2 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.4 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.5 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.7 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-10-09T05:00:01.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.8 is stuck undersized for 70s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.9 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.b is activating+undersized+degraded+remapped, acting [1,6] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.c is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.d is stuck undersized for 61s, current state active+recovering+undersized+remapped, last acting [6,5] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.e is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,6] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.f is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.10 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.12 is active+recovering+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.13 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T05:00:01.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.14 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.15 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-10-09T05:00:01.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.16 is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.17 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-10-09T05:00:01.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.18 is stuck undersized for 64s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-10-09T05:00:01.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: pg 14.19 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:01.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pgmap v1858: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 38 op/s; 81521/216468 objects degraded (37.660%); 21252/216468 objects misplaced (9.818%) 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: Health detail: HEALTH_WARN Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: [WRN] PG_DEGRADED: Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 1.4 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.0 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.1 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.2 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.3 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.4 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.5 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.7 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.8 is stuck undersized for 70s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.9 is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.b is activating+undersized+degraded+remapped, acting [1,6] 2023-10-09T05:00:01.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.c is stuck undersized for 60s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,4] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.d is stuck undersized for 61s, current state active+recovering+undersized+remapped, last acting [6,5] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.e is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,6] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.f is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.10 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.12 is active+recovering+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.13 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.14 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2023-10-09T05:00:01.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.15 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,7] 2023-10-09T05:00:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.16 is active+recovery_wait+undersized+degraded+remapped, acting [1,3] 2023-10-09T05:00:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.17 is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2023-10-09T05:00:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.18 is stuck undersized for 64s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-10-09T05:00:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: pg 14.19 is stuck undersized for 61s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3] 2023-10-09T05:00:01.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 81521/216468 objects degraded (37.660%), 25 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:01.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:00:01.907 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:01.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:01.976 INFO:teuthology.orchestra.run.smithi029.stdout:moved 27 / 327 (8.25688%) 2023-10-09T05:00:01.976 INFO:teuthology.orchestra.run.smithi029.stdout:avg 46.7143 2023-10-09T05:00:01.976 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 20.6655 -> 25.3079 (expected baseline 6.32778) 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 19 -> 16 pgs (0.406728 -> 0.342508 * mean) 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 74 -> 77 pgs (1.5841 -> 1.64832 * mean) 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:oload 110 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 3 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0355 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.0390 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 weight 0.5315 -> 0.3312 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 weight 0.7004 -> 0.5959 2023-10-09T05:00:01.977 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 weight 0.4904 -> 0.3550 2023-10-09T05:00:01.978 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 weight 0.8585 -> 0.7489 2023-10-09T05:00:02.017 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:02.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:02.048 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:02.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:02.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:02.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:01 smithi029 ceph-mon[216903]: from='client.37299 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'23266','4':'49078','5':'21705','7':'39052'}"}]: dispatch 2023-10-09T05:00:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:01 smithi029 ceph-mon[223858]: from='client.37299 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'23266','4':'49078','5':'21705','7':'39052'}"}]: dispatch 2023-10-09T05:00:02.208 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:02.239 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:01 smithi177 ceph-mon[195797]: from='client.37299 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:02.240 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'23266','4':'49078','5':'21705','7':'39052'}"}]: dispatch 2023-10-09T05:00:02.243 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:02.297 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:02.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:02.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:02.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:02.434 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:03.147 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:03.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: pgmap v1859: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 81323/215652 objects degraded (37.710%); 21252/215652 objects misplaced (9.855%) 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'23266','4':'49078','5':'21705','7':'39052'}"}]': finished 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[216903]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.159 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:00:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:00:02] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2023-10-09T05:00:03.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: pgmap v1859: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 81323/215652 objects degraded (37.710%); 21252/215652 objects misplaced (9.855%) 2023-10-09T05:00:03.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'23266','4':'49078','5':'21705','7':'39052'}"}]': finished 2023-10-09T05:00:03.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T05:00:03.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:02 smithi029 ceph-mon[223858]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.259 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:03.262 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-09T05:00:03.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-09T05:00:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: pgmap v1859: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 81323/215652 objects degraded (37.710%); 21252/215652 objects misplaced (9.855%) 2023-10-09T05:00:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'23266','4':'49078','5':'21705','7':'39052'}"}]': finished 2023-10-09T05:00:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: osdmap e558: 8 total, 8 up, 7 in 2023-10-09T05:00:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:02 smithi177 ceph-mon[195797]: failed to encode map e558 with expected crc 2023-10-09T05:00:03.289 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:03.334 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:03.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:03.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:03.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:03.554 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:03.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T05:00:03.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[216903]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:03.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:03 smithi029 ceph-mon[223858]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: osdmap e559: 8 total, 8 up, 7 in 2023-10-09T05:00:04.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:04.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:03 smithi177 ceph-mon[195797]: failed to encode map e559 with expected crc 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[216903]: pgmap v1862: 109 pgs: 1 remapped+peering, 10 peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 81211/215205 objects degraded (37.737%); 21252/215205 objects misplaced (9.875%) 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[216903]: failed to encode map e560 with expected crc 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[216903]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[216903]: failed to encode map e560 with expected crc 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[223858]: pgmap v1862: 109 pgs: 1 remapped+peering, 10 peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 81211/215205 objects degraded (37.737%); 21252/215205 objects misplaced (9.875%) 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[223858]: failed to encode map e560 with expected crc 2023-10-09T05:00:05.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[223858]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T05:00:05.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:04 smithi029 ceph-mon[223858]: failed to encode map e560 with expected crc 2023-10-09T05:00:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:04 smithi177 ceph-mon[195797]: pgmap v1862: 109 pgs: 1 remapped+peering, 10 peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 81211/215205 objects degraded (37.737%); 21252/215205 objects misplaced (9.875%) 2023-10-09T05:00:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:04 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-09T05:00:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:04 smithi177 ceph-mon[195797]: failed to encode map e560 with expected crc 2023-10-09T05:00:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:04 smithi177 ceph-mon[195797]: osdmap e560: 8 total, 8 up, 7 in 2023-10-09T05:00:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:04 smithi177 ceph-mon[195797]: failed to encode map e560 with expected crc 2023-10-09T05:00:05.297 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:05.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:05.327 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:00:05.337 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:05.403 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:05.431 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:05.481 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:05.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:05.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:05.613 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:05.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:05.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:05.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:05.775 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:05.801 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:05.851 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:05.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:05.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:05.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:06.007 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:06.066 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:06.067 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:06.093 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:06.093 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:05 smithi029 ceph-mon[223858]: failed to encode map e560 with expected crc 2023-10-09T05:00:06.093 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3417723931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:06.093 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:05 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 81211/215205 objects degraded (37.737%), 24 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:06.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:05 smithi029 ceph-mon[216903]: failed to encode map e560 with expected crc 2023-10-09T05:00:06.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3417723931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:06.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:05 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 81211/215205 objects degraded (37.737%), 24 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:06.220 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:06.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:06.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:05 smithi177 ceph-mon[195797]: failed to encode map e560 with expected crc 2023-10-09T05:00:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3417723931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:05 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 81211/215205 objects degraded (37.737%), 24 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:06.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:06.356 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:06.557 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:06.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:06.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:06.690 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:06 smithi177 ceph-mon[195797]: pgmap v1864: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 10 peering, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 85165/214794 objects degraded (39.650%); 17245/214794 objects misplaced (8.029%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:00:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:06 smithi029 ceph-mon[216903]: pgmap v1864: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 10 peering, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 85165/214794 objects degraded (39.650%); 17245/214794 objects misplaced (8.029%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:00:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:06 smithi029 ceph-mon[223858]: pgmap v1864: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 10 peering, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 85165/214794 objects degraded (39.650%); 17245/214794 objects misplaced (8.029%); 3.2 MiB/s, 50 objects/s recovering 2023-10-09T05:00:08.256 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:00:08.280 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:00:08.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:00:08.301 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:08.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:08.309 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:00:08.331 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:00:08.366 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:00:08.399 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:00:08.409 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:08.434 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:00:08.435 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:08.436 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.436 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.436 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.436 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.436 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.437 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:00:08.479 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:08.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:08.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:08.617 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:09.071 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:09.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:09.176 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:09.202 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:09.245 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:09.248 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:08 smithi177 ceph-mon[195797]: pgmap v1865: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 10 peering, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 85165/214794 objects degraded (39.650%); 17245/214794 objects misplaced (8.029%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:00:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:09.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:09.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:09.376 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:09.419 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:09.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:08 smithi029 ceph-mon[216903]: pgmap v1865: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 10 peering, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 85165/214794 objects degraded (39.650%); 17245/214794 objects misplaced (8.029%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:00:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:08 smithi029 ceph-mon[223858]: pgmap v1865: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 10 peering, 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 47 op/s; 85165/214794 objects degraded (39.650%); 17245/214794 objects misplaced (8.029%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:00:09.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:09.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:09.524 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:09.551 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:09.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:09.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:09.703 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:09.730 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:09.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:09.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:09.881 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:09.908 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:09.955 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:09.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:09.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:10.042 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:00:10.060 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:10.094 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1951215055' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:00:10.094 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:00:10.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1951215055' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:00:10.094 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:00:10.098 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:10.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1951215055' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:00:10.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:00:10.994 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:10.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:11.099 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:11.126 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: pgmap v1866: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 65 op/s; 92842/214164 objects degraded (43.351%); 13398/214164 objects misplaced (6.256%); 3.7 MiB/s, 66 objects/s recovering 2023-10-09T05:00:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:00:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:11.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 92842/214164 objects degraded (43.351%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:11.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: pgmap v1866: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 65 op/s; 92842/214164 objects degraded (43.351%); 13398/214164 objects misplaced (6.256%); 3.7 MiB/s, 66 objects/s recovering 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T05:00:11.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 92842/214164 objects degraded (43.351%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:11.174 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:11.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: pgmap v1866: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 65 op/s; 92842/214164 objects degraded (43.351%); 13398/214164 objects misplaced (6.256%); 3.7 MiB/s, 66 objects/s recovering 2023-10-09T05:00:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:00:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:00:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: osdmap e561: 8 total, 8 up, 7 in 2023-10-09T05:00:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: failed to encode map e561 with expected crc 2023-10-09T05:00:11.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 92842/214164 objects degraded (43.351%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:11.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:11.313 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:11.825 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:11.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:11.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:11.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:00:11.852 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:11.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:11.973 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:12.216 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:12.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:12.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:12.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:12.373 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:12.587 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:12.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:12.638 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2023-10-09T05:00:12.638 INFO:teuthology.orchestra.run.smithi029.stdout:avg 46.7143 2023-10-09T05:00:12.638 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 25.3079 -> 25.3079 (expected baseline 6.32778) 2023-10-09T05:00:12.638 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 16 -> 16 pgs (0.342508 -> 0.342508 * mean) 2023-10-09T05:00:12.638 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 77 -> 77 pgs (1.64832 -> 1.64832 * mean) 2023-10-09T05:00:12.638 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:00:12.639 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T05:00:12.639 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2023-10-09T05:00:12.639 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T05:00:12.639 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0356 2023-10-09T05:00:12.639 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.3559 2023-10-09T05:00:12.640 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T05:00:12.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:12.718 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:12.766 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:12.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:12.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:12.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:12.901 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:12.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:00:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:00:12] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[216903]: pgmap v1868: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 44 op/s; 92842/214131 objects degraded (43.358%); 13398/214131 objects misplaced (6.257%); 3.3 MiB/s, 59 objects/s recovering 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[223858]: pgmap v1868: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 44 op/s; 92842/214131 objects degraded (43.358%); 13398/214131 objects misplaced (6.257%); 3.3 MiB/s, 59 objects/s recovering 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:12.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:00:12.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:12 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:12.954 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:12.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:12 smithi177 ceph-mon[195797]: pgmap v1868: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 44 op/s; 92842/214131 objects degraded (43.358%); 13398/214131 objects misplaced (6.257%); 3.3 MiB/s, 59 objects/s recovering 2023-10-09T05:00:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:00:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:00:13.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:12 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:13.057 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:13.087 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:13.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:00:13.915 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:13 smithi029 ceph-mon[223858]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:13.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:13.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:13.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:13 smithi029 ceph-mon[216903]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:13.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:13.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:13 smithi177 ceph-mon[195797]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:14.040 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:14.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:14.145 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:14.173 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:14.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:14.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:14.324 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:00:14.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-10-09T05:00:14.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:14.349 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:14.377 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:14.491 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:14.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:14.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:14.595 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:14.622 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:14.668 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:14.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:14.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:14.835 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:14.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:14 smithi029 ceph-mon[223858]: pgmap v1869: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 38 op/s; 92840/214107 objects degraded (43.361%); 13398/214107 objects misplaced (6.258%); 2.7 MiB/s, 49 objects/s recovering 2023-10-09T05:00:14.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:14 smithi029 ceph-mon[216903]: pgmap v1869: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 38 op/s; 92840/214107 objects degraded (43.361%); 13398/214107 objects misplaced (6.258%); 2.7 MiB/s, 49 objects/s recovering 2023-10-09T05:00:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:14 smithi177 ceph-mon[195797]: pgmap v1869: 109 pgs: 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 38 op/s; 92840/214107 objects degraded (43.361%); 13398/214107 objects misplaced (6.258%); 2.7 MiB/s, 49 objects/s recovering 2023-10-09T05:00:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:15.048 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:15.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:15.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:15.177 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:15.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:15.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:15.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:15.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:15.339 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:00:15.339 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:00:15.339 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:00:15.340 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:00:15.340 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:00:15.356 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:15.359 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:15.411 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:15.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:15.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:15.517 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:15.555 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:16.562 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:16.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:16.671 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:16.700 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:16.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:16.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:16.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:16 smithi029 ceph-mon[216903]: pgmap v1870: 109 pgs: 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 92838/214038 objects degraded (43.375%); 11706/214038 objects misplaced (5.469%); 5.0 MiB/s, 87 objects/s recovering 2023-10-09T05:00:16.851 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 92838/214038 objects degraded (43.375%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:16.851 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:16.883 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:16.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:16.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:16 smithi177 ceph-mon[195797]: pgmap v1870: 109 pgs: 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 92838/214038 objects degraded (43.375%); 11706/214038 objects misplaced (5.469%); 5.0 MiB/s, 87 objects/s recovering 2023-10-09T05:00:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 92838/214038 objects degraded (43.375%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:17.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:17.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:16 smithi029 ceph-mon[223858]: pgmap v1870: 109 pgs: 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 92838/214038 objects degraded (43.375%); 11706/214038 objects misplaced (5.469%); 5.0 MiB/s, 87 objects/s recovering 2023-10-09T05:00:17.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 92838/214038 objects degraded (43.375%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:17.125 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:17.314 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:17.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:17.358 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:00:17.428 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:17.540 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:17.712 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:17.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:17.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:17.905 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:18.505 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:00:18.506 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:00:18.507 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.507 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.507 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.507 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.507 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.507 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.508 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:00:18.508 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:00:19.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:18 smithi029 ceph-mon[216903]: pgmap v1871: 109 pgs: 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 92838/214038 objects degraded (43.375%); 11706/214038 objects misplaced (5.469%); 5.0 MiB/s, 87 objects/s recovering 2023-10-09T05:00:19.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2055177998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:19.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:18 smithi029 ceph-mon[223858]: pgmap v1871: 109 pgs: 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 92838/214038 objects degraded (43.375%); 11706/214038 objects misplaced (5.469%); 5.0 MiB/s, 87 objects/s recovering 2023-10-09T05:00:19.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2055177998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:19.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:18 smithi177 ceph-mon[195797]: pgmap v1871: 109 pgs: 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 25 op/s; 92838/214038 objects degraded (43.375%); 11706/214038 objects misplaced (5.469%); 5.0 MiB/s, 87 objects/s recovering 2023-10-09T05:00:19.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2055177998' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:19.287 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:19.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:19.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:19.393 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:19.522 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:19.889 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:19.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:19.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:20.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:20.039 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:20.063 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:20.098 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:20.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:20.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:20.205 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:20.234 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:20.289 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:20.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:20.394 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:20.443 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:20.469 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:20.689 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:20.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:20.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:20.903 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:21.064 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:20 smithi029 ceph-mon[223858]: pgmap v1872: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 92555/213060 objects degraded (43.441%); 10397/213060 objects misplaced (4.880%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:00:21.064 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:20 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 92555/213060 objects degraded (43.441%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:21.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:20 smithi029 ceph-mon[216903]: pgmap v1872: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 92555/213060 objects degraded (43.441%); 10397/213060 objects misplaced (4.880%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:00:21.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:20 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 92555/213060 objects degraded (43.441%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:20 smithi177 ceph-mon[195797]: pgmap v1872: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 92555/213060 objects degraded (43.441%); 10397/213060 objects misplaced (4.880%); 7.2 MiB/s, 115 objects/s recovering 2023-10-09T05:00:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:20 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 92555/213060 objects degraded (43.441%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:21.292 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:21.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:21.397 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:21.427 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:21.484 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:21.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:21.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:21.630 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:21.683 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:21.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:21.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:21.816 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:21.858 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:21.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:21.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:21.944 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:00:21.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:21.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3486004534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:21.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:21.999 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:22.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:22.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:22.064 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:00:22.163 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:22.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3486004534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:22.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3486004534' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:22.192 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:22.242 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:22.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:22.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:22.377 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:22.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:22.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:22.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:22.557 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:23.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:00:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:00:22] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2023-10-09T05:00:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: pgmap v1873: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 39 op/s; 92555/212754 objects degraded (43.503%); 10397/212754 objects misplaced (4.887%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:00:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:00:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T05:00:23.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[223858]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: pgmap v1873: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 39 op/s; 92555/212754 objects degraded (43.503%); 10397/212754 objects misplaced (4.887%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:22 smithi029 ceph-mon[216903]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.259 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:23.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: pgmap v1873: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 39 op/s; 92555/212754 objects degraded (43.503%); 10397/212754 objects misplaced (4.887%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:00:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:00:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: osdmap e562: 8 total, 8 up, 7 in 2023-10-09T05:00:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:22 smithi177 ceph-mon[195797]: failed to encode map e562 with expected crc 2023-10-09T05:00:23.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:23.416 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:23.481 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:23.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:23.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:23.619 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:24.129 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:24.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:24.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:24.276 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:24.429 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:24.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:24.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:24.533 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:24.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:24.563 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:24.610 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:24.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:24.716 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:24.743 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:24.793 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:24.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:24.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:24.930 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:24.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:24.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:24.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:avg 46.7143 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 25.3079 -> 25.3079 (expected baseline 6.32778) 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 16 -> 16 pgs (0.342508 -> 0.342508 * mean) 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 77 -> 77 pgs (1.64832 -> 1.64832 * mean) 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 1 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T05:00:25.004 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0357 2023-10-09T05:00:25.005 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.3569 2023-10-09T05:00:25.012 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T05:00:25.082 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:25.108 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:25.130 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:24 smithi029 ceph-mon[216903]: pgmap v1875: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 66 op/s; 92416/212106 objects degraded (43.571%); 10397/212106 objects misplaced (4.902%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:00:25.131 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:24 smithi029 ceph-mon[223858]: pgmap v1875: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 66 op/s; 92416/212106 objects degraded (43.571%); 10397/212106 objects misplaced (4.902%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:00:25.149 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:25.149 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:25.150 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:25.153 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:24 smithi177 ceph-mon[195797]: pgmap v1875: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 66 op/s; 92416/212106 objects degraded (43.571%); 10397/212106 objects misplaced (4.902%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:00:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:25.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:25.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:25.282 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:25.322 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:00:25.331 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:25.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:25.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:25.496 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:25 smithi029 ceph-mon[216903]: from='client.56836 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:25 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 92416/212106 objects degraded (43.571%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:25 smithi029 ceph-mon[216903]: osdmap e563: 8 total, 8 up, 7 in 2023-10-09T05:00:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:25 smithi029 ceph-mon[223858]: from='client.56836 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:25 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 92416/212106 objects degraded (43.571%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:26.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:25 smithi029 ceph-mon[223858]: osdmap e563: 8 total, 8 up, 7 in 2023-10-09T05:00:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:25 smithi177 ceph-mon[195797]: from='client.56836 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:25 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 92416/212106 objects degraded (43.571%), 30 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:25 smithi177 ceph-mon[195797]: osdmap e563: 8 total, 8 up, 7 in 2023-10-09T05:00:26.838 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:26.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:26.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:27.015 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:27.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:26 smithi029 ceph-mon[223858]: pgmap v1876: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 92239/211365 objects degraded (43.640%); 8435/211365 objects misplaced (3.991%); 9.2 MiB/s, 149 objects/s recovering 2023-10-09T05:00:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:26 smithi029 ceph-mon[223858]: failed to encode map e563 with expected crc 2023-10-09T05:00:27.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:26 smithi029 ceph-mon[223858]: osdmap e564: 8 total, 8 up, 7 in 2023-10-09T05:00:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:26 smithi029 ceph-mon[216903]: pgmap v1876: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 92239/211365 objects degraded (43.640%); 8435/211365 objects misplaced (3.991%); 9.2 MiB/s, 149 objects/s recovering 2023-10-09T05:00:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:26 smithi029 ceph-mon[216903]: failed to encode map e563 with expected crc 2023-10-09T05:00:27.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:26 smithi029 ceph-mon[216903]: osdmap e564: 8 total, 8 up, 7 in 2023-10-09T05:00:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:26 smithi177 ceph-mon[195797]: pgmap v1876: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 92239/211365 objects degraded (43.640%); 8435/211365 objects misplaced (3.991%); 9.2 MiB/s, 149 objects/s recovering 2023-10-09T05:00:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:26 smithi177 ceph-mon[195797]: failed to encode map e563 with expected crc 2023-10-09T05:00:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:26 smithi177 ceph-mon[195797]: osdmap e564: 8 total, 8 up, 7 in 2023-10-09T05:00:28.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:27 smithi029 ceph-mon[216903]: failed to encode map e564 with expected crc 2023-10-09T05:00:28.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:27 smithi029 ceph-mon[216903]: failed to encode map e564 with expected crc 2023-10-09T05:00:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:27 smithi029 ceph-mon[223858]: failed to encode map e564 with expected crc 2023-10-09T05:00:28.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:27 smithi029 ceph-mon[223858]: failed to encode map e564 with expected crc 2023-10-09T05:00:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:27 smithi177 ceph-mon[195797]: failed to encode map e564 with expected crc 2023-10-09T05:00:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:27 smithi177 ceph-mon[195797]: failed to encode map e564 with expected crc 2023-10-09T05:00:28.847 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:28.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:28.957 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:28.986 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:29.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:29.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:29.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:29.173 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:28 smithi177 ceph-mon[195797]: pgmap v1879: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 92239/211365 objects degraded (43.640%); 8435/211365 objects misplaced (3.991%); 9.4 MiB/s, 152 objects/s recovering 2023-10-09T05:00:29.174 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:29.175 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:29.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:28 smithi029 ceph-mon[216903]: pgmap v1879: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 92239/211365 objects degraded (43.640%); 8435/211365 objects misplaced (3.991%); 9.4 MiB/s, 152 objects/s recovering 2023-10-09T05:00:29.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:29.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:28 smithi029 ceph-mon[223858]: pgmap v1879: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 23 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 92239/211365 objects degraded (43.640%); 8435/211365 objects misplaced (3.991%); 9.4 MiB/s, 152 objects/s recovering 2023-10-09T05:00:29.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:29.225 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:29.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:29.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:29.332 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:29.361 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:29.405 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:29.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:29.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:29.510 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:29.540 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:29.620 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:29.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:29.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:29.766 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:30.333 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-09T05:00:30.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-09T05:00:30.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:00:30.568 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:30.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:30.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:30.734 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:30.827 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:31.103 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:31.112 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:30 smithi029 ceph-mon[223858]: pgmap v1880: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 90041/210705 objects degraded (42.733%); 9747/210705 objects misplaced (4.626%); 9.6 MiB/s, 170 objects/s recovering 2023-10-09T05:00:31.112 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:30 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 90041/210705 objects degraded (42.733%), 29 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:31.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:30 smithi029 ceph-mon[216903]: pgmap v1880: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 90041/210705 objects degraded (42.733%); 9747/210705 objects misplaced (4.626%); 9.6 MiB/s, 170 objects/s recovering 2023-10-09T05:00:31.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:30 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 90041/210705 objects degraded (42.733%), 29 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:31.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:31.215 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:31.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:30 smithi177 ceph-mon[195797]: pgmap v1880: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 90041/210705 objects degraded (42.733%); 9747/210705 objects misplaced (4.626%); 9.6 MiB/s, 170 objects/s recovering 2023-10-09T05:00:31.238 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:30 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 90041/210705 objects degraded (42.733%), 29 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:31.241 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:31.287 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:31.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:31.393 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:31.504 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:31.525 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:00:31.568 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:00:31.601 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:31.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:00:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:31.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:31.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:00:31.656 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:00:31.682 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:00:31.705 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:31.708 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:00:31.733 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:00:31.736 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:00:31.749 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:31.861 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:32.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3511273963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:32.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3511273963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3511273963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:33.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:32 smithi029 ceph-mon[216903]: pgmap v1881: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 89871/210153 objects degraded (42.765%); 9747/210153 objects misplaced (4.638%); 9.5 MiB/s, 167 objects/s recovering 2023-10-09T05:00:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:32 smithi029 ceph-mon[223858]: pgmap v1881: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 89871/210153 objects degraded (42.765%); 9747/210153 objects misplaced (4.638%); 9.5 MiB/s, 167 objects/s recovering 2023-10-09T05:00:33.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:00:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:00:32] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2023-10-09T05:00:33.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:32 smithi177 ceph-mon[195797]: pgmap v1881: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 89871/210153 objects degraded (42.765%); 9747/210153 objects misplaced (4.638%); 9.5 MiB/s, 167 objects/s recovering 2023-10-09T05:00:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:34 smithi177 ceph-mon[195797]: pgmap v1882: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 89495/209634 objects degraded (42.691%); 9187/209634 objects misplaced (4.382%); 5.7 MiB/s, 104 objects/s recovering 2023-10-09T05:00:35.367 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:34 smithi029 ceph-mon[223858]: pgmap v1882: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 89495/209634 objects degraded (42.691%); 9187/209634 objects misplaced (4.382%); 5.7 MiB/s, 104 objects/s recovering 2023-10-09T05:00:35.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:34 smithi029 ceph-mon[216903]: pgmap v1882: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 72 op/s; 89495/209634 objects degraded (42.691%); 9187/209634 objects misplaced (4.382%); 5.7 MiB/s, 104 objects/s recovering 2023-10-09T05:00:36.037 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:36.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:36.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:36.171 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:36.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:36.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:36.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:35 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 89495/209634 objects degraded (42.691%), 29 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:36.320 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:36.362 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:35 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 89495/209634 objects degraded (42.691%), 29 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:36.369 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:35 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 89495/209634 objects degraded (42.691%), 29 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:36.371 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:36.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:36.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:36.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:36.588 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:36.682 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:36.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:36.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:36.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:36.813 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:36.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:36.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:36.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:36.991 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:36 smithi029 ceph-mon[216903]: pgmap v1883: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 89288/208860 objects degraded (42.750%); 9164/208860 objects misplaced (4.388%); 5.5 MiB/s, 99 objects/s recovering 2023-10-09T05:00:37.041 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:37.124 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:37.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:37.230 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:37.259 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:36 smithi029 ceph-mon[223858]: pgmap v1883: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 89288/208860 objects degraded (42.750%); 9164/208860 objects misplaced (4.388%); 5.5 MiB/s, 99 objects/s recovering 2023-10-09T05:00:37.262 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:37.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:36 smithi177 ceph-mon[195797]: pgmap v1883: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 89288/208860 objects degraded (42.750%); 9164/208860 objects misplaced (4.388%); 5.5 MiB/s, 99 objects/s recovering 2023-10-09T05:00:37.349 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:37.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:37.455 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:37.509 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:38.039 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:38.057 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:38.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:38.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:38.201 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:38.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:38.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:38.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:38.351 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:38.448 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:38.900 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:38.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:39.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:39.067 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:39.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:39 smithi029 ceph-mon[223858]: pgmap v1884: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 89288/208860 objects degraded (42.750%); 9164/208860 objects misplaced (4.388%); 5.0 MiB/s, 89 objects/s recovering 2023-10-09T05:00:39.117 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:39 smithi029 ceph-mon[216903]: pgmap v1884: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 89288/208860 objects degraded (42.750%); 9164/208860 objects misplaced (4.388%); 5.0 MiB/s, 89 objects/s recovering 2023-10-09T05:00:39.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:39 smithi177 ceph-mon[195797]: pgmap v1884: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 89288/208860 objects degraded (42.750%); 9164/208860 objects misplaced (4.388%); 5.0 MiB/s, 89 objects/s recovering 2023-10-09T05:00:39.985 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:00:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3417349408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:00:40.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:40 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:00:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3417349408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:00:40.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:40 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:00:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3417349408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:00:40.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:40 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:00:40.542 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:00:41.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: pgmap v1885: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 88820/208182 objects degraded (42.665%); 8443/208182 objects misplaced (4.056%); 6.7 MiB/s, 115 objects/s recovering 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: osdmap e565: 8 total, 8 up, 7 in 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: failed to encode map e565 with expected crc 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: failed to encode map e565 with expected crc 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 88820/208182 objects degraded (42.665%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: pgmap v1885: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 88820/208182 objects degraded (42.665%); 8443/208182 objects misplaced (4.056%); 6.7 MiB/s, 115 objects/s recovering 2023-10-09T05:00:41.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:41.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:00:41.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: osdmap e565: 8 total, 8 up, 7 in 2023-10-09T05:00:41.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: failed to encode map e565 with expected crc 2023-10-09T05:00:41.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: failed to encode map e565 with expected crc 2023-10-09T05:00:41.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 88820/208182 objects degraded (42.665%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: pgmap v1885: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 88820/208182 objects degraded (42.665%); 8443/208182 objects misplaced (4.056%); 6.7 MiB/s, 115 objects/s recovering 2023-10-09T05:00:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:00:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:00:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: osdmap e565: 8 total, 8 up, 7 in 2023-10-09T05:00:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: failed to encode map e565 with expected crc 2023-10-09T05:00:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: failed to encode map e565 with expected crc 2023-10-09T05:00:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 88820/208182 objects degraded (42.665%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:42.415 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:42.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:42.521 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:42.853 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:43.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:43 smithi029 ceph-mon[216903]: pgmap v1887: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 84 op/s; 88631/207630 objects degraded (42.687%); 8443/207630 objects misplaced (4.066%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:00:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:00:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:00:42] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2023-10-09T05:00:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:43 smithi029 ceph-mon[223858]: pgmap v1887: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 84 op/s; 88631/207630 objects degraded (42.687%); 8443/207630 objects misplaced (4.066%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:00:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:43 smithi177 ceph-mon[195797]: pgmap v1887: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 84 op/s; 88631/207630 objects degraded (42.687%); 8443/207630 objects misplaced (4.066%); 6.3 MiB/s, 100 objects/s recovering 2023-10-09T05:00:44.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:45.053 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:45.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:45.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:45.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:45.190 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:45.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:45.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:45.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:45.347 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:45.371 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:45 smithi177 ceph-mon[195797]: pgmap v1888: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 83 op/s; 88233/207120 objects degraded (42.600%); 7707/207120 objects misplaced (3.721%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T05:00:45.373 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:45.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:45 smithi029 ceph-mon[216903]: pgmap v1888: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 83 op/s; 88233/207120 objects degraded (42.600%); 7707/207120 objects misplaced (3.721%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T05:00:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:45 smithi029 ceph-mon[223858]: pgmap v1888: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 83 op/s; 88233/207120 objects degraded (42.600%); 7707/207120 objects misplaced (3.721%); 6.8 MiB/s, 108 objects/s recovering 2023-10-09T05:00:45.421 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:45.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:45.527 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:45.553 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:45.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:45.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:45.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:45.704 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:45.732 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:45.774 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:45.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:45.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:45.879 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:45.906 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:45.947 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:45.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:46.055 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:46.151 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 88233/207120 objects degraded (42.600%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:46.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 88233/207120 objects degraded (42.600%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:46.156 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:46.430 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:46.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:46.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:46.451 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 88233/207120 objects degraded (42.600%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:46.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:46.573 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:46.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:46.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:46.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:46.720 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:46.747 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:moved 0 / 327 (0%) 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:avg 46.7143 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:stddev 25.3079 -> 25.3079 (expected baseline 6.32778) 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:min osd.2 with 16 -> 16 pgs (0.342508 -> 0.342508 * mean) 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:max osd.1 with 77 -> 77 pgs (1.64832 -> 1.64832 * mean) 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:oload 1000 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:max_change 0.05 2023-10-09T05:00:46.786 INFO:teuthology.orchestra.run.smithi029.stdout:max_change_osds 4 2023-10-09T05:00:46.787 INFO:teuthology.orchestra.run.smithi029.stdout:average_utilization 0.0357 2023-10-09T05:00:46.787 INFO:teuthology.orchestra.run.smithi029.stdout:overload_utilization 0.3570 2023-10-09T05:00:46.792 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:46.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:46.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:46.797 INFO:teuthology.orchestra.run.smithi029.stderr:no change 2023-10-09T05:00:46.887 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:00:46.887 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:00:46.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.1 565/546 [PushOp(14:878caf41:::benchmark_data_smithi029_550670_object14044:head, version: 444'858, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:878caf41:::benchmark_data_smithi029_550670_object14044:head@444'858, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:00:46.885932+0000", 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 8.3968999999999995e-05, 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013299399999999999, 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "queued for pg", 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.885932+0000", 2023-10-09T05:00:46.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.885932+0000", 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.885930+0000", 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.885963+0000", 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2372000000000001e-05 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.885982+0000", 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9556e-05 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.885996+0000", 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3174e-05 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:86709 14.13 14:cd2bf81a:::benchmark_data_smithi029_550670_object9098:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e565)", 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:00:46.867280+0000", 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.018735631999999999, 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018847681000000002, 2023-10-09T05:00:46.891 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86709 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.867280+0000", 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.867280+0000", 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.867278+0000", 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:00:46.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.867288+0000", 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2089999999999994e-06 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.867305+0000", 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7711e-05 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.867318+0000", 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2248e-05 2023-10-09T05:00:46.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.868342+0000", 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010247259999999999 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.874022+0000", 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056798769999999998 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:00:46.895 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:86668 14.3 14:c67644b1:::benchmark_data_smithi029_550670_object9057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e565)", 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:00:46.291118+0000", 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.59489810799999998, 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.595081215, 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:00:46.896 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 86668 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.291118+0000", 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.291118+0000", 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.291115+0000", 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.291127+0000", 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1606999999999999e-05 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.291154+0000", 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7166999999999999e-05 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.291167+0000", 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2513e-05 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.292910+0000", 2023-10-09T05:00:46.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017438659999999999 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.297145+0000", 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042343550000000004 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.352290+0000", 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.055145211 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.354205+0000", 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001915019 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.732913+0000", 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.37870746500000002 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.736581+0000", 2023-10-09T05:00:46.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036683779999999999 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.744057+0000", 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007476145 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.753259+0000", 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092021259999999997 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.753670+0000", 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041104100000000001 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 565/546 [PushOp(14:cd2bf81a:::benchmark_data_smithi029_550670_object9098:head, version: 444'572, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:cd2bf81a:::benchmark_data_smithi029_550670_object9098:head@444'572, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:00:46.875058+0000", 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.01095775, 2023-10-09T05:00:46.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011245170000000001, 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.875058+0000", 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:00:46.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.875058+0000", 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.875057+0000", 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.875141+0000", 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4302999999999996e-05 2023-10-09T05:00:46.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.875163+0000", 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1365e-05 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.875180+0000", 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7020999999999998e-05 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.876101+0000", 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00092160300000000001 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:00:46.876861+0000", 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00075931000000000002 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:00:46.907 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T05:00:46.908 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 4 2023-10-09T05:00:46.908 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:00:46.908 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:46.935 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:46.978 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:46.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:46.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:47.081 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:47 smithi029 ceph-mon[216903]: pgmap v1889: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 88217/206625 objects degraded (42.694%); 7685/206625 objects misplaced (3.719%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T05:00:47.081 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:47 smithi029 ceph-mon[223858]: pgmap v1889: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 88217/206625 objects degraded (42.694%); 7685/206625 objects misplaced (3.719%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T05:00:47.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:47.169 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:47.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:47 smithi177 ceph-mon[195797]: pgmap v1889: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 88217/206625 objects degraded (42.694%); 7685/206625 objects misplaced (3.719%); 6.1 MiB/s, 98 objects/s recovering 2023-10-09T05:00:47.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:00:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:48 smithi029 ceph-mon[216903]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:48 smithi029 ceph-mon[223858]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:48 smithi177 ceph-mon[195797]: from='client.37347 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-09T05:00:49.357 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:49 smithi029 ceph-mon[216903]: pgmap v1890: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 88003/206265 objects degraded (42.665%); 7685/206265 objects misplaced (3.726%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T05:00:49.358 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:49 smithi029 ceph-mon[223858]: pgmap v1890: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 88003/206265 objects degraded (42.665%); 7685/206265 objects misplaced (3.726%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T05:00:49.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:49 smithi177 ceph-mon[195797]: pgmap v1890: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 88003/206265 objects degraded (42.665%); 7685/206265 objects misplaced (3.726%); 6.9 MiB/s, 109 objects/s recovering 2023-10-09T05:00:50.546 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:50.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:50.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:50.703 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:51.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:51 smithi029 ceph-mon[223858]: pgmap v1891: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 87905/205626 objects degraded (42.750%); 6687/205626 objects misplaced (3.252%); 8.9 MiB/s, 152 objects/s recovering 2023-10-09T05:00:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 87905/205626 objects degraded (42.750%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:51 smithi029 ceph-mon[223858]: osdmap e566: 8 total, 8 up, 7 in 2023-10-09T05:00:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:51 smithi029 ceph-mon[216903]: pgmap v1891: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 87905/205626 objects degraded (42.750%); 6687/205626 objects misplaced (3.252%); 8.9 MiB/s, 152 objects/s recovering 2023-10-09T05:00:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 87905/205626 objects degraded (42.750%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:51 smithi029 ceph-mon[216903]: osdmap e566: 8 total, 8 up, 7 in 2023-10-09T05:00:51.526 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:51 smithi177 ceph-mon[195797]: pgmap v1891: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 87905/205626 objects degraded (42.750%); 6687/205626 objects misplaced (3.252%); 8.9 MiB/s, 152 objects/s recovering 2023-10-09T05:00:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 87905/205626 objects degraded (42.750%), 29 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:51 smithi177 ceph-mon[195797]: osdmap e566: 8 total, 8 up, 7 in 2023-10-09T05:00:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:51.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:51.630 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:51.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:00:51.719 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:51.917 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:51.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:51.942 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:52.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:52.135 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:52.265 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:52.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:52.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:52.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:52.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:52 smithi177 ceph-mon[195797]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:52 smithi177 ceph-mon[195797]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:52 smithi177 ceph-mon[195797]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:52 smithi177 ceph-mon[195797]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:52 smithi177 ceph-mon[195797]: osdmap e567: 8 total, 8 up, 7 in 2023-10-09T05:00:52.408 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:52 smithi177 ceph-mon[195797]: failed to encode map e567 with expected crc 2023-10-09T05:00:52.411 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[223858]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[223858]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[223858]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[223858]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[223858]: osdmap e567: 8 total, 8 up, 7 in 2023-10-09T05:00:52.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[223858]: failed to encode map e567 with expected crc 2023-10-09T05:00:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[216903]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[216903]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[216903]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[216903]: failed to encode map e566 with expected crc 2023-10-09T05:00:52.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[216903]: osdmap e567: 8 total, 8 up, 7 in 2023-10-09T05:00:52.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:52 smithi029 ceph-mon[216903]: failed to encode map e567 with expected crc 2023-10-09T05:00:52.460 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:52.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:52.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:52.544 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:00:52.567 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:52.568 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:00:52.593 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:52.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:00:52.633 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:00:52.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:52.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:52.654 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:00:52.680 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:00:52.712 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:00:52.740 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:00:52.743 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.743 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.743 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.743 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.743 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.744 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.744 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.744 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:00:52.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:52.775 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:52.808 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:52.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:52.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:52.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:52.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:52.931 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:00:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:00:52] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2023-10-09T05:00:52.957 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:53.007 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:53.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:53.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:53.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:53.157 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:53.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: pgmap v1893: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 87693/205026 objects degraded (42.772%); 6687/205026 objects misplaced (3.262%); 8.9 MiB/s, 153 objects/s recovering 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1167582399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: pgmap v1893: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 87693/205026 objects degraded (42.772%); 6687/205026 objects misplaced (3.262%); 8.9 MiB/s, 153 objects/s recovering 2023-10-09T05:00:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1167582399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: pgmap v1893: 109 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 22 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 86 op/s; 87693/205026 objects degraded (42.772%); 6687/205026 objects misplaced (3.262%); 8.9 MiB/s, 153 objects/s recovering 2023-10-09T05:00:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: failed to encode map e567 with expected crc 2023-10-09T05:00:53.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1167582399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:00:54.092 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:54.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:54.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:54.235 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:54.722 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:54.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:54.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:54.856 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:54.900 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:54.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:55.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:55.053 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:55.186 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:55.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:55.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:55.323 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:55 smithi029 ceph-mon[223858]: pgmap v1895: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 81554/204468 objects degraded (39.886%); 12016/204468 objects misplaced (5.877%); 11 MiB/s, 195 objects/s recovering 2023-10-09T05:00:55.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:55 smithi029 ceph-mon[216903]: pgmap v1895: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 81554/204468 objects degraded (39.886%); 12016/204468 objects misplaced (5.877%); 11 MiB/s, 195 objects/s recovering 2023-10-09T05:00:55.332 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:55 smithi177 ceph-mon[195797]: pgmap v1895: 109 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 81554/204468 objects degraded (39.886%); 12016/204468 objects misplaced (5.877%); 11 MiB/s, 195 objects/s recovering 2023-10-09T05:00:55.732 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:55.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:55.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:00:55.888 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:55.965 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:00:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:55.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:00:56.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:00:56.106 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:56.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:00:56.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:56.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:00:56.255 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:56.281 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:56.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:56.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 81554/204468 objects degraded (39.886%), 27 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1520812216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:56.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 81554/204468 objects degraded (39.886%), 27 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:56.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1520812216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:56.434 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:56.494 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6, 4] out_osds: [0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:00:56.494 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:00:56.494 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_9 2023-10-09T05:00:56.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:00:56.520 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 81554/204468 objects degraded (39.886%), 27 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T05:00:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1520812216' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:00:56.669 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:56.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:56.780 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:00:56.866 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:56.949 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:00:56.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:00:56.992 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:00:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:56.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:00:57.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:57.125 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:57.173 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:57.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:57.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:57.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:57.306 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:57.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:57.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:00:57.449 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:00:57.449 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:00:57.468 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:57.492 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:57 smithi177 ceph-mon[195797]: pgmap v1896: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 81522/203949 objects degraded (39.972%); 11983/203949 objects misplaced (5.875%); 10 MiB/s, 184 objects/s recovering 2023-10-09T05:00:57.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:57 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:00:57.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1520812216' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:00:57.493 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:57 smithi177 ceph-mon[195797]: osdmap e568: 8 total, 8 up, 7 in 2023-10-09T05:00:57.495 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:57.541 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:57.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:57.649 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:57.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[216903]: pgmap v1896: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 81522/203949 objects degraded (39.972%); 11983/203949 objects misplaced (5.875%); 10 MiB/s, 184 objects/s recovering 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1520812216' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[216903]: osdmap e568: 8 total, 8 up, 7 in 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[223858]: pgmap v1896: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 81522/203949 objects degraded (39.972%); 11983/203949 objects misplaced (5.875%); 10 MiB/s, 184 objects/s recovering 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1520812216' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:00:57.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:57 smithi029 ceph-mon[223858]: osdmap e568: 8 total, 8 up, 7 in 2023-10-09T05:00:57.675 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:57.718 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:57.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:57.829 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:57.876 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:58 smithi177 ceph-mon[195797]: failed to encode map e568 with expected crc 2023-10-09T05:00:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:58 smithi177 ceph-mon[195797]: failed to encode map e568 with expected crc 2023-10-09T05:00:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:58 smithi029 ceph-mon[216903]: failed to encode map e568 with expected crc 2023-10-09T05:00:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:58 smithi029 ceph-mon[216903]: failed to encode map e568 with expected crc 2023-10-09T05:00:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:58 smithi029 ceph-mon[223858]: failed to encode map e568 with expected crc 2023-10-09T05:00:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:58 smithi029 ceph-mon[223858]: failed to encode map e568 with expected crc 2023-10-09T05:00:58.741 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:58.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:58.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:00:58.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:00:58.933 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:58.967 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:59.024 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:00:59.033 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:00:59.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:59.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:00:59.137 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:00:59.163 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:59.206 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:00:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:59.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:00:59.263 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:59 smithi029 ceph-mon[223858]: pgmap v1898: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 81312/203601 objects degraded (39.937%); 11983/203601 objects misplaced (5.886%); 5.4 MiB/s, 93 objects/s recovering 2023-10-09T05:00:59.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:59.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:00:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:59.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:59 smithi029 ceph-mon[216903]: pgmap v1898: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 81312/203601 objects degraded (39.937%); 11983/203601 objects misplaced (5.886%); 5.4 MiB/s, 93 objects/s recovering 2023-10-09T05:00:59.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:59.264 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:00:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:59.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:00:59.339 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:59.383 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:00:59.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:59.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:00:59.488 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:00:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:59 smithi177 ceph-mon[195797]: pgmap v1898: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 81312/203601 objects degraded (39.937%); 11983/203601 objects misplaced (5.886%); 5.4 MiB/s, 93 objects/s recovering 2023-10-09T05:00:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:00:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:00:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:00:59.547 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:59.771 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:00:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:59.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:00:59.876 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:00:59.902 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:00:59.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:00:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:00:59.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:00.049 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:00.076 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:00.123 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:00.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:00.228 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:00.260 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:00.310 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:00.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:00.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:00.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:00.473 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:00.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:01:00.856 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:00.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:00.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:00.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:00.995 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:01 smithi177 ceph-mon[195797]: pgmap v1899: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 81197/202803 objects degraded (40.037%); 11870/202803 objects misplaced (5.853%); 5.4 MiB/s, 92 objects/s recovering 2023-10-09T05:01:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 81197/202803 objects degraded (40.037%), 26 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:01.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:01 smithi029 ceph-mon[216903]: pgmap v1899: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 81197/202803 objects degraded (40.037%); 11870/202803 objects misplaced (5.853%); 5.4 MiB/s, 92 objects/s recovering 2023-10-09T05:01:01.542 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 81197/202803 objects degraded (40.037%), 26 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:01.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:01 smithi029 ceph-mon[223858]: pgmap v1899: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 92 op/s; 81197/202803 objects degraded (40.037%); 11870/202803 objects misplaced (5.853%); 5.4 MiB/s, 92 objects/s recovering 2023-10-09T05:01:01.543 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 81197/202803 objects degraded (40.037%), 26 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:01.612 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:01.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:01.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:01.801 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:02.001 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:02.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:02.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:02.032 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:02.107 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:02.135 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:02.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:02.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:02.195 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:02.290 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:02.337 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:02.429 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:01:02.431 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:01:02.473 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:02.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:02.540 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:01:02.569 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:01:02.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:02.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:01:02.613 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:02.630 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:01:02.654 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:01:02.664 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:02.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:02.674 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:01:02.702 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:01:02.726 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:01:02.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.727 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.728 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.728 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:01:02.770 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:02.803 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:03.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:01:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:01:02] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.43.0" 2023-10-09T05:01:03.496 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:03 smithi029 ceph-mon[216903]: pgmap v1900: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 80984/202224 objects degraded (40.047%); 11870/202224 objects misplaced (5.870%); 5.2 MiB/s, 88 objects/s recovering 2023-10-09T05:01:03.496 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:03 smithi029 ceph-mon[216903]: from='client.47252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:01:03.496 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:03 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2241319325' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:03.498 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:03 smithi029 ceph-mon[223858]: pgmap v1900: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 80984/202224 objects degraded (40.047%); 11870/202224 objects misplaced (5.870%); 5.2 MiB/s, 88 objects/s recovering 2023-10-09T05:01:03.498 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:03 smithi029 ceph-mon[223858]: from='client.47252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:01:03.499 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:03 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2241319325' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:03 smithi177 ceph-mon[195797]: pgmap v1900: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 80984/202224 objects degraded (40.047%); 11870/202224 objects misplaced (5.870%); 5.2 MiB/s, 88 objects/s recovering 2023-10-09T05:01:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:03 smithi177 ceph-mon[195797]: from='client.47252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:01:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:03 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2241319325' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:03.908 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1900,"stamp":"2023-10-09T05:01:01.618805+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4416608184,"num_objects":67408,"num_object_clones":0,"num_object_copies":202224,"num_objects_missing_on_primary":24624,"num_objects_missing":24624,"num_objects_degraded":80984,"num_objects_misplaced":11870,"num_objects_unfound":0,"num_objects_dirty":67408,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":88134,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42379,"num_bytes_recovered":2699148600,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126078,"ondisk_log_size":126078,"up":327,"acting":305,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":484,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23266384,"kb_used_data":16159192,"kb_used_omap":26,"kb_used_meta":7107109,"kb_avail":632949680,"statfs":{"total":671965249536,"available":648140472320,"internally_reserved":0,"allocated":16547012608,"data_stored":16534011533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26724,"internal_metadata":7277680540},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-61210624,"num_objects":-934,"num_object_clones":0,"num_object_copies":-2802,"num_objects_missing_on_primary":-251,"num_objects_missing":-251,"num_objects_degraded":-6709,"num_objects_misplaced":5183,"num_objects_unfound":0,"num_objects_dirty":-934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":934,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":859,"num_bytes_recovered":52822570,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.710767"},"pg_stats":[{"pgid":"14.17","version":"568'5311","reported_seq":14000,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:57.193504+0000","last_change":"2023-10-09T04:59:46.171318+0000","last_active":"2023-10-09T05:00:57.193504+0000","last_peered":"2023-10-09T05:00:57.193504+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:59:46.097475+0000","last_became_peered":"2023-10-09T04:59:46.097475+0000","last_unstale":"2023-10-09T05:00:57.193504+0000","last_undegraded":"2023-10-09T04:59:37.772753+0000","last_fullsized":"2023-10-09T04:59:34.824225+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5311,"log_dups_size":0,"ondisk_log_size":5311,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138772480,"num_objects":2118,"num_object_clones":0,"num_object_copies":6354,"num_objects_missing_on_primary":74,"num_objects_missing":74,"num_objects_degraded":2962,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2770,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2117,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":6},{"shards":"1,2,4,5","objects":1740},{"shards":"1,4","objects":-168},{"shards":"2,4,5","objects":375}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.929504+0000","last_change":"2023-10-09T04:58:57.067059+0000","last_active":"2023-10-09T05:00:51.929504+0000","last_peered":"2023-10-09T05:00:51.929504+0000","last_clean":"2023-10-09T05:00:51.929504+0000","last_became_active":"2023-10-09T04:58:57.066168+0000","last_became_peered":"2023-10-09T04:58:57.066168+0000","last_unstale":"2023-10-09T05:00:51.929504+0000","last_undegraded":"2023-10-09T05:00:51.929504+0000","last_fullsized":"2023-10-09T05:00:51.929504+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:23.404016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":182,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.925708+0000","last_change":"2023-10-09T04:58:57.034175+0000","last_active":"2023-10-09T05:00:51.925708+0000","last_peered":"2023-10-09T05:00:51.925708+0000","last_clean":"2023-10-09T05:00:51.925708+0000","last_became_active":"2023-10-09T04:58:57.033913+0000","last_became_peered":"2023-10-09T04:58:57.033913+0000","last_unstale":"2023-10-09T05:00:51.925708+0000","last_undegraded":"2023-10-09T05:00:51.925708+0000","last_fullsized":"2023-10-09T05:00:51.925708+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:50.157531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"568'5312","reported_seq":8162,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:59.411991+0000","last_change":"2023-10-09T04:59:32.066581+0000","last_active":"2023-10-09T05:00:59.411991+0000","last_peered":"2023-10-09T05:00:59.411991+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:59:32.059503+0000","last_became_peered":"2023-10-09T04:59:32.059503+0000","last_unstale":"2023-10-09T05:00:59.411991+0000","last_undegraded":"2023-10-09T04:59:31.931681+0000","last_fullsized":"2023-10-09T04:59:31.829224+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5312,"log_dups_size":0,"ondisk_log_size":5312,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136806400,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3032,"num_objects_misplaced":2226,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2773,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":839,"num_bytes_recovered":54951936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[7,1],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"1,7","objects":17},{"shards":"2,3,4,6,7","objects":322},{"shards":"2,3,4,7","objects":349},{"shards":"2,3,6,7","objects":65},{"shards":"2,4,7","objects":1335}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":110,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.922412+0000","last_change":"2023-10-09T04:58:07.889785+0000","last_active":"2023-10-09T05:00:51.922412+0000","last_peered":"2023-10-09T05:00:51.922412+0000","last_clean":"2023-10-09T05:00:51.922412+0000","last_became_active":"2023-10-09T04:58:07.889365+0000","last_became_peered":"2023-10-09T04:58:07.889365+0000","last_unstale":"2023-10-09T05:00:51.922412+0000","last_undegraded":"2023-10-09T05:00:51.922412+0000","last_fullsized":"2023-10-09T05:00:51.922412+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:14.043179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":42,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923073+0000","last_change":"2023-10-09T05:00:03.051680+0000","last_active":"2023-10-09T05:00:51.923073+0000","last_peered":"2023-10-09T05:00:51.923073+0000","last_clean":"2023-10-09T05:00:51.923073+0000","last_became_active":"2023-10-09T05:00:03.051310+0000","last_became_peered":"2023-10-09T05:00:03.051310+0000","last_unstale":"2023-10-09T05:00:51.923073+0000","last_undegraded":"2023-10-09T05:00:51.923073+0000","last_fullsized":"2023-10-09T05:00:51.923073+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:27.928211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"568'5289","reported_seq":12718,"reported_epoch":568,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:00:57.370059+0000","last_change":"2023-10-09T05:00:05.955878+0000","last_active":"2023-10-09T05:00:57.370059+0000","last_peered":"2023-10-09T05:00:57.370059+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T05:00:05.445792+0000","last_became_peered":"2023-10-09T05:00:05.445792+0000","last_unstale":"2023-10-09T05:00:57.370059+0000","last_undegraded":"2023-10-09T05:00:05.302303+0000","last_fullsized":"2023-10-09T05:00:57.370059+0000","mapping_epoch":559,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5043,"log_dups_size":0,"ondisk_log_size":5043,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138412032,"num_objects":2112,"num_object_clones":0,"num_object_copies":6336,"num_objects_missing_on_primary":93,"num_objects_missing":93,"num_objects_degraded":1798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2749,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1863,"num_bytes_recovered":116916224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":761},{"shards":"3,4,5","objects":422},{"shards":"4,5,7","objects":929}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":168,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920444+0000","last_change":"2023-10-09T05:00:03.007586+0000","last_active":"2023-10-09T05:00:51.920444+0000","last_peered":"2023-10-09T05:00:51.920444+0000","last_clean":"2023-10-09T05:00:51.920444+0000","last_became_active":"2023-10-09T05:00:03.006873+0000","last_became_peered":"2023-10-09T05:00:03.006873+0000","last_unstale":"2023-10-09T05:00:51.920444+0000","last_undegraded":"2023-10-09T05:00:51.920444+0000","last_fullsized":"2023-10-09T05:00:51.920444+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:10.127952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":64,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.931946+0000","last_change":"2023-10-09T04:58:07.895975+0000","last_active":"2023-10-09T05:00:51.931946+0000","last_peered":"2023-10-09T05:00:51.931946+0000","last_clean":"2023-10-09T05:00:51.931946+0000","last_became_active":"2023-10-09T04:58:07.893663+0000","last_became_peered":"2023-10-09T04:58:07.893663+0000","last_unstale":"2023-10-09T05:00:51.931946+0000","last_undegraded":"2023-10-09T05:00:51.931946+0000","last_fullsized":"2023-10-09T05:00:51.931946+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:35.053280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":568,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923529+0000","last_change":"2023-10-09T05:00:02.991301+0000","last_active":"2023-10-09T05:00:51.923529+0000","last_peered":"2023-10-09T05:00:51.923529+0000","last_clean":"2023-10-09T05:00:51.923529+0000","last_became_active":"2023-10-09T05:00:02.990987+0000","last_became_peered":"2023-10-09T05:00:02.990987+0000","last_unstale":"2023-10-09T05:00:51.923529+0000","last_undegraded":"2023-10-09T05:00:51.923529+0000","last_fullsized":"2023-10-09T05:00:51.923529+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:08.164323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":168,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920387+0000","last_change":"2023-10-09T04:58:07.920414+0000","last_active":"2023-10-09T05:00:51.920387+0000","last_peered":"2023-10-09T05:00:51.920387+0000","last_clean":"2023-10-09T05:00:51.920387+0000","last_became_active":"2023-10-09T04:58:07.920149+0000","last_became_peered":"2023-10-09T04:58:07.920149+0000","last_unstale":"2023-10-09T05:00:51.920387+0000","last_undegraded":"2023-10-09T05:00:51.920387+0000","last_fullsized":"2023-10-09T05:00:51.920387+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:34.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":484,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.940084+0000","last_change":"2023-10-09T05:00:03.038038+0000","last_active":"2023-10-09T05:00:51.940084+0000","last_peered":"2023-10-09T05:00:51.940084+0000","last_clean":"2023-10-09T05:00:51.940084+0000","last_became_active":"2023-10-09T05:00:03.037433+0000","last_became_peered":"2023-10-09T05:00:03.037433+0000","last_unstale":"2023-10-09T05:00:51.940084+0000","last_undegraded":"2023-10-09T05:00:51.940084+0000","last_fullsized":"2023-10-09T05:00:51.940084+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:15.535445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923282+0000","last_change":"2023-10-09T04:59:30.857503+0000","last_active":"2023-10-09T05:00:51.923282+0000","last_peered":"2023-10-09T05:00:51.923282+0000","last_clean":"2023-10-09T05:00:51.923282+0000","last_became_active":"2023-10-09T04:59:30.855741+0000","last_became_peered":"2023-10-09T04:59:30.855741+0000","last_unstale":"2023-10-09T05:00:51.923282+0000","last_undegraded":"2023-10-09T05:00:51.923282+0000","last_fullsized":"2023-10-09T05:00:51.923282+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:28.636962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"568'5444","reported_seq":6979,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:00.635810+0000","last_change":"2023-10-09T04:59:55.112316+0000","last_active":"2023-10-09T05:01:00.635810+0000","last_peered":"2023-10-09T05:01:00.635810+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:58:59.753906+0000","last_became_peered":"2023-10-09T04:58:59.753906+0000","last_unstale":"2023-10-09T05:01:00.635810+0000","last_undegraded":"2023-10-09T04:59:55.091481+0000","last_fullsized":"2023-10-09T04:58:54.779150+0000","mapping_epoch":535,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3908,"log_dups_size":0,"ondisk_log_size":3908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141492224,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":2133,"num_objects_missing":2133,"num_objects_degraded":3919,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2835,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":340,"num_bytes_recovered":22282240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2","objects":26},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1702}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":204,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.925621+0000","last_change":"2023-10-09T04:59:31.023495+0000","last_active":"2023-10-09T05:00:51.925621+0000","last_peered":"2023-10-09T05:00:51.925621+0000","last_clean":"2023-10-09T05:00:51.925621+0000","last_became_active":"2023-10-09T04:59:30.947798+0000","last_became_peered":"2023-10-09T04:59:30.947798+0000","last_unstale":"2023-10-09T05:00:51.925621+0000","last_undegraded":"2023-10-09T05:00:51.925621+0000","last_fullsized":"2023-10-09T05:00:51.925621+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:35.811495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":134,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.926334+0000","last_change":"2023-10-09T04:58:07.886648+0000","last_active":"2023-10-09T05:00:51.926334+0000","last_peered":"2023-10-09T05:00:51.926334+0000","last_clean":"2023-10-09T05:00:51.926334+0000","last_became_active":"2023-10-09T04:58:07.886050+0000","last_became_peered":"2023-10-09T04:58:07.886050+0000","last_unstale":"2023-10-09T05:00:51.926334+0000","last_undegraded":"2023-10-09T05:00:51.926334+0000","last_fullsized":"2023-10-09T05:00:51.926334+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:08.968095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"568'5278","reported_seq":16994,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:58.976396+0000","last_change":"2023-10-09T04:59:32.057503+0000","last_active":"2023-10-09T05:00:58.976396+0000","last_peered":"2023-10-09T05:00:58.976396+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:58:58.100530+0000","last_became_peered":"2023-10-09T04:58:58.100530+0000","last_unstale":"2023-10-09T05:00:58.976396+0000","last_undegraded":"2023-10-09T04:59:32.057314+0000","last_fullsized":"2023-10-09T04:58:57.986120+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5278,"log_dups_size":0,"ondisk_log_size":5278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140869632,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":66,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2740,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2884,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":2150}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":50,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930319+0000","last_change":"2023-10-09T04:59:29.995103+0000","last_active":"2023-10-09T05:00:51.930319+0000","last_peered":"2023-10-09T05:00:51.930319+0000","last_clean":"2023-10-09T05:00:51.930319+0000","last_became_active":"2023-10-09T04:58:57.210019+0000","last_became_peered":"2023-10-09T04:58:57.210019+0000","last_unstale":"2023-10-09T05:00:51.930319+0000","last_undegraded":"2023-10-09T05:00:51.930319+0000","last_fullsized":"2023-10-09T05:00:51.930319+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:53.045092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930040+0000","last_change":"2023-10-09T04:58:57.063972+0000","last_active":"2023-10-09T05:00:51.930040+0000","last_peered":"2023-10-09T05:00:51.930040+0000","last_clean":"2023-10-09T05:00:51.930040+0000","last_became_active":"2023-10-09T04:58:57.063636+0000","last_became_peered":"2023-10-09T04:58:57.063636+0000","last_unstale":"2023-10-09T05:00:51.930040+0000","last_undegraded":"2023-10-09T05:00:51.930040+0000","last_fullsized":"2023-10-09T05:00:51.930040+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:19.332300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":46,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930085+0000","last_change":"2023-10-09T04:59:29.960299+0000","last_active":"2023-10-09T05:00:51.930085+0000","last_peered":"2023-10-09T05:00:51.930085+0000","last_clean":"2023-10-09T05:00:51.930085+0000","last_became_active":"2023-10-09T04:58:57.075811+0000","last_became_peered":"2023-10-09T04:58:57.075811+0000","last_unstale":"2023-10-09T05:00:51.930085+0000","last_undegraded":"2023-10-09T05:00:51.930085+0000","last_fullsized":"2023-10-09T05:00:51.930085+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:45.388764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923353+0000","last_change":"2023-10-09T04:55:19.974453+0000","last_active":"2023-10-09T05:00:51.923353+0000","last_peered":"2023-10-09T05:00:51.923353+0000","last_clean":"2023-10-09T05:00:51.923353+0000","last_became_active":"2023-10-09T04:55:19.973871+0000","last_became_peered":"2023-10-09T04:55:19.973871+0000","last_unstale":"2023-10-09T05:00:51.923353+0000","last_undegraded":"2023-10-09T05:00:51.923353+0000","last_fullsized":"2023-10-09T05:00:51.923353+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:47.239847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":33,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920614+0000","last_change":"2023-10-09T04:59:30.847978+0000","last_active":"2023-10-09T05:00:51.920614+0000","last_peered":"2023-10-09T05:00:51.920614+0000","last_clean":"2023-10-09T05:00:51.920614+0000","last_became_active":"2023-10-09T04:59:30.847631+0000","last_became_peered":"2023-10-09T04:59:30.847631+0000","last_unstale":"2023-10-09T05:00:51.920614+0000","last_undegraded":"2023-10-09T05:00:51.920614+0000","last_fullsized":"2023-10-09T05:00:51.920614+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:23.563332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":473,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.925406+0000","last_change":"2023-10-09T04:59:30.948799+0000","last_active":"2023-10-09T05:00:51.925406+0000","last_peered":"2023-10-09T05:00:51.925406+0000","last_clean":"2023-10-09T05:00:51.925406+0000","last_became_active":"2023-10-09T04:59:30.948277+0000","last_became_peered":"2023-10-09T04:59:30.948277+0000","last_unstale":"2023-10-09T05:00:51.925406+0000","last_undegraded":"2023-10-09T05:00:51.925406+0000","last_fullsized":"2023-10-09T05:00:51.925406+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:25.028468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.929023+0000","last_change":"2023-10-09T04:58:56.993529+0000","last_active":"2023-10-09T05:00:51.929023+0000","last_peered":"2023-10-09T05:00:51.929023+0000","last_clean":"2023-10-09T05:00:51.929023+0000","last_became_active":"2023-10-09T04:58:56.993185+0000","last_became_peered":"2023-10-09T04:58:56.993185+0000","last_unstale":"2023-10-09T05:00:51.929023+0000","last_undegraded":"2023-10-09T05:00:51.929023+0000","last_fullsized":"2023-10-09T05:00:51.929023+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:36.864637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923421+0000","last_change":"2023-10-09T04:58:07.882093+0000","last_active":"2023-10-09T05:00:51.923421+0000","last_peered":"2023-10-09T05:00:51.923421+0000","last_clean":"2023-10-09T05:00:51.923421+0000","last_became_active":"2023-10-09T04:58:07.881272+0000","last_became_peered":"2023-10-09T04:58:07.881272+0000","last_unstale":"2023-10-09T05:00:51.923421+0000","last_undegraded":"2023-10-09T05:00:51.923421+0000","last_fullsized":"2023-10-09T05:00:51.923421+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:09.422315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":162,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932784+0000","last_change":"2023-10-09T04:58:07.908453+0000","last_active":"2023-10-09T05:00:51.932784+0000","last_peered":"2023-10-09T05:00:51.932784+0000","last_clean":"2023-10-09T05:00:51.932784+0000","last_became_active":"2023-10-09T04:58:07.904674+0000","last_became_peered":"2023-10-09T04:58:07.904674+0000","last_unstale":"2023-10-09T05:00:51.932784+0000","last_undegraded":"2023-10-09T05:00:51.932784+0000","last_fullsized":"2023-10-09T05:00:51.932784+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:18.595337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923419+0000","last_change":"2023-10-09T04:59:30.859544+0000","last_active":"2023-10-09T05:00:51.923419+0000","last_peered":"2023-10-09T05:00:51.923419+0000","last_clean":"2023-10-09T05:00:51.923419+0000","last_became_active":"2023-10-09T04:59:30.859269+0000","last_became_peered":"2023-10-09T04:59:30.859269+0000","last_unstale":"2023-10-09T05:00:51.923419+0000","last_undegraded":"2023-10-09T05:00:51.923419+0000","last_fullsized":"2023-10-09T05:00:51.923419+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:07:14.815137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"568'5231","reported_seq":8276,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:59.073097+0000","last_change":"2023-10-09T04:59:37.563341+0000","last_active":"2023-10-09T05:00:59.073097+0000","last_peered":"2023-10-09T05:00:59.073097+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T04:59:37.503937+0000","last_became_peered":"2023-10-09T04:59:37.503937+0000","last_unstale":"2023-10-09T05:00:59.073097+0000","last_undegraded":"2023-10-09T04:59:34.538086+0000","last_fullsized":"2023-10-09T04:59:31.829741+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5231,"log_dups_size":0,"ondisk_log_size":5231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136511488,"num_objects":2083,"num_object_clones":0,"num_object_copies":6249,"num_objects_missing_on_primary":338,"num_objects_missing":338,"num_objects_degraded":4125,"num_objects_misplaced":1623,"num_objects_unfound":0,"num_objects_dirty":2083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2732,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":406,"num_bytes_recovered":26574848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[7,1],"avail_no_missing":["4"],"object_location_counts":[{"shards":"1,4,5,6,7","objects":65},{"shards":"1,7","objects":-54},{"shards":"3,4,5","objects":224},{"shards":"3,4,5,6","objects":46},{"shards":"3,4,5,6,7","objects":1},{"shards":"4,5,6,7","objects":1},{"shards":"4,5,7","objects":1638}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":58,"reported_epoch":567,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:00:51.942377+0000","last_change":"2023-10-09T05:00:03.006739+0000","last_active":"2023-10-09T05:00:51.942377+0000","last_peered":"2023-10-09T05:00:51.942377+0000","last_clean":"2023-10-09T04:59:56.906793+0000","last_became_active":"2023-10-09T05:00:03.006377+0000","last_became_peered":"2023-10-09T05:00:03.006377+0000","last_unstale":"2023-10-09T05:00:51.942377+0000","last_undegraded":"2023-10-09T05:00:02.989790+0000","last_fullsized":"2023-10-09T05:00:51.942377+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:05.241750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":593,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932453+0000","last_change":"2023-10-09T04:59:30.861892+0000","last_active":"2023-10-09T05:00:51.932453+0000","last_peered":"2023-10-09T05:00:51.932453+0000","last_clean":"2023-10-09T05:00:51.932453+0000","last_became_active":"2023-10-09T04:59:30.861571+0000","last_became_peered":"2023-10-09T04:59:30.861571+0000","last_unstale":"2023-10-09T05:00:51.932453+0000","last_undegraded":"2023-10-09T05:00:51.932453+0000","last_fullsized":"2023-10-09T05:00:51.932453+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:28.783652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"568'5295","reported_seq":7182,"reported_epoch":568,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:00:58.007676+0000","last_change":"2023-10-09T05:00:50.815372+0000","last_active":"2023-10-09T05:00:58.007676+0000","last_peered":"2023-10-09T05:00:58.007676+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:59:55.013898+0000","last_became_peered":"2023-10-09T04:59:55.013898+0000","last_unstale":"2023-10-09T05:00:58.007676+0000","last_undegraded":"2023-10-09T05:00:58.007676+0000","last_fullsized":"2023-10-09T04:59:31.949346+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5295,"log_dups_size":0,"ondisk_log_size":5295,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137691136,"num_objects":2101,"num_object_clones":0,"num_object_copies":6303,"num_objects_missing_on_primary":1644,"num_objects_missing":1644,"num_objects_degraded":0,"num_objects_misplaced":5722,"num_objects_unfound":0,"num_objects_dirty":2101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2763,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":21823488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":7},{"shards":"1,3","objects":36},{"shards":"2,3,5,7","objects":369},{"shards":"2,5,7","objects":1644},{"shards":"3,5,6","objects":45}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":411,"reported_epoch":567,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:00:51.939909+0000","last_change":"2023-10-09T05:00:25.182219+0000","last_active":"2023-10-09T05:00:51.939909+0000","last_peered":"2023-10-09T05:00:51.939909+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:00:03.037168+0000","last_became_peered":"2023-10-09T05:00:03.037168+0000","last_unstale":"2023-10-09T05:00:51.939909+0000","last_undegraded":"2023-10-09T05:00:25.180458+0000","last_fullsized":"2023-10-09T05:00:51.939909+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:38.583814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","4","5","7"],"object_location_counts":[{"shards":"1,4,5,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":50,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.928948+0000","last_change":"2023-10-09T04:59:30.853667+0000","last_active":"2023-10-09T05:00:51.928948+0000","last_peered":"2023-10-09T05:00:51.928948+0000","last_clean":"2023-10-09T05:00:51.928948+0000","last_became_active":"2023-10-09T04:59:30.853265+0000","last_became_peered":"2023-10-09T04:59:30.853265+0000","last_unstale":"2023-10-09T05:00:51.928948+0000","last_undegraded":"2023-10-09T05:00:51.928948+0000","last_fullsized":"2023-10-09T05:00:51.928948+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:04.088823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"568'5300","reported_seq":5616,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:59.596614+0000","last_change":"2023-10-09T05:00:50.799653+0000","last_active":"2023-10-09T05:00:59.596614+0000","last_peered":"2023-10-09T05:00:59.596614+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:58.136774+0000","last_became_peered":"2023-10-09T04:58:58.136774+0000","last_unstale":"2023-10-09T05:00:59.596614+0000","last_undegraded":"2023-10-09T04:58:58.128621+0000","last_fullsized":"2023-10-09T04:58:57.986571+0000","mapping_epoch":538,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4762,"log_dups_size":0,"ondisk_log_size":4762,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137265152,"num_objects":2094,"num_object_clones":0,"num_object_copies":6282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2528,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2768,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1736704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":2029},{"shards":"2,5,6","objects":55},{"shards":"4,5","objects":-227}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":186,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.934881+0000","last_change":"2023-10-09T04:59:31.025595+0000","last_active":"2023-10-09T05:00:51.934881+0000","last_peered":"2023-10-09T05:00:51.934881+0000","last_clean":"2023-10-09T05:00:51.934881+0000","last_became_active":"2023-10-09T04:59:30.946523+0000","last_became_peered":"2023-10-09T04:59:30.946523+0000","last_unstale":"2023-10-09T05:00:51.934881+0000","last_undegraded":"2023-10-09T05:00:51.934881+0000","last_fullsized":"2023-10-09T05:00:51.934881+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:05.945423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":74,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.933108+0000","last_change":"2023-10-09T04:59:30.854424+0000","last_active":"2023-10-09T05:00:51.933108+0000","last_peered":"2023-10-09T05:00:51.933108+0000","last_clean":"2023-10-09T05:00:51.933108+0000","last_became_active":"2023-10-09T04:59:30.853989+0000","last_became_peered":"2023-10-09T04:59:30.853989+0000","last_unstale":"2023-10-09T05:00:51.933108+0000","last_undegraded":"2023-10-09T05:00:51.933108+0000","last_fullsized":"2023-10-09T05:00:51.933108+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:30.423138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"568'5274","reported_seq":15836,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T05:00:57.769756+0000","last_change":"2023-10-09T05:00:51.946565+0000","last_active":"2023-10-09T05:00:57.769756+0000","last_peered":"2023-10-09T05:00:57.769756+0000","last_clean":"2023-10-09T05:00:57.769756+0000","last_became_active":"2023-10-09T05:00:51.939927+0000","last_became_peered":"2023-10-09T05:00:51.939927+0000","last_unstale":"2023-10-09T05:00:57.769756+0000","last_undegraded":"2023-10-09T05:00:57.769756+0000","last_fullsized":"2023-10-09T05:00:57.769756+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138936320,"num_objects":2120,"num_object_clones":0,"num_object_copies":6360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2739,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3303,"num_bytes_recovered":206536704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":628,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923471+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:00:51.923471+0000","last_peered":"2023-10-09T05:00:51.923471+0000","last_clean":"2023-10-09T05:00:51.923471+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:00:51.923471+0000","last_undegraded":"2023-10-09T05:00:51.923471+0000","last_fullsized":"2023-10-09T05:00:51.923471+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.929293+0000","last_change":"2023-10-09T04:59:02.687690+0000","last_active":"2023-10-09T05:00:51.929293+0000","last_peered":"2023-10-09T05:00:51.929293+0000","last_clean":"2023-10-09T05:00:51.929293+0000","last_became_active":"2023-10-09T04:59:02.687418+0000","last_became_peered":"2023-10-09T04:59:02.687418+0000","last_unstale":"2023-10-09T05:00:51.929293+0000","last_undegraded":"2023-10-09T05:00:51.929293+0000","last_fullsized":"2023-10-09T05:00:51.929293+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:21.317269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1076,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932111+0000","last_change":"2023-10-09T04:59:33.900445+0000","last_active":"2023-10-09T05:00:51.932111+0000","last_peered":"2023-10-09T05:00:51.932111+0000","last_clean":"2023-10-09T05:00:51.932111+0000","last_became_active":"2023-10-09T04:59:33.899975+0000","last_became_peered":"2023-10-09T04:59:33.899975+0000","last_unstale":"2023-10-09T05:00:51.932111+0000","last_undegraded":"2023-10-09T05:00:51.932111+0000","last_fullsized":"2023-10-09T05:00:51.932111+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"568'5142","reported_seq":6782,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:59.844263+0000","last_change":"2023-10-09T04:59:40.909184+0000","last_active":"2023-10-09T05:00:59.844263+0000","last_peered":"2023-10-09T05:00:59.844263+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:59:39.104756+0000","last_became_peered":"2023-10-09T04:59:39.104756+0000","last_unstale":"2023-10-09T05:00:59.844263+0000","last_undegraded":"2023-10-09T04:59:33.724244+0000","last_fullsized":"2023-10-09T04:59:33.707563+0000","mapping_epoch":546,"log_start":"444'386","ondisk_log_start":"444'386","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4756,"log_dups_size":0,"ondisk_log_size":4756,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133070848,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":1994,"num_objects_missing":1994,"num_objects_degraded":5785,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2685,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":328,"num_bytes_recovered":21495808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"1,3","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":1598}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":137,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.997682+0000","last_change":"2023-10-09T04:59:29.960112+0000","last_active":"2023-10-09T05:00:51.997682+0000","last_peered":"2023-10-09T05:00:51.997682+0000","last_clean":"2023-10-09T05:00:51.997682+0000","last_became_active":"2023-10-09T04:59:02.687355+0000","last_became_peered":"2023-10-09T04:59:02.687355+0000","last_unstale":"2023-10-09T05:00:51.997682+0000","last_undegraded":"2023-10-09T05:00:51.997682+0000","last_fullsized":"2023-10-09T05:00:51.997682+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:03.680594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930329+0000","last_change":"2023-10-09T04:58:57.210297+0000","last_active":"2023-10-09T05:00:51.930329+0000","last_peered":"2023-10-09T05:00:51.930329+0000","last_clean":"2023-10-09T05:00:51.930329+0000","last_became_active":"2023-10-09T04:58:57.209978+0000","last_became_peered":"2023-10-09T04:58:57.209978+0000","last_unstale":"2023-10-09T05:00:51.930329+0000","last_undegraded":"2023-10-09T05:00:51.930329+0000","last_fullsized":"2023-10-09T05:00:51.930329+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:58.875418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1081,"reported_epoch":567,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:51.932717+0000","last_change":"2023-10-09T04:59:31.881787+0000","last_active":"2023-10-09T05:00:51.932717+0000","last_peered":"2023-10-09T05:00:51.932717+0000","last_clean":"2023-10-09T04:59:25.875698+0000","last_became_active":"2023-10-09T04:59:31.880964+0000","last_became_peered":"2023-10-09T04:59:31.880964+0000","last_unstale":"2023-10-09T05:00:51.932717+0000","last_undegraded":"2023-10-09T04:59:31.831404+0000","last_fullsized":"2023-10-09T04:59:31.830955+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1","2","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":1},{"shards":"1,6","objects":-1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"568'5283","reported_seq":13973,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:57.106898+0000","last_change":"2023-10-09T04:59:46.171466+0000","last_active":"2023-10-09T05:00:57.106898+0000","last_peered":"2023-10-09T05:00:57.106898+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:59:46.091981+0000","last_became_peered":"2023-10-09T04:59:46.091981+0000","last_unstale":"2023-10-09T05:00:57.106898+0000","last_undegraded":"2023-10-09T04:59:34.803920+0000","last_fullsized":"2023-10-09T04:59:31.843578+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5283,"log_dups_size":0,"ondisk_log_size":5283,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140607488,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":93,"num_objects_missing":93,"num_objects_degraded":2980,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2741,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2115,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":1},{"shards":"1,2,4,5","objects":1781},{"shards":"1,4","objects":-149},{"shards":"2,4,5","objects":360}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":74,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.933045+0000","last_change":"2023-10-09T04:58:57.060419+0000","last_active":"2023-10-09T05:00:51.933045+0000","last_peered":"2023-10-09T05:00:51.933045+0000","last_clean":"2023-10-09T05:00:51.933045+0000","last_became_active":"2023-10-09T04:58:57.059935+0000","last_became_peered":"2023-10-09T04:58:57.059935+0000","last_unstale":"2023-10-09T05:00:51.933045+0000","last_undegraded":"2023-10-09T05:00:51.933045+0000","last_fullsized":"2023-10-09T05:00:51.933045+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:44.313186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":343,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923655+0000","last_change":"2023-10-09T04:59:30.952535+0000","last_active":"2023-10-09T05:00:51.923655+0000","last_peered":"2023-10-09T05:00:51.923655+0000","last_clean":"2023-10-09T05:00:51.923655+0000","last_became_active":"2023-10-09T04:59:30.952185+0000","last_became_peered":"2023-10-09T04:59:30.952185+0000","last_unstale":"2023-10-09T05:00:51.923655+0000","last_undegraded":"2023-10-09T05:00:51.923655+0000","last_fullsized":"2023-10-09T05:00:51.923655+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:00.478193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.4","version":"568'5282","reported_seq":6869,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:59.082247+0000","last_change":"2023-10-09T04:59:32.066582+0000","last_active":"2023-10-09T05:00:59.082247+0000","last_peered":"2023-10-09T05:00:59.082247+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:59:32.057983+0000","last_became_peered":"2023-10-09T04:59:32.057983+0000","last_unstale":"2023-10-09T05:00:59.082247+0000","last_undegraded":"2023-10-09T04:59:31.912599+0000","last_fullsized":"2023-10-09T04:59:31.833438+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5282,"log_dups_size":0,"ondisk_log_size":5282,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138772480,"num_objects":2117,"num_object_clones":0,"num_object_copies":6351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3834,"num_objects_misplaced":2299,"num_objects_unfound":0,"num_objects_dirty":2117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2742,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20480000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[7,1],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"1,7","objects":-17},{"shards":"2,3,4,7","objects":364},{"shards":"2,3,6,7","objects":46},{"shards":"2,4,7","objects":1724}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.928802+0000","last_change":"2023-10-09T04:58:57.557023+0000","last_active":"2023-10-09T05:00:51.928802+0000","last_peered":"2023-10-09T05:00:51.928802+0000","last_clean":"2023-10-09T05:00:51.928802+0000","last_became_active":"2023-10-09T04:58:57.556703+0000","last_became_peered":"2023-10-09T04:58:57.556703+0000","last_unstale":"2023-10-09T05:00:51.928802+0000","last_undegraded":"2023-10-09T05:00:51.928802+0000","last_fullsized":"2023-10-09T05:00:51.928802+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:44.046576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930780+0000","last_change":"2023-10-09T04:58:57.071995+0000","last_active":"2023-10-09T05:00:51.930780+0000","last_peered":"2023-10-09T05:00:51.930780+0000","last_clean":"2023-10-09T05:00:51.930780+0000","last_became_active":"2023-10-09T04:58:57.070855+0000","last_became_peered":"2023-10-09T04:58:57.070855+0000","last_unstale":"2023-10-09T05:00:51.930780+0000","last_undegraded":"2023-10-09T05:00:51.930780+0000","last_fullsized":"2023-10-09T05:00:51.930780+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:29.469962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1085,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.933346+0000","last_change":"2023-10-09T04:59:34.572197+0000","last_active":"2023-10-09T05:00:51.933346+0000","last_peered":"2023-10-09T05:00:51.933346+0000","last_clean":"2023-10-09T05:00:51.933346+0000","last_became_active":"2023-10-09T04:59:34.571477+0000","last_became_peered":"2023-10-09T04:59:34.571477+0000","last_unstale":"2023-10-09T05:00:51.933346+0000","last_undegraded":"2023-10-09T05:00:51.933346+0000","last_fullsized":"2023-10-09T05:00:51.933346+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"568'5302","reported_seq":12759,"reported_epoch":568,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:00:57.782557+0000","last_change":"2023-10-09T05:00:06.492042+0000","last_active":"2023-10-09T05:00:57.782557+0000","last_peered":"2023-10-09T05:00:57.782557+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T05:00:05.977194+0000","last_became_peered":"2023-10-09T05:00:05.977194+0000","last_unstale":"2023-10-09T05:00:57.782557+0000","last_undegraded":"2023-10-09T05:00:05.416161+0000","last_fullsized":"2023-10-09T05:00:57.782557+0000","mapping_epoch":559,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5056,"log_dups_size":0,"ondisk_log_size":5056,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137560064,"num_objects":2099,"num_object_clones":0,"num_object_copies":6297,"num_objects_missing_on_primary":85,"num_objects_missing":85,"num_objects_degraded":1797,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2762,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1876,"num_bytes_recovered":117833728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":781},{"shards":"3,4,5","objects":405},{"shards":"4,5,7","objects":913}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":42,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.919934+0000","last_change":"2023-10-09T05:00:03.007784+0000","last_active":"2023-10-09T05:00:51.919934+0000","last_peered":"2023-10-09T05:00:51.919934+0000","last_clean":"2023-10-09T05:00:51.919934+0000","last_became_active":"2023-10-09T05:00:03.007280+0000","last_became_peered":"2023-10-09T05:00:03.007280+0000","last_unstale":"2023-10-09T05:00:51.919934+0000","last_undegraded":"2023-10-09T05:00:51.919934+0000","last_fullsized":"2023-10-09T05:00:51.919934+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:41.087401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.971049+0000","last_change":"2023-10-09T04:58:57.066851+0000","last_active":"2023-10-09T05:00:51.971049+0000","last_peered":"2023-10-09T05:00:51.971049+0000","last_clean":"2023-10-09T05:00:51.971049+0000","last_became_active":"2023-10-09T04:58:57.066522+0000","last_became_peered":"2023-10-09T04:58:57.066522+0000","last_unstale":"2023-10-09T05:00:51.971049+0000","last_undegraded":"2023-10-09T05:00:51.971049+0000","last_fullsized":"2023-10-09T05:00:51.971049+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:48.448579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1091,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932034+0000","last_change":"2023-10-09T04:59:55.153783+0000","last_active":"2023-10-09T05:00:51.932034+0000","last_peered":"2023-10-09T05:00:51.932034+0000","last_clean":"2023-10-09T05:00:51.932034+0000","last_became_active":"2023-10-09T04:59:55.153450+0000","last_became_peered":"2023-10-09T04:59:55.153450+0000","last_unstale":"2023-10-09T05:00:51.932034+0000","last_undegraded":"2023-10-09T05:00:51.932034+0000","last_fullsized":"2023-10-09T05:00:51.932034+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"568'5443","reported_seq":6972,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:00.767260+0000","last_change":"2023-10-09T04:59:31.036614+0000","last_active":"2023-10-09T05:01:00.767260+0000","last_peered":"2023-10-09T05:01:00.767260+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:58:52.441046+0000","last_became_peered":"2023-10-09T04:58:52.441046+0000","last_unstale":"2023-10-09T05:01:00.767260+0000","last_undegraded":"2023-10-09T04:59:31.029253+0000","last_fullsized":"2023-10-09T04:58:48.896671+0000","mapping_epoch":496,"log_start":"445'1536","ondisk_log_start":"445'1536","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3907,"log_dups_size":0,"ondisk_log_size":3907,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141557760,"num_objects":2160,"num_object_clones":0,"num_object_copies":6480,"num_objects_missing_on_primary":2186,"num_objects_missing":2186,"num_objects_degraded":4038,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2834,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":339,"num_bytes_recovered":22216704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"1,2","objects":-26},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":1780}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":126,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.926718+0000","last_change":"2023-10-09T04:56:53.920990+0000","last_active":"2023-10-09T05:00:51.926718+0000","last_peered":"2023-10-09T05:00:51.926718+0000","last_clean":"2023-10-09T05:00:51.926718+0000","last_became_active":"2023-10-09T04:56:53.920700+0000","last_became_peered":"2023-10-09T04:56:53.920700+0000","last_unstale":"2023-10-09T05:00:51.926718+0000","last_undegraded":"2023-10-09T05:00:51.926718+0000","last_fullsized":"2023-10-09T05:00:51.926718+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:06.039757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":505,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.925510+0000","last_change":"2023-10-09T05:00:03.068377+0000","last_active":"2023-10-09T05:00:51.925510+0000","last_peered":"2023-10-09T05:00:51.925510+0000","last_clean":"2023-10-09T05:00:51.925510+0000","last_became_active":"2023-10-09T05:00:03.068136+0000","last_became_peered":"2023-10-09T05:00:03.068136+0000","last_unstale":"2023-10-09T05:00:51.925510+0000","last_undegraded":"2023-10-09T05:00:51.925510+0000","last_fullsized":"2023-10-09T05:00:51.925510+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:25.397131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":176,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.926267+0000","last_change":"2023-10-09T05:00:03.024450+0000","last_active":"2023-10-09T05:00:51.926267+0000","last_peered":"2023-10-09T05:00:51.926267+0000","last_clean":"2023-10-09T05:00:51.926267+0000","last_became_active":"2023-10-09T05:00:03.024135+0000","last_became_peered":"2023-10-09T05:00:03.024135+0000","last_unstale":"2023-10-09T05:00:51.926267+0000","last_undegraded":"2023-10-09T05:00:51.926267+0000","last_fullsized":"2023-10-09T05:00:51.926267+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:40.696966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1074,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932022+0000","last_change":"2023-10-09T04:59:33.900400+0000","last_active":"2023-10-09T05:00:51.932022+0000","last_peered":"2023-10-09T05:00:51.932022+0000","last_clean":"2023-10-09T05:00:51.932022+0000","last_became_active":"2023-10-09T04:59:33.899972+0000","last_became_peered":"2023-10-09T04:59:33.899972+0000","last_unstale":"2023-10-09T05:00:51.932022+0000","last_undegraded":"2023-10-09T05:00:51.932022+0000","last_fullsized":"2023-10-09T05:00:51.932022+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"568'5417","reported_seq":16011,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:57.397364+0000","last_change":"2023-10-09T05:00:03.237538+0000","last_active":"2023-10-09T05:00:57.397364+0000","last_peered":"2023-10-09T05:00:57.397364+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:00:03.142517+0000","last_became_peered":"2023-10-09T05:00:03.142517+0000","last_unstale":"2023-10-09T05:00:57.397364+0000","last_undegraded":"2023-10-09T05:00:03.014197+0000","last_fullsized":"2023-10-09T05:00:02.989747+0000","mapping_epoch":558,"log_start":"445'1381","ondisk_log_start":"445'1381","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4036,"log_dups_size":0,"ondisk_log_size":4036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:39.361359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273088512,"num_objects":4167,"num_object_clones":0,"num_object_copies":12501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4025,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5417,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4790,"num_bytes_recovered":304742400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,3,4,6","objects":170},{"shards":"2,4,6","objects":111},{"shards":"2,4,6,7","objects":3181},{"shards":"2,6","objects":435}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":622,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.924758+0000","last_change":"2023-10-09T04:58:07.914635+0000","last_active":"2023-10-09T05:00:51.924758+0000","last_peered":"2023-10-09T05:00:51.924758+0000","last_clean":"2023-10-09T05:00:51.924758+0000","last_became_active":"2023-10-09T04:58:07.913894+0000","last_became_peered":"2023-10-09T04:58:07.913894+0000","last_unstale":"2023-10-09T05:00:51.924758+0000","last_undegraded":"2023-10-09T05:00:51.924758+0000","last_fullsized":"2023-10-09T05:00:51.924758+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:32.545025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930001+0000","last_change":"2023-10-09T04:58:57.063810+0000","last_active":"2023-10-09T05:00:51.930001+0000","last_peered":"2023-10-09T05:00:51.930001+0000","last_clean":"2023-10-09T05:00:51.930001+0000","last_became_active":"2023-10-09T04:58:57.063532+0000","last_became_peered":"2023-10-09T04:58:57.063532+0000","last_unstale":"2023-10-09T05:00:51.930001+0000","last_undegraded":"2023-10-09T05:00:51.930001+0000","last_fullsized":"2023-10-09T05:00:51.930001+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:01.777681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":152,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.926649+0000","last_change":"2023-10-09T04:59:30.852320+0000","last_active":"2023-10-09T05:00:51.926649+0000","last_peered":"2023-10-09T05:00:51.926649+0000","last_clean":"2023-10-09T05:00:51.926649+0000","last_became_active":"2023-10-09T04:59:30.852046+0000","last_became_peered":"2023-10-09T04:59:30.852046+0000","last_unstale":"2023-10-09T05:00:51.926649+0000","last_undegraded":"2023-10-09T05:00:51.926649+0000","last_fullsized":"2023-10-09T05:00:51.926649+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:36.807391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1085,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.933185+0000","last_change":"2023-10-09T04:59:34.572474+0000","last_active":"2023-10-09T05:00:51.933185+0000","last_peered":"2023-10-09T05:00:51.933185+0000","last_clean":"2023-10-09T05:00:51.933185+0000","last_became_active":"2023-10-09T04:59:34.571777+0000","last_became_peered":"2023-10-09T04:59:34.571777+0000","last_unstale":"2023-10-09T05:00:51.933185+0000","last_undegraded":"2023-10-09T05:00:51.933185+0000","last_fullsized":"2023-10-09T05:00:51.933185+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"568'5443","reported_seq":7394,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:58.764492+0000","last_change":"2023-10-09T05:00:03.069862+0000","last_active":"2023-10-09T05:00:58.764492+0000","last_peered":"2023-10-09T05:00:58.764492+0000","last_clean":"2023-10-09T04:54:44.521216+0000","last_became_active":"2023-10-09T04:59:07.457064+0000","last_became_peered":"2023-10-09T04:59:07.457064+0000","last_unstale":"2023-10-09T05:00:58.764492+0000","last_undegraded":"2023-10-09T05:00:03.057001+0000","last_fullsized":"2023-10-09T04:58:57.981736+0000","mapping_epoch":538,"log_start":"445'1507","ondisk_log_start":"445'1507","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3936,"log_dups_size":0,"ondisk_log_size":3936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:35.050351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272891928,"num_objects":4165,"num_object_clones":0,"num_object_copies":12495,"num_objects_missing_on_primary":3797,"num_objects_missing":3797,"num_objects_degraded":3962,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4165,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5443,"num_write_kb":307393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":885,"num_bytes_recovered":57999360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":3723},{"shards":"5,6","objects":442}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":33,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920403+0000","last_change":"2023-10-09T04:59:30.855844+0000","last_active":"2023-10-09T05:00:51.920403+0000","last_peered":"2023-10-09T05:00:51.920403+0000","last_clean":"2023-10-09T05:00:51.920403+0000","last_became_active":"2023-10-09T04:59:30.855493+0000","last_became_peered":"2023-10-09T04:59:30.855493+0000","last_unstale":"2023-10-09T05:00:51.920403+0000","last_undegraded":"2023-10-09T05:00:51.920403+0000","last_fullsized":"2023-10-09T05:00:51.920403+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:39.038360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.976905+0000","last_change":"2023-10-09T05:00:02.997768+0000","last_active":"2023-10-09T05:00:51.976905+0000","last_peered":"2023-10-09T05:00:51.976905+0000","last_clean":"2023-10-09T05:00:51.976905+0000","last_became_active":"2023-10-09T05:00:02.997451+0000","last_became_peered":"2023-10-09T05:00:02.997451+0000","last_unstale":"2023-10-09T05:00:51.976905+0000","last_undegraded":"2023-10-09T05:00:51.976905+0000","last_fullsized":"2023-10-09T05:00:51.976905+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:18.292200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":134,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.971188+0000","last_change":"2023-10-09T05:00:03.018405+0000","last_active":"2023-10-09T05:00:51.971188+0000","last_peered":"2023-10-09T05:00:51.971188+0000","last_clean":"2023-10-09T05:00:51.971188+0000","last_became_active":"2023-10-09T05:00:03.018051+0000","last_became_peered":"2023-10-09T05:00:03.018051+0000","last_unstale":"2023-10-09T05:00:51.971188+0000","last_undegraded":"2023-10-09T05:00:51.971188+0000","last_fullsized":"2023-10-09T05:00:51.971188+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:26.400152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"568'5418","reported_seq":8162,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:01.228367+0000","last_change":"2023-10-09T05:00:14.084542+0000","last_active":"2023-10-09T05:01:01.228367+0000","last_peered":"2023-10-09T05:01:01.228367+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:00:13.871981+0000","last_became_peered":"2023-10-09T05:00:13.871981+0000","last_unstale":"2023-10-09T05:01:01.228367+0000","last_undegraded":"2023-10-09T05:00:08.191997+0000","last_fullsized":"2023-10-09T05:00:04.000077+0000","mapping_epoch":559,"log_start":"445'1927","ondisk_log_start":"445'1927","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3491,"log_dups_size":0,"ondisk_log_size":3491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:27.881660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271319040,"num_objects":4140,"num_object_clones":0,"num_object_copies":12420,"num_objects_missing_on_primary":4275,"num_objects_missing":4275,"num_objects_degraded":7888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5418,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1280,"num_bytes_recovered":83886080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[1,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,3,5,7","objects":10},{"shards":"2,3,5,7","objects":137},{"shards":"2,5,7","objects":3216},{"shards":"3,5,7","objects":776},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":409,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.980722+0000","last_change":"2023-10-09T05:00:03.028225+0000","last_active":"2023-10-09T05:00:51.980722+0000","last_peered":"2023-10-09T05:00:51.980722+0000","last_clean":"2023-10-09T05:00:51.980722+0000","last_became_active":"2023-10-09T05:00:03.027843+0000","last_became_peered":"2023-10-09T05:00:03.027843+0000","last_unstale":"2023-10-09T05:00:51.980722+0000","last_undegraded":"2023-10-09T05:00:51.980722+0000","last_fullsized":"2023-10-09T05:00:51.980722+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:39.724975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1091,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932537+0000","last_change":"2023-10-09T04:59:33.901001+0000","last_active":"2023-10-09T05:00:51.932537+0000","last_peered":"2023-10-09T05:00:51.932537+0000","last_clean":"2023-10-09T05:00:51.932537+0000","last_became_active":"2023-10-09T04:59:33.900627+0000","last_became_peered":"2023-10-09T04:59:33.900627+0000","last_unstale":"2023-10-09T05:00:51.932537+0000","last_undegraded":"2023-10-09T05:00:51.932537+0000","last_fullsized":"2023-10-09T05:00:51.932537+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":34,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920660+0000","last_change":"2023-10-09T04:59:30.856383+0000","last_active":"2023-10-09T05:00:51.920660+0000","last_peered":"2023-10-09T05:00:51.920660+0000","last_clean":"2023-10-09T05:00:51.920660+0000","last_became_active":"2023-10-09T04:59:30.855682+0000","last_became_peered":"2023-10-09T04:59:30.855682+0000","last_unstale":"2023-10-09T05:00:51.920660+0000","last_undegraded":"2023-10-09T05:00:51.920660+0000","last_fullsized":"2023-10-09T05:00:51.920660+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.088239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.935155+0000","last_change":"2023-10-09T04:58:56.999352+0000","last_active":"2023-10-09T05:00:51.935155+0000","last_peered":"2023-10-09T05:00:51.935155+0000","last_clean":"2023-10-09T05:00:51.935155+0000","last_became_active":"2023-10-09T04:58:56.999048+0000","last_became_peered":"2023-10-09T04:58:56.999048+0000","last_unstale":"2023-10-09T05:00:51.935155+0000","last_undegraded":"2023-10-09T05:00:51.935155+0000","last_fullsized":"2023-10-09T05:00:51.935155+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:53.056597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1094,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.933925+0000","last_change":"2023-10-09T04:59:57.034671+0000","last_active":"2023-10-09T05:00:51.933925+0000","last_peered":"2023-10-09T05:00:51.933925+0000","last_clean":"2023-10-09T05:00:51.933925+0000","last_became_active":"2023-10-09T04:59:57.034309+0000","last_became_peered":"2023-10-09T04:59:57.034309+0000","last_unstale":"2023-10-09T05:00:51.933925+0000","last_undegraded":"2023-10-09T05:00:51.933925+0000","last_fullsized":"2023-10-09T05:00:51.933925+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"568'5516","reported_seq":10488,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:57.348148+0000","last_change":"2023-10-09T04:59:32.037813+0000","last_active":"2023-10-09T05:00:57.348148+0000","last_peered":"2023-10-09T05:00:57.348148+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T04:58:59.452514+0000","last_became_peered":"2023-10-09T04:58:59.452514+0000","last_unstale":"2023-10-09T05:00:57.348148+0000","last_undegraded":"2023-10-09T04:58:59.441276+0000","last_fullsized":"2023-10-09T04:58:59.025474+0000","mapping_epoch":539,"log_start":"445'1463","ondisk_log_start":"445'1463","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4053,"log_dups_size":0,"ondisk_log_size":4053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:08.028846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277479424,"num_objects":4234,"num_object_clones":0,"num_object_copies":12702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5516,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2204,"num_bytes_recovered":129499136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[2,4],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":2},{"shards":"1,2,4,5","objects":1196},{"shards":"2,4","objects":-285},{"shards":"2,4,5","objects":2943}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":117,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.976783+0000","last_change":"2023-10-09T04:56:55.417824+0000","last_active":"2023-10-09T05:00:51.976783+0000","last_peered":"2023-10-09T05:00:51.976783+0000","last_clean":"2023-10-09T05:00:51.976783+0000","last_became_active":"2023-10-09T04:56:55.417430+0000","last_became_peered":"2023-10-09T04:56:55.417430+0000","last_unstale":"2023-10-09T05:00:51.976783+0000","last_undegraded":"2023-10-09T05:00:51.976783+0000","last_fullsized":"2023-10-09T05:00:51.976783+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:22.097654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":33,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920487+0000","last_change":"2023-10-09T04:59:30.875340+0000","last_active":"2023-10-09T05:00:51.920487+0000","last_peered":"2023-10-09T05:00:51.920487+0000","last_clean":"2023-10-09T05:00:51.920487+0000","last_became_active":"2023-10-09T04:59:30.875026+0000","last_became_peered":"2023-10-09T04:59:30.875026+0000","last_unstale":"2023-10-09T05:00:51.920487+0000","last_undegraded":"2023-10-09T05:00:51.920487+0000","last_fullsized":"2023-10-09T05:00:51.920487+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:12.350534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":128,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.939724+0000","last_change":"2023-10-09T05:00:03.037746+0000","last_active":"2023-10-09T05:00:51.939724+0000","last_peered":"2023-10-09T05:00:51.939724+0000","last_clean":"2023-10-09T05:00:51.939724+0000","last_became_active":"2023-10-09T05:00:03.036701+0000","last_became_peered":"2023-10-09T05:00:03.036701+0000","last_unstale":"2023-10-09T05:00:51.939724+0000","last_undegraded":"2023-10-09T05:00:51.939724+0000","last_fullsized":"2023-10-09T05:00:51.939724+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:11.897581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1094,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.932563+0000","last_change":"2023-10-09T04:59:41.847534+0000","last_active":"2023-10-09T05:00:51.932563+0000","last_peered":"2023-10-09T05:00:51.932563+0000","last_clean":"2023-10-09T05:00:51.932563+0000","last_became_active":"2023-10-09T04:59:41.847139+0000","last_became_peered":"2023-10-09T04:59:41.847139+0000","last_unstale":"2023-10-09T05:00:51.932563+0000","last_undegraded":"2023-10-09T05:00:51.932563+0000","last_fullsized":"2023-10-09T05:00:51.932563+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":551,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"568'5305","reported_seq":16997,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:59.515626+0000","last_change":"2023-10-09T04:58:59.000602+0000","last_active":"2023-10-09T05:00:59.515626+0000","last_peered":"2023-10-09T05:00:59.515626+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:58:58.998253+0000","last_became_peered":"2023-10-09T04:58:58.998253+0000","last_unstale":"2023-10-09T05:00:59.515626+0000","last_undegraded":"2023-10-09T04:58:58.992982+0000","last_fullsized":"2023-10-09T04:58:58.986976+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139100160,"num_objects":2122,"num_object_clones":0,"num_object_copies":6366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":73,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2766,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2872,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":2122}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":41,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.930726+0000","last_change":"2023-10-09T04:58:57.034371+0000","last_active":"2023-10-09T05:00:51.930726+0000","last_peered":"2023-10-09T05:00:51.930726+0000","last_clean":"2023-10-09T05:00:51.930726+0000","last_became_active":"2023-10-09T04:58:57.034021+0000","last_became_peered":"2023-10-09T04:58:57.034021+0000","last_unstale":"2023-10-09T05:00:51.930726+0000","last_undegraded":"2023-10-09T05:00:51.930726+0000","last_fullsized":"2023-10-09T05:00:51.930726+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:22.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":134,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.926581+0000","last_change":"2023-10-09T04:59:30.852166+0000","last_active":"2023-10-09T05:00:51.926581+0000","last_peered":"2023-10-09T05:00:51.926581+0000","last_clean":"2023-10-09T05:00:51.926581+0000","last_became_active":"2023-10-09T04:59:30.851555+0000","last_became_peered":"2023-10-09T04:59:30.851555+0000","last_unstale":"2023-10-09T05:00:51.926581+0000","last_undegraded":"2023-10-09T05:00:51.926581+0000","last_fullsized":"2023-10-09T05:00:51.926581+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:00.921200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":399,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923797+0000","last_change":"2023-10-09T05:00:03.109676+0000","last_active":"2023-10-09T05:00:51.923797+0000","last_peered":"2023-10-09T05:00:51.923797+0000","last_clean":"2023-10-09T05:00:51.923797+0000","last_became_active":"2023-10-09T05:00:03.109229+0000","last_became_peered":"2023-10-09T05:00:03.109229+0000","last_unstale":"2023-10-09T05:00:51.923797+0000","last_undegraded":"2023-10-09T05:00:51.923797+0000","last_fullsized":"2023-10-09T05:00:51.923797+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:32.681498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1089,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.935080+0000","last_change":"2023-10-09T04:59:33.919822+0000","last_active":"2023-10-09T05:00:51.935080+0000","last_peered":"2023-10-09T05:00:51.935080+0000","last_clean":"2023-10-09T05:00:51.935080+0000","last_became_active":"2023-10-09T04:59:33.919386+0000","last_became_peered":"2023-10-09T04:59:33.919386+0000","last_unstale":"2023-10-09T05:00:51.935080+0000","last_undegraded":"2023-10-09T05:00:51.935080+0000","last_fullsized":"2023-10-09T05:00:51.935080+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"568'5362","reported_seq":21674,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T05:00:59.644306+0000","last_change":"2023-10-09T04:58:39.912315+0000","last_active":"2023-10-09T05:00:59.644306+0000","last_peered":"2023-10-09T05:00:59.644306+0000","last_clean":"2023-10-09T05:00:59.644306+0000","last_became_active":"2023-10-09T04:58:39.903645+0000","last_became_peered":"2023-10-09T04:58:39.903645+0000","last_unstale":"2023-10-09T05:00:59.644306+0000","last_undegraded":"2023-10-09T05:00:59.644306+0000","last_fullsized":"2023-10-09T05:00:59.644306+0000","mapping_epoch":526,"log_start":"444'100","ondisk_log_start":"444'100","created":440,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5262,"log_dups_size":0,"ondisk_log_size":5262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:27.838809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273940480,"num_objects":4180,"num_object_clones":0,"num_object_copies":12540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5362,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7458,"num_bytes_recovered":477888512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":166,"reported_epoch":567,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:00:51.923834+0000","last_change":"2023-10-09T05:00:03.015330+0000","last_active":"2023-10-09T05:00:51.923834+0000","last_peered":"2023-10-09T05:00:51.923834+0000","last_clean":"2023-10-09T04:59:56.922992+0000","last_became_active":"2023-10-09T05:00:03.015081+0000","last_became_peered":"2023-10-09T05:00:03.015081+0000","last_unstale":"2023-10-09T05:00:51.923834+0000","last_undegraded":"2023-10-09T05:00:02.978496+0000","last_fullsized":"2023-10-09T05:00:51.923834+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:50.144636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":154,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.940160+0000","last_change":"2023-10-09T04:58:57.063270+0000","last_active":"2023-10-09T05:00:51.940160+0000","last_peered":"2023-10-09T05:00:51.940160+0000","last_clean":"2023-10-09T05:00:51.940160+0000","last_became_active":"2023-10-09T04:58:57.062873+0000","last_became_peered":"2023-10-09T04:58:57.062873+0000","last_unstale":"2023-10-09T05:00:51.940160+0000","last_undegraded":"2023-10-09T05:00:51.940160+0000","last_fullsized":"2023-10-09T05:00:51.940160+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:58.612148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":33,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.919860+0000","last_change":"2023-10-09T04:59:30.860452+0000","last_active":"2023-10-09T05:00:51.919860+0000","last_peered":"2023-10-09T05:00:51.919860+0000","last_clean":"2023-10-09T05:00:51.919860+0000","last_became_active":"2023-10-09T04:59:30.859944+0000","last_became_peered":"2023-10-09T04:59:30.859944+0000","last_unstale":"2023-10-09T05:00:51.919860+0000","last_undegraded":"2023-10-09T05:00:51.919860+0000","last_fullsized":"2023-10-09T05:00:51.919860+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:01.173959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1065,"reported_epoch":567,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:51.935684+0000","last_change":"2023-10-09T04:59:31.846857+0000","last_active":"2023-10-09T05:00:51.935684+0000","last_peered":"2023-10-09T05:00:51.935684+0000","last_clean":"2023-10-09T04:59:25.885390+0000","last_became_active":"2023-10-09T04:59:31.846529+0000","last_became_peered":"2023-10-09T04:59:31.846529+0000","last_unstale":"2023-10-09T05:00:51.935684+0000","last_undegraded":"2023-10-09T04:59:31.832883+0000","last_fullsized":"2023-10-09T04:59:31.831034+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1138389,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1","2","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":1},{"shards":"1,6","objects":-1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"568'5631","reported_seq":8427,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:01.127397+0000","last_change":"2023-10-09T04:59:55.409660+0000","last_active":"2023-10-09T05:01:01.127397+0000","last_peered":"2023-10-09T05:01:01.127397+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T04:59:55.037735+0000","last_became_peered":"2023-10-09T04:59:55.037735+0000","last_unstale":"2023-10-09T05:01:01.127397+0000","last_undegraded":"2023-10-09T04:59:44.912466+0000","last_fullsized":"2023-10-09T04:59:31.829930+0000","mapping_epoch":546,"log_start":"445'2096","ondisk_log_start":"445'2096","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3535,"log_dups_size":0,"ondisk_log_size":3535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:51.517413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283705344,"num_objects":4329,"num_object_clones":0,"num_object_copies":12987,"num_objects_missing_on_primary":4098,"num_objects_missing":4098,"num_objects_degraded":8900,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5631,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1219,"num_bytes_recovered":79888384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"1,2,4,7","objects":230},{"shards":"1,6","objects":1},{"shards":"2,4,7","objects":3972},{"shards":"2,6,7","objects":125}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":64,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.931810+0000","last_change":"2023-10-09T04:58:07.881659+0000","last_active":"2023-10-09T05:00:51.931810+0000","last_peered":"2023-10-09T05:00:51.931810+0000","last_clean":"2023-10-09T05:00:51.931810+0000","last_became_active":"2023-10-09T04:58:07.881399+0000","last_became_peered":"2023-10-09T04:58:07.881399+0000","last_unstale":"2023-10-09T05:00:51.931810+0000","last_undegraded":"2023-10-09T05:00:51.931810+0000","last_fullsized":"2023-10-09T05:00:51.931810+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:31.736513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":195,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.940252+0000","last_change":"2023-10-09T04:59:29.958563+0000","last_active":"2023-10-09T05:00:51.940252+0000","last_peered":"2023-10-09T05:00:51.940252+0000","last_clean":"2023-10-09T05:00:51.940252+0000","last_became_active":"2023-10-09T04:58:58.015415+0000","last_became_peered":"2023-10-09T04:58:58.015415+0000","last_unstale":"2023-10-09T05:00:51.940252+0000","last_undegraded":"2023-10-09T05:00:51.940252+0000","last_fullsized":"2023-10-09T05:00:51.940252+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:12.808436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.928883+0000","last_change":"2023-10-09T05:00:03.002429+0000","last_active":"2023-10-09T05:00:51.928883+0000","last_peered":"2023-10-09T05:00:51.928883+0000","last_clean":"2023-10-09T05:00:51.928883+0000","last_became_active":"2023-10-09T05:00:03.002074+0000","last_became_peered":"2023-10-09T05:00:03.002074+0000","last_unstale":"2023-10-09T05:00:51.928883+0000","last_undegraded":"2023-10-09T05:00:51.928883+0000","last_fullsized":"2023-10-09T05:00:51.928883+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:38.429397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"568'5139","reported_seq":6779,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:00.835798+0000","last_change":"2023-10-09T04:59:40.916464+0000","last_active":"2023-10-09T05:01:00.835798+0000","last_peered":"2023-10-09T05:01:00.835798+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:59:40.797700+0000","last_became_peered":"2023-10-09T04:59:40.797700+0000","last_unstale":"2023-10-09T05:01:00.835798+0000","last_undegraded":"2023-10-09T04:59:34.069489+0000","last_fullsized":"2023-10-09T04:59:34.063514+0000","mapping_epoch":546,"log_start":"444'386","ondisk_log_start":"444'386","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133267456,"num_objects":2034,"num_object_clones":0,"num_object_copies":6102,"num_objects_missing_on_primary":2070,"num_objects_missing":2070,"num_objects_degraded":5983,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2683,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":325,"num_bytes_recovered":21299200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"1,3","objects":-36},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":1683}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.929957+0000","last_change":"2023-10-09T04:58:59.185788+0000","last_active":"2023-10-09T05:00:51.929957+0000","last_peered":"2023-10-09T05:00:51.929957+0000","last_clean":"2023-10-09T05:00:51.929957+0000","last_became_active":"2023-10-09T04:58:59.185338+0000","last_became_peered":"2023-10-09T04:58:59.185338+0000","last_unstale":"2023-10-09T05:00:51.929957+0000","last_undegraded":"2023-10-09T05:00:51.929957+0000","last_fullsized":"2023-10-09T05:00:51.929957+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:19.825485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":115,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.926498+0000","last_change":"2023-10-09T04:58:57.064053+0000","last_active":"2023-10-09T05:00:51.926498+0000","last_peered":"2023-10-09T05:00:51.926498+0000","last_clean":"2023-10-09T05:00:51.926498+0000","last_became_active":"2023-10-09T04:58:57.063696+0000","last_became_peered":"2023-10-09T04:58:57.063696+0000","last_unstale":"2023-10-09T05:00:51.926498+0000","last_undegraded":"2023-10-09T05:00:51.926498+0000","last_fullsized":"2023-10-09T05:00:51.926498+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:30.110906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"568'5273","reported_seq":9986,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T05:00:57.390466+0000","last_change":"2023-10-09T04:59:55.883965+0000","last_active":"2023-10-09T05:00:57.390466+0000","last_peered":"2023-10-09T05:00:57.390466+0000","last_clean":"2023-10-09T05:00:57.390466+0000","last_became_active":"2023-10-09T04:59:55.880766+0000","last_became_peered":"2023-10-09T04:59:55.880766+0000","last_unstale":"2023-10-09T05:00:57.390466+0000","last_undegraded":"2023-10-09T05:00:57.390466+0000","last_fullsized":"2023-10-09T05:00:57.390466+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5273,"log_dups_size":0,"ondisk_log_size":5273,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139001856,"num_objects":2121,"num_object_clones":0,"num_object_copies":6363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2738,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":663,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923558+0000","last_change":"2023-10-09T04:59:30.021044+0000","last_active":"2023-10-09T05:00:51.923558+0000","last_peered":"2023-10-09T05:00:51.923558+0000","last_clean":"2023-10-09T05:00:51.923558+0000","last_became_active":"2023-10-09T04:58:57.166878+0000","last_became_peered":"2023-10-09T04:58:57.166878+0000","last_unstale":"2023-10-09T05:00:51.923558+0000","last_undegraded":"2023-10-09T05:00:51.923558+0000","last_fullsized":"2023-10-09T05:00:51.923558+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.097934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":340,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:53.065424+0000","last_change":"2023-10-09T05:00:03.122991+0000","last_active":"2023-10-09T05:00:53.065424+0000","last_peered":"2023-10-09T05:00:53.065424+0000","last_clean":"2023-10-09T05:00:53.065424+0000","last_became_active":"2023-10-09T05:00:03.122557+0000","last_became_peered":"2023-10-09T05:00:03.122557+0000","last_unstale":"2023-10-09T05:00:53.065424+0000","last_undegraded":"2023-10-09T05:00:53.065424+0000","last_fullsized":"2023-10-09T05:00:53.065424+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:54.753971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.10","version":"568'5311","reported_seq":5626,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:58.608819+0000","last_change":"2023-10-09T05:00:25.185018+0000","last_active":"2023-10-09T05:00:58.608819+0000","last_peered":"2023-10-09T05:00:58.608819+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:58.134873+0000","last_became_peered":"2023-10-09T04:58:58.134873+0000","last_unstale":"2023-10-09T05:00:58.608819+0000","last_undegraded":"2023-10-09T04:58:58.123193+0000","last_fullsized":"2023-10-09T04:58:58.004066+0000","mapping_epoch":538,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4773,"log_dups_size":0,"ondisk_log_size":4773,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136544256,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2536,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2780,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":1736704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":2027},{"shards":"2,5,6","objects":61},{"shards":"4,5","objects":-256}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":632,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.924106+0000","last_change":"2023-10-09T04:59:30.019132+0000","last_active":"2023-10-09T05:00:51.924106+0000","last_peered":"2023-10-09T05:00:51.924106+0000","last_clean":"2023-10-09T05:00:51.924106+0000","last_became_active":"2023-10-09T04:58:56.996966+0000","last_became_peered":"2023-10-09T04:58:56.996966+0000","last_unstale":"2023-10-09T05:00:51.924106+0000","last_undegraded":"2023-10-09T05:00:51.924106+0000","last_fullsized":"2023-10-09T05:00:51.924106+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:43.116063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.920843+0000","last_change":"2023-10-09T04:59:30.856551+0000","last_active":"2023-10-09T05:00:51.920843+0000","last_peered":"2023-10-09T05:00:51.920843+0000","last_clean":"2023-10-09T05:00:51.920843+0000","last_became_active":"2023-10-09T04:59:30.856103+0000","last_became_peered":"2023-10-09T04:59:30.856103+0000","last_unstale":"2023-10-09T05:00:51.920843+0000","last_undegraded":"2023-10-09T05:00:51.920843+0000","last_fullsized":"2023-10-09T05:00:51.920843+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:16.595682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"568'5276","reported_seq":7046,"reported_epoch":568,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:00:57.642273+0000","last_change":"2023-10-09T04:59:55.375337+0000","last_active":"2023-10-09T05:00:57.642273+0000","last_peered":"2023-10-09T05:00:57.642273+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:59:55.016369+0000","last_became_peered":"2023-10-09T04:59:55.016369+0000","last_unstale":"2023-10-09T05:00:57.642273+0000","last_undegraded":"2023-10-09T04:59:33.814536+0000","last_fullsized":"2023-10-09T04:59:31.938954+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5276,"log_dups_size":0,"ondisk_log_size":5276,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138936320,"num_objects":2120,"num_object_clones":0,"num_object_copies":6360,"num_objects_missing_on_primary":1744,"num_objects_missing":1744,"num_objects_degraded":6081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2744,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":308,"num_bytes_recovered":20119552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":9},{"shards":"1,3","objects":-28},{"shards":"2,3,5,7","objects":346},{"shards":"2,5,7","objects":1744},{"shards":"3,5,6","objects":49}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":168,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.923421+0000","last_change":"2023-10-09T04:59:30.857229+0000","last_active":"2023-10-09T05:00:51.923421+0000","last_peered":"2023-10-09T05:00:51.923421+0000","last_clean":"2023-10-09T05:00:51.923421+0000","last_became_active":"2023-10-09T04:59:30.856946+0000","last_became_peered":"2023-10-09T04:59:30.856946+0000","last_unstale":"2023-10-09T05:00:51.923421+0000","last_undegraded":"2023-10-09T05:00:51.923421+0000","last_fullsized":"2023-10-09T05:00:51.923421+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:31.040095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":158,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.924850+0000","last_change":"2023-10-09T05:00:02.991182+0000","last_active":"2023-10-09T05:00:51.924850+0000","last_peered":"2023-10-09T05:00:51.924850+0000","last_clean":"2023-10-09T05:00:51.924850+0000","last_became_active":"2023-10-09T05:00:02.990840+0000","last_became_peered":"2023-10-09T05:00:02.990840+0000","last_unstale":"2023-10-09T05:00:51.924850+0000","last_undegraded":"2023-10-09T05:00:51.924850+0000","last_fullsized":"2023-10-09T05:00:51.924850+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:55.114402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"568'5202","reported_seq":16514,"reported_epoch":568,"state":"active+clean","last_fresh":"2023-10-09T05:00:58.382014+0000","last_change":"2023-10-09T05:00:26.900201+0000","last_active":"2023-10-09T05:00:58.382014+0000","last_peered":"2023-10-09T05:00:58.382014+0000","last_clean":"2023-10-09T05:00:58.382014+0000","last_became_active":"2023-10-09T05:00:26.894254+0000","last_became_peered":"2023-10-09T05:00:26.894254+0000","last_unstale":"2023-10-09T05:00:58.382014+0000","last_undegraded":"2023-10-09T05:00:58.382014+0000","last_fullsized":"2023-10-09T05:00:58.382014+0000","mapping_epoch":563,"log_start":"444'341","ondisk_log_start":"444'341","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138412032,"num_objects":2112,"num_object_clones":0,"num_object_copies":6336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2703,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2662,"num_bytes_recovered":161120256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":561,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.931666+0000","last_change":"2023-10-09T05:00:50.797283+0000","last_active":"2023-10-09T05:00:51.931666+0000","last_peered":"2023-10-09T05:00:51.931666+0000","last_clean":"2023-10-09T05:00:51.931666+0000","last_became_active":"2023-10-09T05:00:03.009228+0000","last_became_peered":"2023-10-09T05:00:03.009228+0000","last_unstale":"2023-10-09T05:00:51.931666+0000","last_undegraded":"2023-10-09T05:00:51.931666+0000","last_fullsized":"2023-10-09T05:00:51.931666+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:07.706140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":40,"reported_epoch":567,"state":"active+clean","last_fresh":"2023-10-09T05:00:51.928731+0000","last_change":"2023-10-09T04:58:59.821723+0000","last_active":"2023-10-09T05:00:51.928731+0000","last_peered":"2023-10-09T05:00:51.928731+0000","last_clean":"2023-10-09T05:00:51.928731+0000","last_became_active":"2023-10-09T04:58:59.821428+0000","last_became_peered":"2023-10-09T04:58:59.821428+0000","last_unstale":"2023-10-09T05:00:51.928731+0000","last_undegraded":"2023-10-09T05:00:51.928731+0000","last_fullsized":"2023-10-09T05:00:51.928731+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:15.048879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":4416012312,"num_objects":67384,"num_object_clones":0,"num_object_copies":202152,"num_objects_missing_on_primary":24624,"num_objects_missing":24624,"num_objects_degraded":80978,"num_objects_misplaced":11870,"num_objects_unfound":0,"num_objects_dirty":67384,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":87834,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42220,"num_bytes_recovered":2692939776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16692756480,"data_stored":16692740192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124252,"ondisk_log_size":124252,"up":78,"acting":58,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":31134,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31051,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":6177690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":31,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234645,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3755232,"kb_used_data":2241880,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":89989920,"statfs":{"total":95995035648,"available":92149678080,"internally_reserved":0,"allocated":2295685120,"data_stored":2294354390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.46000000000000002,"15min":0.96199999999999997},"min":{"1min":0.29299999999999998,"5min":0.27800000000000002,"15min":0.224},"max":{"1min":3.742,"5min":3.742,"15min":14.970000000000001},"last":0.433},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.55800000000000005,"15min":1.391},"min":{"1min":0.31,"5min":0.26300000000000001,"15min":0.249},"max":{"1min":3.319,"5min":3.319,"15min":21.216000000000001},"last":0.42099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:00:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.53300000000000003,"15min":3.0329999999999999},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.246},"max":{"1min":2.3290000000000002,"5min":2.9980000000000002,"15min":128.601},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.748,"15min":2.984},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":1.9039999999999999,"5min":8.1920000000000002,"15min":128.381},"last":0.40200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":10.281000000000001,"5min":2.9039999999999999,"15min":5.2009999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":154.76400000000001,"5min":154.76400000000001,"15min":207.96600000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":10.458,"5min":3,"15min":5.3949999999999996},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.29699999999999999},"max":{"1min":154.80000000000001,"5min":154.80000000000001,"15min":207.97900000000001},"last":0.76200000000000001}]},{"osd":3,"last update":"Mon Oct 9 04:59:58 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.6020000000000001,"15min":1.706},"min":{"1min":0.41099999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.048,"5min":74.259,"15min":74.259},"last":0.5},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.5169999999999999,"15min":1.9470000000000001},"min":{"1min":0.38200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2490000000000001,"5min":74.100999999999999,"15min":74.100999999999999},"last":6.7999999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.30199999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.60099999999999998,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.347,"5min":3.347,"15min":3.347},"last":0.34100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.49299999999999999,"15min":1.6830000000000001},"min":{"1min":0.33100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":131.84899999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.61799999999999999,"15min":2.1549999999999998},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.0880000000000001,"5min":7.0880000000000001,"15min":135.74199999999999},"last":0.44700000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.51400000000000001,"15min":1.111},"min":{"1min":0.38800000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81699999999999995,"5min":0.90000000000000002,"15min":10.047000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.63,"15min":1.397},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.1799999999999997,"5min":7.1799999999999997,"15min":21.321999999999999},"last":0.38700000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496277,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1690664,"kb_used_data":788384,"kb_used_omap":0,"kb_used_meta":902272,"kb_avail":92054488,"statfs":{"total":95995035648,"available":94263795712,"internally_reserved":0,"allocated":807305216,"data_stored":805966639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":923926528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:00 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.48799999999999999,"15min":0.84499999999999997},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":0.89600000000000002,"5min":3.2109999999999999,"15min":9.4979999999999993},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.54800000000000004,"15min":1.0900000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26700000000000002},"max":{"1min":2.3039999999999998,"5min":6.1980000000000004,"15min":16.574000000000002},"last":0.35799999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:00:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.8400000000000001,"15min":5.4240000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":13.621,"5min":90.278999999999996,"15min":260.738},"last":1.7070000000000001},{"interface":"front","average":{"1min":1.524,"5min":1.843,"15min":5.1130000000000004},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.26300000000000001},"max":{"1min":13.756,"5min":90.414000000000001,"15min":261.26100000000002},"last":1.7949999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.024,"15min":2.915},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.706,"5min":9.2759999999999998,"15min":133.535},"last":1.8120000000000001},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.0449999999999999,"15min":3.0720000000000001},"min":{"1min":0.36199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.7370000000000001,"5min":9.2639999999999993,"15min":133.51900000000001},"last":1.7809999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:00:00 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":4.2969999999999997,"15min":2.6930000000000001},"min":{"1min":0.40500000000000003,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":5,"5min":147.43600000000001,"15min":147.43600000000001},"last":1.8919999999999999},{"interface":"front","average":{"1min":1.032,"5min":4.4050000000000002,"15min":2.859},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.9590000000000001,"5min":147.41999999999999,"15min":147.41999999999999},"last":1.8819999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:00:42 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.45600000000000002,"15min":0.46500000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":1.7290000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66700000000000004,"5min":2.411,"15min":2.411},"last":1.8500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:00:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.54300000000000004,"15min":1.952},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":1.119,"5min":3.4180000000000001,"15min":83.168999999999997},"last":1.837},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.55100000000000005,"15min":2.0089999999999999},"min":{"1min":0.378,"5min":0.254,"15min":0.254},"max":{"1min":0.66400000000000003,"5min":2.3879999999999999,"15min":83.126000000000005},"last":1.863}]},{"osd":7,"last update":"Mon Oct 9 05:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.497,"15min":1.351},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":0.76300000000000001,"5min":0.80700000000000005,"15min":36.204999999999998},"last":1.8260000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.53000000000000003,"15min":1.3260000000000001},"min":{"1min":0.40200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.64700000000000002,"5min":3.3969999999999998,"15min":10.661},"last":1.909}]}]},{"osd":1,"up_from":118,"seq":506806141350,"num_pgs":87,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2285868,"kb_used_data":1090660,"kb_used_omap":4,"kb_used_meta":1195195,"kb_avail":91459284,"statfs":{"total":95995035648,"available":93654306816,"internally_reserved":0,"allocated":1116835840,"data_stored":1113967342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1223880590},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.196,"15min":1.1339999999999999},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":8.0999999999999996,"5min":11.763999999999999,"15min":11.763999999999999},"last":0.23300000000000001},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.0549999999999999,"15min":1.149},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":7.032,"5min":7.032,"15min":10.765000000000001},"last":0.49199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.8,"15min":3.7530000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.21099999999999999},"max":{"1min":7.1909999999999998,"5min":63.143999999999998,"15min":263.089},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.107,"5min":1.6459999999999999,"15min":3.702},"min":{"1min":0.32400000000000001,"5min":0.254,"15min":0.222},"max":{"1min":7.3810000000000002,"5min":66.198999999999998,"15min":254.50899999999999},"last":0.52500000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:00:05 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.585,"15min":1.73},"min":{"1min":0.34100000000000003,"5min":0.28699999999999998,"15min":0.26500000000000001},"max":{"1min":4.3040000000000003,"5min":39.289000000000001,"15min":77.819999999999993},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.377,"15min":1.776},"min":{"1min":0.315,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":2.7530000000000001,"5min":14.212,"15min":77.796000000000006},"last":0.33100000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:00:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.6279999999999999,"15min":1.79},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":9.7699999999999996,"5min":19.634,"15min":19.634},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.232,"5min":1.2949999999999999,"15min":1.3109999999999999},"min":{"1min":0.38800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":7.298,"5min":13.045,"15min":13.045},"last":0.313}]},{"osd":5,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.002,"15min":1.534},"min":{"1min":0.42299999999999999,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":7.79,"5min":7.79,"15min":9.782},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.907,"5min":1.244,"15min":1.659},"min":{"1min":0.379,"5min":0.379,"15min":0.315},"max":{"1min":9.7430000000000003,"5min":19.597999999999999,"15min":20.021000000000001},"last":0.61799999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.113,"15min":1.46},"min":{"1min":0.39700000000000002,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":7.4370000000000003,"5min":7.4370000000000003,"15min":9.7840000000000007},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.407,"15min":1.736},"min":{"1min":0.44800000000000001,"5min":0.38,"15min":0.32400000000000001},"max":{"1min":9.6639999999999997,"5min":19.548999999999999,"15min":20.164999999999999},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.4279999999999999,"15min":2.2450000000000001},"min":{"1min":0.48799999999999999,"5min":0.373,"15min":0.35899999999999999},"max":{"1min":9.7040000000000006,"5min":19.664999999999999,"15min":59.972000000000001},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.252,"15min":2.0150000000000001},"min":{"1min":0.52200000000000002,"5min":0.39300000000000002,"15min":0.378},"max":{"1min":7.335,"5min":9.8710000000000004,"15min":60.011000000000003},"last":0.40500000000000003}]}]},{"osd":0,"up_from":0,"seq":390842024430,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772705,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4531380,"kb_used_data":3741932,"kb_used_omap":7,"kb_used_meta":789432,"kb_avail":89213772,"statfs":{"total":95995035648,"available":91354902528,"internally_reserved":0,"allocated":3831738368,"data_stored":3829278734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":808378449},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.133,"15min":1.3280000000000001},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.188},"max":{"1min":5.4580000000000002,"5min":13.042999999999999,"15min":25.899999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.385,"5min":0.85499999999999998,"15min":1.0640000000000001},"min":{"1min":0.316,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":7.8010000000000002,"5min":12.161,"15min":16.550000000000001},"last":0.313}]},{"osd":1,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":2.016,"15min":3.2930000000000001},"min":{"1min":0.27000000000000002,"5min":0.25800000000000001,"15min":0.20499999999999999},"max":{"1min":7.7649999999999997,"5min":42.799999999999997,"15min":170.172},"last":5.5670000000000002},{"interface":"front","average":{"1min":1.3819999999999999,"5min":2.1389999999999998,"15min":3.4700000000000002},"min":{"1min":0.29399999999999998,"5min":0.254,"15min":0.219},"max":{"1min":8.0960000000000001,"5min":42.524999999999999,"15min":170.511},"last":0.55700000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.921,"15min":1.6759999999999999},"min":{"1min":0.34100000000000003,"5min":0.30199999999999999,"15min":0.248},"max":{"1min":2.9980000000000002,"5min":42.557000000000002,"15min":45.664000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.872,"5min":1.9690000000000001,"15min":1.7789999999999999},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":5.5090000000000003,"5min":42.293999999999997,"15min":45.686999999999998},"last":0.35399999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.75800000000000001,"15min":0.622},"min":{"1min":0.376,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.9660000000000002,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":0.76900000000000002,"15min":0.64800000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8620000000000001,"5min":7.8620000000000001,"15min":7.8620000000000001},"last":0.51000000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.79600000000000004,"15min":3.5950000000000002},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.9299999999999997,"5min":7.9299999999999997,"15min":272.04700000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.93799999999999994,"15min":3.653},"min":{"1min":0.38,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":2.79,"5min":10.427,"15min":271.89699999999999},"last":0.61899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.94899999999999995,"15min":1.4039999999999999},"min":{"1min":0.42799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.5720000000000001,"5min":8.3469999999999995,"15min":16.425999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.84599999999999997,"15min":1.24},"min":{"1min":0.42299999999999999,"5min":0.38700000000000001,"15min":0.35199999999999998},"max":{"1min":7.9000000000000004,"5min":7.9000000000000004,"15min":10.279999999999999},"last":0.435}]},{"osd":7,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.161,"15min":1.73},"min":{"1min":0.45100000000000001,"5min":0.38600000000000001,"15min":0.34300000000000003},"max":{"1min":5.5439999999999996,"5min":8.3949999999999996,"15min":16.462},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.355,"5min":1.02,"15min":1.548},"min":{"1min":0.45400000000000001,"5min":0.42199999999999999,"15min":0.373},"max":{"1min":7.9870000000000001,"5min":9.0640000000000001,"15min":10.211},"last":0.47899999999999998}]}]},{"osd":3,"up_from":142,"seq":609885356402,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2195880,"kb_used_data":1383200,"kb_used_omap":2,"kb_used_meta":812669,"kb_avail":91549272,"statfs":{"total":95995035648,"available":93746454528,"internally_reserved":0,"allocated":1416396800,"data_stored":1414059260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":832173109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.58999999999999997,"15min":0.83799999999999997},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.19},"max":{"1min":2.9409999999999998,"5min":3.0219999999999998,"15min":11.817},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.80000000000000004,"15min":1.103},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.22},"max":{"1min":2.9780000000000002,"5min":6.5229999999999997,"15min":16.358000000000001},"last":3.1779999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:00:01 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":0.92000000000000004,"15min":2.9369999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":7.1740000000000004,"5min":8.5199999999999996,"15min":125.233},"last":0.496},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.997,"15min":2.8300000000000001},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.21299999999999999},"max":{"1min":7.2300000000000004,"5min":11.44,"15min":126.102},"last":0.63500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:00:26 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":2.8260000000000001,"15min":3.0529999999999999},"min":{"1min":0.30199999999999999,"5min":0.29599999999999999,"15min":0.23499999999999999},"max":{"1min":15.606999999999999,"5min":212.065,"15min":212.065},"last":0.372},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.8599999999999999,"15min":3.2080000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.22600000000000001},"max":{"1min":4.2140000000000004,"5min":211.98400000000001,"15min":211.98400000000001},"last":3.3479999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:00:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.024,"15min":0.98399999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.7679999999999998,"5min":4.9109999999999996,"15min":4.9109999999999996},"last":3.3170000000000002},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.2030000000000001,"15min":1.2529999999999999},"min":{"1min":0.35799999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.8879999999999999,"5min":8.5779999999999994,"15min":8.5779999999999994},"last":3.4750000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:00:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.78100000000000003,"15min":1.4930000000000001},"min":{"1min":0.38,"5min":0.35299999999999998,"15min":0.30599999999999999},"max":{"1min":0.92300000000000004,"5min":11.368,"15min":28.302},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.101,"15min":1.6870000000000001},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":7.3040000000000003,"5min":24.986999999999998,"15min":28.280000000000001},"last":3.2559999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:00:01 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.79800000000000004,"15min":1.2869999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.32900000000000001},"max":{"1min":2.8239999999999998,"5min":5.1589999999999998,"15min":16.439},"last":3.2890000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.194,"15min":1.6220000000000001},"min":{"1min":0.44400000000000001,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":5.9359999999999999,"5min":24.916,"15min":24.916},"last":3.5139999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:00:01 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.171,"15min":1.8420000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.375},"max":{"1min":6.0119999999999996,"5min":25.004000000000001,"15min":25.004000000000001},"last":3.5459999999999998},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.71499999999999997,"15min":1.347},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":7.2409999999999997,"5min":7.2409999999999997,"15min":12.486000000000001},"last":0.73999999999999999}]}]},{"osd":4,"up_from":507,"seq":2177548419113,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3576744,"kb_used_data":3179936,"kb_used_omap":11,"kb_used_meta":396788,"kb_avail":90168408,"statfs":{"total":95995035648,"available":92332449792,"internally_reserved":0,"allocated":3256254464,"data_stored":3254915618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":406311816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.80200000000000005,"15min":0.81399999999999995},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9379999999999999,"5min":7.9020000000000001,"15min":7.9020000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.71999999999999997,"15min":0.69699999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.226,"5min":5.4770000000000003,"15min":5.4770000000000003},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":5.6269999999999998,"15min":7.5119999999999996},"min":{"1min":0.28000000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2000000000000002,"5min":147.49600000000001,"15min":147.49600000000001},"last":2.839},{"interface":"front","average":{"1min":0.995,"5min":5.7709999999999999,"15min":7.7370000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.8639999999999999,"5min":147.51300000000001,"15min":147.51300000000001},"last":0.36199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.028,"15min":0.95799999999999996},"min":{"1min":0.34300000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.5620000000000003,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.498},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.23,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.766,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.45300000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.377,"15min":1.1619999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.8099999999999996,"5min":24.379000000000001,"15min":24.379000000000001},"last":2.669},{"interface":"front","average":{"1min":1.4430000000000001,"5min":2.2679999999999998,"15min":2.3919999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.2610000000000001,"5min":24.23,"15min":24.23},"last":1.3080000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.40799999999999997,"15min":0.40500000000000003},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.53300000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50600000000000001,"15min":0.48999999999999999},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":1.8879999999999999,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.47599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.83499999999999996,"15min":0.74099999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67300000000000004,"5min":22.350999999999999,"15min":22.350999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.77400000000000002,"15min":0.60999999999999999},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.702,"5min":22.433,"15min":22.433},"last":0.38700000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.308},"max":{"1min":1.9179999999999999,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.65200000000000002,"15min":0.71899999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70099999999999996,"5min":4.5949999999999998,"15min":4.5949999999999998},"last":0.371}]}]},{"osd":5,"up_from":38,"seq":163208757921,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5230616,"kb_used_data":3733200,"kb_used_omap":0,"kb_used_meta":1497408,"kb_avail":88514536,"statfs":{"total":95995035648,"available":90638884864,"internally_reserved":0,"allocated":3822796800,"data_stored":3821469550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1533345792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:14 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.7430000000000001,"15min":1.617},"min":{"1min":0.28000000000000003,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.55000000000000004,"5min":106.465,"15min":106.465},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.8959999999999999,"15min":1.637},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.23799999999999999},"max":{"1min":1.1479999999999999,"5min":106.54600000000001,"15min":106.54600000000001},"last":0.35799999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:00:14 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.69299999999999995,"15min":4.6509999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":3.3210000000000002,"5min":3.5840000000000001,"15min":211.85400000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.72999999999999998,"15min":4.8559999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.255},"max":{"1min":3.8820000000000001,"5min":11.249000000000001,"15min":211.435},"last":0.39900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:00:32 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":1.2529999999999999,"15min":2.831},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":47.262999999999998,"5min":47.262999999999998,"15min":86.488},"last":0.41899999999999998},{"interface":"front","average":{"1min":3.488,"5min":1.5589999999999999,"15min":3.0630000000000002},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":47.408000000000001,"5min":47.408000000000001,"15min":86.510999999999996},"last":3.3580000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.121,"15min":1.823},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.31900000000000001},"max":{"1min":4.0549999999999997,"5min":105.879,"15min":105.879},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":2.278,"15min":1.6439999999999999},"min":{"1min":0.44500000000000001,"5min":0.316,"15min":0.316},"max":{"1min":4.0830000000000002,"5min":105.625,"15min":105.625},"last":3.722}]},{"osd":4,"last update":"Mon Oct 9 05:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.53700000000000003,"15min":0.56499999999999995},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.1679999999999999,"5min":2.2360000000000002,"15min":2.2360000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.47499999999999998,"15min":0.48199999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73299999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.436}]},{"osd":6,"last update":"Mon Oct 9 05:00:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.54200000000000004,"15min":0.98599999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.0659999999999998,"5min":4.0659999999999998,"15min":10.295999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.49399999999999999,"15min":1.103},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.3240000000000001,"5min":2.234,"15min":13.083},"last":0.5}]},{"osd":7,"last update":"Mon Oct 9 05:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.55600000000000005,"15min":2.3380000000000001},"min":{"1min":0.38300000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.8540000000000001,"5min":2.8540000000000001,"15min":110.521},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56899999999999995,"15min":2.4159999999999999},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.29499999999999998},"max":{"1min":1.7310000000000001,"5min":2.266,"15min":110.746},"last":0.54100000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":197722112,"data_stored":197722112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1108148224,"data_stored":1108148224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3823505408,"data_stored":3823501336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1409482752,"data_stored":1409482752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3250196480,"data_stored":3250192408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3816361984,"data_stored":3816357912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":800788480,"data_stored":800784408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2286551040,"data_stored":2286551040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:01:03.922 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pgp_num 26 2023-10-09T05:01:04.018 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:04.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:04.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:04.180 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:04.600 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:04.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:04.706 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:04.745 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:05 smithi177 ceph-mon[195797]: pgmap v1901: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80927/201786 objects degraded (40.105%); 11798/201786 objects misplaced (5.847%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:05.587 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:05 smithi029 ceph-mon[223858]: pgmap v1901: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80927/201786 objects degraded (40.105%); 11798/201786 objects misplaced (5.847%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:05.587 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:05 smithi029 ceph-mon[216903]: pgmap v1901: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80927/201786 objects degraded (40.105%); 11798/201786 objects misplaced (5.847%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:05.911 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:05.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:05.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:06.016 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:06.044 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:06.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:06.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:06.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:06.202 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:06.255 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:06.365 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:06.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:06.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:06.474 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:06.499 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80927/201786 objects degraded (40.105%), 26 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:06.502 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:06.526 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:06.552 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:06.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:06.657 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80927/201786 objects degraded (40.105%), 26 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80927/201786 objects degraded (40.105%), 26 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:06.684 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:06.736 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:06.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:06.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:06.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:06.869 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:06.914 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:06.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:07.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:07.053 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:07.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:07.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:07.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:07.310 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:07 smithi029 ceph-mon[216903]: pgmap v1902: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80897/201264 objects degraded (40.194%); 11768/201264 objects misplaced (5.847%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:01:07.311 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:07 smithi029 ceph-mon[223858]: pgmap v1902: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80897/201264 objects degraded (40.194%); 11768/201264 objects misplaced (5.847%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:01:07.311 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75785 14.1 14:8b563722:::benchmark_data_smithi029_550670_object75784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581965+0000", 2023-10-09T05:01:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 381.80407967600001, 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 12.481576762, 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75785 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581965+0000", 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581965+0000", 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581964+0000", 2023-10-09T05:01:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581992+0000", 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7560999999999999e-05 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582001+0000", 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2410000000000001e-06 2023-10-09T05:01:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582014+0000", 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2964e-05 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582129+0000", 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115244 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582138+0000", 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0499999999999997e-06 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:57.982937+0000", 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 12.400798625 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:57.990054+0000", 2023-10-09T05:01:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071166620000000002 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.008517+0000", 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018463080999999999 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.021868+0000", 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01335124 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.042048+0000", 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020179806000000002 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.044195+0000", 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021467069999999999 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.047476+0000", 2023-10-09T05:01:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032809219999999999 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.063542+0000", 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016066225999999999 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75806 14.9 e454/449)", 2023-10-09T05:01:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.525967+0000", 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.86007744099999, 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.546416377, 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525967+0000", 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525967+0000", 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525966+0000", 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526032+0000", 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6188999999999999e-05 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526047+0000", 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4789e-05 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526056+0000", 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6300000000000004e-06 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.042096+0000", 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.516040319 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.042131+0000", 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4876000000000003e-05 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072334+0000", 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030203741999999999 2023-10-09T05:01:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072384+0000", 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9141999999999998e-05 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75808 14.9 e454/449)", 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.529907+0000", 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.856137494, 2023-10-09T05:01:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.547706969, 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529907+0000", 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529907+0000", 2023-10-09T05:01:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529906+0000", 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529941+0000", 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4762999999999999e-05 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529954+0000", 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3297e-05 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529963+0000", 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3389999999999999e-06 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.063586+0000", 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.533623241000001 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.063618+0000", 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1579999999999999e-05 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077574+0000", 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013955778 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077614+0000", 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0300999999999999e-05 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75811 14.9 e454/449)", 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.544466+0000", 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.84157888999999, 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.533182881, 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544466+0000", 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544466+0000", 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544465+0000", 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544531+0000", 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5989999999999997e-05 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544543+0000", 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1668000000000001e-05 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544550+0000", 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8689999999999998e-06 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.068317+0000", 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.523767645 2023-10-09T05:01:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.068341+0000", 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3470999999999999e-05 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077621+0000", 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092804050000000003 2023-10-09T05:01:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077649+0000", 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7525000000000002e-05 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75815 14.1 14:83dd8a20:::benchmark_data_smithi029_550670_object75814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.548948+0000", 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.83709661500001, 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.711953803, 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75815 2023-10-09T05:01:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548948+0000", 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548948+0000", 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548946+0000", 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548972+0000", 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5953999999999999e-05 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548991+0000", 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8706e-05 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.549004+0000", 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2743e-05 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.101680+0000", 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.552676115000001 2023-10-09T05:01:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.114195+0000", 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012514759 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.134726+0000", 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020531269000000001 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.216945+0000", 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.082219406999999994 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.221124+0000", 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041786879999999998 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.222701+0000", 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015774960000000001 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.230571+0000", 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078695970000000007 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.260902+0000", 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030330778999999999 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75824 14.1 14:86e81355:::benchmark_data_smithi029_550670_object75823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.566240+0000", 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.81980490000001, 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.729266398, 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75824 2023-10-09T05:01:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566240+0000", 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566240+0000", 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566238+0000", 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566260+0000", 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1810999999999998e-05 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566270+0000", 2023-10-09T05:01:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6870000000000006e-06 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566277+0000", 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.481e-06 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.147946+0000", 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.581669004 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.169373+0000", 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021426062999999999 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.199762+0000", 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030389360000000001 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.262198+0000", 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062436396999999998 2023-10-09T05:01:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.266913+0000", 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047142499999999997 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.271230+0000", 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043171829999999996 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.277067+0000", 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005837582 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.295506+0000", 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01843887 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75829 14.1 14:8b2b9eaf:::benchmark_data_smithi029_550670_object75828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.571726+0000", 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.81431831100002, 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.818256821, 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75829 2023-10-09T05:01:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571726+0000", 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571726+0000", 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571724+0000", 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571762+0000", 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8056000000000003e-05 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571776+0000", 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.349e-05 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571786+0000", 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8719999999999997e-06 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.296115+0000", 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.724328866 2023-10-09T05:01:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.304023+0000", 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079088049999999997 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.315603+0000", 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011579145000000001 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.358599+0000", 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042996389000000003 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.360933+0000", 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023335909999999999 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.363428+0000", 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024950300000000001 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.366629+0000", 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032013940000000002 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.389983+0000", 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.02335419 2023-10-09T05:01:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75832 14.9 e454/449)", 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.582541+0000", 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.80350386999999, 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.518063569000001, 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582541+0000", 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582541+0000", 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582540+0000", 2023-10-09T05:01:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582578+0000", 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8028000000000003e-05 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582594+0000", 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6560999999999999e-05 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582603+0000", 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3489999999999992e-06 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072417+0000", 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.489814439 2023-10-09T05:01:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072445+0000", 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7645999999999999e-05 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.100571+0000", 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028126284000000001 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.100604+0000", 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3399000000000003e-05 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75834 14.9 e454/449)", 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.585970+0000", 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.80007508300002, 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.628231904, 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585970+0000", 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585970+0000", 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585969+0000", 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586023+0000", 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4001e-05 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586033+0000", 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0575e-05 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586040+0000", 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0829999999999998e-06 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077690+0000", 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.491649827 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077716+0000", 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5324999999999999e-05 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214158+0000", 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.136442024 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214201+0000", 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3862000000000001e-05 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75837 14.1 14:8c05572a:::benchmark_data_smithi029_550670_object75836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.601333+0000", 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.78471153200002, 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.873180149, 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75837 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601333+0000", 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601333+0000", 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601332+0000", 2023-10-09T05:01:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601361+0000", 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9600000000000001e-05 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601376+0000", 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4403000000000001e-05 2023-10-09T05:01:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601386+0000", 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0416e-05 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.323704+0000", 2023-10-09T05:01:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.72231845 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.332083+0000", 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083783480000000007 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:01:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.346710+0000", 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014627693000000001 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.362819+0000", 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016108062999999999 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.392608+0000", 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029789645999999999 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.445466+0000", 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.052858166999999998 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.451631+0000", 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061645199999999997 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.474513+0000", 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022882422999999999 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75840 14.9 e454/449)", 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.614574+0000", 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.771470846, 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.599658032000001, 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614574+0000", 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614574+0000", 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614573+0000", 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614607+0000", 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3590999999999997e-05 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614616+0000", 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.081e-06 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614624+0000", 2023-10-09T05:01:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4540000000000007e-06 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.083376+0000", 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.468752077 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.083408+0000", 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2087e-05 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214204+0000", 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13079591800000001 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214232+0000", 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7529e-05 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPushReply(14.3 467/464 [PushReplyOp(14:cffce7f4:::benchmark_data_smithi029_550670_object77452:head)])", 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:36.986257+0000", 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 330.39978812700002, 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9163757380000002, 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986257+0000", 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986257+0000", 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986255+0000", 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986265+0000", 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9890000000000004e-06 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986281+0000", 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5446000000000001e-05 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986291+0000", 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8539999999999992e-06 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:43.883633+0000", 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8973423409999999 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:43.902632+0000", 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018999194 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79398 14.3 14:c5eeef8d:::benchmark_data_smithi029_550670_object1787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:10.796523+0000", 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 296.58952213499998, 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6241380140000001, 2023-10-09T05:01:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79398 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796523+0000", 2023-10-09T05:01:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796523+0000", 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796520+0000", 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796533+0000", 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2126e-05 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796548+0000", 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5633000000000001e-05 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796564+0000", 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5344000000000001e-05 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.219053+0000", 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.422489797 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.235375+0000", 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016321806000000001 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.331532+0000", 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.096156778999999998 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.337123+0000", 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055913389999999999 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.158303+0000", 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8211796690000002 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.168693+0000", 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010389667999999999 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.180089+0000", 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011396702999999999 2023-10-09T05:01:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.256103+0000", 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076013678000000001 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.395645+0000", 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13954192900000001 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.397350+0000", 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001704672 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.401058+0000", 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037084610000000001 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.420661+0000", 2023-10-09T05:01:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019602482000000001 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79458 14.3 14:c243e4b1:::benchmark_data_smithi029_550670_object1847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:11.471770+0000", 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 295.91427481800002, 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5716450159999997, 2023-10-09T05:01:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79458 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471770+0000", 2023-10-09T05:01:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471770+0000", 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471768+0000", 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471775+0000", 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2960000000000001e-06 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471790+0000", 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4711e-05 2023-10-09T05:01:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471799+0000", 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8880000000000003e-06 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.722387+0000", 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25058769800000003 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.736403+0000", 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014016631 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.835566+0000", 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099163167999999996 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.839398+0000", 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038317640000000001 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.899006+0000", 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0596074309999999 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.906843+0000", 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078377759999999994 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.917496+0000", 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010652557999999999 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.945373+0000", 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027876995000000002 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.021672+0000", 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076299464999999997 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.024686+0000", 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030134469999999998 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.028127+0000", 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034407800000000001 2023-10-09T05:01:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.043415+0000", 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015288339999999999 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79468 14.3 14:cf2d94b1:::benchmark_data_smithi029_550670_object1857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:11.676314+0000", 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 295.70973020399998, 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3917630189999999, 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79468 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676314+0000", 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676314+0000", 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676312+0000", 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676321+0000", 2023-10-09T05:01:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.166e-06 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676337+0000", 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5852e-05 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676346+0000", 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9590000000000001e-06 2023-10-09T05:01:07.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.774451+0000", 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.09810518 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.786991+0000", 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012539385 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.898062+0000", 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.111071581 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.909583+0000", 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011521103 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.991195+0000", 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0816113939999998 2023-10-09T05:01:07.492 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.996599+0000", 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054041219999999999 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.006568+0000", 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099691250000000006 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.022256+0000", 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015688437999999999 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.045384+0000", 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023128057 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.047287+0000", 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001902513 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.050149+0000", 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002862065 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.068077+0000", 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017928436999999998 2023-10-09T05:01:07.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79489 14.3 14:c00d8f05:::benchmark_data_smithi029_550670_object1878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:12.219889+0000", 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 295.16615546499997, 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4242194270000001, 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.496 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79489 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219889+0000", 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219889+0000", 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219887+0000", 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219896+0000", 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8710000000000003e-06 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219915+0000", 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9021e-05 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219928+0000", 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2598e-05 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.273993+0000", 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054065509999999997 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.293072+0000", 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019078660000000001 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.529355+0000", 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23628290199999999 2023-10-09T05:01:07.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.537965+0000", 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086095359999999992 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.443954+0000", 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9059896649999999 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.452576+0000", 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086220489999999997 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.463464+0000", 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010887732000000001 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.528955+0000", 2023-10-09T05:01:07.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.065491266000000006 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.605316+0000", 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076360474999999997 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.608931+0000", 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036152419999999998 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.612435+0000", 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035043959999999999 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.644109+0000", 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031673289 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79493 14.3 14:c64f621f:::benchmark_data_smithi029_550670_object1882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:12.269365+0000", 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 295.11667931300002, 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4045552749999999, 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79493 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.505 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269365+0000", 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269365+0000", 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269364+0000", 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269371+0000", 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8739999999999997e-06 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269385+0000", 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4453999999999999e-05 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269393+0000", 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3969999999999997e-06 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.609689+0000", 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34029656000000003 2023-10-09T05:01:07.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.619319+0000", 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096294910000000004 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.718361+0000", 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099042179999999994 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.509 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.722461+0000", 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041005870000000002 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.573673+0000", 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8512119760000001 2023-10-09T05:01:07.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.581260+0000", 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075868860000000002 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.596270+0000", 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015009972 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.605958+0000", 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.009687279 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.649286+0000", 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043328558000000003 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.652757+0000", 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034707100000000001 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.657267+0000", 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045104250000000002 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.673921+0000", 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016653411 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79501 14.3 14:cfa44866:::benchmark_data_smithi029_550670_object1890:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:12.343757+0000", 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 295.04228730800003, 2023-10-09T05:01:07.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5847279009999999, 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79501 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343757+0000", 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343757+0000", 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343755+0000", 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343763+0000", 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3429999999999998e-06 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343776+0000", 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3511e-05 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343785+0000", 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6260000000000001e-06 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.620575+0000", 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27679018100000002 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.640756+0000", 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020180974000000001 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.756643+0000", 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.115886903 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.760712+0000", 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040691850000000003 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.796144+0000", 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0354321290000001 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.801378+0000", 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052342739999999997 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.810629+0000", 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092507800000000001 2023-10-09T05:01:07.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.865118+0000", 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054488254999999999 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.896139+0000", 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031021271999999999 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.898233+0000", 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020940500000000001 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.901551+0000", 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033177020000000001 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.928485+0000", 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026934738 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80535 14.3 14:cc98a142:::benchmark_data_smithi029_550670_object2924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:01:07.524 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 181.13611493799999, 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 40.148030116999998, 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80535 2023-10-09T05:01:07.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249927+0000", 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249939+0000", 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1640999999999999e-05 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249963+0000", 2023-10-09T05:01:07.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3714999999999999e-05 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249976+0000", 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3699e-05 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.632019+0000", 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38204312600000001 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.653132+0000", 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021113060999999999 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.779084+0000", 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12595120000000001 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.797955+0000", 2023-10-09T05:01:07.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018871354 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103793+0000", 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30583778099999998 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103806+0000", 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3494e-05 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844372+0000", 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.74056582000000004 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844384+0000", 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2394e-05 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.144509+0000", 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 38.300124437999997 2023-10-09T05:01:07.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.170256+0000", 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025747190999999999 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.299425+0000", 2023-10-09T05:01:07.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12916846400000001 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.304976+0000", 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055519480000000001 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.314789+0000", 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098120340000000007 2023-10-09T05:01:07.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.351252+0000", 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036463151999999999 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.353456+0000", 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022039910000000002 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.356260+0000", 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028044680000000001 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.358811+0000", 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025511090000000002 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.365432+0000", 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066203989999999999 2023-10-09T05:01:07.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.397960+0000", 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032528230999999998 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80543 14.3 14:cf2f7cf9:::benchmark_data_smithi029_550670_object2932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:01:07.537 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 180.856774976, 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 39.820873380999998, 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80543 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:01:07.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529268+0000", 2023-10-09T05:01:07.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529279+0000", 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1258e-05 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529301+0000", 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1912e-05 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529314+0000", 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3030000000000001e-05 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.655681+0000", 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12636664 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:01:07.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.679149+0000", 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023468648000000002 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.887190+0000", 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20804069 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.911521+0000", 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0243314 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103710+0000", 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19218885399999999 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103727+0000", 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6439999999999998e-05 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844324+0000", 2023-10-09T05:01:07.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.740597536 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844341+0000", 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7054999999999998e-05 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.011168+0000", 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 38.166826931000003 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for readable", 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.028691+0000", 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017522685 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.108149+0000", 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.079458184000000001 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.127919+0000", 2023-10-09T05:01:07.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019770056000000001 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.216507+0000", 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.088587677000000004 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.223312+0000", 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068052829999999996 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.233101+0000", 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097893839999999999 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.265633+0000", 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032531428000000001 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.323546+0000", 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057912784000000002 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.325561+0000", 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020157949999999999 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.327379+0000", 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001817639 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.330515+0000", 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031356869999999998 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.350143+0000", 2023-10-09T05:01:07.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019628362 2023-10-09T05:01:07.550 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.550 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.550 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.550 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:01:07.550 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:01:07.550 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:01:07.551 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:07 smithi177 ceph-mon[195797]: pgmap v1902: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80897/201264 objects degraded (40.194%); 11768/201264 objects misplaced (5.847%); 2.7 MiB/s, 42 objects/s recovering 2023-10-09T05:01:07.769 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:07.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:07.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:07.901 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:07.942 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:07.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:08.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:08.079 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:08.274 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:01:08.389 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:01:08.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3368256614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3368256614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3368256614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: pgmap v1903: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 80717/200931 objects degraded (40.172%); 11768/200931 objects misplaced (5.857%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:01:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:01:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:09 smithi177 ceph-mon[195797]: osdmap e569: 8 total, 8 up, 7 in 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: pgmap v1903: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 80717/200931 objects degraded (40.172%); 11768/200931 objects misplaced (5.857%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[216903]: osdmap e569: 8 total, 8 up, 7 in 2023-10-09T05:01:09.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: pgmap v1903: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 80717/200931 objects degraded (40.172%); 11768/200931 objects misplaced (5.857%); 2.5 MiB/s, 39 objects/s recovering 2023-10-09T05:01:09.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:09.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: failed to encode map e569 with expected crc 2023-10-09T05:01:09.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:01:09.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:09 smithi029 ceph-mon[223858]: osdmap e569: 8 total, 8 up, 7 in 2023-10-09T05:01:09.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:01:09.942 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:09.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:09.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:10.006 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:10.047 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:10.078 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:10.135 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:10.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:10.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:10.183 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:10.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:10.289 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:10.416 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:10.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:10.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:10.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:10.556 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:10.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:10.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:10.865 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:10.917 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:11.033 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:11.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:11.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:11.138 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:11.171 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:11.223 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:11.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:11.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:11.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[216903]: pgmap v1905: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 80611/200235 objects degraded (40.258%); 11712/200235 objects misplaced (5.849%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:01:11.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2093653136' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:11.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:11.271 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80611/200235 objects degraded (40.258%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:11.292 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 14 pgp_num to 26 2023-10-09T05:01:11.327 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:11.368 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:11.442 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:11.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:11 smithi177 ceph-mon[195797]: pgmap v1905: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 80611/200235 objects degraded (40.258%); 11712/200235 objects misplaced (5.849%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:01:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2093653136' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80611/200235 objects degraded (40.258%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:11.555 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:11.583 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[223858]: pgmap v1905: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 80611/200235 objects degraded (40.258%); 11712/200235 objects misplaced (5.849%); 2.4 MiB/s, 37 objects/s recovering 2023-10-09T05:01:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2093653136' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:11.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80611/200235 objects degraded (40.258%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:11.771 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:11.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:11.874 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:11.907 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:11.971 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:11.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:11.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:12.036 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:01:12.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:12.152 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:12.356 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:01:12.356 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:01:12.356 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:01:12.356 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:01:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:01:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:12 smithi177 ceph-mon[195797]: osdmap e570: 8 total, 8 up, 7 in 2023-10-09T05:01:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:12 smithi177 ceph-mon[195797]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:12 smithi177 ceph-mon[195797]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:12 smithi177 ceph-mon[195797]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:12 smithi177 ceph-mon[195797]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:01:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[216903]: osdmap e570: 8 total, 8 up, 7 in 2023-10-09T05:01:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[216903]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[216903]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[216903]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[216903]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:01:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[223858]: osdmap e570: 8 total, 8 up, 7 in 2023-10-09T05:01:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[223858]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[223858]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[223858]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:12 smithi029 ceph-mon[223858]: failed to encode map e570 with expected crc 2023-10-09T05:01:12.742 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:12.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:12.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:12.905 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:13.031 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:01:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:01:12] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.43.0" 2023-10-09T05:01:13.047 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:13.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:13.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:13.157 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:13.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:13.292 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:01:13.347 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:13.348 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:13.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:13.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:13.424 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:01:13.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:13.463 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:01:13.488 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:01:13.502 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:13.521 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:01:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:13 smithi177 ceph-mon[195797]: pgmap v1907: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 80611/199992 objects degraded (40.307%); 11712/199992 objects misplaced (5.856%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:01:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:13.564 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:01:13.573 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:13.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:13.592 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:01:13.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:01:13.647 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:01:13.648 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.648 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.648 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.648 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.649 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.649 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.649 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.649 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:01:13.651 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:13 smithi029 ceph-mon[216903]: pgmap v1907: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 80611/199992 objects degraded (40.307%); 11712/199992 objects misplaced (5.856%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:01:13.652 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:13.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:13 smithi029 ceph-mon[223858]: pgmap v1907: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 74 op/s; 80611/199992 objects degraded (40.307%); 11712/199992 objects misplaced (5.856%); 1.6 MiB/s, 26 objects/s recovering 2023-10-09T05:01:13.652 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:01:13.682 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:13.711 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:13.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:13.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:13.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:13.880 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:13.924 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:13.952 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:14.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:14.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:14.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:14.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:14.188 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:14.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:14.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:14.348 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:14.395 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/462226843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:14.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:14.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:14.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/462226843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:14.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:14.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/462226843' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:15.110 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:15.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:15.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:15.241 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:15.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:15 smithi029 ceph-mon[216903]: pgmap v1908: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80334/199107 objects degraded (40.347%); 11700/199107 objects misplaced (5.876%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:15.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:15 smithi029 ceph-mon[223858]: pgmap v1908: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80334/199107 objects degraded (40.347%); 11700/199107 objects misplaced (5.876%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:15 smithi177 ceph-mon[195797]: pgmap v1908: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 89 op/s; 80334/199107 objects degraded (40.347%); 11700/199107 objects misplaced (5.876%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:15.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:15.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:15.761 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:15.795 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:15.886 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:15.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:15.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:15.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:16.047 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:16.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:16.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:16.254 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:16.254 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:16.301 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:16.411 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:16.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80334/199107 objects degraded (40.347%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:16.529 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:16.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:16.633 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:16.665 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:16.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80334/199107 objects degraded (40.347%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:16.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80334/199107 objects degraded (40.347%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:17.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:01:17.320 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:17.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:17.349 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:01:17.427 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:17.464 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:17.571 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:17.571 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:17 smithi177 ceph-mon[195797]: pgmap v1909: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 80296/198582 objects degraded (40.435%); 11673/198582 objects misplaced (5.878%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:01:17.572 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1762657739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:17.572 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:17 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:17.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:17.649 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:17 smithi029 ceph-mon[216903]: pgmap v1909: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 80296/198582 objects degraded (40.435%); 11673/198582 objects misplaced (5.878%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:01:17.649 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1762657739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:17.649 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:17 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:17.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:17 smithi029 ceph-mon[223858]: pgmap v1909: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 80296/198582 objects degraded (40.435%); 11673/198582 objects misplaced (5.878%); 2.0 MiB/s, 32 objects/s recovering 2023-10-09T05:01:17.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1762657739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:17.650 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:17 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:17.674 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:17.710 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:17.749 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:17.961 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6, 4] out_osds: [0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:01:17.961 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:01:17.962 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_9 2023-10-09T05:01:17.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:01:18.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:01:18.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:18.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:01:18.323 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: osdmap e571: 8 total, 8 up, 7 in 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[216903]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: osdmap e571: 8 total, 8 up, 7 in 2023-10-09T05:01:18.324 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:18 smithi029 ceph-mon[223858]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.692 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:18.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:18.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:18.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:01:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: osdmap e571: 8 total, 8 up, 7 in 2023-10-09T05:01:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:18 smithi177 ceph-mon[195797]: failed to encode map e571 with expected crc 2023-10-09T05:01:18.799 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:18.827 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:19.258 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:19.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:19.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:19.406 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:19.409 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:19.457 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:19.458 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:19 smithi177 ceph-mon[195797]: pgmap v1911: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 80069/198183 objects degraded (40.402%); 11673/198183 objects misplaced (5.890%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:01:19.458 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:19.458 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:19.490 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:19.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:19.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:19.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:19.634 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:19.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:19 smithi029 ceph-mon[216903]: pgmap v1911: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 80069/198183 objects degraded (40.402%); 11673/198183 objects misplaced (5.890%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:01:19.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:19.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:19.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:19 smithi029 ceph-mon[223858]: pgmap v1911: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 85 op/s; 80069/198183 objects degraded (40.402%); 11673/198183 objects misplaced (5.890%); 2.5 MiB/s, 40 objects/s recovering 2023-10-09T05:01:19.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:19.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:19.684 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:19.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:19.703 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:19.703 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:19.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:19.843 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:19.922 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:19.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:20.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:20.075 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:20.503 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:01:20.898 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:20.915 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:01:21.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:21 smithi029 ceph-mon[223858]: pgmap v1912: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 79954/197229 objects degraded (40.539%); 11623/197229 objects misplaced (5.893%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:01:21.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 79954/197229 objects degraded (40.539%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:21.445 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:21 smithi177 ceph-mon[195797]: pgmap v1912: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 79954/197229 objects degraded (40.539%); 11623/197229 objects misplaced (5.893%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:01:21.446 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 79954/197229 objects degraded (40.539%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:21.480 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:21.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:21 smithi029 ceph-mon[216903]: pgmap v1912: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 110 op/s; 79954/197229 objects degraded (40.539%); 11623/197229 objects misplaced (5.893%); 3.1 MiB/s, 50 objects/s recovering 2023-10-09T05:01:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 79954/197229 objects degraded (40.539%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:21.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:21.627 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:22.438 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:22.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:22.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:22.573 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:22.573 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:22.629 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:22.837 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:22.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:22.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:22.894 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:22.902 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:01:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:01:22] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2023-10-09T05:01:22.952 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:22.983 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:23.065 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:01:23.107 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:01:23.137 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:01:23.179 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:01:23.200 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:01:23.220 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:01:23.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:01:23.242 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:01:23.252 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:23.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:23.271 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.290 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:01:23.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:23.387 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:23.437 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:23.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:23.489 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:01:23.493 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: pgmap v1913: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 79954/196917 objects degraded (40.603%); 11623/196917 objects misplaced (5.902%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2834909251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:23.521 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: pgmap v1913: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 79954/196917 objects degraded (40.603%); 11623/196917 objects misplaced (5.902%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2834909251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:23.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: pgmap v1913: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 79954/196917 objects degraded (40.603%); 11623/196917 objects misplaced (5.902%); 2.6 MiB/s, 41 objects/s recovering 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2834909251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:01:23.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:23.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:23.577 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:23.623 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:23.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:23.637 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:23.727 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:23.757 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:25.331 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:01:25.332 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:01:25.332 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:01:25.332 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:01:25.332 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:01:25.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:25 smithi177 ceph-mon[195797]: from='client.56893 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:01:25.362 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:25 smithi177 ceph-mon[195797]: pgmap v1914: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 79668/196050 objects degraded (40.637%); 11604/196050 objects misplaced (5.919%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:25.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:25 smithi029 ceph-mon[223858]: from='client.56893 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:01:25.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:25 smithi029 ceph-mon[223858]: pgmap v1914: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 79668/196050 objects degraded (40.637%); 11604/196050 objects misplaced (5.919%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:25 smithi029 ceph-mon[216903]: from='client.56893 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:01:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:25 smithi029 ceph-mon[216903]: pgmap v1914: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 79668/196050 objects degraded (40.637%); 11604/196050 objects misplaced (5.919%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:25.821 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:25.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:25.927 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:25.950 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1913,"stamp":"2023-10-09T05:01:21.627415+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4300675000,"num_objects":65639,"num_object_clones":0,"num_object_copies":196917,"num_objects_missing_on_primary":24259,"num_objects_missing":24259,"num_objects_degraded":79954,"num_objects_misplaced":11623,"num_objects_unfound":0,"num_objects_dirty":65639,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":89903,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43062,"num_bytes_recovered":2743909688,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127147,"ondisk_log_size":127147,"up":327,"acting":305,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":23025664,"kb_used_data":15871752,"kb_used_omap":26,"kb_used_meta":7153829,"kb_avail":633190400,"statfs":{"total":671965249536,"available":648386969600,"internally_reserved":0,"allocated":16252674048,"data_stored":16239583903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26724,"internal_metadata":7325521820},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,2,3,1,6,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-67174400,"num_objects":-1025,"num_object_clones":0,"num_object_copies":-3075,"num_objects_missing_on_primary":-219,"num_objects_missing":-219,"num_objects_degraded":-657,"num_objects_misplaced":-89,"num_objects_unfound":0,"num_objects_dirty":-1025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1025,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":27525120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004328"},"pg_stats":[{"pgid":"14.17","version":"570'5373","reported_seq":14065,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:17.426194+0000","last_change":"2023-10-09T04:59:46.171318+0000","last_active":"2023-10-09T05:01:17.426194+0000","last_peered":"2023-10-09T05:01:17.426194+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:59:46.097475+0000","last_became_peered":"2023-10-09T04:59:46.097475+0000","last_unstale":"2023-10-09T05:01:17.426194+0000","last_undegraded":"2023-10-09T04:59:37.772753+0000","last_fullsized":"2023-10-09T04:59:34.824225+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5373,"log_dups_size":0,"ondisk_log_size":5373,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134709248,"num_objects":2056,"num_object_clones":0,"num_object_copies":6168,"num_objects_missing_on_primary":74,"num_objects_missing":74,"num_objects_degraded":2962,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2832,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2117,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":6},{"shards":"1,2,4,5","objects":1678},{"shards":"1,4","objects":-230},{"shards":"2,4,5","objects":375}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.372134+0000","last_change":"2023-10-09T04:58:57.067059+0000","last_active":"2023-10-09T05:01:11.372134+0000","last_peered":"2023-10-09T05:01:11.372134+0000","last_clean":"2023-10-09T05:01:11.372134+0000","last_became_active":"2023-10-09T04:58:57.066168+0000","last_became_peered":"2023-10-09T04:58:57.066168+0000","last_unstale":"2023-10-09T05:01:11.372134+0000","last_undegraded":"2023-10-09T05:01:11.372134+0000","last_fullsized":"2023-10-09T05:01:11.372134+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:02:23.404016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":185,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.333944+0000","last_change":"2023-10-09T04:58:57.034175+0000","last_active":"2023-10-09T05:01:11.333944+0000","last_peered":"2023-10-09T05:01:11.333944+0000","last_clean":"2023-10-09T05:01:11.333944+0000","last_became_active":"2023-10-09T04:58:57.033913+0000","last_became_peered":"2023-10-09T04:58:57.033913+0000","last_unstale":"2023-10-09T05:01:11.333944+0000","last_undegraded":"2023-10-09T05:01:11.333944+0000","last_fullsized":"2023-10-09T05:01:11.333944+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:15:50.157531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"571'5357","reported_seq":8300,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:18.975061+0000","last_change":"2023-10-09T04:59:32.066581+0000","last_active":"2023-10-09T05:01:18.975061+0000","last_peered":"2023-10-09T05:01:18.975061+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:59:32.059503+0000","last_became_peered":"2023-10-09T04:59:32.059503+0000","last_unstale":"2023-10-09T05:01:18.975061+0000","last_undegraded":"2023-10-09T04:59:31.931681+0000","last_fullsized":"2023-10-09T04:59:31.829224+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133857280,"num_objects":2043,"num_object_clones":0,"num_object_copies":6129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3032,"num_objects_misplaced":2181,"num_objects_unfound":0,"num_objects_dirty":2043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2818,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":884,"num_bytes_recovered":57901056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[7,1],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"1,7","objects":17},{"shards":"2,3,4,6,7","objects":277},{"shards":"2,3,4,7","objects":349},{"shards":"2,3,6,7","objects":65},{"shards":"2,4,7","objects":1335}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":112,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.256353+0000","last_change":"2023-10-09T04:58:07.889785+0000","last_active":"2023-10-09T05:01:08.256353+0000","last_peered":"2023-10-09T05:01:08.256353+0000","last_clean":"2023-10-09T05:01:08.256353+0000","last_became_active":"2023-10-09T04:58:07.889365+0000","last_became_peered":"2023-10-09T04:58:07.889365+0000","last_unstale":"2023-10-09T05:01:08.256353+0000","last_undegraded":"2023-10-09T05:01:08.256353+0000","last_fullsized":"2023-10-09T05:01:08.256353+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:44:14.043179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":45,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.334225+0000","last_change":"2023-10-09T05:00:03.051680+0000","last_active":"2023-10-09T05:01:11.334225+0000","last_peered":"2023-10-09T05:01:11.334225+0000","last_clean":"2023-10-09T05:01:11.334225+0000","last_became_active":"2023-10-09T05:00:03.051310+0000","last_became_peered":"2023-10-09T05:00:03.051310+0000","last_unstale":"2023-10-09T05:01:11.334225+0000","last_undegraded":"2023-10-09T05:01:11.334225+0000","last_fullsized":"2023-10-09T05:01:11.334225+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:33:27.928211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"570'5342","reported_seq":12880,"reported_epoch":570,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:01:16.981647+0000","last_change":"2023-10-09T05:00:05.955878+0000","last_active":"2023-10-09T05:01:16.981647+0000","last_peered":"2023-10-09T05:01:16.981647+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T05:00:05.445792+0000","last_became_peered":"2023-10-09T05:00:05.445792+0000","last_unstale":"2023-10-09T05:01:16.981647+0000","last_undegraded":"2023-10-09T05:00:05.302303+0000","last_fullsized":"2023-10-09T05:01:16.981647+0000","mapping_epoch":559,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134938624,"num_objects":2059,"num_object_clones":0,"num_object_copies":6177,"num_objects_missing_on_primary":93,"num_objects_missing":93,"num_objects_degraded":1745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2802,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1916,"num_bytes_recovered":120389632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":708},{"shards":"3,4,5","objects":422},{"shards":"4,5,7","objects":929}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":171,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332078+0000","last_change":"2023-10-09T05:00:03.007586+0000","last_active":"2023-10-09T05:01:11.332078+0000","last_peered":"2023-10-09T05:01:11.332078+0000","last_clean":"2023-10-09T05:01:11.332078+0000","last_became_active":"2023-10-09T05:00:03.006873+0000","last_became_peered":"2023-10-09T05:00:03.006873+0000","last_unstale":"2023-10-09T05:01:11.332078+0000","last_undegraded":"2023-10-09T05:01:11.332078+0000","last_fullsized":"2023-10-09T05:01:11.332078+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:10.127952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":67,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.367254+0000","last_change":"2023-10-09T04:58:07.895975+0000","last_active":"2023-10-09T05:01:11.367254+0000","last_peered":"2023-10-09T05:01:11.367254+0000","last_clean":"2023-10-09T05:01:11.367254+0000","last_became_active":"2023-10-09T04:58:07.893663+0000","last_became_peered":"2023-10-09T04:58:07.893663+0000","last_unstale":"2023-10-09T05:01:11.367254+0000","last_undegraded":"2023-10-09T05:01:11.367254+0000","last_fullsized":"2023-10-09T05:01:11.367254+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:28:35.053280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":571,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.335030+0000","last_change":"2023-10-09T05:00:02.991301+0000","last_active":"2023-10-09T05:01:11.335030+0000","last_peered":"2023-10-09T05:01:11.335030+0000","last_clean":"2023-10-09T05:01:11.335030+0000","last_became_active":"2023-10-09T05:00:02.990987+0000","last_became_peered":"2023-10-09T05:00:02.990987+0000","last_unstale":"2023-10-09T05:01:11.335030+0000","last_undegraded":"2023-10-09T05:01:11.335030+0000","last_fullsized":"2023-10-09T05:01:11.335030+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:53:08.164323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":171,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.331976+0000","last_change":"2023-10-09T04:58:07.920414+0000","last_active":"2023-10-09T05:01:11.331976+0000","last_peered":"2023-10-09T05:01:11.331976+0000","last_clean":"2023-10-09T05:01:11.331976+0000","last_became_active":"2023-10-09T04:58:07.920149+0000","last_became_peered":"2023-10-09T04:58:07.920149+0000","last_unstale":"2023-10-09T05:01:11.331976+0000","last_undegraded":"2023-10-09T05:01:11.331976+0000","last_fullsized":"2023-10-09T05:01:11.331976+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:34.057529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":486,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.270068+0000","last_change":"2023-10-09T05:00:03.038038+0000","last_active":"2023-10-09T05:01:08.270068+0000","last_peered":"2023-10-09T05:01:08.270068+0000","last_clean":"2023-10-09T05:01:08.270068+0000","last_became_active":"2023-10-09T05:00:03.037433+0000","last_became_peered":"2023-10-09T05:00:03.037433+0000","last_unstale":"2023-10-09T05:01:08.270068+0000","last_undegraded":"2023-10-09T05:01:08.270068+0000","last_fullsized":"2023-10-09T05:01:08.270068+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:15.535445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332642+0000","last_change":"2023-10-09T04:59:30.857503+0000","last_active":"2023-10-09T05:01:11.332642+0000","last_peered":"2023-10-09T05:01:11.332642+0000","last_clean":"2023-10-09T05:01:11.332642+0000","last_became_active":"2023-10-09T04:59:30.855741+0000","last_became_peered":"2023-10-09T04:59:30.855741+0000","last_unstale":"2023-10-09T05:01:11.332642+0000","last_undegraded":"2023-10-09T05:01:11.332642+0000","last_fullsized":"2023-10-09T05:01:11.332642+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:28.636962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"570'5485","reported_seq":7146,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:16.733925+0000","last_change":"2023-10-09T04:59:55.112316+0000","last_active":"2023-10-09T05:01:16.733925+0000","last_peered":"2023-10-09T05:01:16.733925+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:58:59.753906+0000","last_became_peered":"2023-10-09T04:58:59.753906+0000","last_unstale":"2023-10-09T05:01:16.733925+0000","last_undegraded":"2023-10-09T04:59:55.091481+0000","last_fullsized":"2023-10-09T04:58:54.779150+0000","mapping_epoch":535,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3949,"log_dups_size":0,"ondisk_log_size":3949,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138805248,"num_objects":2118,"num_object_clones":0,"num_object_copies":6354,"num_objects_missing_on_primary":2091,"num_objects_missing":2091,"num_objects_degraded":3836,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2876,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":381,"num_bytes_recovered":24969216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2","objects":26},{"shards":"1,2,5,7","objects":1},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1660}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":207,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.333877+0000","last_change":"2023-10-09T04:59:31.023495+0000","last_active":"2023-10-09T05:01:11.333877+0000","last_peered":"2023-10-09T05:01:11.333877+0000","last_clean":"2023-10-09T05:01:11.333877+0000","last_became_active":"2023-10-09T04:59:30.947798+0000","last_became_peered":"2023-10-09T04:59:30.947798+0000","last_unstale":"2023-10-09T05:01:11.333877+0000","last_undegraded":"2023-10-09T05:01:11.333877+0000","last_fullsized":"2023-10-09T05:01:11.333877+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:35.811495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.259433+0000","last_change":"2023-10-09T04:58:07.886648+0000","last_active":"2023-10-09T05:01:08.259433+0000","last_peered":"2023-10-09T05:01:08.259433+0000","last_clean":"2023-10-09T05:01:08.259433+0000","last_became_active":"2023-10-09T04:58:07.886050+0000","last_became_peered":"2023-10-09T04:58:07.886050+0000","last_unstale":"2023-10-09T05:01:08.259433+0000","last_undegraded":"2023-10-09T05:01:08.259433+0000","last_fullsized":"2023-10-09T05:01:08.259433+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:08.968095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"571'5348","reported_seq":17067,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:18.753605+0000","last_change":"2023-10-09T04:59:32.057503+0000","last_active":"2023-10-09T05:01:18.753605+0000","last_peered":"2023-10-09T05:01:18.753605+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:58:58.100530+0000","last_became_peered":"2023-10-09T04:58:58.100530+0000","last_unstale":"2023-10-09T05:01:18.753605+0000","last_undegraded":"2023-10-09T04:59:32.057314+0000","last_fullsized":"2023-10-09T04:58:57.986120+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136282112,"num_objects":2080,"num_object_clones":0,"num_object_copies":6240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":66,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2810,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2884,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":2080}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":53,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.372414+0000","last_change":"2023-10-09T04:59:29.995103+0000","last_active":"2023-10-09T05:01:11.372414+0000","last_peered":"2023-10-09T05:01:11.372414+0000","last_clean":"2023-10-09T05:01:11.372414+0000","last_became_active":"2023-10-09T04:58:57.210019+0000","last_became_peered":"2023-10-09T04:58:57.210019+0000","last_unstale":"2023-10-09T05:01:11.372414+0000","last_undegraded":"2023-10-09T05:01:11.372414+0000","last_fullsized":"2023-10-09T05:01:11.372414+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:53.045092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371542+0000","last_change":"2023-10-09T04:58:57.063972+0000","last_active":"2023-10-09T05:01:11.371542+0000","last_peered":"2023-10-09T05:01:11.371542+0000","last_clean":"2023-10-09T05:01:11.371542+0000","last_became_active":"2023-10-09T04:58:57.063636+0000","last_became_peered":"2023-10-09T04:58:57.063636+0000","last_unstale":"2023-10-09T05:01:11.371542+0000","last_undegraded":"2023-10-09T05:01:11.371542+0000","last_fullsized":"2023-10-09T05:01:11.371542+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:19:19.332300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":49,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371761+0000","last_change":"2023-10-09T04:59:29.960299+0000","last_active":"2023-10-09T05:01:11.371761+0000","last_peered":"2023-10-09T05:01:11.371761+0000","last_clean":"2023-10-09T05:01:11.371761+0000","last_became_active":"2023-10-09T04:58:57.075811+0000","last_became_peered":"2023-10-09T04:58:57.075811+0000","last_unstale":"2023-10-09T05:01:11.371761+0000","last_undegraded":"2023-10-09T05:01:11.371761+0000","last_fullsized":"2023-10-09T05:01:11.371761+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:45.388764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":137,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332785+0000","last_change":"2023-10-09T04:55:19.974453+0000","last_active":"2023-10-09T05:01:11.332785+0000","last_peered":"2023-10-09T05:01:11.332785+0000","last_clean":"2023-10-09T05:01:11.332785+0000","last_became_active":"2023-10-09T04:55:19.973871+0000","last_became_peered":"2023-10-09T04:55:19.973871+0000","last_unstale":"2023-10-09T05:01:11.332785+0000","last_undegraded":"2023-10-09T05:01:11.332785+0000","last_fullsized":"2023-10-09T05:01:11.332785+0000","mapping_epoch":463,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:47.239847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332889+0000","last_change":"2023-10-09T04:59:30.847978+0000","last_active":"2023-10-09T05:01:11.332889+0000","last_peered":"2023-10-09T05:01:11.332889+0000","last_clean":"2023-10-09T05:01:11.332889+0000","last_became_active":"2023-10-09T04:59:30.847631+0000","last_became_peered":"2023-10-09T04:59:30.847631+0000","last_unstale":"2023-10-09T05:01:11.332889+0000","last_undegraded":"2023-10-09T05:01:11.332889+0000","last_fullsized":"2023-10-09T05:01:11.332889+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:50:23.563332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":476,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.333679+0000","last_change":"2023-10-09T04:59:30.948799+0000","last_active":"2023-10-09T05:01:11.333679+0000","last_peered":"2023-10-09T05:01:11.333679+0000","last_clean":"2023-10-09T05:01:11.333679+0000","last_became_active":"2023-10-09T04:59:30.948277+0000","last_became_peered":"2023-10-09T04:59:30.948277+0000","last_unstale":"2023-10-09T05:01:11.333679+0000","last_undegraded":"2023-10-09T05:01:11.333679+0000","last_fullsized":"2023-10-09T05:01:11.333679+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:25.028468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.372075+0000","last_change":"2023-10-09T04:58:56.993529+0000","last_active":"2023-10-09T05:01:11.372075+0000","last_peered":"2023-10-09T05:01:11.372075+0000","last_clean":"2023-10-09T05:01:11.372075+0000","last_became_active":"2023-10-09T04:58:56.993185+0000","last_became_peered":"2023-10-09T04:58:56.993185+0000","last_unstale":"2023-10-09T05:01:11.372075+0000","last_undegraded":"2023-10-09T05:01:11.372075+0000","last_fullsized":"2023-10-09T05:01:11.372075+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:36.864637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":179,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.334584+0000","last_change":"2023-10-09T04:58:07.882093+0000","last_active":"2023-10-09T05:01:11.334584+0000","last_peered":"2023-10-09T05:01:11.334584+0000","last_clean":"2023-10-09T05:01:11.334584+0000","last_became_active":"2023-10-09T04:58:07.881272+0000","last_became_peered":"2023-10-09T04:58:07.881272+0000","last_unstale":"2023-10-09T05:01:11.334584+0000","last_undegraded":"2023-10-09T05:01:11.334584+0000","last_fullsized":"2023-10-09T05:01:11.334584+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:38:09.422315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":165,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366600+0000","last_change":"2023-10-09T04:58:07.908453+0000","last_active":"2023-10-09T05:01:11.366600+0000","last_peered":"2023-10-09T05:01:11.366600+0000","last_clean":"2023-10-09T05:01:11.366600+0000","last_became_active":"2023-10-09T04:58:07.904674+0000","last_became_peered":"2023-10-09T04:58:07.904674+0000","last_unstale":"2023-10-09T05:01:11.366600+0000","last_undegraded":"2023-10-09T05:01:11.366600+0000","last_fullsized":"2023-10-09T05:01:11.366600+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:38:18.595337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.334899+0000","last_change":"2023-10-09T04:59:30.859544+0000","last_active":"2023-10-09T05:01:11.334899+0000","last_peered":"2023-10-09T05:01:11.334899+0000","last_clean":"2023-10-09T05:01:11.334899+0000","last_became_active":"2023-10-09T04:59:30.859269+0000","last_became_peered":"2023-10-09T04:59:30.859269+0000","last_unstale":"2023-10-09T05:01:11.334899+0000","last_undegraded":"2023-10-09T05:01:11.334899+0000","last_fullsized":"2023-10-09T05:01:11.334899+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:07:14.815137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"571'5291","reported_seq":8339,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:19.126361+0000","last_change":"2023-10-09T04:59:37.563341+0000","last_active":"2023-10-09T05:01:19.126361+0000","last_peered":"2023-10-09T05:01:19.126361+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T04:59:37.503937+0000","last_became_peered":"2023-10-09T04:59:37.503937+0000","last_unstale":"2023-10-09T05:01:19.126361+0000","last_undegraded":"2023-10-09T04:59:34.538086+0000","last_fullsized":"2023-10-09T04:59:31.829741+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":338,"num_objects_missing":338,"num_objects_degraded":4125,"num_objects_misplaced":1623,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2792,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":406,"num_bytes_recovered":26574848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[7,1],"avail_no_missing":["4"],"object_location_counts":[{"shards":"1,4,5,6,7","objects":5},{"shards":"1,7","objects":-114},{"shards":"3,4,5","objects":224},{"shards":"3,4,5,6","objects":46},{"shards":"3,4,5,6,7","objects":1},{"shards":"4,5,6,7","objects":1},{"shards":"4,5,7","objects":1638}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":61,"reported_epoch":570,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:01:11.388593+0000","last_change":"2023-10-09T05:00:03.006739+0000","last_active":"2023-10-09T05:01:11.388593+0000","last_peered":"2023-10-09T05:01:11.388593+0000","last_clean":"2023-10-09T04:59:56.906793+0000","last_became_active":"2023-10-09T05:00:03.006377+0000","last_became_peered":"2023-10-09T05:00:03.006377+0000","last_unstale":"2023-10-09T05:01:11.388593+0000","last_undegraded":"2023-10-09T05:00:02.989790+0000","last_fullsized":"2023-10-09T05:01:11.388593+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:05.241750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":["4","1","5"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":596,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366068+0000","last_change":"2023-10-09T04:59:30.861892+0000","last_active":"2023-10-09T05:01:11.366068+0000","last_peered":"2023-10-09T05:01:11.366068+0000","last_clean":"2023-10-09T05:01:11.366068+0000","last_became_active":"2023-10-09T04:59:30.861571+0000","last_became_peered":"2023-10-09T04:59:30.861571+0000","last_unstale":"2023-10-09T05:01:11.366068+0000","last_undegraded":"2023-10-09T05:01:11.366068+0000","last_fullsized":"2023-10-09T05:01:11.366068+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:28.783652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"571'5342","reported_seq":7411,"reported_epoch":571,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:01:17.913014+0000","last_change":"2023-10-09T05:00:50.815372+0000","last_active":"2023-10-09T05:01:17.913014+0000","last_peered":"2023-10-09T05:01:17.913014+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:59:55.013898+0000","last_became_peered":"2023-10-09T04:59:55.013898+0000","last_unstale":"2023-10-09T05:01:17.913014+0000","last_undegraded":"2023-10-09T05:01:17.913014+0000","last_fullsized":"2023-10-09T04:59:31.949346+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5342,"log_dups_size":0,"ondisk_log_size":5342,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134610944,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":1598,"num_objects_missing":1598,"num_objects_degraded":0,"num_objects_misplaced":5585,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2810,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":379,"num_bytes_recovered":24707072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":7},{"shards":"1,3","objects":33},{"shards":"2,3,5,6,7","objects":3},{"shards":"2,3,5,7","objects":368},{"shards":"2,5,7","objects":1598},{"shards":"3,5,6","objects":45}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":413,"reported_epoch":569,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:01:08.269957+0000","last_change":"2023-10-09T05:00:25.182219+0000","last_active":"2023-10-09T05:01:08.269957+0000","last_peered":"2023-10-09T05:01:08.269957+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:00:03.037168+0000","last_became_peered":"2023-10-09T05:00:03.037168+0000","last_unstale":"2023-10-09T05:01:08.269957+0000","last_undegraded":"2023-10-09T05:00:25.180458+0000","last_fullsized":"2023-10-09T05:01:08.269957+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:28:38.583814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","4","5","7"],"object_location_counts":[{"shards":"1,4,5,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":53,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371998+0000","last_change":"2023-10-09T04:59:30.853667+0000","last_active":"2023-10-09T05:01:11.371998+0000","last_peered":"2023-10-09T05:01:11.371998+0000","last_clean":"2023-10-09T05:01:11.371998+0000","last_became_active":"2023-10-09T04:59:30.853265+0000","last_became_peered":"2023-10-09T04:59:30.853265+0000","last_unstale":"2023-10-09T05:01:11.371998+0000","last_undegraded":"2023-10-09T05:01:11.371998+0000","last_fullsized":"2023-10-09T05:01:11.371998+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:59:04.088823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"571'5362","reported_seq":5681,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:18.707484+0000","last_change":"2023-10-09T05:00:50.799653+0000","last_active":"2023-10-09T05:01:18.707484+0000","last_peered":"2023-10-09T05:01:18.707484+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:58.136774+0000","last_became_peered":"2023-10-09T04:58:58.136774+0000","last_unstale":"2023-10-09T05:01:18.707484+0000","last_undegraded":"2023-10-09T04:58:58.128621+0000","last_fullsized":"2023-10-09T04:58:57.986571+0000","mapping_epoch":538,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4824,"log_dups_size":0,"ondisk_log_size":4824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133201920,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2528,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2830,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1736704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":1967},{"shards":"2,5,6","objects":55},{"shards":"4,5","objects":-289}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":189,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.399885+0000","last_change":"2023-10-09T04:59:31.025595+0000","last_active":"2023-10-09T05:01:11.399885+0000","last_peered":"2023-10-09T05:01:11.399885+0000","last_clean":"2023-10-09T05:01:11.399885+0000","last_became_active":"2023-10-09T04:59:30.946523+0000","last_became_peered":"2023-10-09T04:59:30.946523+0000","last_unstale":"2023-10-09T05:01:11.399885+0000","last_undegraded":"2023-10-09T05:01:11.399885+0000","last_fullsized":"2023-10-09T05:01:11.399885+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:05.945423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":77,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366044+0000","last_change":"2023-10-09T04:59:30.854424+0000","last_active":"2023-10-09T05:01:11.366044+0000","last_peered":"2023-10-09T05:01:11.366044+0000","last_clean":"2023-10-09T05:01:11.366044+0000","last_became_active":"2023-10-09T04:59:30.853989+0000","last_became_peered":"2023-10-09T04:59:30.853989+0000","last_unstale":"2023-10-09T05:01:11.366044+0000","last_undegraded":"2023-10-09T05:01:11.366044+0000","last_fullsized":"2023-10-09T05:01:11.366044+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:41:30.423138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"571'5335","reported_seq":15900,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-09T05:01:18.072740+0000","last_change":"2023-10-09T05:00:51.946565+0000","last_active":"2023-10-09T05:01:18.072740+0000","last_peered":"2023-10-09T05:01:18.072740+0000","last_clean":"2023-10-09T05:01:18.072740+0000","last_became_active":"2023-10-09T05:00:51.939927+0000","last_became_peered":"2023-10-09T05:00:51.939927+0000","last_unstale":"2023-10-09T05:01:18.072740+0000","last_undegraded":"2023-10-09T05:01:18.072740+0000","last_fullsized":"2023-10-09T05:01:18.072740+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134938624,"num_objects":2059,"num_object_clones":0,"num_object_copies":6177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2800,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3303,"num_bytes_recovered":206536704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":631,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.360232+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:01:11.360232+0000","last_peered":"2023-10-09T05:01:11.360232+0000","last_clean":"2023-10-09T05:01:11.360232+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:01:11.360232+0000","last_undegraded":"2023-10-09T05:01:11.360232+0000","last_fullsized":"2023-10-09T05:01:11.360232+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":143,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.263126+0000","last_change":"2023-10-09T04:59:02.687690+0000","last_active":"2023-10-09T05:01:08.263126+0000","last_peered":"2023-10-09T05:01:08.263126+0000","last_clean":"2023-10-09T05:01:08.263126+0000","last_became_active":"2023-10-09T04:59:02.687418+0000","last_became_peered":"2023-10-09T04:59:02.687418+0000","last_unstale":"2023-10-09T05:01:08.263126+0000","last_undegraded":"2023-10-09T05:01:08.263126+0000","last_fullsized":"2023-10-09T05:01:08.263126+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:21.317269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1079,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.367304+0000","last_change":"2023-10-09T04:59:33.900445+0000","last_active":"2023-10-09T05:01:11.367304+0000","last_peered":"2023-10-09T05:01:11.367304+0000","last_clean":"2023-10-09T05:01:11.367304+0000","last_became_active":"2023-10-09T04:59:33.899975+0000","last_became_peered":"2023-10-09T04:59:33.899975+0000","last_unstale":"2023-10-09T05:01:11.367304+0000","last_undegraded":"2023-10-09T05:01:11.367304+0000","last_fullsized":"2023-10-09T05:01:11.367304+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"570'5181","reported_seq":6979,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:16.018799+0000","last_change":"2023-10-09T04:59:40.909184+0000","last_active":"2023-10-09T05:01:16.018799+0000","last_peered":"2023-10-09T05:01:16.018799+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:59:39.104756+0000","last_became_peered":"2023-10-09T04:59:39.104756+0000","last_unstale":"2023-10-09T05:01:16.018799+0000","last_undegraded":"2023-10-09T04:59:33.724244+0000","last_fullsized":"2023-10-09T04:59:33.707563+0000","mapping_epoch":546,"log_start":"444'386","ondisk_log_start":"444'386","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4795,"log_dups_size":0,"ondisk_log_size":4795,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130514944,"num_objects":1991,"num_object_clones":0,"num_object_copies":5973,"num_objects_missing_on_primary":1955,"num_objects_missing":1955,"num_objects_degraded":5668,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2724,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":24051712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"1,3","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":1559}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":141,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:13.185678+0000","last_change":"2023-10-09T04:59:29.960112+0000","last_active":"2023-10-09T05:01:13.185678+0000","last_peered":"2023-10-09T05:01:13.185678+0000","last_clean":"2023-10-09T05:01:13.185678+0000","last_became_active":"2023-10-09T04:59:02.687355+0000","last_became_peered":"2023-10-09T04:59:02.687355+0000","last_unstale":"2023-10-09T05:01:13.185678+0000","last_undegraded":"2023-10-09T05:01:13.185678+0000","last_fullsized":"2023-10-09T05:01:13.185678+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:24:03.680594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.372469+0000","last_change":"2023-10-09T04:58:57.210297+0000","last_active":"2023-10-09T05:01:11.372469+0000","last_peered":"2023-10-09T05:01:11.372469+0000","last_clean":"2023-10-09T05:01:11.372469+0000","last_became_active":"2023-10-09T04:58:57.209978+0000","last_became_peered":"2023-10-09T04:58:57.209978+0000","last_unstale":"2023-10-09T05:01:11.372469+0000","last_undegraded":"2023-10-09T05:01:11.372469+0000","last_fullsized":"2023-10-09T05:01:11.372469+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:58.875418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1084,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:11.366336+0000","last_change":"2023-10-09T04:59:31.881787+0000","last_active":"2023-10-09T05:01:11.366336+0000","last_peered":"2023-10-09T05:01:11.366336+0000","last_clean":"2023-10-09T04:59:25.875698+0000","last_became_active":"2023-10-09T04:59:31.880964+0000","last_became_peered":"2023-10-09T04:59:31.880964+0000","last_unstale":"2023-10-09T05:01:11.366336+0000","last_undegraded":"2023-10-09T04:59:31.831404+0000","last_fullsized":"2023-10-09T04:59:31.830955+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1","2","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":1},{"shards":"1,6","objects":-1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"570'5337","reported_seq":14030,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:17.041615+0000","last_change":"2023-10-09T04:59:46.171466+0000","last_active":"2023-10-09T05:01:17.041615+0000","last_peered":"2023-10-09T05:01:17.041615+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T04:59:46.091981+0000","last_became_peered":"2023-10-09T04:59:46.091981+0000","last_unstale":"2023-10-09T05:01:17.041615+0000","last_undegraded":"2023-10-09T04:59:34.803920+0000","last_fullsized":"2023-10-09T04:59:31.843578+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137068544,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":93,"num_objects_missing":93,"num_objects_degraded":2980,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2795,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2115,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":1},{"shards":"1,2,4,5","objects":1727},{"shards":"1,4","objects":-203},{"shards":"2,4,5","objects":360}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.365966+0000","last_change":"2023-10-09T04:58:57.060419+0000","last_active":"2023-10-09T05:01:11.365966+0000","last_peered":"2023-10-09T05:01:11.365966+0000","last_clean":"2023-10-09T05:01:11.365966+0000","last_became_active":"2023-10-09T04:58:57.059935+0000","last_became_peered":"2023-10-09T04:58:57.059935+0000","last_unstale":"2023-10-09T05:01:11.365966+0000","last_undegraded":"2023-10-09T05:01:11.365966+0000","last_fullsized":"2023-10-09T05:01:11.365966+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:05:44.313186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":345,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.256876+0000","last_change":"2023-10-09T04:59:30.952535+0000","last_active":"2023-10-09T05:01:08.256876+0000","last_peered":"2023-10-09T05:01:08.256876+0000","last_clean":"2023-10-09T05:01:08.256876+0000","last_became_active":"2023-10-09T04:59:30.952185+0000","last_became_peered":"2023-10-09T04:59:30.952185+0000","last_unstale":"2023-10-09T05:01:08.256876+0000","last_undegraded":"2023-10-09T05:01:08.256876+0000","last_fullsized":"2023-10-09T05:01:08.256876+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:29:00.478193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.4","version":"571'5347","reported_seq":7197,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:19.131756+0000","last_change":"2023-10-09T04:59:32.066582+0000","last_active":"2023-10-09T05:01:19.131756+0000","last_peered":"2023-10-09T05:01:19.131756+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T04:59:32.057983+0000","last_became_peered":"2023-10-09T04:59:32.057983+0000","last_unstale":"2023-10-09T05:01:19.131756+0000","last_undegraded":"2023-10-09T04:59:31.912599+0000","last_fullsized":"2023-10-09T04:59:31.833438+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5347,"log_dups_size":0,"ondisk_log_size":5347,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134512640,"num_objects":2052,"num_object_clones":0,"num_object_copies":6156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3704,"num_objects_misplaced":2234,"num_objects_unfound":0,"num_objects_dirty":2052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2807,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":377,"num_bytes_recovered":24739840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[7,1],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"1,7","objects":-17},{"shards":"2,3,4,7","objects":364},{"shards":"2,3,6,7","objects":46},{"shards":"2,4,7","objects":1659}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371852+0000","last_change":"2023-10-09T04:58:57.557023+0000","last_active":"2023-10-09T05:01:11.371852+0000","last_peered":"2023-10-09T05:01:11.371852+0000","last_clean":"2023-10-09T05:01:11.371852+0000","last_became_active":"2023-10-09T04:58:57.556703+0000","last_became_peered":"2023-10-09T04:58:57.556703+0000","last_unstale":"2023-10-09T05:01:11.371852+0000","last_undegraded":"2023-10-09T05:01:11.371852+0000","last_fullsized":"2023-10-09T05:01:11.371852+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:05:44.046576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371910+0000","last_change":"2023-10-09T04:58:57.071995+0000","last_active":"2023-10-09T05:01:11.371910+0000","last_peered":"2023-10-09T05:01:11.371910+0000","last_clean":"2023-10-09T05:01:11.371910+0000","last_became_active":"2023-10-09T04:58:57.070855+0000","last_became_peered":"2023-10-09T04:58:57.070855+0000","last_unstale":"2023-10-09T05:01:11.371910+0000","last_undegraded":"2023-10-09T05:01:11.371910+0000","last_fullsized":"2023-10-09T05:01:11.371910+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:14:29.469962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1088,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366305+0000","last_change":"2023-10-09T04:59:34.572197+0000","last_active":"2023-10-09T05:01:11.366305+0000","last_peered":"2023-10-09T05:01:11.366305+0000","last_clean":"2023-10-09T05:01:11.366305+0000","last_became_active":"2023-10-09T04:59:34.571477+0000","last_became_peered":"2023-10-09T04:59:34.571477+0000","last_unstale":"2023-10-09T05:01:11.366305+0000","last_undegraded":"2023-10-09T05:01:11.366305+0000","last_fullsized":"2023-10-09T05:01:11.366305+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"570'5350","reported_seq":12906,"reported_epoch":570,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:01:17.330092+0000","last_change":"2023-10-09T05:00:06.492042+0000","last_active":"2023-10-09T05:01:17.330092+0000","last_peered":"2023-10-09T05:01:17.330092+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T05:00:05.977194+0000","last_became_peered":"2023-10-09T05:00:05.977194+0000","last_unstale":"2023-10-09T05:01:17.330092+0000","last_undegraded":"2023-10-09T05:00:05.416161+0000","last_fullsized":"2023-10-09T05:01:17.330092+0000","mapping_epoch":559,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5104,"log_dups_size":0,"ondisk_log_size":5104,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134414336,"num_objects":2051,"num_object_clones":0,"num_object_copies":6153,"num_objects_missing_on_primary":85,"num_objects_missing":85,"num_objects_degraded":1749,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2810,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1924,"num_bytes_recovered":120979456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":733},{"shards":"3,4,5","objects":405},{"shards":"4,5,7","objects":913}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":45,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.331504+0000","last_change":"2023-10-09T05:00:03.007784+0000","last_active":"2023-10-09T05:01:11.331504+0000","last_peered":"2023-10-09T05:01:11.331504+0000","last_clean":"2023-10-09T05:01:11.331504+0000","last_became_active":"2023-10-09T05:00:03.007280+0000","last_became_peered":"2023-10-09T05:00:03.007280+0000","last_unstale":"2023-10-09T05:01:11.331504+0000","last_undegraded":"2023-10-09T05:01:11.331504+0000","last_fullsized":"2023-10-09T05:01:11.331504+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:21:41.087401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":133,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.364018+0000","last_change":"2023-10-09T04:58:57.066851+0000","last_active":"2023-10-09T05:01:08.364018+0000","last_peered":"2023-10-09T05:01:08.364018+0000","last_clean":"2023-10-09T05:01:08.364018+0000","last_became_active":"2023-10-09T04:58:57.066522+0000","last_became_peered":"2023-10-09T04:58:57.066522+0000","last_unstale":"2023-10-09T05:01:08.364018+0000","last_undegraded":"2023-10-09T05:01:08.364018+0000","last_fullsized":"2023-10-09T05:01:08.364018+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:20:48.448579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1094,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.367337+0000","last_change":"2023-10-09T04:59:55.153783+0000","last_active":"2023-10-09T05:01:11.367337+0000","last_peered":"2023-10-09T05:01:11.367337+0000","last_clean":"2023-10-09T05:01:11.367337+0000","last_became_active":"2023-10-09T04:59:55.153450+0000","last_became_peered":"2023-10-09T04:59:55.153450+0000","last_unstale":"2023-10-09T05:01:11.367337+0000","last_undegraded":"2023-10-09T05:01:11.367337+0000","last_fullsized":"2023-10-09T05:01:11.367337+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":555,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"570'5485","reported_seq":7142,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:16.419788+0000","last_change":"2023-10-09T04:59:31.036614+0000","last_active":"2023-10-09T05:01:16.419788+0000","last_peered":"2023-10-09T05:01:16.419788+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T04:58:52.441046+0000","last_became_peered":"2023-10-09T04:58:52.441046+0000","last_unstale":"2023-10-09T05:01:16.419788+0000","last_undegraded":"2023-10-09T04:59:31.029253+0000","last_fullsized":"2023-10-09T04:58:48.896671+0000","mapping_epoch":496,"log_start":"445'1536","ondisk_log_start":"445'1536","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3949,"log_dups_size":0,"ondisk_log_size":3949,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138805248,"num_objects":2118,"num_object_clones":0,"num_object_copies":6354,"num_objects_missing_on_primary":2144,"num_objects_missing":2144,"num_objects_degraded":3954,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2876,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":381,"num_bytes_recovered":24969216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"1,2","objects":-26},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":1738}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":128,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.260303+0000","last_change":"2023-10-09T04:56:53.920990+0000","last_active":"2023-10-09T05:01:08.260303+0000","last_peered":"2023-10-09T05:01:08.260303+0000","last_clean":"2023-10-09T05:01:08.260303+0000","last_became_active":"2023-10-09T04:56:53.920700+0000","last_became_peered":"2023-10-09T04:56:53.920700+0000","last_unstale":"2023-10-09T05:01:08.260303+0000","last_undegraded":"2023-10-09T05:01:08.260303+0000","last_fullsized":"2023-10-09T05:01:08.260303+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:25:06.039757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":508,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.333790+0000","last_change":"2023-10-09T05:00:03.068377+0000","last_active":"2023-10-09T05:01:11.333790+0000","last_peered":"2023-10-09T05:01:11.333790+0000","last_clean":"2023-10-09T05:01:11.333790+0000","last_became_active":"2023-10-09T05:00:03.068136+0000","last_became_peered":"2023-10-09T05:00:03.068136+0000","last_unstale":"2023-10-09T05:01:11.333790+0000","last_undegraded":"2023-10-09T05:01:11.333790+0000","last_fullsized":"2023-10-09T05:01:11.333790+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:52:25.397131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":178,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.259350+0000","last_change":"2023-10-09T05:00:03.024450+0000","last_active":"2023-10-09T05:01:08.259350+0000","last_peered":"2023-10-09T05:01:08.259350+0000","last_clean":"2023-10-09T05:01:08.259350+0000","last_became_active":"2023-10-09T05:00:03.024135+0000","last_became_peered":"2023-10-09T05:00:03.024135+0000","last_unstale":"2023-10-09T05:01:08.259350+0000","last_undegraded":"2023-10-09T05:01:08.259350+0000","last_fullsized":"2023-10-09T05:01:08.259350+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:00:40.696966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1077,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.367215+0000","last_change":"2023-10-09T04:59:33.900400+0000","last_active":"2023-10-09T05:01:11.367215+0000","last_peered":"2023-10-09T05:01:11.367215+0000","last_clean":"2023-10-09T05:01:11.367215+0000","last_became_active":"2023-10-09T04:59:33.899972+0000","last_became_peered":"2023-10-09T04:59:33.899972+0000","last_unstale":"2023-10-09T05:01:11.367215+0000","last_undegraded":"2023-10-09T05:01:11.367215+0000","last_fullsized":"2023-10-09T05:01:11.367215+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"571'5549","reported_seq":16146,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:17.568465+0000","last_change":"2023-10-09T05:00:03.237538+0000","last_active":"2023-10-09T05:01:17.568465+0000","last_peered":"2023-10-09T05:01:17.568465+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:00:03.142517+0000","last_became_peered":"2023-10-09T05:00:03.142517+0000","last_unstale":"2023-10-09T05:01:17.568465+0000","last_undegraded":"2023-10-09T05:00:03.014197+0000","last_fullsized":"2023-10-09T05:00:02.989747+0000","mapping_epoch":558,"log_start":"445'1481","ondisk_log_start":"445'1481","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4068,"log_dups_size":0,"ondisk_log_size":4068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:48:39.361359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264437760,"num_objects":4035,"num_object_clones":0,"num_object_copies":12105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4025,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5549,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4790,"num_bytes_recovered":304742400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,3,4,6","objects":170},{"shards":"2,4,6","objects":111},{"shards":"2,4,6,7","objects":3049},{"shards":"2,6","objects":303}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":625,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.360364+0000","last_change":"2023-10-09T04:58:07.914635+0000","last_active":"2023-10-09T05:01:11.360364+0000","last_peered":"2023-10-09T05:01:11.360364+0000","last_clean":"2023-10-09T05:01:11.360364+0000","last_became_active":"2023-10-09T04:58:07.913894+0000","last_became_peered":"2023-10-09T04:58:07.913894+0000","last_unstale":"2023-10-09T05:01:11.360364+0000","last_undegraded":"2023-10-09T05:01:11.360364+0000","last_fullsized":"2023-10-09T05:01:11.360364+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:45:32.545025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371517+0000","last_change":"2023-10-09T04:58:57.063810+0000","last_active":"2023-10-09T05:01:11.371517+0000","last_peered":"2023-10-09T05:01:11.371517+0000","last_clean":"2023-10-09T05:01:11.371517+0000","last_became_active":"2023-10-09T04:58:57.063532+0000","last_became_peered":"2023-10-09T04:58:57.063532+0000","last_unstale":"2023-10-09T05:01:11.371517+0000","last_undegraded":"2023-10-09T05:01:11.371517+0000","last_fullsized":"2023-10-09T05:01:11.371517+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:47:01.777681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":154,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.260214+0000","last_change":"2023-10-09T04:59:30.852320+0000","last_active":"2023-10-09T05:01:08.260214+0000","last_peered":"2023-10-09T05:01:08.260214+0000","last_clean":"2023-10-09T05:01:08.260214+0000","last_became_active":"2023-10-09T04:59:30.852046+0000","last_became_peered":"2023-10-09T04:59:30.852046+0000","last_unstale":"2023-10-09T05:01:08.260214+0000","last_undegraded":"2023-10-09T05:01:08.260214+0000","last_fullsized":"2023-10-09T05:01:08.260214+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:22:36.807391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1088,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366128+0000","last_change":"2023-10-09T04:59:34.572474+0000","last_active":"2023-10-09T05:01:11.366128+0000","last_peered":"2023-10-09T05:01:11.366128+0000","last_clean":"2023-10-09T05:01:11.366128+0000","last_became_active":"2023-10-09T04:59:34.571777+0000","last_became_peered":"2023-10-09T04:59:34.571777+0000","last_unstale":"2023-10-09T05:01:11.366128+0000","last_undegraded":"2023-10-09T05:01:11.366128+0000","last_fullsized":"2023-10-09T05:01:11.366128+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"571'5554","reported_seq":7508,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:19.014846+0000","last_change":"2023-10-09T05:00:03.069862+0000","last_active":"2023-10-09T05:01:19.014846+0000","last_peered":"2023-10-09T05:01:19.014846+0000","last_clean":"2023-10-09T04:54:44.521216+0000","last_became_active":"2023-10-09T04:59:07.457064+0000","last_became_peered":"2023-10-09T04:59:07.457064+0000","last_unstale":"2023-10-09T05:01:19.014846+0000","last_undegraded":"2023-10-09T05:00:03.057001+0000","last_fullsized":"2023-10-09T04:58:57.981736+0000","mapping_epoch":538,"log_start":"445'1607","ondisk_log_start":"445'1607","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3947,"log_dups_size":0,"ondisk_log_size":3947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:35.050351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265617432,"num_objects":4054,"num_object_clones":0,"num_object_copies":12162,"num_objects_missing_on_primary":3797,"num_objects_missing":3797,"num_objects_degraded":3962,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4054,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5554,"num_write_kb":307393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":885,"num_bytes_recovered":57999360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":3723},{"shards":"5,6","objects":331}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332043+0000","last_change":"2023-10-09T04:59:30.855844+0000","last_active":"2023-10-09T05:01:11.332043+0000","last_peered":"2023-10-09T05:01:11.332043+0000","last_clean":"2023-10-09T05:01:11.332043+0000","last_became_active":"2023-10-09T04:59:30.855493+0000","last_became_peered":"2023-10-09T04:59:30.855493+0000","last_unstale":"2023-10-09T05:01:11.332043+0000","last_undegraded":"2023-10-09T05:01:11.332043+0000","last_fullsized":"2023-10-09T05:01:11.332043+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:46:39.038360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":135,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.265971+0000","last_change":"2023-10-09T05:00:02.997768+0000","last_active":"2023-10-09T05:01:08.265971+0000","last_peered":"2023-10-09T05:01:08.265971+0000","last_clean":"2023-10-09T05:01:08.265971+0000","last_became_active":"2023-10-09T05:00:02.997451+0000","last_became_peered":"2023-10-09T05:00:02.997451+0000","last_unstale":"2023-10-09T05:01:08.265971+0000","last_undegraded":"2023-10-09T05:01:08.265971+0000","last_fullsized":"2023-10-09T05:01:08.265971+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:14:18.292200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":136,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.364114+0000","last_change":"2023-10-09T05:00:03.018405+0000","last_active":"2023-10-09T05:01:08.364114+0000","last_peered":"2023-10-09T05:01:08.364114+0000","last_clean":"2023-10-09T05:01:08.364114+0000","last_became_active":"2023-10-09T05:00:03.018051+0000","last_became_peered":"2023-10-09T05:00:03.018051+0000","last_unstale":"2023-10-09T05:01:08.364114+0000","last_undegraded":"2023-10-09T05:01:08.364114+0000","last_fullsized":"2023-10-09T05:01:08.364114+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:56:26.400152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"570'5512","reported_seq":8439,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:17.121053+0000","last_change":"2023-10-09T05:00:14.084542+0000","last_active":"2023-10-09T05:01:17.121053+0000","last_peered":"2023-10-09T05:01:17.121053+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:00:13.871981+0000","last_became_peered":"2023-10-09T05:00:13.871981+0000","last_unstale":"2023-10-09T05:01:17.121053+0000","last_undegraded":"2023-10-09T05:00:08.191997+0000","last_fullsized":"2023-10-09T05:00:04.000077+0000","mapping_epoch":559,"log_start":"445'2027","ondisk_log_start":"445'2027","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3485,"log_dups_size":0,"ondisk_log_size":3485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:26:27.881660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265158656,"num_objects":4046,"num_object_clones":0,"num_object_copies":12138,"num_objects_missing_on_primary":4188,"num_objects_missing":4188,"num_objects_degraded":7801,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5512,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1375,"num_bytes_recovered":90112000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[1,5],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,3,5,7","objects":2},{"shards":"1,5","objects":1},{"shards":"2,3,5,7","objects":50},{"shards":"2,5,7","objects":3216},{"shards":"3,5,7","objects":776},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":412,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.317588+0000","last_change":"2023-10-09T05:00:03.028225+0000","last_active":"2023-10-09T05:01:11.317588+0000","last_peered":"2023-10-09T05:01:11.317588+0000","last_clean":"2023-10-09T05:01:11.317588+0000","last_became_active":"2023-10-09T05:00:03.027843+0000","last_became_peered":"2023-10-09T05:00:03.027843+0000","last_unstale":"2023-10-09T05:01:11.317588+0000","last_undegraded":"2023-10-09T05:01:11.317588+0000","last_fullsized":"2023-10-09T05:01:11.317588+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:12:39.724975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1094,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366156+0000","last_change":"2023-10-09T04:59:33.901001+0000","last_active":"2023-10-09T05:01:11.366156+0000","last_peered":"2023-10-09T05:01:11.366156+0000","last_clean":"2023-10-09T05:01:11.366156+0000","last_became_active":"2023-10-09T04:59:33.900627+0000","last_became_peered":"2023-10-09T04:59:33.900627+0000","last_unstale":"2023-10-09T05:01:11.366156+0000","last_undegraded":"2023-10-09T05:01:11.366156+0000","last_fullsized":"2023-10-09T05:01:11.366156+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":37,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332966+0000","last_change":"2023-10-09T04:59:30.856383+0000","last_active":"2023-10-09T05:01:11.332966+0000","last_peered":"2023-10-09T05:01:11.332966+0000","last_clean":"2023-10-09T05:01:11.332966+0000","last_became_active":"2023-10-09T04:59:30.855682+0000","last_became_peered":"2023-10-09T04:59:30.855682+0000","last_unstale":"2023-10-09T05:01:11.332966+0000","last_undegraded":"2023-10-09T05:01:11.332966+0000","last_fullsized":"2023-10-09T05:01:11.332966+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.088239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":75,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.400190+0000","last_change":"2023-10-09T04:58:56.999352+0000","last_active":"2023-10-09T05:01:11.400190+0000","last_peered":"2023-10-09T05:01:11.400190+0000","last_clean":"2023-10-09T05:01:11.400190+0000","last_became_active":"2023-10-09T04:58:56.999048+0000","last_became_peered":"2023-10-09T04:58:56.999048+0000","last_unstale":"2023-10-09T05:01:11.400190+0000","last_undegraded":"2023-10-09T05:01:11.400190+0000","last_fullsized":"2023-10-09T05:01:11.400190+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:53.056597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1097,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.369746+0000","last_change":"2023-10-09T04:59:57.034671+0000","last_active":"2023-10-09T05:01:11.369746+0000","last_peered":"2023-10-09T05:01:11.369746+0000","last_clean":"2023-10-09T05:01:11.369746+0000","last_became_active":"2023-10-09T04:59:57.034309+0000","last_became_peered":"2023-10-09T04:59:57.034309+0000","last_unstale":"2023-10-09T05:01:11.369746+0000","last_undegraded":"2023-10-09T05:01:11.369746+0000","last_fullsized":"2023-10-09T05:01:11.369746+0000","mapping_epoch":556,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"571'5643","reported_seq":10618,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:17.590210+0000","last_change":"2023-10-09T04:59:32.037813+0000","last_active":"2023-10-09T05:01:17.590210+0000","last_peered":"2023-10-09T05:01:17.590210+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T04:58:59.452514+0000","last_became_peered":"2023-10-09T04:58:59.452514+0000","last_unstale":"2023-10-09T05:01:17.590210+0000","last_undegraded":"2023-10-09T04:58:59.441276+0000","last_fullsized":"2023-10-09T04:58:59.025474+0000","mapping_epoch":539,"log_start":"445'1563","ondisk_log_start":"445'1563","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4080,"log_dups_size":0,"ondisk_log_size":4080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:16:08.028846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269156352,"num_objects":4107,"num_object_clones":0,"num_object_copies":12321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2204,"num_bytes_recovered":129499136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[2,4],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":2},{"shards":"1,2,4,5","objects":1069},{"shards":"2,4","objects":-412},{"shards":"2,4,5","objects":2943}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":119,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.265889+0000","last_change":"2023-10-09T04:56:55.417824+0000","last_active":"2023-10-09T05:01:08.265889+0000","last_peered":"2023-10-09T05:01:08.265889+0000","last_clean":"2023-10-09T05:01:08.265889+0000","last_became_active":"2023-10-09T04:56:55.417430+0000","last_became_peered":"2023-10-09T04:56:55.417430+0000","last_unstale":"2023-10-09T05:01:08.265889+0000","last_undegraded":"2023-10-09T05:01:08.265889+0000","last_fullsized":"2023-10-09T05:01:08.265889+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:22.097654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332131+0000","last_change":"2023-10-09T04:59:30.875340+0000","last_active":"2023-10-09T05:01:11.332131+0000","last_peered":"2023-10-09T05:01:11.332131+0000","last_clean":"2023-10-09T05:01:11.332131+0000","last_became_active":"2023-10-09T04:59:30.875026+0000","last_became_peered":"2023-10-09T04:59:30.875026+0000","last_unstale":"2023-10-09T05:01:11.332131+0000","last_undegraded":"2023-10-09T05:01:11.332131+0000","last_fullsized":"2023-10-09T05:01:11.332131+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:43:12.350534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":131,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.284147+0000","last_change":"2023-10-09T05:00:03.037746+0000","last_active":"2023-10-09T05:01:11.284147+0000","last_peered":"2023-10-09T05:01:11.284147+0000","last_clean":"2023-10-09T05:01:11.284147+0000","last_became_active":"2023-10-09T05:00:03.036701+0000","last_became_peered":"2023-10-09T05:00:03.036701+0000","last_unstale":"2023-10-09T05:01:11.284147+0000","last_undegraded":"2023-10-09T05:01:11.284147+0000","last_fullsized":"2023-10-09T05:01:11.284147+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:11:11.897581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1097,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366667+0000","last_change":"2023-10-09T04:59:41.847534+0000","last_active":"2023-10-09T05:01:11.366667+0000","last_peered":"2023-10-09T05:01:11.366667+0000","last_clean":"2023-10-09T05:01:11.366667+0000","last_became_active":"2023-10-09T04:59:41.847139+0000","last_became_peered":"2023-10-09T04:59:41.847139+0000","last_unstale":"2023-10-09T05:01:11.366667+0000","last_undegraded":"2023-10-09T05:01:11.366667+0000","last_fullsized":"2023-10-09T05:01:11.366667+0000","mapping_epoch":550,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":551,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"571'5367","reported_seq":17062,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:19.110084+0000","last_change":"2023-10-09T04:58:59.000602+0000","last_active":"2023-10-09T05:01:19.110084+0000","last_peered":"2023-10-09T05:01:19.110084+0000","last_clean":"2023-10-09T04:56:31.960470+0000","last_became_active":"2023-10-09T04:58:58.998253+0000","last_became_peered":"2023-10-09T04:58:58.998253+0000","last_unstale":"2023-10-09T05:01:19.110084+0000","last_undegraded":"2023-10-09T04:58:58.992982+0000","last_fullsized":"2023-10-09T04:58:58.986976+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135036928,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":73,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2828,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2872,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"3,5","objects":2060}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":44,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.372937+0000","last_change":"2023-10-09T04:58:57.034371+0000","last_active":"2023-10-09T05:01:11.372937+0000","last_peered":"2023-10-09T05:01:11.372937+0000","last_clean":"2023-10-09T05:01:11.372937+0000","last_became_active":"2023-10-09T04:58:57.034021+0000","last_became_peered":"2023-10-09T04:58:57.034021+0000","last_unstale":"2023-10-09T05:01:11.372937+0000","last_undegraded":"2023-10-09T05:01:11.372937+0000","last_fullsized":"2023-10-09T05:01:11.372937+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:02:22.688678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.260125+0000","last_change":"2023-10-09T04:59:30.852166+0000","last_active":"2023-10-09T05:01:08.260125+0000","last_peered":"2023-10-09T05:01:08.260125+0000","last_clean":"2023-10-09T05:01:08.260125+0000","last_became_active":"2023-10-09T04:59:30.851555+0000","last_became_peered":"2023-10-09T04:59:30.851555+0000","last_unstale":"2023-10-09T05:01:08.260125+0000","last_undegraded":"2023-10-09T05:01:08.260125+0000","last_fullsized":"2023-10-09T05:01:08.260125+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:46:00.921200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":401,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.258342+0000","last_change":"2023-10-09T05:00:03.109676+0000","last_active":"2023-10-09T05:01:08.258342+0000","last_peered":"2023-10-09T05:01:08.258342+0000","last_clean":"2023-10-09T05:01:08.258342+0000","last_became_active":"2023-10-09T05:00:03.109229+0000","last_became_peered":"2023-10-09T05:00:03.109229+0000","last_unstale":"2023-10-09T05:01:08.258342+0000","last_undegraded":"2023-10-09T05:01:08.258342+0000","last_fullsized":"2023-10-09T05:01:08.258342+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:30:32.681498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1092,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.400112+0000","last_change":"2023-10-09T04:59:33.919822+0000","last_active":"2023-10-09T05:01:11.400112+0000","last_peered":"2023-10-09T05:01:11.400112+0000","last_clean":"2023-10-09T05:01:11.400112+0000","last_became_active":"2023-10-09T04:59:33.919386+0000","last_became_peered":"2023-10-09T04:59:33.919386+0000","last_unstale":"2023-10-09T05:01:11.400112+0000","last_undegraded":"2023-10-09T05:01:11.400112+0000","last_fullsized":"2023-10-09T05:01:11.400112+0000","mapping_epoch":548,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":549,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"571'5478","reported_seq":21793,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-09T05:01:19.220701+0000","last_change":"2023-10-09T04:58:39.912315+0000","last_active":"2023-10-09T05:01:19.220701+0000","last_peered":"2023-10-09T05:01:19.220701+0000","last_clean":"2023-10-09T05:01:19.220701+0000","last_became_active":"2023-10-09T04:58:39.903645+0000","last_became_peered":"2023-10-09T04:58:39.903645+0000","last_unstale":"2023-10-09T05:01:19.220701+0000","last_undegraded":"2023-10-09T05:01:19.220701+0000","last_fullsized":"2023-10-09T05:01:19.220701+0000","mapping_epoch":526,"log_start":"444'300","ondisk_log_start":"444'300","created":440,"last_epoch_clean":527,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:13:27.838809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266338304,"num_objects":4064,"num_object_clones":0,"num_object_copies":12192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5478,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7458,"num_bytes_recovered":477888512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":170,"reported_epoch":570,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:01:11.335360+0000","last_change":"2023-10-09T05:00:03.015330+0000","last_active":"2023-10-09T05:01:11.335360+0000","last_peered":"2023-10-09T05:01:11.335360+0000","last_clean":"2023-10-09T04:59:56.922992+0000","last_became_active":"2023-10-09T05:00:03.015081+0000","last_became_peered":"2023-10-09T05:00:03.015081+0000","last_unstale":"2023-10-09T05:01:11.335360+0000","last_undegraded":"2023-10-09T05:00:02.978496+0000","last_fullsized":"2023-10-09T05:01:11.335360+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:50.144636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":156,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.270111+0000","last_change":"2023-10-09T04:58:57.063270+0000","last_active":"2023-10-09T05:01:08.270111+0000","last_peered":"2023-10-09T05:01:08.270111+0000","last_clean":"2023-10-09T05:01:08.270111+0000","last_became_active":"2023-10-09T04:58:57.062873+0000","last_became_peered":"2023-10-09T04:58:57.062873+0000","last_unstale":"2023-10-09T05:01:08.270111+0000","last_undegraded":"2023-10-09T05:01:08.270111+0000","last_fullsized":"2023-10-09T05:01:08.270111+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:43:58.612148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":36,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.331498+0000","last_change":"2023-10-09T04:59:30.860452+0000","last_active":"2023-10-09T05:01:11.331498+0000","last_peered":"2023-10-09T05:01:11.331498+0000","last_clean":"2023-10-09T05:01:11.331498+0000","last_became_active":"2023-10-09T04:59:30.859944+0000","last_became_peered":"2023-10-09T04:59:30.859944+0000","last_unstale":"2023-10-09T05:01:11.331498+0000","last_undegraded":"2023-10-09T05:01:11.331498+0000","last_fullsized":"2023-10-09T05:01:11.331498+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:17:01.173959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1068,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:11.367064+0000","last_change":"2023-10-09T04:59:31.846857+0000","last_active":"2023-10-09T05:01:11.367064+0000","last_peered":"2023-10-09T05:01:11.367064+0000","last_clean":"2023-10-09T04:59:25.885390+0000","last_became_active":"2023-10-09T04:59:31.846529+0000","last_became_peered":"2023-10-09T04:59:31.846529+0000","last_unstale":"2023-10-09T05:01:11.367064+0000","last_undegraded":"2023-10-09T04:59:31.832883+0000","last_fullsized":"2023-10-09T04:59:31.831034+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1138389,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1],"avail_no_missing":["6","1","2","7"],"object_location_counts":[{"shards":"1,2,6,7","objects":1},{"shards":"1,6","objects":-1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"570'5735","reported_seq":8739,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:16.887378+0000","last_change":"2023-10-09T04:59:55.409660+0000","last_active":"2023-10-09T05:01:16.887378+0000","last_peered":"2023-10-09T05:01:16.887378+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T04:59:55.037735+0000","last_became_peered":"2023-10-09T04:59:55.037735+0000","last_unstale":"2023-10-09T05:01:16.887378+0000","last_undegraded":"2023-10-09T04:59:44.912466+0000","last_fullsized":"2023-10-09T04:59:31.829930+0000","mapping_epoch":546,"log_start":"445'2196","ondisk_log_start":"445'2196","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3539,"log_dups_size":0,"ondisk_log_size":3539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:35:51.517413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276889600,"num_objects":4225,"num_object_clones":0,"num_object_copies":12675,"num_objects_missing_on_primary":4098,"num_objects_missing":4098,"num_objects_degraded":8797,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5735,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1322,"num_bytes_recovered":86638592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"1,2,4,7","objects":127},{"shards":"2,4,7","objects":3972},{"shards":"2,6,7","objects":125}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":67,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.367123+0000","last_change":"2023-10-09T04:58:07.881659+0000","last_active":"2023-10-09T05:01:11.367123+0000","last_peered":"2023-10-09T05:01:11.367123+0000","last_clean":"2023-10-09T05:01:11.367123+0000","last_became_active":"2023-10-09T04:58:07.881399+0000","last_became_peered":"2023-10-09T04:58:07.881399+0000","last_unstale":"2023-10-09T05:01:11.367123+0000","last_undegraded":"2023-10-09T05:01:11.367123+0000","last_fullsized":"2023-10-09T05:01:11.367123+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:42:31.736513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":197,"reported_epoch":569,"state":"active+clean","last_fresh":"2023-10-09T05:01:08.270183+0000","last_change":"2023-10-09T04:59:29.958563+0000","last_active":"2023-10-09T05:01:08.270183+0000","last_peered":"2023-10-09T05:01:08.270183+0000","last_clean":"2023-10-09T05:01:08.270183+0000","last_became_active":"2023-10-09T04:58:58.015415+0000","last_became_peered":"2023-10-09T04:58:58.015415+0000","last_unstale":"2023-10-09T05:01:08.270183+0000","last_undegraded":"2023-10-09T05:01:08.270183+0000","last_fullsized":"2023-10-09T05:01:08.270183+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:17:12.808436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371929+0000","last_change":"2023-10-09T05:00:03.002429+0000","last_active":"2023-10-09T05:01:11.371929+0000","last_peered":"2023-10-09T05:01:11.371929+0000","last_clean":"2023-10-09T05:01:11.371929+0000","last_became_active":"2023-10-09T05:00:03.002074+0000","last_became_peered":"2023-10-09T05:00:03.002074+0000","last_unstale":"2023-10-09T05:01:11.371929+0000","last_undegraded":"2023-10-09T05:01:11.371929+0000","last_fullsized":"2023-10-09T05:01:11.371929+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:47:38.429397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"570'5189","reported_seq":7032,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:16.857178+0000","last_change":"2023-10-09T04:59:40.916464+0000","last_active":"2023-10-09T05:01:16.857178+0000","last_peered":"2023-10-09T05:01:16.857178+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T04:59:40.797700+0000","last_became_peered":"2023-10-09T04:59:40.797700+0000","last_unstale":"2023-10-09T05:01:16.857178+0000","last_undegraded":"2023-10-09T04:59:34.069489+0000","last_fullsized":"2023-10-09T04:59:34.063514+0000","mapping_epoch":546,"log_start":"444'386","ondisk_log_start":"444'386","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4803,"log_dups_size":0,"ondisk_log_size":4803,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129990656,"num_objects":1984,"num_object_clones":0,"num_object_copies":5952,"num_objects_missing_on_primary":2019,"num_objects_missing":2019,"num_objects_degraded":5832,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2733,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":375,"num_bytes_recovered":24576000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"1,2,4,5","objects":1},{"shards":"1,3","objects":-36},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":1632}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.371485+0000","last_change":"2023-10-09T04:58:59.185788+0000","last_active":"2023-10-09T05:01:11.371485+0000","last_peered":"2023-10-09T05:01:11.371485+0000","last_clean":"2023-10-09T05:01:11.371485+0000","last_became_active":"2023-10-09T04:58:59.185338+0000","last_became_peered":"2023-10-09T04:58:59.185338+0000","last_unstale":"2023-10-09T05:01:11.371485+0000","last_undegraded":"2023-10-09T05:01:11.371485+0000","last_fullsized":"2023-10-09T05:01:11.371485+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:22:19.825485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":118,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.268617+0000","last_change":"2023-10-09T04:58:57.064053+0000","last_active":"2023-10-09T05:01:11.268617+0000","last_peered":"2023-10-09T05:01:11.268617+0000","last_clean":"2023-10-09T05:01:11.268617+0000","last_became_active":"2023-10-09T04:58:57.063696+0000","last_became_peered":"2023-10-09T04:58:57.063696+0000","last_unstale":"2023-10-09T05:01:11.268617+0000","last_undegraded":"2023-10-09T05:01:11.268617+0000","last_fullsized":"2023-10-09T05:01:11.268617+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:31:30.110906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"570'5324","reported_seq":10040,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:16.564207+0000","last_change":"2023-10-09T04:59:55.883965+0000","last_active":"2023-10-09T05:01:16.564207+0000","last_peered":"2023-10-09T05:01:16.564207+0000","last_clean":"2023-10-09T05:01:16.564207+0000","last_became_active":"2023-10-09T04:59:55.880766+0000","last_became_peered":"2023-10-09T04:59:55.880766+0000","last_unstale":"2023-10-09T05:01:16.564207+0000","last_undegraded":"2023-10-09T05:01:16.564207+0000","last_fullsized":"2023-10-09T05:01:16.564207+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135659520,"num_objects":2070,"num_object_clones":0,"num_object_copies":6210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2789,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":666,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.360316+0000","last_change":"2023-10-09T04:59:30.021044+0000","last_active":"2023-10-09T05:01:11.360316+0000","last_peered":"2023-10-09T05:01:11.360316+0000","last_clean":"2023-10-09T05:01:11.360316+0000","last_became_active":"2023-10-09T04:58:57.166878+0000","last_became_peered":"2023-10-09T04:58:57.166878+0000","last_unstale":"2023-10-09T05:01:11.360316+0000","last_undegraded":"2023-10-09T05:01:11.360316+0000","last_fullsized":"2023-10-09T05:01:11.360316+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:33:33.097934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":343,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.331827+0000","last_change":"2023-10-09T05:00:03.122991+0000","last_active":"2023-10-09T05:01:11.331827+0000","last_peered":"2023-10-09T05:01:11.331827+0000","last_clean":"2023-10-09T05:01:11.331827+0000","last_became_active":"2023-10-09T05:00:03.122557+0000","last_became_peered":"2023-10-09T05:00:03.122557+0000","last_unstale":"2023-10-09T05:01:11.331827+0000","last_undegraded":"2023-10-09T05:01:11.331827+0000","last_fullsized":"2023-10-09T05:01:11.331827+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:04:54.753971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.10","version":"571'5366","reported_seq":5684,"reported_epoch":571,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:19.328062+0000","last_change":"2023-10-09T05:00:25.185018+0000","last_active":"2023-10-09T05:01:19.328062+0000","last_peered":"2023-10-09T05:01:19.328062+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T04:58:58.134873+0000","last_became_peered":"2023-10-09T04:58:58.134873+0000","last_unstale":"2023-10-09T05:01:19.328062+0000","last_undegraded":"2023-10-09T04:58:58.123193+0000","last_fullsized":"2023-10-09T04:58:58.004066+0000","mapping_epoch":538,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4828,"log_dups_size":0,"ondisk_log_size":4828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132939776,"num_objects":2029,"num_object_clones":0,"num_object_copies":6087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2536,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2835,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":1736704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,4],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":1972},{"shards":"2,5,6","objects":61},{"shards":"4,5","objects":-311}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":635,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.363475+0000","last_change":"2023-10-09T04:59:30.019132+0000","last_active":"2023-10-09T05:01:11.363475+0000","last_peered":"2023-10-09T05:01:11.363475+0000","last_clean":"2023-10-09T05:01:11.363475+0000","last_became_active":"2023-10-09T04:58:56.996966+0000","last_became_peered":"2023-10-09T04:58:56.996966+0000","last_unstale":"2023-10-09T05:01:11.363475+0000","last_undegraded":"2023-10-09T05:01:11.363475+0000","last_fullsized":"2023-10-09T05:01:11.363475+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:43.116063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":189,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.333281+0000","last_change":"2023-10-09T04:59:30.856551+0000","last_active":"2023-10-09T05:01:11.333281+0000","last_peered":"2023-10-09T05:01:11.333281+0000","last_clean":"2023-10-09T05:01:11.333281+0000","last_became_active":"2023-10-09T04:59:30.856103+0000","last_became_peered":"2023-10-09T04:59:30.856103+0000","last_unstale":"2023-10-09T05:01:11.333281+0000","last_undegraded":"2023-10-09T05:01:11.333281+0000","last_fullsized":"2023-10-09T05:01:11.333281+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:22:16.595682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"570'5334","reported_seq":7340,"reported_epoch":570,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:01:16.511210+0000","last_change":"2023-10-09T04:59:55.375337+0000","last_active":"2023-10-09T05:01:16.511210+0000","last_peered":"2023-10-09T05:01:16.511210+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T04:59:55.016369+0000","last_became_peered":"2023-10-09T04:59:55.016369+0000","last_unstale":"2023-10-09T05:01:16.511210+0000","last_undegraded":"2023-10-09T04:59:33.814536+0000","last_fullsized":"2023-10-09T04:59:31.938954+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135135232,"num_objects":2062,"num_object_clones":0,"num_object_copies":6186,"num_objects_missing_on_primary":1686,"num_objects_missing":1686,"num_objects_degraded":5907,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2802,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23920640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1,3","objects":9},{"shards":"1,3","objects":-28},{"shards":"2,3,5,7","objects":346},{"shards":"2,5,7","objects":1686},{"shards":"3,5,6","objects":49}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":171,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.334514+0000","last_change":"2023-10-09T04:59:30.857229+0000","last_active":"2023-10-09T05:01:11.334514+0000","last_peered":"2023-10-09T05:01:11.334514+0000","last_clean":"2023-10-09T05:01:11.334514+0000","last_became_active":"2023-10-09T04:59:30.856946+0000","last_became_peered":"2023-10-09T04:59:30.856946+0000","last_unstale":"2023-10-09T05:01:11.334514+0000","last_undegraded":"2023-10-09T05:01:11.334514+0000","last_fullsized":"2023-10-09T05:01:11.334514+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:18:31.040095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":161,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.332726+0000","last_change":"2023-10-09T05:00:02.991182+0000","last_active":"2023-10-09T05:01:11.332726+0000","last_peered":"2023-10-09T05:01:11.332726+0000","last_clean":"2023-10-09T05:01:11.332726+0000","last_became_active":"2023-10-09T05:00:02.990840+0000","last_became_peered":"2023-10-09T05:00:02.990840+0000","last_unstale":"2023-10-09T05:01:11.332726+0000","last_undegraded":"2023-10-09T05:01:11.332726+0000","last_fullsized":"2023-10-09T05:01:11.332726+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:55.114402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"571'5262","reported_seq":16577,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-09T05:01:19.290284+0000","last_change":"2023-10-09T05:00:26.900201+0000","last_active":"2023-10-09T05:01:19.290284+0000","last_peered":"2023-10-09T05:01:19.290284+0000","last_clean":"2023-10-09T05:01:19.290284+0000","last_became_active":"2023-10-09T05:00:26.894254+0000","last_became_peered":"2023-10-09T05:00:26.894254+0000","last_unstale":"2023-10-09T05:01:19.290284+0000","last_undegraded":"2023-10-09T05:01:19.290284+0000","last_fullsized":"2023-10-09T05:01:19.290284+0000","mapping_epoch":563,"log_start":"444'341","ondisk_log_start":"444'341","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4921,"log_dups_size":0,"ondisk_log_size":4921,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134479872,"num_objects":2052,"num_object_clones":0,"num_object_copies":6156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2763,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2662,"num_bytes_recovered":161120256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":564,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.366994+0000","last_change":"2023-10-09T05:00:50.797283+0000","last_active":"2023-10-09T05:01:11.366994+0000","last_peered":"2023-10-09T05:01:11.366994+0000","last_clean":"2023-10-09T05:01:11.366994+0000","last_became_active":"2023-10-09T05:00:03.009228+0000","last_became_peered":"2023-10-09T05:00:03.009228+0000","last_unstale":"2023-10-09T05:01:11.366994+0000","last_undegraded":"2023-10-09T05:01:11.366994+0000","last_fullsized":"2023-10-09T05:01:11.366994+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":565,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:38:07.706140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":43,"reported_epoch":570,"state":"active+clean","last_fresh":"2023-10-09T05:01:11.369206+0000","last_change":"2023-10-09T04:58:59.821723+0000","last_active":"2023-10-09T05:01:11.369206+0000","last_peered":"2023-10-09T05:01:11.369206+0000","last_clean":"2023-10-09T05:01:11.369206+0000","last_became_active":"2023-10-09T04:58:59.821428+0000","last_became_peered":"2023-10-09T04:58:59.821428+0000","last_unstale":"2023-10-09T05:01:11.369206+0000","last_undegraded":"2023-10-09T05:01:11.369206+0000","last_fullsized":"2023-10-09T05:01:11.369206+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:37:15.048879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":4300079128,"num_objects":65615,"num_object_clones":0,"num_object_copies":196845,"num_objects_missing_on_primary":24259,"num_objects_missing":24259,"num_objects_degraded":79948,"num_objects_misplaced":11623,"num_objects_unfound":0,"num_objects_dirty":65615,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":89603,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42903,"num_bytes_recovered":2737700864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16393519104,"data_stored":16393502816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125321,"ondisk_log_size":125321,"up":78,"acting":58,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":138,"num_bytes_recovered":31134,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31051,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":6177690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":31,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234649,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3718340,"kb_used_data":2200892,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":90026812,"statfs":{"total":95995035648,"available":92187455488,"internally_reserved":0,"allocated":2253713408,"data_stored":2252371515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45500000000000002,"15min":0.88200000000000001},"min":{"1min":0.313,"5min":0.27800000000000002,"15min":0.224},"max":{"1min":0.92300000000000004,"5min":3.742,"15min":14.970000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.505,"5min":0.56000000000000005,"15min":1.3089999999999999},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.1379999999999999,"5min":3.319,"15min":21.216000000000001},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:01:11 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.56000000000000005,"15min":2.5539999999999998},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.246},"max":{"1min":2.6299999999999999,"5min":2.9980000000000002,"15min":128.601},"last":1.004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.68899999999999995,"15min":2.5699999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26900000000000002},"max":{"1min":1.837,"5min":8.1920000000000002,"15min":128.381},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":10.281000000000001,"5min":2.9039999999999999,"15min":5.2009999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.29499999999999998},"max":{"1min":154.76400000000001,"5min":154.76400000000001,"15min":207.96600000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":10.458,"5min":3,"15min":5.3949999999999996},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.29699999999999999},"max":{"1min":154.80000000000001,"5min":154.80000000000001,"15min":207.97900000000001},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:01:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":0.999,"15min":1.6599999999999999},"min":{"1min":0.40799999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.5940000000000003,"5min":7.5940000000000003,"15min":74.259},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.99,"5min":1.0660000000000001,"15min":1.8959999999999999},"min":{"1min":0.311,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.6210000000000004,"5min":7.6210000000000004,"15min":74.100999999999999},"last":4.0359999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.30199999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.60099999999999998,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.69299999999999995,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.347,"5min":3.347,"15min":3.347},"last":0.58099999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.49299999999999999,"15min":1.6830000000000001},"min":{"1min":0.33100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":131.84899999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.61799999999999999,"15min":2.1549999999999998},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.0880000000000001,"5min":7.0880000000000001,"15min":135.74199999999999},"last":0.47299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:00:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.51400000000000001,"15min":1.111},"min":{"1min":0.38800000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.81699999999999995,"5min":0.90000000000000002,"15min":10.047000000000001},"last":0.441},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.63,"15min":1.397},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.1799999999999997,"5min":7.1799999999999997,"15min":21.321999999999999},"last":0.50900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496281,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1661132,"kb_used_data":754756,"kb_used_omap":0,"kb_used_meta":906368,"kb_avail":92084020,"statfs":{"total":95995035648,"available":94294036480,"internally_reserved":0,"allocated":772870144,"data_stored":771520404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928120832},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.55900000000000005,"15min":0.80400000000000005},"min":{"1min":0.27500000000000002,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":5.9900000000000002,"5min":5.9900000000000002,"15min":7.8540000000000001},"last":0.371},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.60499999999999998,"15min":1.0329999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.8149999999999999,"5min":6.1980000000000004,"15min":16.574000000000002},"last":0.51900000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:01:00 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":2.3530000000000002,"15min":5.5099999999999998},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":31.457999999999998,"5min":90.278999999999996,"15min":260.738},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.6389999999999998,"5min":2.2549999999999999,"15min":5.1630000000000003},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":31.488,"5min":90.414000000000001,"15min":261.26100000000002},"last":1.113}]},{"osd":2,"last update":"Mon Oct 9 05:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.024,"15min":2.915},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.706,"5min":9.2759999999999998,"15min":133.535},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.0449999999999999,"15min":3.0720000000000001},"min":{"1min":0.36199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.7370000000000001,"5min":9.2639999999999993,"15min":133.51900000000001},"last":0.48399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:01:00 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":3.133,"15min":2.2690000000000001},"min":{"1min":0.34399999999999997,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":4.6959999999999997,"5min":147.43600000000001,"15min":147.43600000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.2989999999999999,"5min":3.3410000000000002,"15min":2.516},"min":{"1min":0.40300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.9169999999999998,"5min":147.41999999999999,"15min":147.41999999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:00:42 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.45600000000000002,"15min":0.46500000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.496,"15min":0.49399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66700000000000004,"5min":2.411,"15min":2.411},"last":0.35499999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:01:00 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.56699999999999995,"15min":1.95},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.28299999999999997},"max":{"1min":3.5550000000000002,"5min":3.5550000000000002,"15min":83.168999999999997},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.52100000000000002,"15min":1.9450000000000001},"min":{"1min":0.32200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":1.863,"5min":2.3879999999999999,"15min":83.126000000000005},"last":0.47199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:00:33 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.497,"15min":1.351},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":0.76300000000000001,"5min":0.80700000000000005,"15min":36.204999999999998},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.53000000000000003,"15min":1.3260000000000001},"min":{"1min":0.40200000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.64700000000000002,"5min":3.3969999999999998,"15min":10.661},"last":0.45000000000000001}]}]},{"osd":1,"up_from":118,"seq":506806141353,"num_pgs":87,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2278220,"kb_used_data":1060868,"kb_used_omap":4,"kb_used_meta":1217339,"kb_avail":91466932,"statfs":{"total":95995035648,"available":93662138368,"internally_reserved":0,"allocated":1086328832,"data_stored":1083445528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1246556046},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,2,3,1,6,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.196,"15min":1.1339999999999999},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":8.0999999999999996,"5min":11.763999999999999,"15min":11.763999999999999},"last":1.931},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.0549999999999999,"15min":1.149},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":7.032,"5min":7.032,"15min":10.765000000000001},"last":0.376}]},{"osd":2,"last update":"Mon Oct 9 05:00:36 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.8,"15min":3.7530000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.21099999999999999},"max":{"1min":7.1909999999999998,"5min":63.143999999999998,"15min":263.089},"last":1.974},{"interface":"front","average":{"1min":1.107,"5min":1.6459999999999999,"15min":3.702},"min":{"1min":0.32400000000000001,"5min":0.254,"15min":0.222},"max":{"1min":7.3810000000000002,"5min":66.198999999999998,"15min":254.50899999999999},"last":2.9489999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:01:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":1.762,"15min":1.6000000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26500000000000001},"max":{"1min":9.9760000000000009,"5min":39.289000000000001,"15min":77.819999999999993},"last":2.895},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.385,"15min":1.613},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":7.0860000000000003,"5min":14.212,"15min":77.796000000000006},"last":0.57999999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:00:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.6279999999999999,"15min":1.79},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":9.7699999999999996,"5min":19.634,"15min":19.634},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.232,"5min":1.2949999999999999,"15min":1.3109999999999999},"min":{"1min":0.38800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":7.298,"5min":13.045,"15min":13.045},"last":0.60699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.002,"15min":1.534},"min":{"1min":0.42299999999999999,"5min":0.34599999999999997,"15min":0.33000000000000002},"max":{"1min":7.79,"5min":7.79,"15min":9.782},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.907,"5min":1.244,"15min":1.659},"min":{"1min":0.379,"5min":0.379,"15min":0.315},"max":{"1min":9.7430000000000003,"5min":19.597999999999999,"15min":20.021000000000001},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.113,"15min":1.46},"min":{"1min":0.39700000000000002,"5min":0.35699999999999998,"15min":0.34399999999999997},"max":{"1min":7.4370000000000003,"5min":7.4370000000000003,"15min":9.7840000000000007},"last":0.628},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.407,"15min":1.736},"min":{"1min":0.44800000000000001,"5min":0.38,"15min":0.32400000000000001},"max":{"1min":9.6639999999999997,"5min":19.548999999999999,"15min":20.164999999999999},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:00:24 2023","interfaces":[{"interface":"back","average":{"1min":1.98,"5min":1.4279999999999999,"15min":2.2450000000000001},"min":{"1min":0.48799999999999999,"5min":0.373,"15min":0.35899999999999999},"max":{"1min":9.7040000000000006,"5min":19.664999999999999,"15min":59.972000000000001},"last":1.6719999999999999},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.252,"15min":2.0150000000000001},"min":{"1min":0.52200000000000002,"5min":0.39300000000000002,"15min":0.378},"max":{"1min":7.335,"5min":9.8710000000000004,"15min":60.011000000000003},"last":1.6839999999999999}]}]},{"osd":0,"up_from":0,"seq":390842024434,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":221,"seq":949187772709,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4512724,"kb_used_data":3719180,"kb_used_omap":7,"kb_used_meta":793528,"kb_avail":89232428,"statfs":{"total":95995035648,"available":91374006272,"internally_reserved":0,"allocated":3808440320,"data_stored":3805965880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":812572753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.133,"15min":1.3280000000000001},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.188},"max":{"1min":5.4580000000000002,"5min":13.042999999999999,"15min":25.899999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.385,"5min":0.85499999999999998,"15min":1.0640000000000001},"min":{"1min":0.316,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":7.8010000000000002,"5min":12.161,"15min":16.550000000000001},"last":6.8680000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":2.016,"15min":3.2930000000000001},"min":{"1min":0.27000000000000002,"5min":0.25800000000000001,"15min":0.20499999999999999},"max":{"1min":7.7649999999999997,"5min":42.799999999999997,"15min":170.172},"last":2.282},{"interface":"front","average":{"1min":1.3819999999999999,"5min":2.1389999999999998,"15min":3.4700000000000002},"min":{"1min":0.29399999999999998,"5min":0.254,"15min":0.219},"max":{"1min":8.0960000000000001,"5min":42.524999999999999,"15min":170.511},"last":0.64800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.921,"15min":1.6759999999999999},"min":{"1min":0.34100000000000003,"5min":0.30199999999999999,"15min":0.248},"max":{"1min":2.9980000000000002,"5min":42.557000000000002,"15min":45.664000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.872,"5min":1.9690000000000001,"15min":1.7789999999999999},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":5.5090000000000003,"5min":42.293999999999997,"15min":45.686999999999998},"last":0.374}]},{"osd":4,"last update":"Mon Oct 9 05:00:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.75800000000000001,"15min":0.622},"min":{"1min":0.376,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.9660000000000002,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":2.1549999999999998},{"interface":"front","average":{"1min":1.3240000000000001,"5min":0.76900000000000002,"15min":0.64800000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.8620000000000001,"5min":7.8620000000000001,"15min":7.8620000000000001},"last":2.2509999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.79600000000000004,"15min":3.5950000000000002},"min":{"1min":0.35799999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.9299999999999997,"5min":7.9299999999999997,"15min":272.04700000000003},"last":2.21},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.93799999999999994,"15min":3.653},"min":{"1min":0.38,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":2.79,"5min":10.427,"15min":271.89699999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.94899999999999995,"15min":1.4039999999999999},"min":{"1min":0.42799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.5720000000000001,"5min":8.3469999999999995,"15min":16.425999999999998},"last":7.6820000000000004},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.84599999999999997,"15min":1.24},"min":{"1min":0.42299999999999999,"5min":0.38700000000000001,"15min":0.35199999999999998},"max":{"1min":7.9000000000000004,"5min":7.9000000000000004,"15min":10.279999999999999},"last":2.1150000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:00:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.161,"15min":1.73},"min":{"1min":0.45100000000000001,"5min":0.38600000000000001,"15min":0.34300000000000003},"max":{"1min":5.5439999999999996,"5min":8.3949999999999996,"15min":16.462},"last":7.7380000000000004},{"interface":"front","average":{"1min":1.355,"5min":1.02,"15min":1.548},"min":{"1min":0.45400000000000001,"5min":0.42199999999999999,"15min":0.373},"max":{"1min":7.9870000000000001,"5min":9.0640000000000001,"15min":10.211},"last":2.1840000000000002}]}]},{"osd":3,"up_from":142,"seq":609885356406,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2167060,"kb_used_data":1346188,"kb_used_omap":2,"kb_used_meta":820861,"kb_avail":91578092,"statfs":{"total":95995035648,"available":93775966208,"internally_reserved":0,"allocated":1378496512,"data_stored":1376140445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":840561717},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:01:04 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":0.82199999999999995,"15min":0.81899999999999995},"min":{"1min":0.28299999999999997,"5min":0.22800000000000001,"15min":0.19},"max":{"1min":10.167,"5min":10.167,"15min":11.817},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.95,"5min":1.103,"15min":0.99399999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.23499999999999999},"max":{"1min":5.1609999999999996,"5min":6.5229999999999997,"15min":9.4580000000000002},"last":0.35899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:01:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":1.2090000000000001,"15min":2.7210000000000001},"min":{"1min":0.23899999999999999,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":10.145,"5min":10.145,"15min":125.233},"last":0.249},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.2989999999999999,"15min":2.5680000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.21299999999999999},"max":{"1min":26.692,"5min":26.692,"15min":126.102},"last":0.44500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:00:26 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":2.8260000000000001,"15min":3.0529999999999999},"min":{"1min":0.30199999999999999,"5min":0.29599999999999999,"15min":0.23499999999999999},"max":{"1min":15.606999999999999,"5min":212.065,"15min":212.065},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.8599999999999999,"15min":3.2080000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.22600000000000001},"max":{"1min":4.2140000000000004,"5min":211.98400000000001,"15min":211.98400000000001},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:00:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.024,"15min":0.98399999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.7679999999999998,"5min":4.9109999999999996,"15min":4.9109999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.2030000000000001,"15min":1.2529999999999999},"min":{"1min":0.35799999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.8879999999999999,"5min":8.5779999999999994,"15min":8.5779999999999994},"last":0.48499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:01:04 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.748,"15min":1.341},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.30599999999999999},"max":{"1min":4.4160000000000004,"5min":11.368,"15min":28.302},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.262,"15min":1.5649999999999999},"min":{"1min":0.372,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":4.7270000000000003,"5min":24.986999999999998,"15min":28.280000000000001},"last":0.46100000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:01:04 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":1.0449999999999999,"15min":1.1739999999999999},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.32900000000000001},"max":{"1min":4.6699999999999999,"5min":5.1589999999999998,"15min":8.7859999999999996},"last":0.504},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.335,"15min":1.532},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.35299999999999998},"max":{"1min":4.6479999999999997,"5min":24.916,"15min":24.916},"last":0.58699999999999997}]},{"osd":7,"last update":"Mon Oct 9 05:01:04 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.3129999999999999,"15min":1.76},"min":{"1min":0.439,"5min":0.39800000000000002,"15min":0.375},"max":{"1min":4.5590000000000002,"5min":25.004000000000001,"15min":25.004000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.77400000000000002,"15min":1.2310000000000001},"min":{"1min":0.51000000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":4.4569999999999999,"5min":7.2409999999999997,"15min":10.092000000000001},"last":0.56299999999999994}]}]},{"osd":4,"up_from":507,"seq":2177548419117,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3497280,"kb_used_data":3100472,"kb_used_omap":11,"kb_used_meta":396788,"kb_avail":90247872,"statfs":{"total":95995035648,"available":92413820928,"internally_reserved":0,"allocated":3174883328,"data_stored":3173537040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":406311816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.80200000000000005,"15min":0.81399999999999995},"min":{"1min":0.29299999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9379999999999999,"5min":7.9020000000000001,"15min":7.9020000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.71999999999999997,"15min":0.69699999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.226,"5min":5.4770000000000003,"15min":5.4770000000000003},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":5.6269999999999998,"15min":7.5119999999999996},"min":{"1min":0.28000000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2000000000000002,"5min":147.49600000000001,"15min":147.49600000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.995,"5min":5.7709999999999999,"15min":7.7370000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.8639999999999999,"5min":147.51300000000001,"15min":147.51300000000001},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.028,"15min":0.95799999999999996},"min":{"1min":0.34300000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.5620000000000003,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.23,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.766,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.377,"15min":1.1619999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.8099999999999996,"5min":24.379000000000001,"15min":24.379000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.4430000000000001,"5min":2.2679999999999998,"15min":2.3919999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":6.2610000000000001,"5min":24.23,"15min":24.23},"last":0.42699999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.40799999999999997,"15min":0.40500000000000003},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.53300000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50600000000000001,"15min":0.48999999999999999},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":1.8879999999999999,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.61199999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.83499999999999996,"15min":0.74099999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67300000000000004,"5min":22.350999999999999,"15min":22.350999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.77400000000000002,"15min":0.60999999999999999},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.702,"5min":22.433,"15min":22.433},"last":0.503}]},{"osd":7,"last update":"Mon Oct 9 05:00:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36299999999999999,"5min":0.308,"15min":0.308},"max":{"1min":1.9179999999999999,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":1.008},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.65200000000000002,"15min":0.71899999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70099999999999996,"5min":4.5949999999999998,"15min":4.5949999999999998},"last":0.74199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757925,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5190908,"kb_used_data":3689396,"kb_used_omap":0,"kb_used_meta":1501504,"kb_avail":88554244,"statfs":{"total":95995035648,"available":90679545856,"internally_reserved":0,"allocated":3777941504,"data_stored":3776603091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1537540096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:01:16 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.55700000000000005,"15min":1.5589999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":7.5650000000000004,"5min":7.5650000000000004,"15min":106.465},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.70099999999999996,"15min":1.5700000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23799999999999999},"max":{"1min":2.9329999999999998,"5min":11.052,"15min":106.54600000000001},"last":0.51200000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:01:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":0.88300000000000001,"15min":4.2830000000000004},"min":{"1min":0.29499999999999998,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":6.7869999999999999,"5min":6.7869999999999999,"15min":211.85400000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.3759999999999999,"5min":0.90800000000000003,"15min":4.4980000000000002},"min":{"1min":0.35499999999999998,"5min":0.26900000000000002,"15min":0.255},"max":{"1min":7.5819999999999999,"5min":11.249000000000001,"15min":211.435},"last":0.46700000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:00:32 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":1.2529999999999999,"15min":2.831},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":47.262999999999998,"5min":47.262999999999998,"15min":86.488},"last":1.6930000000000001},{"interface":"front","average":{"1min":3.488,"5min":1.5589999999999999,"15min":3.0630000000000002},"min":{"1min":0.36899999999999999,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":47.408000000000001,"5min":47.408000000000001,"15min":86.510999999999996},"last":1.788}]},{"osd":3,"last update":"Mon Oct 9 05:00:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.121,"15min":1.823},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.31900000000000001},"max":{"1min":4.0549999999999997,"5min":105.879,"15min":105.879},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.3819999999999999,"5min":2.278,"15min":1.6439999999999999},"min":{"1min":0.44500000000000001,"5min":0.316,"15min":0.316},"max":{"1min":4.0830000000000002,"5min":105.625,"15min":105.625},"last":1.7190000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:00:39 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.53700000000000003,"15min":0.56499999999999995},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.1679999999999999,"5min":2.2360000000000002,"15min":2.2360000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.47499999999999998,"15min":0.48199999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73299999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.435}]},{"osd":6,"last update":"Mon Oct 9 05:00:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.54200000000000004,"15min":0.98599999999999999},"min":{"1min":0.29999999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.0659999999999998,"5min":4.0659999999999998,"15min":10.295999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.49399999999999999,"15min":1.103},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.3240000000000001,"5min":2.234,"15min":13.083},"last":0.625}]},{"osd":7,"last update":"Mon Oct 9 05:00:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.55600000000000005,"15min":2.3380000000000001},"min":{"1min":0.38300000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.8540000000000001,"5min":2.8540000000000001,"15min":110.521},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56899999999999995,"15min":2.4159999999999999},"min":{"1min":0.35799999999999998,"5min":0.34599999999999997,"15min":0.29499999999999998},"max":{"1min":1.7310000000000001,"5min":2.266,"15min":110.746},"last":0.63600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":197722112,"data_stored":197722112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1076363264,"data_stored":1076363264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3799126016,"data_stored":3799121944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1371275264,"data_stored":1371275264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3165130752,"data_stored":3165126680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3770748928,"data_stored":3770744856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":766513152,"data_stored":766509080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2246639616,"data_stored":2246639616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:01:25.957 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pgp_num 26 2023-10-09T05:01:25.996 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:26.063 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:26.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:26.093 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:01:26.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:26.209 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:26.342 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:26.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:26.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:26.447 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:26.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 79668/196050 objects degraded (40.637%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:26.480 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:26.536 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:26.564 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:26.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:26.645 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:26.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 79668/196050 objects degraded (40.637%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:26.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 79668/196050 objects degraded (40.637%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:26.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:26.798 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:27.013 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:27.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:27.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:27.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:27.148 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:27.181 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:01:27.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:27.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:27.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:27.322 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:01:27.330 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:27.375 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:27.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:27.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:27 smithi029 ceph-mon[216903]: pgmap v1915: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 79623/195540 objects degraded (40.720%); 11569/195540 objects misplaced (5.916%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:27.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1459919980' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:01:27.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:27 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:01:27.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:27 smithi029 ceph-mon[223858]: pgmap v1915: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 79623/195540 objects degraded (40.720%); 11569/195540 objects misplaced (5.916%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:27.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1459919980' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:01:27.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:27 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:01:27.478 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:27.501 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:27 smithi177 ceph-mon[195797]: pgmap v1915: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 79623/195540 objects degraded (40.720%); 11569/195540 objects misplaced (5.916%); 2.7 MiB/s, 43 objects/s recovering 2023-10-09T05:01:27.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1459919980' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:01:27.502 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:27 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:01:27.504 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:27.706 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:27.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:27.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:27.814 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:27.844 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:27.943 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:27.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:28.048 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:28.420 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:01:28.420 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:01:28.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78136 14.d 14:b962615d:::benchmark_data_smithi029_550670_object525:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:53.119162+0000", 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 335.075714656, 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.807735605, 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.421 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78136 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119162+0000", 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119162+0000", 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119160+0000", 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119167+0000", 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1029999999999998e-06 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119178+0000", 2023-10-09T05:01:28.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1148999999999999e-05 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119184+0000", 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0719999999999996e-06 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119233+0000", 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9398999999999999e-05 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.119241+0000", 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.807e-06 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.132867+0000", 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013626289999999999 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:01:28.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.132876+0000", 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4959999999999996e-06 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.293474+0000", 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1605982830000001 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.305201+0000", 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011727044000000001 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.388919+0000", 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.083718230000000005 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.409031+0000", 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020111832 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.722309+0000", 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31327797200000002 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.732693+0000", 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010383580999999999 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 2,5", 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.755958+0000", 2023-10-09T05:01:28.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023265058000000002 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.892521+0000", 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.13656343500000001 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.894039+0000", 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001517914 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.896846+0000", 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028064349999999999 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.901496+0000", 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046509139999999999 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.926897+0000", 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025400763999999999 2023-10-09T05:01:28.430 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(14.2 507/499 [PushOp(14:46d6768f:::benchmark_data_smithi029_550670_object12344:head, version: 444'757, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:46d6768f:::benchmark_data_smithi029_550670_object12344:head@444'757, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:57:37.986655+0000", 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 230.208221221, 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5003260059999999, 2023-10-09T05:01:28.431 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986655+0000", 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986655+0000", 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986654+0000", 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986682+0000", 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8099999999999999e-05 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986698+0000", 2023-10-09T05:01:28.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5310000000000001e-05 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:37.986706+0000", 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7530000000000007e-06 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:40.486950+0000", 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5002433499999999 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:57:40.486981+0000", 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1350999999999998e-05 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:81761 14.d 14:b4dc8e48:::benchmark_data_smithi029_550670_object4150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:21.124734+0000", 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 127.070142555, 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4717528630000001, 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.435 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 81761 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.124734+0000", 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.124734+0000", 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.124732+0000", 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.124740+0000", 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4429999999999997e-06 2023-10-09T05:01:28.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.124754+0000", 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4691e-05 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.124761+0000", 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.562e-06 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.222093+0000", 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.097331988999999994 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.229597+0000", 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0075041700000000001 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.330035+0000", 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.100437756 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.336259+0000", 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062241650000000003 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.516645+0000", 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1803859489999999 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.522122+0000", 2023-10-09T05:01:28.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054767710000000001 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.532927+0000", 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010805176999999999 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.554799+0000", 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021871696999999999 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.580189+0000", 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025390578 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.582010+0000", 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001821018 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.584952+0000", 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029421669999999999 2023-10-09T05:01:28.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.596487+0000", 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011534134 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:81763 14.d 14:b3c35cde:::benchmark_data_smithi029_550670_object4152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:21.129790+0000", 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 127.06508611300001, 2023-10-09T05:01:28.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.369824473, 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 81763 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.129790+0000", 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.129790+0000", 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.129789+0000", 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.129797+0000", 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9049999999999997e-06 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.129812+0000", 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5603999999999998e-05 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.129819+0000", 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4500000000000001e-06 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.230475+0000", 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.10065647899999999 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.236973+0000", 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064972340000000002 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.298933+0000", 2023-10-09T05:01:28.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.061960042999999999 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.305126+0000", 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061929560000000003 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.401971+0000", 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.096845724 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.408640+0000", 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066682349999999998 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.420935+0000", 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012295653 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.472797+0000", 2023-10-09T05:01:28.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.051861529000000003 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.475655+0000", 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028585799999999999 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.478231+0000", 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025755270000000002 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.481693+0000", 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003462555 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.499615+0000", 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017921295 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:81790 14.d 14:b11ec0b0:::benchmark_data_smithi029_550670_object4179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:21.544118+0000", 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 126.650757958, 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4468223650000001, 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 81790 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.544118+0000", 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.544118+0000", 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.544117+0000", 2023-10-09T05:01:28.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.544124+0000", 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5860000000000001e-06 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.544138+0000", 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4715000000000001e-05 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.544144+0000", 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1320000000000002e-06 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.579241+0000", 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.035096967999999999 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.585496+0000", 2023-10-09T05:01:28.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062556440000000003 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.677436+0000", 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.091939832999999999 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.683621+0000", 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061848279999999999 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.822155+0000", 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1385339489999999 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.832013+0000", 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0098580920000000006 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.848052+0000", 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016038976999999999 2023-10-09T05:01:28.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.915119+0000", 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.067067134 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.915769+0000", 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00065030700000000003 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.969277+0000", 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.053507335000000003 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.976281+0000", 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0070042409999999996 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.990941+0000", 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014659838999999999 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.459 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:81835 14.d 14:b657a51d:::benchmark_data_smithi029_550670_object4224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:21.944702+0000", 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 126.25017477900001, 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.459181133, 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.460 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 81835 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.944702+0000", 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.944702+0000", 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.944700+0000", 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.944708+0000", 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0010000000000001e-06 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.944723+0000", 2023-10-09T05:01:28.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5882000000000001e-05 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.944731+0000", 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3760000000000002e-06 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.971102+0000", 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026371057 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:21.976963+0000", 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058614380000000001 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.017930+0000", 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.040966632000000003 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:22.024279+0000", 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063486660000000002 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.287489+0000", 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2632099669999999 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.293089+0000", 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056001879999999999 2023-10-09T05:01:28.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.305889+0000", 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012800545 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.378187+0000", 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.072297273999999995 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.380638+0000", 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024518420000000001 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.381469+0000", 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083088400000000005 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.385679+0000", 2023-10-09T05:01:28.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042097879999999999 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:23.403883+0000", 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018203519000000001 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82218 14.d 14:b73b344d:::benchmark_data_smithi029_550670_object4607:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:24.924369+0000", 2023-10-09T05:01:28.468 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 123.270507162, 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5028485869999999, 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82218 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:24.924369+0000", 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:24.924369+0000", 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:24.924367+0000", 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:28.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:24.924375+0000", 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4680000000000003e-06 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:24.924390+0000", 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5252000000000001e-05 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:24.924397+0000", 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5379999999999999e-06 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.052592+0000", 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.12819562900000001 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.062142+0000", 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0095502060000000003 2023-10-09T05:01:28.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.158163+0000", 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.096020559000000005 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.165012+0000", 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068493620000000003 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.328550+0000", 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1635380099999999 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.333468+0000", 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049177880000000002 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.344047+0000", 2023-10-09T05:01:28.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010579039 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.380837+0000", 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.036789596000000001 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.402128+0000", 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021291338999999999 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.412245+0000", 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010116916 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.415103+0000", 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028579410000000001 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.427218+0000", 2023-10-09T05:01:28.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012114812000000001 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82236 14.d 14:bce9f147:::benchmark_data_smithi029_550670_object4625:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:25.081659+0000", 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 123.11321755500001, 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.466199118, 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82236 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.081659+0000", 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.081659+0000", 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.081657+0000", 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.081665+0000", 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8320000000000006e-06 2023-10-09T05:01:28.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.081681+0000", 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6087999999999999e-05 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.081690+0000", 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3959999999999995e-06 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.167498+0000", 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.085807474999999994 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.173865+0000", 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063673769999999996 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.298248+0000", 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.12438292500000001 2023-10-09T05:01:28.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.305045+0000", 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067968079999999997 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.480116+0000", 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1750714179999999 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.485341+0000", 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052243810000000002 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.495553+0000", 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010212512999999999 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.528959+0000", 2023-10-09T05:01:28.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.033405915000000001 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.530954+0000", 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019946460000000001 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.532646+0000", 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001692432 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.535395+0000", 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00274873 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.547858+0000", 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01246302 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82241 14.d 14:b5ea6670:::benchmark_data_smithi029_550670_object4630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:25.113525+0000", 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 123.081351751, 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.414847975, 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82241 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.486 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.113525+0000", 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.113525+0000", 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.113523+0000", 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.113530+0000", 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2749999999999998e-06 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.113545+0000", 2023-10-09T05:01:28.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4656e-05 2023-10-09T05:01:28.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.113553+0000", 2023-10-09T05:01:28.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.537e-06 2023-10-09T05:01:28.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.174894+0000", 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.061341537000000002 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.181411+0000", 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006516488 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.248351+0000", 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.066940551000000001 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.258865+0000", 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010513521 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.441399+0000", 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1825342860000001 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.446764+0000", 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053645899999999998 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.457791+0000", 2023-10-09T05:01:28.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011027056 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.503692+0000", 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.045901338999999999 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.507278+0000", 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035855309999999999 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.507877+0000", 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00059956300000000001 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.510841+0000", 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002964155 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.528373+0000", 2023-10-09T05:01:28.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017531221999999999 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82266 14.d 14:b88664c7:::benchmark_data_smithi029_550670_object4655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:25.320368+0000", 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 122.87450856300001, 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3969675289999999, 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82266 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.320368+0000", 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.320368+0000", 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.320366+0000", 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.320373+0000", 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9110000000000004e-06 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.320387+0000", 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4087000000000001e-05 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.320395+0000", 2023-10-09T05:01:28.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3799999999999996e-06 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.330505+0000", 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010110401999999999 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336434+0000", 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059291860000000004 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.400824+0000", 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.064389185000000002 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.407091+0000", 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062677790000000002 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.561748+0000", 2023-10-09T05:01:28.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1546568530000001 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.566895+0000", 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005147118 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.576762+0000", 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.009866596 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.620622+0000", 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.043859796999999999 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.622386+0000", 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001764322 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.698967+0000", 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.076581040000000003 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.702647+0000", 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036802050000000002 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.717335+0000", 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014688013 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.503 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82268 14.d 14:b2e2f4a2:::benchmark_data_smithi029_550670_object4657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:25.336256+0000", 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 122.85862070899999, 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3975721169999999, 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82268 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336256+0000", 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336256+0000", 2023-10-09T05:01:28.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336255+0000", 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336261+0000", 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7880000000000003e-06 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336270+0000", 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1160000000000005e-06 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.336274+0000", 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3429999999999998e-06 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.428225+0000", 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.091950968999999994 2023-10-09T05:01:28.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.434418+0000", 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061930630000000004 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.503494+0000", 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.069075873999999995 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:25.509597+0000", 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061029659999999996 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.597618+0000", 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.088021224 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.602658+0000", 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0050402169999999996 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.612638+0000", 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0099798960000000003 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.624063+0000", 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011424273 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.625790+0000", 2023-10-09T05:01:28.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017275649999999999 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.717934+0000", 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.092144023000000005 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.720877+0000", 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002942582 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.733828+0000", 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012951008 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82352 14.d 14:b46dd18a:::benchmark_data_smithi029_550670_object4741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:26.087465+0000", 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 122.107411493, 2023-10-09T05:01:28.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.440878914, 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82352 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.087465+0000", 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.087465+0000", 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.087464+0000", 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.087470+0000", 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4540000000000004e-06 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.087483+0000", 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3385e-05 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.087491+0000", 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3900000000000004e-06 2023-10-09T05:01:28.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.196811+0000", 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.10932027499999999 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.203830+0000", 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0070191159999999997 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.262337+0000", 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.058507067000000003 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.269116+0000", 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0067787689999999996 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.429499+0000", 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.16038299 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.434804+0000", 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053045829999999999 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.446807+0000", 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012003181999999999 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.509329+0000", 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.062521880000000002 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.510101+0000", 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00077194099999999995 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.511919+0000", 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001818835 2023-10-09T05:01:28.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.514909+0000", 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029892819999999998 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.528344+0000", 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013435018999999999 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82361 14.d 14:bc86382a:::benchmark_data_smithi029_550670_object4750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:26.178077+0000", 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 122.016798917, 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3723857159999999, 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82361 2023-10-09T05:01:28.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.178077+0000", 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.178077+0000", 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.178076+0000", 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.178084+0000", 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3699999999999995e-06 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.178096+0000", 2023-10-09T05:01:28.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2021000000000001e-05 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.178101+0000", 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9350000000000002e-06 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.204681+0000", 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026580125999999999 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.210717+0000", 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060359699999999999 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.292602+0000", 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081884707000000001 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.298847+0000", 2023-10-09T05:01:28.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062449949999999997 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.482167+0000", 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1833195359999999 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.489311+0000", 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0071442090000000003 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.499734+0000", 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.01042357 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.528989+0000", 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029255040999999999 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.531869+0000", 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00287922 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.533755+0000", 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001886539 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.536720+0000", 2023-10-09T05:01:28.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029645840000000001 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.550463+0000", 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013743396 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.529 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82368 14.d 14:bd3b2a89:::benchmark_data_smithi029_550670_object4757:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:26.201185+0000", 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 121.99369121700001, 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.480881366, 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82368 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.201185+0000", 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.201185+0000", 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.201183+0000", 2023-10-09T05:01:28.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.201192+0000", 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5979999999999997e-06 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.201208+0000", 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6354999999999998e-05 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.201217+0000", 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1200000000000002e-06 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.300609+0000", 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.099392751000000001 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.306546+0000", 2023-10-09T05:01:28.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059366590000000004 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.372925+0000", 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.066378693000000002 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.379133+0000", 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062079300000000004 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.566313+0000", 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1871799679999999 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.573198+0000", 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068855100000000001 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.583694+0000", 2023-10-09T05:01:28.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010496145 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.649725+0000", 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.066030579000000006 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.653590+0000", 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038655870000000002 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.656620+0000", 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003029243 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.661136+0000", 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045163010000000003 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.682066+0000", 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020930548 2023-10-09T05:01:28.537 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82369 14.d 14:b040f91e:::benchmark_data_smithi029_550670_object4758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:26.274207+0000", 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 121.920669737, 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4328381880000001, 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.538 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82369 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.274207+0000", 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.274207+0000", 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.274205+0000", 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.274212+0000", 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.199e-06 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.274227+0000", 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4403000000000001e-05 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.274234+0000", 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0299999999999996e-06 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.307958+0000", 2023-10-09T05:01:28.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.033724046000000001 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.313601+0000", 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056429549999999998 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.403999+0000", 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.090398554000000006 2023-10-09T05:01:28.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.410512+0000", 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065124179999999999 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.613667+0000", 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.203154941 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.622011+0000", 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083442470000000008 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.640489+0000", 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018477534 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.684889+0000", 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.044400518999999999 2023-10-09T05:01:28.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.685667+0000", 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00077786999999999995 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.688470+0000", 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002803013 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.691606+0000", 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003136078 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.707045+0000", 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015438729999999999 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.546 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82389 14.d 14:b307c710:::benchmark_data_smithi029_550670_object4778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:26.741338+0000", 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 121.45353866799999, 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4011032619999999, 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82389 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.741338+0000", 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.741338+0000", 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.741336+0000", 2023-10-09T05:01:28.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.741344+0000", 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5029999999999997e-06 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.741365+0000", 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0922999999999999e-05 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.741372+0000", 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2430000000000002e-06 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.773618+0000", 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.032246016000000002 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.779347+0000", 2023-10-09T05:01:28.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057298460000000002 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.850161+0000", 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.070813991000000007 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.857188+0000", 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0070260720000000004 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.058388+0000", 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.201199992 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.063820+0000", 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054326799999999996 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.074563+0000", 2023-10-09T05:01:28.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010742913999999999 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.124882+0000", 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.050318382000000002 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.126906+0000", 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020249439999999999 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.127482+0000", 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057543899999999996 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.130401+0000", 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029186939999999999 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.142441+0000", 2023-10-09T05:01:28.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012040299000000001 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82399 14.d 14:b65bec14:::benchmark_data_smithi029_550670_object4788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:26.811135+0000", 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 121.38374145, 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.356187727, 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.555 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82399 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.811135+0000", 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.811135+0000", 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.811134+0000", 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.811140+0000", 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0159999999999997e-06 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.557 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.811153+0000", 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3189e-05 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.811159+0000", 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1020000000000002e-06 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.883505+0000", 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.072346133000000007 2023-10-09T05:01:28.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.890172+0000", 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066668320000000001 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:26.992245+0000", 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.102073486 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.006930+0000", 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014684792 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.100254+0000", 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0933236639999999 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.105858+0000", 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056043619999999999 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.116019+0000", 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010161099999999999 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.144227+0000", 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028207666999999999 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.144810+0000", 2023-10-09T05:01:28.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00058269300000000001 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.146603+0000", 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001792856 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.149363+0000", 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027605910000000002 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.167323+0000", 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017959385000000001 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82550 14.d 14:b80e4fde:::benchmark_data_smithi029_550670_object4939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:27.959921+0000", 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 120.23495501399999, 2023-10-09T05:01:28.563 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.372074, 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82550 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.959921+0000", 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.959921+0000", 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.959920+0000", 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.959928+0000", 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8650000000000001e-06 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.959943+0000", 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5296999999999999e-05 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:27.959952+0000", 2023-10-09T05:01:28.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0380000000000005e-06 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.003395+0000", 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.043443622000000001 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.008996+0000", 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056008189999999999 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.049507+0000", 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.040510288999999998 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.055973+0000", 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064666339999999998 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.233490+0000", 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.177516537 2023-10-09T05:01:28.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.238829+0000", 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053394030000000004 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.249406+0000", 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010577075999999999 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.304126+0000", 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.054719886000000002 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.305195+0000", 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010685460000000001 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.310673+0000", 2023-10-09T05:01:28.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054786410000000002 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.317253+0000", 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0065802020000000003 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.331995+0000", 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014741781000000001 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.571 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82600 14.d 14:bd83422a:::benchmark_data_smithi029_550670_object4989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:28.494567+0000", 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 119.70030936000001, 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5651281290000001, 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.572 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82600 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.494567+0000", 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.494567+0000", 2023-10-09T05:01:28.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.494565+0000", 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.494574+0000", 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9779999999999994e-06 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.494588+0000", 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3608000000000001e-05 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.494597+0000", 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6119999999999999e-06 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.673399+0000", 2023-10-09T05:01:28.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.17880222000000001 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.680226+0000", 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0068275369999999998 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.816072+0000", 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.13584589499999999 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.823562+0000", 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0074899880000000004 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.957267+0000", 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1337045109999999 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.965571+0000", 2023-10-09T05:01:28.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008304166 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.977893+0000", 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012321752 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.014796+0000", 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.036903313 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.578 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.042952+0000", 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028156173999999999 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.044686+0000", 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001734195 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.047552+0000", 2023-10-09T05:01:28.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028654370000000002 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.059695+0000", 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012143259999999999 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82614 14.d 14:beadb533:::benchmark_data_smithi029_550670_object5003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:28.589172+0000", 2023-10-09T05:01:28.580 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 119.60570407, 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4503363929999999, 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82614 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.589172+0000", 2023-10-09T05:01:28.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.589172+0000", 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.589171+0000", 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.589180+0000", 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6840000000000008e-06 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.589196+0000", 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5610000000000001e-05 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:01:28.583 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.589204+0000", 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7919999999999993e-06 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.680887+0000", 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.091683583999999999 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.687544+0000", 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066571089999999996 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.752532+0000", 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.064987404999999998 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:28.759507+0000", 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069748020000000004 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.883610+0000", 2023-10-09T05:01:28.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1241037119999999 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.894528+0000", 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010917464999999999 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 4,5", 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.906903+0000", 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012375667 2023-10-09T05:01:28.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.988684+0000", 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081780797000000002 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.022300+0000", 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.033615873999999997 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:01:28.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.024155+0000", 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018549560000000001 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.027117+0000", 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029620760000000001 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.039509+0000", 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012391522 2023-10-09T05:01:28.588 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.589 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.589 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.589 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:01:28.589 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:01:28.589 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:28 smithi177 ceph-mon[195797]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.621 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:28.665 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:01:28.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[223858]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: osdmap e572: 8 total, 8 up, 7 in 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:28 smithi029 ceph-mon[216903]: failed to encode map e572 with expected crc 2023-10-09T05:01:28.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:28.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:28.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:28.817 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:29.009 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:29.026 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:01:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:29.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:29.129 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:29.165 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:29.187 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:29.271 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:29.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:29.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:29.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:29.389 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:29.413 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:29.580 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:29.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:29 smithi029 ceph-mon[216903]: pgmap v1917: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 79623/195540 objects degraded (40.720%); 11569/195540 objects misplaced (5.916%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:01:29.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:29.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:29 smithi029 ceph-mon[223858]: pgmap v1917: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 79623/195540 objects degraded (40.720%); 11569/195540 objects misplaced (5.916%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:01:29.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:29.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:29.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:29.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:29 smithi177 ceph-mon[195797]: pgmap v1917: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 79623/195540 objects degraded (40.720%); 11569/195540 objects misplaced (5.916%); 1.9 MiB/s, 30 objects/s recovering 2023-10-09T05:01:29.598 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:29.687 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:29.718 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:29.928 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:29.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:30.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:30.071 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:30.145 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:30.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:30.251 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:30.313 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:30.352 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 14 pgp_num to 26 2023-10-09T05:01:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/347735905' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:30.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:30.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/347735905' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:30.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/347735905' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:30.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-09T05:01:30.784 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:30.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:30.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:30.923 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:30.968 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:30.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:30.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:01:31.007 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:01:31.007 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:01:31.007 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:01:31.007 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:01:31.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:31.099 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:31.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:31.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:31.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:31.417 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: pgmap v1918: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 79252/194295 objects degraded (40.790%); 11450/194295 objects misplaced (5.893%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 79252/194295 objects degraded (40.790%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: pgmap v1918: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 79252/194295 objects degraded (40.790%); 11450/194295 objects misplaced (5.893%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 79252/194295 objects degraded (40.790%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:31.737 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:31.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: pgmap v1918: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 97 op/s; 79252/194295 objects degraded (40.790%); 11450/194295 objects misplaced (5.893%); 2.9 MiB/s, 46 objects/s recovering 2023-10-09T05:01:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-10-09T05:01:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: osdmap e573: 8 total, 8 up, 7 in 2023-10-09T05:01:31.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: failed to encode map e573 with expected crc 2023-10-09T05:01:31.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 79252/194295 objects degraded (40.790%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:31.801 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:01:31.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:31.877 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:31.886 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:31.957 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:31.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:32.068 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:32.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:32.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:32.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:32.173 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:32.253 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:32.282 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:32.342 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:01:32.369 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:01:32.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:32.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:32.391 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:01:32.412 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:01:32.444 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:01:32.472 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:01:32.489 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:32.494 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:01:32.523 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:01:32.528 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.528 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.529 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.529 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.529 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.529 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.529 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.529 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:01:32.532 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1495449365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:32.533 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1495449365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:32.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:32.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1495449365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:33.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:01:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:01:32] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2023-10-09T05:01:33.529 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:33.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:33.621 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:33 smithi177 ceph-mon[195797]: pgmap v1920: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 79252/194058 objects degraded (40.839%); 11450/194058 objects misplaced (5.900%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:01:33.639 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:33.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:33 smithi029 ceph-mon[223858]: pgmap v1920: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 79252/194058 objects degraded (40.839%); 11450/194058 objects misplaced (5.900%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:01:33.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:33 smithi029 ceph-mon[216903]: pgmap v1920: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 82 op/s; 79252/194058 objects degraded (40.839%); 11450/194058 objects misplaced (5.900%); 2.2 MiB/s, 35 objects/s recovering 2023-10-09T05:01:33.698 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:33.889 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:33.889 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:34.398 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:34.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:34.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:34.566 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:34.668 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:01:34.778 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:34.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:34.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:34.915 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:34.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:34.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:34.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:35.066 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:01:35.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:35.116 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:35.202 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:35.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:35.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:35.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:35.348 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:35.404 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:01:35.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:35 smithi177 ceph-mon[195797]: pgmap v1921: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 78938/193161 objects degraded (40.866%); 11384/193161 objects misplaced (5.894%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:01:35.463 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1260022553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:35.502 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:35.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:35.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:35.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:35.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:35 smithi029 ceph-mon[216903]: pgmap v1921: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 78938/193161 objects degraded (40.866%); 11384/193161 objects misplaced (5.894%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:01:35.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1260022553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:35.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:35 smithi029 ceph-mon[223858]: pgmap v1921: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 99 op/s; 78938/193161 objects degraded (40.866%); 11384/193161 objects misplaced (5.894%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:01:35.661 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1260022553' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:01:35.667 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:35.791 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:35.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:35.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:35.927 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:36.239 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:36.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:36.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:36.346 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:36.385 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:36.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:01:36.478 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:36 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:01:36.478 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1260022553' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:01:36.478 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:36 smithi177 ceph-mon[195797]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T05:01:36.478 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:36 smithi177 ceph-mon[195797]: failed to encode map e574 with expected crc 2023-10-09T05:01:36.478 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 78938/193161 objects degraded (40.866%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:36.609 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:36.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:01:36.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1260022553' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:01:36.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[216903]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T05:01:36.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[216903]: failed to encode map e574 with expected crc 2023-10-09T05:01:36.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 78938/193161 objects degraded (40.866%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:36.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:01:36.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1260022553' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:01:36.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[223858]: osdmap e574: 8 total, 8 up, 7 in 2023-10-09T05:01:36.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[223858]: failed to encode map e574 with expected crc 2023-10-09T05:01:36.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 78938/193161 objects degraded (40.866%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:36.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:36.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:36.748 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:36.753 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 1, 6, 4] out_osds: [0] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:01:36.753 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:01:36.753 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [5, 7, 2, 3, 1, 6, 4] 2023-10-09T05:01:36.754 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 1 2023-10-09T05:01:36.779 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:01:36.780 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:01:36.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:36.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:36.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:36.964 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:36.970 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:37.075 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:37.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:37.183 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:37.215 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:37.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:37.511 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:37 smithi177 ceph-mon[195797]: pgmap v1923: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 78867/192465 objects degraded (40.977%); 11323/192465 objects misplaced (5.883%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:01:37.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:37.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:37.595 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:01:37.613 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:37.623 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:37.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:37 smithi029 ceph-mon[216903]: pgmap v1923: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 78867/192465 objects degraded (40.977%); 11323/192465 objects misplaced (5.883%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:01:37.637 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:37 smithi029 ceph-mon[223858]: pgmap v1923: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 78867/192465 objects degraded (40.977%); 11323/192465 objects misplaced (5.883%); 3.9 MiB/s, 62 objects/s recovering 2023-10-09T05:01:37.697 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:37.888 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:37.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:37.993 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:38.019 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:38.061 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:38.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:38.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:38.222 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:38.541 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:38.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:38.650 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:38.680 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:38.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:01:38.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:38.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:38.841 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:38.869 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:38.912 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:38.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:39.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:39.044 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:39.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:39.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:39.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:39.230 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:39.276 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:39.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:39.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:39.413 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:39.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:39 smithi029 ceph-mon[216903]: pgmap v1924: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 76 op/s; 78867/192465 objects degraded (40.977%); 11323/192465 objects misplaced (5.883%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T05:01:39.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:39 smithi029 ceph-mon[223858]: pgmap v1924: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 76 op/s; 78867/192465 objects degraded (40.977%); 11323/192465 objects misplaced (5.883%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T05:01:39.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:39 smithi177 ceph-mon[195797]: pgmap v1924: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 76 op/s; 78867/192465 objects degraded (40.977%); 11323/192465 objects misplaced (5.883%); 2.0 MiB/s, 31 objects/s recovering 2023-10-09T05:01:40.038 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:40.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:40.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:40.199 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:40.222 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:40.339 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:40.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:40.413 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:01:40.448 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:01:40.452 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:40.478 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:40.481 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:01:40.507 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:01:40.525 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:40.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:40.538 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:01:40.551 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:40.573 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:01:40.602 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:01:40.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:40.635 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:01:40.637 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.637 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.638 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.638 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.638 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.638 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.638 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.638 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:01:40.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/646151383' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:40.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/646151383' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:40.679 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:40.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/646151383' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:41.424 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:41.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:41.529 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:41.563 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:41.619 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:41.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:41.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:41.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:41 smithi029 ceph-mon[223858]: pgmap v1925: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 78450/191223 objects degraded (41.025%); 11188/191223 objects misplaced (5.851%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:01:41.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 78450/191223 objects degraded (41.025%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:41 smithi029 ceph-mon[216903]: pgmap v1925: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 78450/191223 objects degraded (41.025%); 11188/191223 objects misplaced (5.851%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:01:41.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 78450/191223 objects degraded (41.025%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:41.724 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:41.749 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:41 smithi177 ceph-mon[195797]: pgmap v1925: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 101 op/s; 78450/191223 objects degraded (41.025%); 11188/191223 objects misplaced (5.851%); 3.3 MiB/s, 53 objects/s recovering 2023-10-09T05:01:41.749 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 78450/191223 objects degraded (41.025%), 26 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:41.751 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:41.795 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:41.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:41.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:41.963 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:42.221 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:42 smithi029 conmon[216878]: 2023-10-09T05:01:42.144+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e574 definitely_dead 0 2023-10-09T05:01:42.418 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.1. 2023-10-09T05:01:42.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/933762525' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:01:42.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:42 smithi029 ceph-mon[216903]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:01:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/933762525' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:01:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:42 smithi029 ceph-mon[223858]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:01:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/933762525' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-10-09T05:01:42.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:42 smithi177 ceph-mon[195797]: Client client.admin marked osd.1 out, while it was still marked up 2023-10-09T05:01:43.078 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:01:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:01:42] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2023-10-09T05:01:43.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[216903]: pgmap v1926: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 78450/190986 objects degraded (41.076%); 11188/190986 objects misplaced (5.858%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:01:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[216903]: failed to encode map e575 with expected crc 2023-10-09T05:01:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/933762525' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:01:43.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[216903]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T05:01:43.477 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:43.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:43.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:43.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:43.646 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:43 smithi177 ceph-mon[195797]: pgmap v1926: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 78450/190986 objects degraded (41.076%); 11188/190986 objects misplaced (5.858%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:01:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:43 smithi177 ceph-mon[195797]: failed to encode map e575 with expected crc 2023-10-09T05:01:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/933762525' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:01:43.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:43 smithi177 ceph-mon[195797]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T05:01:43.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[223858]: pgmap v1926: 109 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded, 21 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 78450/190986 objects degraded (41.076%); 11188/190986 objects misplaced (5.858%); 3.1 MiB/s, 49 objects/s recovering 2023-10-09T05:01:43.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[223858]: failed to encode map e575 with expected crc 2023-10-09T05:01:43.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/933762525' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-10-09T05:01:43.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:43 smithi029 ceph-mon[223858]: osdmap e575: 8 total, 8 up, 6 in 2023-10-09T05:01:44.263 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:44.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:44.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:01:44.399 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[216903]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:44.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:44 smithi029 ceph-mon[223858]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T05:01:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: osdmap e576: 8 total, 8 up, 6 in 2023-10-09T05:01:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: failed to encode map e576 with expected crc 2023-10-09T05:01:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:44.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:44 smithi177 ceph-mon[195797]: osdmap e577: 8 total, 8 up, 6 in 2023-10-09T05:01:44.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:01:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: pgmap v1929: 109 pgs: 11 remapped+peering, 4 unknown, 34 peering, 2 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 47 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 27802/190182 objects degraded (14.619%); 5839/190182 objects misplaced (3.070%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:01:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: 14.a starting backfill to osd.4 from (0'0,0'0] MAX to 574'5625 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: 14.f starting backfill to osd.6 from (0'0,0'0] MAX to 574'5635 2023-10-09T05:01:45.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:45 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 12 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: pgmap v1929: 109 pgs: 11 remapped+peering, 4 unknown, 34 peering, 2 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 47 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 27802/190182 objects degraded (14.619%); 5839/190182 objects misplaced (3.070%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: 14.a starting backfill to osd.4 from (0'0,0'0] MAX to 574'5625 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: 14.f starting backfill to osd.6 from (0'0,0'0] MAX to 574'5635 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 12 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: pgmap v1929: 109 pgs: 11 remapped+peering, 4 unknown, 34 peering, 2 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 47 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 27802/190182 objects degraded (14.619%); 5839/190182 objects misplaced (3.070%); 3.0 MiB/s, 48 objects/s recovering 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: failed to encode map e577 with expected crc 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: 14.a starting backfill to osd.4 from (0'0,0'0] MAX to 574'5625 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: 14.f starting backfill to osd.6 from (0'0,0'0] MAX to 574'5635 2023-10-09T05:01:45.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:45 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 12 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T05:01:46.508 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:01:46.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:46.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:01:46.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:46.655 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:46.719 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:46.723 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:46 smithi177 ceph-mon[195797]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T05:01:46.723 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:46 smithi177 ceph-mon[195797]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:46 smithi177 ceph-mon[195797]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:46 smithi177 ceph-mon[195797]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:46 smithi177 ceph-mon[195797]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.724 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 27802/190182 objects degraded (14.619%), 13 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:46.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:46.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:46.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[223858]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T05:01:46.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[223858]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[223858]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[223858]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[223858]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.748 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 27802/190182 objects degraded (14.619%), 13 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:46.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[216903]: osdmap e578: 8 total, 8 up, 6 in 2023-10-09T05:01:46.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[216903]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[216903]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[216903]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[216903]: failed to encode map e578 with expected crc 2023-10-09T05:01:46.750 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 27802/190182 objects degraded (14.619%), 13 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:46.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:46.855 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:46.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:46.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:47.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:01:47.054 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:47.093 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:01:47.093 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:01:47.093 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T05:01:47.093 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:01:47.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:01:47.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:47.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:01:47.217 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:47.247 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:47.298 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:47.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:47.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:47.423 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:47.501 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:47.651 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:47.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:47.756 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:47 smithi177 ceph-mon[195797]: pgmap v1932: 109 pgs: 27 remapped+peering, 4 unknown, 35 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 34 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 21117/189894 objects degraded (11.120%); 5491/189894 objects misplaced (2.892%); 2.8 MiB/s, 63 objects/s recovering 2023-10-09T05:01:47.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:47 smithi177 ceph-mon[195797]: failed to encode map e579 with expected crc 2023-10-09T05:01:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:47 smithi177 ceph-mon[195797]: osdmap e579: 8 total, 8 up, 6 in 2023-10-09T05:01:47.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:47 smithi177 ceph-mon[195797]: failed to encode map e579 with expected crc 2023-10-09T05:01:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[216903]: pgmap v1932: 109 pgs: 27 remapped+peering, 4 unknown, 35 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 34 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 21117/189894 objects degraded (11.120%); 5491/189894 objects misplaced (2.892%); 2.8 MiB/s, 63 objects/s recovering 2023-10-09T05:01:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[216903]: failed to encode map e579 with expected crc 2023-10-09T05:01:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[216903]: osdmap e579: 8 total, 8 up, 6 in 2023-10-09T05:01:47.804 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[216903]: failed to encode map e579 with expected crc 2023-10-09T05:01:47.804 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[223858]: pgmap v1932: 109 pgs: 27 remapped+peering, 4 unknown, 35 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 34 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 21117/189894 objects degraded (11.120%); 5491/189894 objects misplaced (2.892%); 2.8 MiB/s, 63 objects/s recovering 2023-10-09T05:01:47.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[223858]: failed to encode map e579 with expected crc 2023-10-09T05:01:47.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[223858]: osdmap e579: 8 total, 8 up, 6 in 2023-10-09T05:01:47.805 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:47 smithi029 ceph-mon[223858]: failed to encode map e579 with expected crc 2023-10-09T05:01:47.817 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:48.493 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:01:48.640 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:48.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:48.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:48 smithi177 ceph-mon[195797]: pgmap v1934: 109 pgs: 1 activating+undersized+degraded+remapped, 27 remapped+peering, 4 unknown, 35 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 34 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 25 op/s; 19693/189855 objects degraded (10.373%); 5478/189855 objects misplaced (2.885%); 772 KiB/s, 29 objects/s recovering 2023-10-09T05:01:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1623958457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:48.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:48.786 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:48.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:48.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:48.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:48 smithi029 ceph-mon[216903]: pgmap v1934: 109 pgs: 1 activating+undersized+degraded+remapped, 27 remapped+peering, 4 unknown, 35 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 34 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 25 op/s; 19693/189855 objects degraded (10.373%); 5478/189855 objects misplaced (2.885%); 772 KiB/s, 29 objects/s recovering 2023-10-09T05:01:48.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1623958457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:48.887 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:48.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:48 smithi029 ceph-mon[223858]: pgmap v1934: 109 pgs: 1 activating+undersized+degraded+remapped, 27 remapped+peering, 4 unknown, 35 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 34 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 25 op/s; 19693/189855 objects degraded (10.373%); 5478/189855 objects misplaced (2.885%); 772 KiB/s, 29 objects/s recovering 2023-10-09T05:01:48.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1623958457' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:48.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:01:48.965 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:49.008 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:49.062 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 6, 4] out_osds: [0, 1] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:01:49.062 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:01:49.063 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-09T05:01:49.063 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-09T05:01:49.451 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:01:49.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: osdmap e580: 8 total, 8 up, 6 in 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[216903]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:49.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:01:49.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: osdmap e580: 8 total, 8 up, 6 in 2023-10-09T05:01:49.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:49 smithi029 ceph-mon[223858]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.765 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:01:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:01:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: osdmap e580: 8 total, 8 up, 6 in 2023-10-09T05:01:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:49 smithi177 ceph-mon[195797]: failed to encode map e580 with expected crc 2023-10-09T05:01:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:49.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:49.885 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:49.916 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:49.974 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:49.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:50.084 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:50.114 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:50.167 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:50.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:50.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:50.300 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:50.357 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:50.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:50.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:50.465 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:50.487 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:50 smithi177 ceph-mon[195797]: pgmap v1936: 109 pgs: 3 activating+undersized+degraded+remapped, 22 remapped+peering, 4 unknown, 23 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 46 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 11 op/s; 33244/189855 objects degraded (17.510%); 9343/189855 objects misplaced (4.921%); 6.6 MiB/s, 147 objects/s recovering 2023-10-09T05:01:50.498 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:50.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:50.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:50.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:50.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:50.701 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:50.754 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:50.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:50.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:50.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:50.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:50 smithi029 ceph-mon[216903]: pgmap v1936: 109 pgs: 3 activating+undersized+degraded+remapped, 22 remapped+peering, 4 unknown, 23 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 46 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 11 op/s; 33244/189855 objects degraded (17.510%); 9343/189855 objects misplaced (4.921%); 6.6 MiB/s, 147 objects/s recovering 2023-10-09T05:01:50.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:50 smithi029 ceph-mon[223858]: pgmap v1936: 109 pgs: 3 activating+undersized+degraded+remapped, 22 remapped+peering, 4 unknown, 23 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 46 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 11 op/s; 33244/189855 objects degraded (17.510%); 9343/189855 objects misplaced (4.921%); 6.6 MiB/s, 147 objects/s recovering 2023-10-09T05:01:50.908 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:51.068 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:51.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:51.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:51.173 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:51.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:51.328 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:51.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:51.450 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:51.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 11 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: osdmap e581: 8 total, 8 up, 6 in 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.536 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 33244/189855 objects degraded (17.510%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:51.537 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 11 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:01:51.537 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: osdmap e581: 8 total, 8 up, 6 in 2023-10-09T05:01:51.537 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.537 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.538 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 33244/189855 objects degraded (17.510%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:51.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:51.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 11 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-10-09T05:01:51.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: osdmap e581: 8 total, 8 up, 6 in 2023-10-09T05:01:51.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: failed to encode map e581 with expected crc 2023-10-09T05:01:51.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 33244/189855 objects degraded (17.510%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:51.788 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:51.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:51.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:51.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:01:51.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:51.906 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:01:51.950 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:52.051 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:52.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:52.166 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:52.193 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:52.237 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:52.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:52.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:01:52.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:52.388 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:52.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: osdmap e582: 8 total, 8 up, 6 in 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: pgmap v1939: 109 pgs: 2 activating+degraded+remapped, 1 active+degraded+remapped, 5 active+clean+remapped, 2 activating+remapped, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 4 remapped+peering, 3 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 48368/189855 objects degraded (25.476%); 17254/189855 objects misplaced (9.088%); 6.7 MiB/s, 134 objects/s recovering 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1019213435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: osdmap e582: 8 total, 8 up, 6 in 2023-10-09T05:01:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: failed to encode map e582 with expected crc 2023-10-09T05:01:52.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: pgmap v1939: 109 pgs: 2 activating+degraded+remapped, 1 active+degraded+remapped, 5 active+clean+remapped, 2 activating+remapped, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 4 remapped+peering, 3 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 48368/189855 objects degraded (25.476%); 17254/189855 objects misplaced (9.088%); 6.7 MiB/s, 134 objects/s recovering 2023-10-09T05:01:52.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:52 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1019213435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: osdmap e582: 8 total, 8 up, 6 in 2023-10-09T05:01:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: failed to encode map e582 with expected crc 2023-10-09T05:01:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: failed to encode map e582 with expected crc 2023-10-09T05:01:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: failed to encode map e582 with expected crc 2023-10-09T05:01:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: failed to encode map e582 with expected crc 2023-10-09T05:01:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: pgmap v1939: 109 pgs: 2 activating+degraded+remapped, 1 active+degraded+remapped, 5 active+clean+remapped, 2 activating+remapped, 1 activating+undersized+remapped, 3 activating+undersized+degraded+remapped, 4 remapped+peering, 3 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 2 op/s; 48368/189855 objects degraded (25.476%); 17254/189855 objects misplaced (9.088%); 6.7 MiB/s, 134 objects/s recovering 2023-10-09T05:01:53.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:52 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1019213435' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:01:53.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:01:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:01:52] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2023-10-09T05:01:53.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:53 smithi029 ceph-mon[223858]: failed to encode map e583 with expected crc 2023-10-09T05:01:53.847 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:53 smithi029 ceph-mon[223858]: osdmap e583: 8 total, 8 up, 6 in 2023-10-09T05:01:53.847 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:53 smithi029 ceph-mon[223858]: failed to encode map e583 with expected crc 2023-10-09T05:01:53.847 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:53 smithi029 ceph-mon[216903]: failed to encode map e583 with expected crc 2023-10-09T05:01:53.847 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:53 smithi029 ceph-mon[216903]: osdmap e583: 8 total, 8 up, 6 in 2023-10-09T05:01:53.847 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:53 smithi029 ceph-mon[216903]: failed to encode map e583 with expected crc 2023-10-09T05:01:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:53 smithi177 ceph-mon[195797]: failed to encode map e583 with expected crc 2023-10-09T05:01:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:53 smithi177 ceph-mon[195797]: osdmap e583: 8 total, 8 up, 6 in 2023-10-09T05:01:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:53 smithi177 ceph-mon[195797]: failed to encode map e583 with expected crc 2023-10-09T05:01:54.068 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:54.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:54.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:01:54.200 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:54.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:01:54.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:54.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:01:54.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:54.395 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:54.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[216903]: osdmap e584: 8 total, 8 up, 6 in 2023-10-09T05:01:54.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[216903]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[216903]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[216903]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[216903]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[216903]: pgmap v1942: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+degraded+remapped, 1 active+degraded+remapped, 5 active+clean+remapped, 2 activating+remapped, 1 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 remapped+peering, 3 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 49678/189849 objects degraded (26.167%); 15135/189849 objects misplaced (7.972%); 1.1 MiB/s, 19 objects/s recovering 2023-10-09T05:01:54.664 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:54.673 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:54.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:54.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:54.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:01:54.807 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:54 smithi177 ceph-mon[195797]: osdmap e584: 8 total, 8 up, 6 in 2023-10-09T05:01:54.807 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:54 smithi177 ceph-mon[195797]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.808 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:54 smithi177 ceph-mon[195797]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.808 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:54 smithi177 ceph-mon[195797]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.808 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:54 smithi177 ceph-mon[195797]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.808 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:54 smithi177 ceph-mon[195797]: pgmap v1942: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+degraded+remapped, 1 active+degraded+remapped, 5 active+clean+remapped, 2 activating+remapped, 1 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 remapped+peering, 3 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 49678/189849 objects degraded (26.167%); 15135/189849 objects misplaced (7.972%); 1.1 MiB/s, 19 objects/s recovering 2023-10-09T05:01:54.810 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:01:54.860 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:01:54.864 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:01:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:54.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:01:54.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[223858]: osdmap e584: 8 total, 8 up, 6 in 2023-10-09T05:01:54.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[223858]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[223858]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.884 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[223858]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[223858]: failed to encode map e584 with expected crc 2023-10-09T05:01:54.885 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:54 smithi029 ceph-mon[223858]: pgmap v1942: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+degraded+remapped, 1 active+degraded+remapped, 5 active+clean+remapped, 2 activating+remapped, 1 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 remapped+peering, 3 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 49678/189849 objects degraded (26.167%); 15135/189849 objects misplaced (7.972%); 1.1 MiB/s, 19 objects/s recovering 2023-10-09T05:01:54.970 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:55.012 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:55.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:54 smithi029 conmon[216878]: 2023-10-09T05:01:54.893+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e584 definitely_dead 0 2023-10-09T05:01:56.430 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2092392611' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T05:01:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:56.431 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:56.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:56.432 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.0. 2023-10-09T05:01:56.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2092392611' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T05:01:56.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2092392611' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-09T05:01:56.532 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:01:56.571 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:56.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2092392611' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T05:01:56.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T05:01:56.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: pgmap v1944: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 6 active+clean+remapped, 1 activating+remapped, 4 remapped+peering, 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 49678/189843 objects degraded (26.168%); 15134/189843 objects misplaced (7.972%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 49678/189849 objects degraded (26.167%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T05:01:56.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e586 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e586 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[223858]: failed to encode map e586 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2092392611' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: pgmap v1944: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 6 active+clean+remapped, 1 activating+remapped, 4 remapped+peering, 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 49678/189843 objects degraded (26.168%); 15134/189843 objects misplaced (7.972%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 49678/189849 objects degraded (26.167%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e586 with expected crc 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e586 with expected crc 2023-10-09T05:01:56.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:56 smithi029 ceph-mon[216903]: failed to encode map e586 with expected crc 2023-10-09T05:01:56.879 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:01:57.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2092392611' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-09T05:01:57.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: osdmap e585: 8 total, 8 up, 7 in 2023-10-09T05:01:57.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: pgmap v1944: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 6 active+clean+remapped, 1 activating+remapped, 4 remapped+peering, 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 49678/189843 objects degraded (26.168%); 15134/189843 objects misplaced (7.972%); 1.1 MiB/s, 18 objects/s recovering 2023-10-09T05:01:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-10-09T05:01:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 49678/189849 objects degraded (26.167%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:01:57.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: osdmap e586: 8 total, 8 up, 7 in 2023-10-09T05:01:57.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:56 smithi177 ceph-mon[195797]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.228 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:01:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:57.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:01:57.349 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:01:57.521 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:57.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[223858]: 14.e starting backfill to osd.0 from (0'0,0'0] MAX to 574'5698 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e585 with expected crc 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e586 with expected crc 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:57.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T05:01:57.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:57.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:57.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:57 smithi029 ceph-mon[216903]: 14.e starting backfill to osd.0 from (0'0,0'0] MAX to 574'5698 2023-10-09T05:01:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e585 with expected crc 2023-10-09T05:01:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e586 with expected crc 2023-10-09T05:01:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e586 with expected crc 2023-10-09T05:01:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e586 with expected crc 2023-10-09T05:01:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: osdmap e587: 8 total, 8 up, 7 in 2023-10-09T05:01:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:57 smithi177 ceph-mon[195797]: 14.e starting backfill to osd.0 from (0'0,0'0] MAX to 574'5698 2023-10-09T05:01:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: pgmap v1947: 109 pgs: 6 peering, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 41547/189843 objects degraded (21.885%); 11154/189843 objects misplaced (5.875%); 21 B/s, 0 objects/s recovering 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:58.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e587 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: pgmap v1947: 109 pgs: 6 peering, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 41547/189843 objects degraded (21.885%); 11154/189843 objects misplaced (5.875%); 21 B/s, 0 objects/s recovering 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:58.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:58.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e587 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: pgmap v1947: 109 pgs: 6 peering, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 41547/189843 objects degraded (21.885%); 11154/189843 objects misplaced (5.875%); 21 B/s, 0 objects/s recovering 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: osdmap e588: 8 total, 8 up, 7 in 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:01:59.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:01:59.304 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-09T05:01:59.537 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:01:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:59.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:01:59.641 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:01:59.688 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:59.776 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:01:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:59.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:01:59.835 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:01:59.882 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:01:59.913 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.914 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:01:59 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.917 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:01:59.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e588 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: osdmap e589: 8 total, 8 up, 7 in 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:01:59 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:01:59.986 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:01:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:01:59.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:00.093 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:00.134 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:00.301 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:00.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:00.408 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:00.444 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:00.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:02:00.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:02:00.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[216903]: failed to encode map e589 with expected crc 2023-10-09T05:02:00.659 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[216903]: pgmap v1950: 109 pgs: 6 peering, 2 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 41243/189843 objects degraded (21.725%); 14814/189843 objects misplaced (7.803%); 320 KiB/s, 5 objects/s recovering 2023-10-09T05:02:00.660 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:02:00.660 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:02:00.660 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[223858]: failed to encode map e589 with expected crc 2023-10-09T05:02:00.660 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:00 smithi029 ceph-mon[223858]: pgmap v1950: 109 pgs: 6 peering, 2 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 41243/189843 objects degraded (21.725%); 14814/189843 objects misplaced (7.803%); 320 KiB/s, 5 objects/s recovering 2023-10-09T05:02:00.721 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:00.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:00.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:00.862 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:00.926 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:00.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:00 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:02:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:00 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:02:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:00 smithi177 ceph-mon[195797]: failed to encode map e589 with expected crc 2023-10-09T05:02:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:00 smithi177 ceph-mon[195797]: pgmap v1950: 109 pgs: 6 peering, 2 active+recovering+undersized+remapped, 1 active+clean+remapped, 8 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 75 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 41243/189843 objects degraded (21.725%); 14814/189843 objects misplaced (7.803%); 320 KiB/s, 5 objects/s recovering 2023-10-09T05:02:01.029 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:01.071 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:01.160 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:01.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:01.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:01.301 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:01.356 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:01.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:01.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:01.369 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:02:01.391 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:02:01.415 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:02:01.462 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:01.513 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:01.639 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:02:01.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:01.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:01 smithi177 ceph-mon[195797]: failed to encode map e590 with expected crc 2023-10-09T05:02:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:01 smithi177 ceph-mon[195797]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T05:02:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:01 smithi177 ceph-mon[195797]: failed to encode map e590 with expected crc 2023-10-09T05:02:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3785222516' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:01 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:02:01.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 41243/189843 objects degraded (21.725%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:01.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:01.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[223858]: failed to encode map e590 with expected crc 2023-10-09T05:02:01.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[223858]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T05:02:01.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[223858]: failed to encode map e590 with expected crc 2023-10-09T05:02:01.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3785222516' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:01.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:02:01.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 41243/189843 objects degraded (21.725%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:01.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[216903]: failed to encode map e590 with expected crc 2023-10-09T05:02:01.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[216903]: osdmap e590: 8 total, 8 up, 7 in 2023-10-09T05:02:01.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[216903]: failed to encode map e590 with expected crc 2023-10-09T05:02:01.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3785222516' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:01.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:02:01.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 41243/189843 objects degraded (21.725%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:01.905 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:01.975 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:01.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:01.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:02.080 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:02.117 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:02.205 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:02.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:02.313 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:02.576 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:02.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:02.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3785222516' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:02.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T05:02:02.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[216903]: pgmap v1953: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+remapped, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 63612/189843 objects degraded (33.508%); 14255/189843 objects misplaced (7.509%); 11 MiB/s, 185 objects/s recovering 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3785222516' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:02.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T05:02:02.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.619 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:02 smithi029 ceph-mon[223858]: pgmap v1953: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+remapped, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 63612/189843 objects degraded (33.508%); 14255/189843 objects misplaced (7.509%); 11 MiB/s, 185 objects/s recovering 2023-10-09T05:02:02.896 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:02.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:02:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:02:02] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2023-10-09T05:02:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:02.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:02.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:02.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3785222516' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:02.936 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: osdmap e591: 8 total, 8 up, 7 in 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: failed to encode map e591 with expected crc 2023-10-09T05:02:02.938 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:02 smithi177 ceph-mon[195797]: pgmap v1953: 109 pgs: 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+remapped, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 63612/189843 objects degraded (33.508%); 14255/189843 objects misplaced (7.509%); 11 MiB/s, 185 objects/s recovering 2023-10-09T05:02:03.039 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:03.352 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:03.567 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:03.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:03.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:03.719 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:03.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[216903]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:03.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:03 smithi029 ceph-mon[223858]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:02:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: osdmap e592: 8 total, 8 up, 7 in 2023-10-09T05:02:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:03 smithi177 ceph-mon[195797]: failed to encode map e592 with expected crc 2023-10-09T05:02:04.175 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:04.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:04.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:04.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:04.304 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 6, 4, 0] out_osds: [1] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:02:04.304 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:02:04.305 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-09T05:02:04.305 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-09T05:02:04.305 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-09T05:02:04.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-09T05:02:04.389 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:04.561 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:04.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:04.623 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:04 smithi029 ceph-mon[223858]: pgmap v1955: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 63609/189831 objects degraded (33.508%); 13315/189831 objects misplaced (7.014%); 15 MiB/s, 252 objects/s recovering 2023-10-09T05:02:04.631 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:04 smithi177 ceph-mon[195797]: pgmap v1955: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 63609/189831 objects degraded (33.508%); 13315/189831 objects misplaced (7.014%); 15 MiB/s, 252 objects/s recovering 2023-10-09T05:02:04.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:04.720 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:04.773 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:04.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:04.877 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:04.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:04 smithi029 ceph-mon[216903]: pgmap v1955: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 1 peering, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 63609/189831 objects degraded (33.508%); 13315/189831 objects misplaced (7.014%); 15 MiB/s, 252 objects/s recovering 2023-10-09T05:02:04.965 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:05.106 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:02:05.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:05.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:05.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:05.240 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:05.284 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:06.168 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:06.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:06.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:06.312 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:06.652 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:06.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:06.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:06.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:06.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.834 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: pgmap v1957: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 59679/189798 objects degraded (31.443%); 16979/189798 objects misplaced (8.946%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:02:06.835 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 63609/189831 objects degraded (33.508%), 17 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:06.844 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:06.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: pgmap v1957: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 59679/189798 objects degraded (31.443%); 16979/189798 objects misplaced (8.946%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 63609/189831 objects degraded (33.508%), 17 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: osdmap e593: 8 total, 8 up, 7 in 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: failed to encode map e593 with expected crc 2023-10-09T05:02:06.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: pgmap v1957: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+laggy, 2 activating+undersized+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 59679/189798 objects degraded (31.443%); 16979/189798 objects misplaced (8.946%); 6.1 MiB/s, 97 objects/s recovering 2023-10-09T05:02:06.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 63609/189831 objects degraded (33.508%), 17 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:07.032 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:07.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:07.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:07.182 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:07.259 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:07.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:07.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:07.394 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:07.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:07.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:07.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:07.572 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:07.618 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:07.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:07.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:07.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: osdmap e594: 8 total, 8 up, 7 in 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[216903]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: osdmap e594: 8 total, 8 up, 7 in 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:07 smithi029 ceph-mon[223858]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.721 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:07.748 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:07.797 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:07.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:07.900 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:07.956 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: osdmap e594: 8 total, 8 up, 7 in 2023-10-09T05:02:07.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.957 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:07 smithi177 ceph-mon[195797]: failed to encode map e594 with expected crc 2023-10-09T05:02:07.967 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:08.102 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:08.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:08.212 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:08.318 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:08.506 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:02:08.506 INFO:teuthology.orchestra.run.smithi177.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-09T05:02:08.506 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:02:08.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:08.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:08.637 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:08.666 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:08.712 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:08.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:08.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:08.844 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:08.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:08 smithi029 ceph-mon[216903]: pgmap v1959: 109 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 activating, 5 remapped+peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 59656/189765 objects degraded (31.437%); 16978/189765 objects misplaced (8.947%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:02:08.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:08 smithi029 ceph-mon[223858]: pgmap v1959: 109 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 activating, 5 remapped+peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 59656/189765 objects degraded (31.437%); 16978/189765 objects misplaced (8.947%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:02:09.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:08 smithi177 ceph-mon[195797]: pgmap v1959: 109 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 activating, 5 remapped+peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 59656/189765 objects degraded (31.437%); 16978/189765 objects misplaced (8.947%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:02:09.293 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:09.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:09.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:09.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:09.428 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:09.529 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:09.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:09.636 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:09.667 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:09.722 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:09.723 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:02:09.724 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:02:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:09.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:09.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:09.863 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:09.929 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:09.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:09.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:10.040 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:10.080 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:10.146 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:10.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:10.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:10.256 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:10.297 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:10.394 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:10.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:10.508 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:10.537 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:10.677 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:10.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:10.715 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:10 smithi177 ceph-mon[195797]: pgmap v1960: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 3 remapped+peering, 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 73875/189753 objects degraded (38.932%); 14258/189753 objects misplaced (7.514%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:02:10.738 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:02:10.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:10.875 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:10.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:10 smithi029 ceph-mon[216903]: pgmap v1960: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 3 remapped+peering, 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 73875/189753 objects degraded (38.932%); 14258/189753 objects misplaced (7.514%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:02:10.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:10 smithi029 ceph-mon[223858]: pgmap v1960: 109 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 activating, 3 remapped+peering, 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 73875/189753 objects degraded (38.932%); 14258/189753 objects misplaced (7.514%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:02:10.999 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:11.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:11.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:11.173 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:11.262 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:02:11.281 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:11.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:11.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:11.390 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:11.412 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:02:11.441 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:11.515 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:11.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:11.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:11.618 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:11.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:11.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:11.798 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:11.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:11.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:11.937 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 73875/189753 objects degraded (38.932%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:11.938 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:12.055 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 73875/189753 objects degraded (38.932%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:12.055 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 73875/189753 objects degraded (38.932%), 19 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:02:12.088 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:12.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:12.191 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:12.234 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:12.299 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:12.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:12.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:12.436 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:12.442 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:02:12.473 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:02:12.486 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:12.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:12.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:12.494 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:02:12.514 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.515 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:02:12.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:12.619 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:12.665 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:12.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:12.768 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:12.853 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:12.908 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:02:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:02:12] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2023-10-09T05:02:12.975 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:12.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:12.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:13.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:13.126 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:12 smithi177 ceph-mon[195797]: pgmap v1961: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 70169/189729 objects degraded (36.984%); 17002/189729 objects misplaced (8.961%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:02:13.132 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:13.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:12 smithi029 ceph-mon[216903]: pgmap v1961: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 70169/189729 objects degraded (36.984%); 17002/189729 objects misplaced (8.961%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:02:13.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:12 smithi029 ceph-mon[223858]: pgmap v1961: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 activating, 1 activating+undersized+degraded+remapped, 3 remapped+peering, 1 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 70169/189729 objects degraded (36.984%); 17002/189729 objects misplaced (8.961%); 4.5 MiB/s, 71 objects/s recovering 2023-10-09T05:02:13.228 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:13.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:13.332 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:13.361 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:13.487 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:02:13.488 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:02:13.844 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:13.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:13.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:13.948 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:13.985 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:14.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:14.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:02:14.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:14.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:14.394 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 6, 4, 0] out_osds: [1] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:02:14.394 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:02:14.394 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-10-09T05:02:14.395 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 1 2023-10-09T05:02:14.408 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:14.467 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:14.520 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:14.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:14.579 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:02:14.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:14.655 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:14.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:14.807 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[216903]: pgmap v1962: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 82499/189690 objects degraded (43.491%); 19897/189690 objects misplaced (10.489%); 151 KiB/s, 0 objects/s recovering 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2161021948' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[223858]: pgmap v1962: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 82499/189690 objects degraded (43.491%); 19897/189690 objects misplaced (10.489%); 151 KiB/s, 0 objects/s recovering 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2161021948' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:14.808 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:14 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:14.843 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:02:14.864 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:14.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:14.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:14.968 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:14.995 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:14 smithi177 ceph-mon[195797]: pgmap v1962: 109 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 activating, 3 activating+undersized+degraded+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 82499/189690 objects degraded (43.491%); 19897/189690 objects misplaced (10.489%); 151 KiB/s, 0 objects/s recovering 2023-10-09T05:02:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2161021948' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:14 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:15.040 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:15.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:15.144 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:15.171 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:15.490 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:15.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:15.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:15.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:15.624 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:15.648 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:15.675 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:15.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:15.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:15.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: osdmap e595: 8 total, 8 up, 7 in 2023-10-09T05:02:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:15 smithi177 ceph-mon[195797]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:15.814 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:15.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: osdmap e595: 8 total, 8 up, 7 in 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[216903]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: osdmap e595: 8 total, 8 up, 7 in 2023-10-09T05:02:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:15 smithi029 ceph-mon[223858]: failed to encode map e595 with expected crc 2023-10-09T05:02:15.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:02:16.137 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:16.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:16.243 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:16.278 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:16.403 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:16.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:16.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:16.510 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:16.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:16.585 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:16.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:16.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:16 smithi029 ceph-mon[216903]: pgmap v1964: 109 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 82468/189561 objects degraded (43.505%); 19852/189561 objects misplaced (10.473%); 440 KiB/s, 0 objects/s recovering 2023-10-09T05:02:16.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 82499/189690 objects degraded (43.491%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:16 smithi029 ceph-mon[223858]: pgmap v1964: 109 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 82468/189561 objects degraded (43.505%); 19852/189561 objects misplaced (10.473%); 440 KiB/s, 0 objects/s recovering 2023-10-09T05:02:16.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 82499/189690 objects degraded (43.491%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:16.689 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:16.730 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:02:16.730 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:02:16.731 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:16 smithi177 ceph-mon[195797]: pgmap v1964: 109 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 82468/189561 objects degraded (43.505%); 19852/189561 objects misplaced (10.473%); 440 KiB/s, 0 objects/s recovering 2023-10-09T05:02:16.731 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 82499/189690 objects degraded (43.491%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:16.735 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:16.870 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:16.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:16.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:17.008 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:17.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:17.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:17.133 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:17.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:17.200 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:17.327 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:17.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:17.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:17.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:17.467 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:17.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:17.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:17.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:02:17.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:17.658 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:17.766 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:17.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:17.873 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:17.907 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:18.015 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:18.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:18.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:18.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:02:18.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:18.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:18.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:17 smithi029 conmon[216878]: 2023-10-09T05:02:17.972+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e595 definitely_dead 0 2023-10-09T05:02:18.353 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:18.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:18.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:18.484 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:18.530 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:18.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:18.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:18.563 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:02:18.633 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:18.661 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:18.706 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:18.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:18.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:18.734 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.1. 2023-10-09T05:02:18.810 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:18.873 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:18 smithi029 ceph-mon[223858]: pgmap v1965: 109 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 82460/189543 objects degraded (43.505%); 19698/189543 objects misplaced (10.392%); 857 KiB/s, 11 objects/s recovering 2023-10-09T05:02:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:18 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3642107151' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:02:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:18 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:02:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:18 smithi029 ceph-mon[216903]: pgmap v1965: 109 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 82460/189543 objects degraded (43.505%); 19698/189543 objects misplaced (10.392%); 857 KiB/s, 11 objects/s recovering 2023-10-09T05:02:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:18 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3642107151' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:02:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:18 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:02:18.949 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:18.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:18 smithi177 ceph-mon[195797]: pgmap v1965: 109 pgs: 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 82460/189543 objects degraded (43.505%); 19698/189543 objects misplaced (10.392%); 857 KiB/s, 11 objects/s recovering 2023-10-09T05:02:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:18 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3642107151' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:02:19.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:18 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-10-09T05:02:19.058 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:19.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:19.267 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:19.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:19.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:19.400 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:19.977 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:19.982 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:19.987 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-10-09T05:02:19.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:02:19.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[216903]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.988 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.989 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:19 smithi029 ceph-mon[223858]: failed to encode map e596 with expected crc 2023-10-09T05:02:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:19.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-10-09T05:02:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: osdmap e596: 8 total, 8 up, 8 in 2023-10-09T05:02:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:19 smithi177 ceph-mon[195797]: failed to encode map e596 with expected crc 2023-10-09T05:02:20.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:20.121 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:20.201 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:20.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:20.305 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:20.333 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:20.386 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:20.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:20.491 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:20.531 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:20.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: pgmap v1967: 109 pgs: 1 remapped, 13 peering, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 4 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 65743/189540 objects degraded (34.686%); 19698/189540 objects misplaced (10.393%) 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/986944185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:20.817 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:20.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:20.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:20.984 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: pgmap v1967: 109 pgs: 1 remapped, 13 peering, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 4 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 65743/189540 objects degraded (34.686%); 19698/189540 objects misplaced (10.393%) 2023-10-09T05:02:20.984 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T05:02:20.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/986944185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:20.986 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: pgmap v1967: 109 pgs: 1 remapped, 13 peering, 1 activating+undersized+remapped, 2 active+recovering+undersized+remapped, 1 activating, 2 activating+undersized+degraded+remapped, 4 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 65743/189540 objects degraded (34.686%); 19698/189540 objects misplaced (10.393%) 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: osdmap e597: 8 total, 8 up, 8 in 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.987 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: failed to encode map e597 with expected crc 2023-10-09T05:02:20.988 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/986944185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:20.988 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:21.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:02:21.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:02:21.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:02:21.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:02:21.196 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:21.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:21.301 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:21.356 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:21.455 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:21.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:21.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:21.545 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:02:21.545 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:02:21.563 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:21.596 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:21.648 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:21.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:21.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:21.782 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:21.836 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:21.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: 1.6 continuing backfill to osd.1 from (0'0,82'161] MIN to 82'161 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 65743/189540 objects degraded (34.686%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:21.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e598 with expected crc 2023-10-09T05:02:21.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: 1.6 continuing backfill to osd.1 from (0'0,82'161] MIN to 82'161 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 65743/189540 objects degraded (34.686%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:21 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:21.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:21.967 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:22.011 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:22.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: osdmap e598: 8 total, 8 up, 8 in 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e598 with expected crc 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: 1.6 continuing backfill to osd.1 from (0'0,82'161] MIN to 82'161 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 65743/189540 objects degraded (34.686%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: osdmap e599: 8 total, 8 up, 8 in 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:21 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.115 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:22.161 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:22.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:22.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:22.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:22.391 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:22.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:22.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:22.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:22.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:22.601 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:22.657 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:22.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:22.742 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:22 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.742 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:22 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.742 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:22 smithi177 ceph-mon[195797]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.742 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:22 smithi177 ceph-mon[195797]: pgmap v1971: 109 pgs: 1 remapped, 17 peering, 1 activating, 2 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 67083/189540 objects degraded (35.393%); 12032/189540 objects misplaced (6.348%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:02:22.742 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:22 smithi177 ceph-mon[195797]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T05:02:22.743 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:22 smithi177 ceph-mon[195797]: failed to encode map e600 with expected crc 2023-10-09T05:02:22.761 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:22.796 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:22.853 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[216903]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[216903]: pgmap v1971: 109 pgs: 1 remapped, 17 peering, 1 activating, 2 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 67083/189540 objects degraded (35.393%); 12032/189540 objects misplaced (6.348%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[216903]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[216903]: failed to encode map e600 with expected crc 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[223858]: failed to encode map e599 with expected crc 2023-10-09T05:02:22.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[223858]: pgmap v1971: 109 pgs: 1 remapped, 17 peering, 1 activating, 2 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 59 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 67083/189540 objects degraded (35.393%); 12032/189540 objects misplaced (6.348%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:02:22.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[223858]: osdmap e600: 8 total, 8 up, 8 in 2023-10-09T05:02:22.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:22 smithi029 ceph-mon[223858]: failed to encode map e600 with expected crc 2023-10-09T05:02:22.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:22.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:22.857 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:02:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:02:22] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2023-10-09T05:02:22.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:23.005 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:23.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:02:23.095 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:23.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:23.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:23.242 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:23.330 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:23.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:23.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:23.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:23.513 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:02:23 smithi177 conmon[174387]: logger=cleanup t=2023-10-09T05:02:23.254033818Z level=info msg="Completed cleanup jobs" duration=70.309079ms 2023-10-09T05:02:23.517 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:23.578 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:23.600 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:23.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:23.713 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:23.893 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:24.037 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:24.039 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:23 smithi029 ceph-mon[216903]: failed to encode map e600 with expected crc 2023-10-09T05:02:24.039 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:23 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:24.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:23 smithi029 ceph-mon[223858]: failed to encode map e600 with expected crc 2023-10-09T05:02:24.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:23 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:24.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:24.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:24.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:24.185 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:24.243 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:24.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:24.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:24.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:23 smithi177 ceph-mon[195797]: failed to encode map e600 with expected crc 2023-10-09T05:02:24.287 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:23 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:02:24.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:24.462 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:24.649 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:24.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:24.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:24.840 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:02:24.929 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:24.987 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 2, 3, 6, 4, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:02:24.988 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:02:24.988 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [5, 7, 2, 3, 6, 4, 0, 1] 2023-10-09T05:02:24.988 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-09T05:02:25.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:25.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:25.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:25.136 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:24 smithi177 ceph-mon[195797]: pgmap v1973: 109 pgs: 1 remapped, 17 peering, 1 activating, 2 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68815/189540 objects degraded (36.306%); 12032/189540 objects misplaced (6.348%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:02:25.136 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3748183410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:25.136 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:25.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:24 smithi029 ceph-mon[216903]: pgmap v1973: 109 pgs: 1 remapped, 17 peering, 1 activating, 2 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68815/189540 objects degraded (36.306%); 12032/189540 objects misplaced (6.348%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:02:25.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3748183410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:25.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:25.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:24 smithi029 ceph-mon[223858]: pgmap v1973: 109 pgs: 1 remapped, 17 peering, 1 activating, 2 activating+undersized+degraded+remapped, 14 remapped+peering, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 59 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 68815/189540 objects degraded (36.306%); 12032/189540 objects misplaced (6.348%); 5.3 MiB/s, 85 objects/s recovering 2023-10-09T05:02:25.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3748183410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:25.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:25.236 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:25.407 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:25.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:25.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:25.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:25.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:25.804 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:25 smithi177 ceph-mon[195797]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[216903]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: osdmap e601: 8 total, 8 up, 8 in 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:25 smithi029 ceph-mon[223858]: failed to encode map e601 with expected crc 2023-10-09T05:02:26.452 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:26.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:26.557 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:26.602 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:26.714 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:26.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:26.821 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:26.832 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:26.953 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:02:26.953 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:02:26.953 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T05:02:26.953 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:02:26.967 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:26.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:26 smithi029 ceph-mon[216903]: pgmap v1975: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 remapped, 17 peering, 3 activating, 3 activating+undersized+degraded+remapped, 4 remapped+peering, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65206/189540 objects degraded (34.402%); 15498/189540 objects misplaced (8.177%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:26.984 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 65206/189540 objects degraded (34.402%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:26.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:26 smithi029 ceph-mon[223858]: pgmap v1975: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 remapped, 17 peering, 3 activating, 3 activating+undersized+degraded+remapped, 4 remapped+peering, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65206/189540 objects degraded (34.402%); 15498/189540 objects misplaced (8.177%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:26.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 65206/189540 objects degraded (34.402%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:27.089 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:27.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:26 smithi177 ceph-mon[195797]: pgmap v1975: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 1 remapped, 17 peering, 3 activating, 3 activating+undersized+degraded+remapped, 4 remapped+peering, 2 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65206/189540 objects degraded (34.402%); 15498/189540 objects misplaced (8.177%); 3.4 MiB/s, 54 objects/s recovering 2023-10-09T05:02:27.122 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 65206/189540 objects degraded (34.402%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:27.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:27.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:28.144 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:28.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:28.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:28.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:27 smithi029 ceph-mon[216903]: Health check failed: 1503/63180 objects unfound (2.379%) (OBJECT_UNFOUND) 2023-10-09T05:02:28.155 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:27 smithi029 ceph-mon[223858]: Health check failed: 1503/63180 objects unfound (2.379%) (OBJECT_UNFOUND) 2023-10-09T05:02:28.163 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:02:28.248 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:27 smithi177 ceph-mon[195797]: Health check failed: 1503/63180 objects unfound (2.379%) (OBJECT_UNFOUND) 2023-10-09T05:02:28.400 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:28.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:02:28.657 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:28.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:28.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:28.778 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:28.810 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:28 smithi177 ceph-mon[195797]: pgmap v1976: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 76980/189540 objects degraded (40.614%); 14829/189540 objects misplaced (7.824%); 1503/63180 objects unfound (2.379%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:02:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:28 smithi029 ceph-mon[216903]: pgmap v1976: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 76980/189540 objects degraded (40.614%); 14829/189540 objects misplaced (7.824%); 1503/63180 objects unfound (2.379%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:02:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:29.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:28 smithi029 ceph-mon[223858]: pgmap v1976: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 76980/189540 objects degraded (40.614%); 14829/189540 objects misplaced (7.824%); 1503/63180 objects unfound (2.379%); 2.6 MiB/s, 40 objects/s recovering 2023-10-09T05:02:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:29.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:29.429 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:29.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:29.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:29.641 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:29.847 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:29.908 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:29.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:30.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:30.108 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:30.206 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:30.337 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:30.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:30.459 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:30.480 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 conmon[216878]: 2023-10-09T05:02:30.400+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e601 definitely_dead 0 2023-10-09T05:02:30.644 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:30.726 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.2. 2023-10-09T05:02:30.753 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:30.753 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[216903]: pgmap v1977: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 76980/189540 objects degraded (40.614%); 14829/189540 objects misplaced (7.824%); 1503/63180 objects unfound (2.379%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:02:30.753 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:30.753 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2388859952' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T05:02:30.754 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T05:02:30.754 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[216903]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T05:02:30.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:30.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[223858]: pgmap v1977: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 76980/189540 objects degraded (40.614%); 14829/189540 objects misplaced (7.824%); 1503/63180 objects unfound (2.379%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:02:30.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:30.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2388859952' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T05:02:30.760 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T05:02:30.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:30 smithi029 ceph-mon[223858]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T05:02:30.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:30.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:30.899 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:31.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:31.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:30 smithi177 ceph-mon[195797]: pgmap v1977: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 76980/189540 objects degraded (40.614%); 14829/189540 objects misplaced (7.824%); 1503/63180 objects unfound (2.379%); 2.2 MiB/s, 34 objects/s recovering 2023-10-09T05:02:31.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:30 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:31.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2388859952' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T05:02:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-09T05:02:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:30 smithi177 ceph-mon[195797]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-09T05:02:31.041 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:31.180 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:31.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:31.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:31.299 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:31.377 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:31.475 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:31.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:31.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:31.587 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:31.624 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:31.698 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T05:02:31.698 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T05:02:31.698 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.699 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.785 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:31.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:31.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 76980/189540 objects degraded (40.614%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: osdmap e602: 8 total, 8 up, 7 in 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e602 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 76980/189540 objects degraded (40.614%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:31 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:31.921 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e602 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 76980/189540 objects degraded (40.614%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: osdmap e603: 8 total, 8 up, 7 in 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:32.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:31 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: pgmap v1980: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73395/189540 objects degraded (38.723%); 14065/189540 objects misplaced (7.421%); 1503/63180 objects unfound (2.379%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T05:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:32 smithi177 ceph-mon[195797]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: pgmap v1980: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73395/189540 objects degraded (38.723%); 14065/189540 objects misplaced (7.421%); 1503/63180 objects unfound (2.379%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:02:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T05:02:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[216903]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:02:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:02:32] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2023-10-09T05:02:33.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e603 with expected crc 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: pgmap v1980: 109 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 4 peering, 4 activating, 5 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73395/189540 objects degraded (38.723%); 14065/189540 objects misplaced (7.421%); 1503/63180 objects unfound (2.379%); 5.1 MiB/s, 82 objects/s recovering 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: osdmap e604: 8 total, 8 up, 7 in 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:32 smithi029 ceph-mon[223858]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.326 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:33.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:33.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:33.430 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:33.472 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:33.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:33.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:33.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:33.647 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:33.718 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:33 smithi177 ceph-mon[195797]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:33 smithi177 ceph-mon[195797]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T05:02:33.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:33 smithi029 ceph-mon[223858]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:33 smithi029 ceph-mon[223858]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T05:02:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:33 smithi029 ceph-mon[216903]: failed to encode map e604 with expected crc 2023-10-09T05:02:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:33 smithi029 ceph-mon[216903]: osdmap e605: 8 total, 8 up, 7 in 2023-10-09T05:02:33.943 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:33.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:33.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:34.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:34.083 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:34.562 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:34.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:34.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:02:34.675 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:02:34.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:34.742 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:02:34.763 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:34.781 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:02:34.807 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:02:34.808 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:02:34.808 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:02:34.808 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:02:34.808 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:02:34.821 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.821 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.821 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.821 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.821 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.822 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.822 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.822 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:02:34.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e605 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: pgmap v1983: 109 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 5 peering, 1 activating, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 69047/189540 objects degraded (36.429%); 19615/189540 objects misplaced (10.349%); 1579/63180 objects unfound (2.499%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: Health check update: 1579/63180 objects unfound (2.499%) (OBJECT_UNFOUND) 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:34.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e605 with expected crc 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: pgmap v1983: 109 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 5 peering, 1 activating, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 69047/189540 objects degraded (36.429%); 19615/189540 objects misplaced (10.349%); 1579/63180 objects unfound (2.499%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: Health check update: 1579/63180 objects unfound (2.499%) (OBJECT_UNFOUND) 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:34.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:34.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:34 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:34.873 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:34.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:34.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:34.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e605 with expected crc 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: pgmap v1983: 109 pgs: 2 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 5 peering, 1 activating, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 69047/189540 objects degraded (36.429%); 19615/189540 objects misplaced (10.349%); 1579/63180 objects unfound (2.499%); 4.9 MiB/s, 78 objects/s recovering 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: Health check update: 1579/63180 objects unfound (2.499%) (OBJECT_UNFOUND) 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: osdmap e606: 8 total, 8 up, 7 in 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:35.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:35.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:34 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:35.064 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:36.006 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:02:36.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:02:35 smithi177 conmon[372243]: 2023-10-09T05:02:35.547+0000 7f0812a5d700 -1 osd.4 607 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.36948.0:91941 14.b 14.8693bf2b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e590) 2023-10-09T05:02:36.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e606 with expected crc 2023-10-09T05:02:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: 14.e continuing backfill to osd.0 from (445'1681,594'5704] MIN to 594'5704 2023-10-09T05:02:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T05:02:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/981542026' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e606 with expected crc 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: 14.e continuing backfill to osd.0 from (445'1681,594'5704] MIN to 594'5704 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.032 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/981542026' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:02:36.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:02:36.033 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e606 with expected crc 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: 14.e continuing backfill to osd.0 from (445'1681,594'5704] MIN to 594'5704 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: osdmap e607: 8 total, 8 up, 7 in 2023-10-09T05:02:36.034 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: failed to encode map e607 with expected crc 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/981542026' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:02:36.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:36.777 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:02:36 smithi177 conmon[372243]: 2023-10-09T05:02:36.525+0000 7f0812a5d700 -1 osd.4 608 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.36948.0:91941 14.b 14.8693bf2b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e590) 2023-10-09T05:02:36.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: pgmap v1986: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 3 peering, 2 activating+undersized+degraded+remapped, 3 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 59925/189507 objects degraded (31.622%); 23572/189507 objects misplaced (12.439%); 1579/63169 objects unfound (2.500%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 69047/189540 objects degraded (36.429%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.779 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:36 smithi177 ceph-mon[195797]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:36.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: pgmap v1986: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 3 peering, 2 activating+undersized+degraded+remapped, 3 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 59925/189507 objects degraded (31.622%); 23572/189507 objects misplaced (12.439%); 1579/63169 objects unfound (2.500%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 69047/189540 objects degraded (36.429%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[216903]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:36.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: pgmap v1986: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 3 peering, 2 activating+undersized+degraded+remapped, 3 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 59925/189507 objects degraded (31.622%); 23572/189507 objects misplaced (12.439%); 1579/63169 objects unfound (2.500%); 1.5 MiB/s, 24 objects/s recovering 2023-10-09T05:02:36.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 69047/189540 objects degraded (36.429%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:36.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:36.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:02:36.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: osdmap e608: 8 total, 8 up, 7 in 2023-10-09T05:02:36.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: failed to encode map e608 with expected crc 2023-10-09T05:02:36.980 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:36 smithi029 ceph-mon[223858]: failed to encode map e608 with expected crc 2023-10-09T05:02:37.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:02:37.643 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:37.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:37.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:37.748 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:37.777 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:37.846 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:37.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:37.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:37.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:37 smithi029 ceph-mon[223858]: failed to encode map e608 with expected crc 2023-10-09T05:02:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:37 smithi029 ceph-mon[223858]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:37 smithi029 ceph-mon[216903]: failed to encode map e608 with expected crc 2023-10-09T05:02:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:37 smithi029 ceph-mon[216903]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:37.950 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:37.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:37 smithi177 ceph-mon[195797]: failed to encode map e608 with expected crc 2023-10-09T05:02:37.978 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:37 smithi177 ceph-mon[195797]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:37.979 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:02:37 smithi177 conmon[372243]: 2023-10-09T05:02:37.573+0000 7f0812a5d700 -1 osd.4 608 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.36948.0:91941 14.b 14.8693bf2b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e590) 2023-10-09T05:02:37.982 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:38.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:38.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:38.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:38.062 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:38.150 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:38.183 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:38.243 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:38.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:38.350 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:38.382 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:38.442 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:38.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:38.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:38.547 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 3, 6, 4, 0, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:02:38.547 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:02:38.548 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2023-10-09T05:02:38.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:02:38.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:38.580 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:38.792 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:38.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:38.897 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:38.930 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:02:38 smithi177 conmon[372243]: 2023-10-09T05:02:38.566+0000 7f0812a5d700 -1 osd.4 608 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.36948.0:91941 14.b 14.8693bf2b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e590) 2023-10-09T05:02:38.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:38 smithi177 ceph-mon[195797]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:38.931 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:38 smithi177 ceph-mon[195797]: pgmap v1988: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 2 peering, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 64321/189507 objects degraded (33.941%); 22379/189507 objects misplaced (11.809%); 1579/63169 objects unfound (2.500%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:02:38.933 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:38.991 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:38.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:39.097 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:39.129 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:39.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:38 smithi029 ceph-mon[223858]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:38 smithi029 ceph-mon[223858]: pgmap v1988: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 2 peering, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 64321/189507 objects degraded (33.941%); 22379/189507 objects misplaced (11.809%); 1579/63169 objects unfound (2.500%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:38 smithi029 ceph-mon[216903]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:38 smithi029 ceph-mon[216903]: pgmap v1988: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 1 activating+undersized+remapped, 2 peering, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 2 op/s; 64321/189507 objects degraded (33.941%); 22379/189507 objects misplaced (11.809%); 1579/63169 objects unfound (2.500%); 6.2 MiB/s, 99 objects/s recovering 2023-10-09T05:02:39.190 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:39.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:39.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:39.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:39.325 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:39.556 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:39.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:39.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:39.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:39.700 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:39.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:02:39.773 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:39.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:39.796 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:39.883 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:39.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:39 smithi029 ceph-mon[223858]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:39.913 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:39 smithi029 ceph-mon[216903]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:39.916 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:39.964 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:02:40.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:02:39 smithi177 conmon[372243]: 2023-10-09T05:02:39.548+0000 7f0812a5d700 -1 osd.4 608 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.36948.0:91941 14.b 14.8693bf2b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e590) 2023-10-09T05:02:40.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:39 smithi177 ceph-mon[195797]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:40.147 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:40.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:40.252 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:40.288 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:40.325 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:77555 14.5 e459/457 14:a29673af:::benchmark_data_smithi029_550670_object77554:head v 459'4859, mlcod=0'0)", 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:11.665942+0000", 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 448.65783958200001, 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018888186000000001, 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:02:40.326 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.665942+0000", 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.665942+0000", 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.665942+0000", 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666033+0000", 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0927000000000005e-05 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666050+0000", 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7331999999999999e-05 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666062+0000", 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1446e-05 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666660+0000", 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059858000000000003 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.666692+0000", 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2064999999999999e-05 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.684775+0000", 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018082463 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:11.684830+0000", 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5773999999999998e-05 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.5 464/457 [PushOp(14:a2a6fac7:::benchmark_data_smithi029_550670_object9:head, version: 444'1, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:a2a6fac7:::benchmark_data_smithi029_550670_object9:head@444'1, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:20.039506+0000", 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 440.28427583500002, 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042652549999999997, 2023-10-09T05:02:40.331 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.039506+0000", 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.039506+0000", 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.039505+0000", 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043032+0000", 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035274920000000001 2023-10-09T05:02:40.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043053+0000", 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0726000000000001e-05 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043071+0000", 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8454e-05 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043161+0000", 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0148999999999994e-05 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.043177+0000", 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5755000000000001e-05 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.082159+0000", 2023-10-09T05:02:40.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038981364999999997 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.5 464/457 [PushOp(14:a903ccc8:::benchmark_data_smithi029_550670_object252:head, version: 444'20, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:a903ccc8:::benchmark_data_smithi029_550670_object252:head@444'20, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:20.374281+0000", 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 439.9495005, 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093335684000000002, 2023-10-09T05:02:40.336 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374281+0000", 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374281+0000", 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374280+0000", 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374326+0000", 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5745999999999999e-05 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374359+0000", 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3473999999999998e-05 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374382+0000", 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2942000000000002e-05 2023-10-09T05:02:40.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374480+0000", 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7909000000000001e-05 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.374495+0000", 2023-10-09T05:02:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4647000000000001e-05 2023-10-09T05:02:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:20.467617+0000", 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093122307000000001 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78017 14.3 e472/464 14:cd0d5495:::benchmark_data_smithi029_550670_object406:head v 472'4892, mlcod=444'27)", 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.099347+0000", 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 408.22443523099997, 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.171165717, 2023-10-09T05:02:40.342 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099347+0000", 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099347+0000", 2023-10-09T05:02:40.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099345+0000", 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099355+0000", 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5130000000000002e-06 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099366+0000", 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0995999999999999e-05 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099374+0000", 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0940000000000008e-06 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099428+0000", 2023-10-09T05:02:40.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3730000000000002e-05 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.099461+0000", 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3688e-05 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270455+0000", 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17099336600000001 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270512+0000", 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.753e-05 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78128 14.0 e472/457 14:04025a37:::benchmark_data_smithi029_550670_object517:head v 472'4898, mlcod=0'0)", 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.271779+0000", 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 408.05200281700002, 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021498257, 2023-10-09T05:02:40.347 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271779+0000", 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271779+0000", 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271779+0000", 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271784+0000", 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7350000000000001e-06 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271792+0000", 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8779999999999998e-06 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271799+0000", 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.204e-06 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271857+0000", 2023-10-09T05:02:40.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7596000000000002e-05 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.271883+0000", 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6097000000000001e-05 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.293228+0000", 2023-10-09T05:02:40.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021345151999999999 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.293277+0000", 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9097000000000001e-05 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78533 14.b e475/464 14:d1503dc4:::benchmark_data_smithi029_550670_object922:head v 475'5041, mlcod=444'60)", 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:58.828771+0000", 2023-10-09T05:02:40.352 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 401.49501076799999, 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016809307999999999, 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828771+0000", 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828771+0000", 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828771+0000", 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967296 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828775+0000", 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8410000000000002e-06 2023-10-09T05:02:40.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828780+0000", 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8300000000000003e-06 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.828784+0000", 2023-10-09T05:02:40.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9809999999999997e-06 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.838471+0000", 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096873289999999997 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.838501+0000", 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0340000000000001e-05 2023-10-09T05:02:40.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.845548+0000", 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007046971 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.845580+0000", 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2248e-05 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.357 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:78538 14.b e475/464 14:d6e8583a:::benchmark_data_smithi029_550670_object927:head v 475'5042, mlcod=444'61)", 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:58.869616+0000", 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 401.45416570999998, 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017353989, 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869616+0000", 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869616+0000", 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869616+0000", 2023-10-09T05:02:40.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869621+0000", 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8389999999999997e-06 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869627+0000", 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0800000000000002e-06 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.869632+0000", 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9599999999999999e-06 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.872224+0000", 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025921640000000001 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.872266+0000", 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2097e-05 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.886914+0000", 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014648458 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.362 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:58.886970+0000", 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5702999999999998e-05 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.8 488/464 [PushOp(14:139ead53:::benchmark_data_smithi029_550670_object2324:head, version: 444'155, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:139ead53:::benchmark_data_smithi029_550670_object2324:head@444'155, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.684903+0000", 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.63887931800002, 2023-10-09T05:02:40.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089983719999999993, 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684903+0000", 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684903+0000", 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684901+0000", 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684938+0000", 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7183e-05 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684962+0000", 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3017e-05 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.684980+0000", 2023-10-09T05:02:40.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8731000000000001e-05 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.691174+0000", 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061934370000000004 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.691188+0000", 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3966e-05 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.693901+0000", 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002713268 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:79935 14.8 e488/464 14:139ead53:::benchmark_data_smithi029_550670_object2324:head v 488'5149, mlcod=444'154)", 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.709849+0000", 2023-10-09T05:02:40.368 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.61393262399997, 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015277671, 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709849+0000", 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709849+0000", 2023-10-09T05:02:40.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709847+0000", 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709862+0000", 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4329e-05 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709889+0000", 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7208000000000001e-05 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.709899+0000", 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0189e-05 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.713197+0000", 2023-10-09T05:02:40.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032979020000000001 2023-10-09T05:02:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.713225+0000", 2023-10-09T05:02:40.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8089000000000002e-05 2023-10-09T05:02:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:02:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.725042+0000", 2023-10-09T05:02:40.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011816488 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.725127+0000", 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5196000000000002e-05 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 488/484 [PushOp(14:91bba494:::benchmark_data_smithi029_550670_object9316:head, version: 444'565, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:91bba494:::benchmark_data_smithi029_550670_object9316:head@444'565, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:43.875065+0000", 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 356.44871672699998, 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011887887, 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875065+0000", 2023-10-09T05:02:40.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875065+0000", 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875063+0000", 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.875136+0000", 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2479999999999997e-05 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884072+0000", 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089364830000000003 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884105+0000", 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2656e-05 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884447+0000", 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000341503 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.884461+0000", 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4121000000000001e-05 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:43.886953+0000", 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024922669999999998 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.9 490/484 [PushOp(14:9dbe5c56:::benchmark_data_smithi029_550670_object14195:head, version: 444'870, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:9dbe5c56:::benchmark_data_smithi029_550670_object14195:head@444'870, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:48.524751+0000", 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 351.79903127199998, 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011665902000000001, 2023-10-09T05:02:40.380 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524751+0000", 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524751+0000", 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524750+0000", 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524798+0000", 2023-10-09T05:02:40.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8872000000000002e-05 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524828+0000", 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9504000000000001e-05 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524847+0000", 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9279e-05 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524943+0000", 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5675999999999997e-05 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.524956+0000", 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3382e-05 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:48.536416+0000", 2023-10-09T05:02:40.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011460187 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 597/586 [PushOp(14:28c29c4b:::benchmark_data_smithi029_550670_object22127:head, version: 445'1393, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:28c29c4b:::benchmark_data_smithi029_550670_object22127:head@445'1393, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:20.775644+0000", 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 19.548137825000001, 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019550764000000002, 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.385 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.775644+0000", 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.775644+0000", 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.775643+0000", 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.775703+0000", 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9509000000000002e-05 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.775723+0000", 2023-10-09T05:02:40.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0381e-05 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.775754+0000", 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1301999999999999e-05 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.793713+0000", 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017958811000000002 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.793729+0000", 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5486e-05 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:20.795195+0000", 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014660090000000001 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 599/586 [PushOp(14:29d5b0d9:::benchmark_data_smithi029_550670_object24083:head, version: 445'1499, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:29d5b0d9:::benchmark_data_smithi029_550670_object24083:head@445'1499, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:21.932317+0000", 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 18.391464431999999, 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.110895616, 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:21.932317+0000", 2023-10-09T05:02:40.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:21.932317+0000", 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:21.932317+0000", 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:21.932386+0000", 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9622000000000005e-05 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:22.039801+0000", 2023-10-09T05:02:40.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10741500900000001 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:22.039834+0000", 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2196999999999997e-05 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:22.041698+0000", 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001864264 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:22.041722+0000", 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3699000000000001e-05 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:22.043213+0000", 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014914240000000001 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 604/597 [PushOp(14:cd5b432b:::benchmark_data_smithi029_550670_object15501:head, version: 444'975, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:cd5b432b:::benchmark_data_smithi029_550670_object15501:head@444'975, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.243079+0000", 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 8.0807033050000001, 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.115225333, 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243079+0000", 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243079+0000", 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243078+0000", 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243112+0000", 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4870999999999999e-05 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243132+0000", 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9327e-05 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243149+0000", 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7512000000000001e-05 2023-10-09T05:02:40.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243565+0000", 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00041605100000000002 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.243578+0000", 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3104e-05 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.358304+0000", 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11472547600000001 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 604/586 [PushOp(14:2cdcb18a:::benchmark_data_smithi029_550670_object39990:head, version: 445'2508, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:2cdcb18a:::benchmark_data_smithi029_550670_object39990:head@445'2508, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.258276+0000", 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 8.0655057419999991, 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.100025506, 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.399 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258276+0000", 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258276+0000", 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258275+0000", 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258342+0000", 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7735000000000001e-05 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258369+0000", 2023-10-09T05:02:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6234e-05 2023-10-09T05:02:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258391+0000", 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2249999999999999e-05 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258479+0000", 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8298000000000001e-05 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.258490+0000", 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.118e-05 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.358302+0000", 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099811263999999997 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.404 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 604/586 [PushOp(14:2c7b6cf1:::benchmark_data_smithi029_550670_object40021:head, version: 445'2510, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:2c7b6cf1:::benchmark_data_smithi029_550670_object40021:head@445'2510, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.371003+0000", 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 7.9527792100000001, 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010240546999999999, 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371003+0000", 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371003+0000", 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371002+0000", 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371042+0000", 2023-10-09T05:02:40.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0555e-05 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371057+0000", 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4562000000000001e-05 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371068+0000", 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0882e-05 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371130+0000", 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2757999999999998e-05 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.371139+0000", 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6770000000000001e-06 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.381243+0000", 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010104034 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 604/586 [PushOp(14:2a14092a:::benchmark_data_smithi029_550670_object40346:head, version: 445'2528, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:2a14092a:::benchmark_data_smithi029_550670_object40346:head@445'2528, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.563207+0000", 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 7.7605747630000002, 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017112665999999999, 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.409 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563207+0000", 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563207+0000", 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563206+0000", 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563257+0000", 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0621999999999997e-05 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563270+0000", 2023-10-09T05:02:40.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2887999999999999e-05 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563280+0000", 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8789999999999998e-06 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563336+0000", 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6295000000000002e-05 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.563343+0000", 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2099999999999996e-06 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.580320+0000", 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016976446999999999 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 604/586 [PushOp(14:284e1bb2:::benchmark_data_smithi029_550670_object41161:head, version: 445'2568, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:284e1bb2:::benchmark_data_smithi029_550670_object41161:head@445'2568, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.885032+0000", 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 7.4387501409999999, 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021906885000000001, 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885032+0000", 2023-10-09T05:02:40.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885032+0000", 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885030+0000", 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885086+0000", 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5254000000000003e-05 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885109+0000", 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3300999999999999e-05 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885128+0000", 2023-10-09T05:02:40.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9191e-05 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885211+0000", 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2663999999999993e-05 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.885224+0000", 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3431999999999999e-05 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.906939+0000", 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021714385999999999 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 604/597 [PushOp(14:cee67e59:::benchmark_data_smithi029_550670_object15684:head, version: 444'995, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:cee67e59:::benchmark_data_smithi029_550670_object15684:head@444'995, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.887948+0000", 2023-10-09T05:02:40.418 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 7.4358343050000002, 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019171601999999999, 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.887948+0000", 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.887948+0000", 2023-10-09T05:02:40.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.887947+0000", 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.887978+0000", 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1628000000000003e-05 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.905902+0000", 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017923881999999999 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.905923+0000", 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1073999999999998e-05 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.905998+0000", 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4616000000000005e-05 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.906016+0000", 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8272999999999999e-05 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.907119+0000", 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001102876 2023-10-09T05:02:40.422 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.14 604/586 [PushOp(14:2ea2be63:::benchmark_data_smithi029_550670_object41305:head, version: 445'2578, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:2ea2be63:::benchmark_data_smithi029_550670_object41305:head@445'2578, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:32.978895+0000", 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 7.3448870719999997, 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.049194659000000002, 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.978895+0000", 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.978895+0000", 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.978893+0000", 2023-10-09T05:02:40.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:32.978977+0000", 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4263000000000003e-05 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:33.026408+0000", 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.047431431000000003 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:33.026435+0000", 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6361999999999999e-05 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:33.026519+0000", 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4368000000000003e-05 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:33.026537+0000", 2023-10-09T05:02:40.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7467999999999999e-05 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:33.028089+0000", 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015527869999999999 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:02:40.427 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:02:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:40.428 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:40.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:40.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:40.562 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:40.618 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:40.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:40.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:40.722 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:40.757 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:40.814 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:40.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:40 smithi177 ceph-mon[195797]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:40.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:40 smithi177 ceph-mon[195797]: pgmap v1989: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 64321/189507 objects degraded (33.941%); 22379/189507 objects misplaced (11.809%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:02:40.819 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:40 smithi177 ceph-mon[195797]: Health check cleared: OBJECT_UNFOUND (was: 1579/63169 objects unfound (2.500%)) 2023-10-09T05:02:40.820 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:02:40 smithi177 conmon[372243]: 2023-10-09T05:02:40.548+0000 7f0812a5d700 -1 osd.4 608 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.36948.0:91941 14.b 14.8693bf2b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e590) 2023-10-09T05:02:40.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:40.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:40.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:40 smithi029 ceph-mon[216903]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:40 smithi029 ceph-mon[216903]: pgmap v1989: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 64321/189507 objects degraded (33.941%); 22379/189507 objects misplaced (11.809%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:02:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:40 smithi029 ceph-mon[216903]: Health check cleared: OBJECT_UNFOUND (was: 1579/63169 objects unfound (2.500%)) 2023-10-09T05:02:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:40 smithi029 ceph-mon[223858]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:40 smithi029 ceph-mon[223858]: pgmap v1989: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 3 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 64321/189507 objects degraded (33.941%); 22379/189507 objects misplaced (11.809%); 4.9 MiB/s, 77 objects/s recovering 2023-10-09T05:02:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:40 smithi029 ceph-mon[223858]: Health check cleared: OBJECT_UNFOUND (was: 1579/63169 objects unfound (2.500%)) 2023-10-09T05:02:40.921 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:40.952 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:41.012 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:41.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:41.119 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:41.151 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:41.214 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:41.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:41.323 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:41.374 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:41.480 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:41.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:41.584 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:41.618 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:41.675 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:41.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:41.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:41.782 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:41.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:41 smithi177 ceph-mon[195797]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 64321/189507 objects degraded (33.941%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:42.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:41 smithi029 ceph-mon[216903]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:42.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 64321/189507 objects degraded (33.941%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:41 smithi029 ceph-mon[223858]: 4 slow requests (by type [ 'delayed' : 4 ] most affected pool [ 'unique_pool_9' : 4 ]) 2023-10-09T05:02:42.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 64321/189507 objects degraded (33.941%), 21 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:42.217 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:42.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:42.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:42.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:02:42.270 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:42.324 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:42.353 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:42.355 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:42.415 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:42.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:42.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:42.555 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:42.621 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:42.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:42.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:42.765 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:42.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:42.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:42.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:42.967 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:43.022 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:43.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:43.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:43.162 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:42 smithi177 ceph-mon[195797]: pgmap v1990: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 64321/189507 objects degraded (33.941%); 20927/189507 objects misplaced (11.043%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:02:43.163 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:43.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:42 smithi029 ceph-mon[216903]: pgmap v1990: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 64321/189507 objects degraded (33.941%); 20927/189507 objects misplaced (11.043%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:02:43.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:02:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:02:42] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2023-10-09T05:02:43.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:42 smithi029 ceph-mon[223858]: pgmap v1990: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 64321/189507 objects degraded (33.941%); 20927/189507 objects misplaced (11.043%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:02:43.219 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:43.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:43.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:43.358 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:44.149 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:02:44.157 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:02:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[216903]: pgmap v1991: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20587/189507 objects misplaced (10.863%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T05:02:44.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[216903]: from='client.47372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:02:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[223858]: pgmap v1991: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20587/189507 objects misplaced (10.863%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T05:02:44.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:44 smithi029 ceph-mon[223858]: from='client.47372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:02:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:02:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:44 smithi177 ceph-mon[195797]: pgmap v1991: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20587/189507 objects misplaced (10.863%); 7.9 MiB/s, 127 objects/s recovering 2023-10-09T05:02:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:44 smithi177 ceph-mon[195797]: from='client.47372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:02:45.129 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:45.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:45.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:45.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:45.277 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:45.782 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:45.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:45.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:45.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:45 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:02:45.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:45 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:02:45.920 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:45 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:02:46.226 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:46.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:46.294 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":1991,"stamp":"2023-10-09T05:02:43.665722+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4138801080,"num_objects":63169,"num_object_clones":0,"num_object_copies":189507,"num_objects_missing_on_primary":16812,"num_objects_missing":16812,"num_objects_degraded":64321,"num_objects_misplaced":20587,"num_objects_unfound":0,"num_objects_dirty":63169,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":92373,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48467,"num_bytes_recovered":3080045064,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133176,"ondisk_log_size":133176,"up":327,"acting":306,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":430,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19947872,"kb_used_data":12589224,"kb_used_omap":23,"kb_used_meta":7358568,"kb_avail":636268192,"statfs":{"total":671965249536,"available":651538628608,"internally_reserved":0,"allocated":12891365376,"data_stored":12877226163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24123,"internal_metadata":7535174085},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,4,7],"upper_bound":131072},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1189,"num_objects_missing":-341,"num_objects_degraded":4257,"num_objects_misplaced":974,"num_objects_unfound":-1579,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1104,"num_bytes_recovered":72351744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.684562"},"pg_stats":[{"pgid":"14.17","version":"594'5455","reported_seq":14209,"reported_epoch":601,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:26.434055+0000","last_change":"2023-10-09T05:02:26.267404+0000","last_active":"2023-10-09T05:02:18.781342+0000","last_peered":"2023-10-09T05:02:14.613458+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:02:11.552455+0000","last_became_peered":"2023-10-09T05:02:11.552455+0000","last_unstale":"2023-10-09T05:02:26.434055+0000","last_undegraded":"2023-10-09T05:02:26.255363+0000","last_fullsized":"2023-10-09T05:02:21.937790+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129335296,"num_objects":1974,"num_object_clones":0,"num_object_copies":5922,"num_objects_missing_on_primary":74,"num_objects_missing":74,"num_objects_degraded":2888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2914,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2119,"num_bytes_recovered":137822208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":6},{"shards":"0,4","objects":-3},{"shards":"1,2,4,5","objects":1596},{"shards":"2,4,5","objects":375}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":118,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:33.000012+0000","last_change":"2023-10-09T05:02:31.006786+0000","last_active":"2023-10-09T05:02:33.000012+0000","last_peered":"2023-10-09T05:02:33.000012+0000","last_clean":"2023-10-09T05:02:33.000012+0000","last_became_active":"2023-10-09T05:02:31.006375+0000","last_became_peered":"2023-10-09T05:02:31.006375+0000","last_unstale":"2023-10-09T05:02:33.000012+0000","last_undegraded":"2023-10-09T05:02:33.000012+0000","last_fullsized":"2023-10-09T05:02:33.000012+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:19.297414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":241,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.004406+0000","last_change":"2023-10-09T05:02:19.800713+0000","last_active":"2023-10-09T05:02:35.004406+0000","last_peered":"2023-10-09T05:02:35.004406+0000","last_clean":"2023-10-09T05:02:35.004406+0000","last_became_active":"2023-10-09T05:02:19.800366+0000","last_became_peered":"2023-10-09T05:02:19.800366+0000","last_unstale":"2023-10-09T05:02:35.004406+0000","last_undegraded":"2023-10-09T05:02:35.004406+0000","last_fullsized":"2023-10-09T05:02:35.004406+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:18:55.932253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"594'5440","reported_seq":13666,"reported_epoch":608,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:40.885138+0000","last_change":"2023-10-09T05:02:19.887844+0000","last_active":"2023-10-09T05:02:40.885138+0000","last_peered":"2023-10-09T05:02:40.885138+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:01:57.602815+0000","last_became_peered":"2023-10-09T05:01:57.602815+0000","last_unstale":"2023-10-09T05:02:40.885138+0000","last_undegraded":"2023-10-09T05:01:57.091826+0000","last_fullsized":"2023-10-09T05:01:56.912657+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5440,"log_dups_size":0,"ondisk_log_size":5440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128417792,"num_objects":1960,"num_object_clones":0,"num_object_copies":5880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":1160,"num_objects_unfound":0,"num_objects_dirty":1960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2901,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2232,"num_bytes_recovered":146243584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[7,3],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,3,4,6,7","objects":194},{"shards":"2,3,4,7","objects":349},{"shards":"2,3,6,7","objects":65},{"shards":"2,4,7","objects":67},{"shards":"3,7","objects":1283}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.991435+0000","last_change":"2023-10-09T05:02:19.814888+0000","last_active":"2023-10-09T05:02:34.991435+0000","last_peered":"2023-10-09T05:02:34.991435+0000","last_clean":"2023-10-09T05:02:34.991435+0000","last_became_active":"2023-10-09T05:02:19.814415+0000","last_became_peered":"2023-10-09T05:02:19.814415+0000","last_unstale":"2023-10-09T05:02:34.991435+0000","last_undegraded":"2023-10-09T05:02:34.991435+0000","last_fullsized":"2023-10-09T05:02:34.991435+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:26.156814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":110,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.004637+0000","last_change":"2023-10-09T05:02:19.766218+0000","last_active":"2023-10-09T05:02:35.004637+0000","last_peered":"2023-10-09T05:02:35.004637+0000","last_clean":"2023-10-09T05:02:35.004637+0000","last_became_active":"2023-10-09T05:02:19.765942+0000","last_became_peered":"2023-10-09T05:02:19.765942+0000","last_unstale":"2023-10-09T05:02:35.004637+0000","last_undegraded":"2023-10-09T05:02:35.004637+0000","last_fullsized":"2023-10-09T05:02:35.004637+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:22.240966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"594'5408","reported_seq":13129,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:33.002246+0000","last_change":"2023-10-09T05:02:01.542689+0000","last_active":"2023-10-09T05:02:33.002246+0000","last_peered":"2023-10-09T05:02:33.002246+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T05:02:01.383679+0000","last_became_peered":"2023-10-09T05:02:01.383679+0000","last_unstale":"2023-10-09T05:02:33.002246+0000","last_undegraded":"2023-10-09T05:02:00.476536+0000","last_fullsized":"2023-10-09T05:01:56.915801+0000","mapping_epoch":586,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5162,"log_dups_size":0,"ondisk_log_size":5162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130613248,"num_objects":1993,"num_object_clones":0,"num_object_copies":5979,"num_objects_missing_on_primary":92,"num_objects_missing":92,"num_objects_degraded":3352,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2868,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1982,"num_bytes_recovered":124715008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":642},{"shards":"3,4","objects":39},{"shards":"3,4,5,7","objects":383},{"shards":"4,5,7","objects":929}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":236,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.001709+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:02:35.001709+0000","last_peered":"2023-10-09T05:02:35.001709+0000","last_clean":"2023-10-09T05:02:35.001709+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:02:35.001709+0000","last_undegraded":"2023-10-09T05:02:35.001709+0000","last_fullsized":"2023-10-09T05:02:35.001709+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.008288+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:02:35.008288+0000","last_peered":"2023-10-09T05:02:35.008288+0000","last_clean":"2023-10-09T05:02:35.008288+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:02:35.008288+0000","last_undegraded":"2023-10-09T05:02:35.008288+0000","last_fullsized":"2023-10-09T05:02:35.008288+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":627,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.005552+0000","last_change":"2023-10-09T05:01:55.975095+0000","last_active":"2023-10-09T05:02:35.005552+0000","last_peered":"2023-10-09T05:02:35.005552+0000","last_clean":"2023-10-09T05:02:35.005552+0000","last_became_active":"2023-10-09T05:01:55.974287+0000","last_became_peered":"2023-10-09T05:01:55.974287+0000","last_unstale":"2023-10-09T05:02:35.005552+0000","last_undegraded":"2023-10-09T05:02:35.005552+0000","last_fullsized":"2023-10-09T05:02:35.005552+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:11:45.050499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":227,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.001632+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:02:35.001632+0000","last_peered":"2023-10-09T05:02:35.001632+0000","last_clean":"2023-10-09T05:02:35.001632+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:02:35.001632+0000","last_undegraded":"2023-10-09T05:02:35.001632+0000","last_fullsized":"2023-10-09T05:02:35.001632+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:00:18.381967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006007+0000","last_change":"2023-10-09T05:02:19.753193+0000","last_active":"2023-10-09T05:02:35.006007+0000","last_peered":"2023-10-09T05:02:35.006007+0000","last_clean":"2023-10-09T05:02:35.006007+0000","last_became_active":"2023-10-09T05:02:19.752724+0000","last_became_peered":"2023-10-09T05:02:19.752724+0000","last_unstale":"2023-10-09T05:02:35.006007+0000","last_undegraded":"2023-10-09T05:02:35.006007+0000","last_fullsized":"2023-10-09T05:02:35.006007+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:14:32.211602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.002801+0000","last_change":"2023-10-09T04:59:30.857503+0000","last_active":"2023-10-09T05:02:35.002801+0000","last_peered":"2023-10-09T05:02:35.002801+0000","last_clean":"2023-10-09T05:02:35.002801+0000","last_became_active":"2023-10-09T04:59:30.855741+0000","last_became_peered":"2023-10-09T04:59:30.855741+0000","last_unstale":"2023-10-09T05:02:35.002801+0000","last_undegraded":"2023-10-09T05:02:35.002801+0000","last_fullsized":"2023-10-09T05:02:35.002801+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:28.636962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"594'5567","reported_seq":7536,"reported_epoch":608,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:36.467689+0000","last_change":"2023-10-09T05:02:36.475072+0000","last_active":"2023-10-09T05:02:36.467689+0000","last_peered":"2023-10-09T05:02:36.467689+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:02:36.467689+0000","last_undegraded":"2023-10-09T05:02:32.875126+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4031,"log_dups_size":0,"ondisk_log_size":4031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133431296,"num_objects":2036,"num_object_clones":0,"num_object_copies":6108,"num_objects_missing_on_primary":2012,"num_objects_missing":2012,"num_objects_degraded":6064,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2958,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":463,"num_bytes_recovered":30343168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,3","objects":26},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1579}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":260,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.004329+0000","last_change":"2023-10-09T05:01:56.141170+0000","last_active":"2023-10-09T05:02:35.004329+0000","last_peered":"2023-10-09T05:02:35.004329+0000","last_clean":"2023-10-09T05:02:35.004329+0000","last_became_active":"2023-10-09T05:01:56.035389+0000","last_became_peered":"2023-10-09T05:01:56.035389+0000","last_unstale":"2023-10-09T05:02:35.004329+0000","last_undegraded":"2023-10-09T05:02:35.004329+0000","last_fullsized":"2023-10-09T05:02:35.004329+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:10.457851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":31,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.987208+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:02:34.987208+0000","last_peered":"2023-10-09T05:02:34.987208+0000","last_clean":"2023-10-09T05:02:34.987208+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:02:34.987208+0000","last_undegraded":"2023-10-09T05:02:34.987208+0000","last_fullsized":"2023-10-09T05:02:34.987208+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"591'5412","reported_seq":17324,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.021782+0000","last_change":"2023-10-09T05:01:57.956021+0000","last_active":"2023-10-09T05:02:35.021782+0000","last_peered":"2023-10-09T05:02:35.021782+0000","last_clean":"2023-10-09T05:02:35.021782+0000","last_became_active":"2023-10-09T05:01:57.950253+0000","last_became_peered":"2023-10-09T05:01:57.950253+0000","last_unstale":"2023-10-09T05:02:35.021782+0000","last_undegraded":"2023-10-09T05:02:35.021782+0000","last_fullsized":"2023-10-09T05:02:35.021782+0000","mapping_epoch":587,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5412,"log_dups_size":0,"ondisk_log_size":5412,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132087808,"num_objects":2016,"num_object_clones":0,"num_object_copies":6048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2874,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2950,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":72,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:33.833081+0000","last_change":"2023-10-09T04:59:29.995103+0000","last_active":"2023-10-09T05:02:33.833081+0000","last_peered":"2023-10-09T05:02:33.833081+0000","last_clean":"2023-10-09T05:02:33.833081+0000","last_became_active":"2023-10-09T04:58:57.210019+0000","last_became_peered":"2023-10-09T04:58:57.210019+0000","last_unstale":"2023-10-09T05:02:33.833081+0000","last_undegraded":"2023-10-09T05:02:33.833081+0000","last_fullsized":"2023-10-09T05:02:33.833081+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:08:53.045092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":84,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T05:02:20.549463+0000","last_change":"2023-10-09T05:02:20.549109+0000","last_active":"2023-10-09T05:02:20.549463+0000","last_peered":"2023-10-09T05:02:20.549463+0000","last_clean":"2023-10-09T05:02:20.549463+0000","last_became_active":"2023-10-09T05:02:20.548790+0000","last_became_peered":"2023-10-09T05:02:20.548790+0000","last_unstale":"2023-10-09T05:02:20.549463+0000","last_undegraded":"2023-10-09T05:02:20.549463+0000","last_fullsized":"2023-10-09T05:02:20.549463+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:36:26.223755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":70,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T05:02:20.544849+0000","last_change":"2023-10-09T04:59:29.960299+0000","last_active":"2023-10-09T05:02:20.544849+0000","last_peered":"2023-10-09T05:02:20.544849+0000","last_clean":"2023-10-09T05:02:20.544849+0000","last_became_active":"2023-10-09T04:58:57.075811+0000","last_became_peered":"2023-10-09T04:58:57.075811+0000","last_unstale":"2023-10-09T05:02:20.544849+0000","last_undegraded":"2023-10-09T05:02:20.544849+0000","last_fullsized":"2023-10-09T05:02:20.544849+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:28:45.388764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":193,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.004103+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:02:35.004103+0000","last_peered":"2023-10-09T05:02:35.004103+0000","last_clean":"2023-10-09T05:02:35.004103+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:02:35.004103+0000","last_undegraded":"2023-10-09T05:02:35.004103+0000","last_fullsized":"2023-10-09T05:02:35.004103+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.003113+0000","last_change":"2023-10-09T05:02:19.779800+0000","last_active":"2023-10-09T05:02:35.003113+0000","last_peered":"2023-10-09T05:02:35.003113+0000","last_clean":"2023-10-09T05:02:35.003113+0000","last_became_active":"2023-10-09T05:02:19.779451+0000","last_became_peered":"2023-10-09T05:02:19.779451+0000","last_unstale":"2023-10-09T05:02:35.003113+0000","last_undegraded":"2023-10-09T05:02:35.003113+0000","last_fullsized":"2023-10-09T05:02:35.003113+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:33.306803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":514,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.004157+0000","last_change":"2023-10-09T04:59:30.948799+0000","last_active":"2023-10-09T05:02:35.004157+0000","last_peered":"2023-10-09T05:02:35.004157+0000","last_clean":"2023-10-09T05:02:35.004157+0000","last_became_active":"2023-10-09T04:59:30.948277+0000","last_became_peered":"2023-10-09T04:59:30.948277+0000","last_unstale":"2023-10-09T05:02:35.004157+0000","last_undegraded":"2023-10-09T05:02:35.004157+0000","last_fullsized":"2023-10-09T05:02:35.004157+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:49:25.028468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:32.995824+0000","last_change":"2023-10-09T04:58:56.993529+0000","last_active":"2023-10-09T05:02:32.995824+0000","last_peered":"2023-10-09T05:02:32.995824+0000","last_clean":"2023-10-09T05:02:32.995824+0000","last_became_active":"2023-10-09T04:58:56.993185+0000","last_became_peered":"2023-10-09T04:58:56.993185+0000","last_unstale":"2023-10-09T05:02:32.995824+0000","last_undegraded":"2023-10-09T05:02:32.995824+0000","last_fullsized":"2023-10-09T05:02:32.995824+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:02:36.864637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":235,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.005177+0000","last_change":"2023-10-09T05:02:31.008925+0000","last_active":"2023-10-09T05:02:35.005177+0000","last_peered":"2023-10-09T05:02:35.005177+0000","last_clean":"2023-10-09T05:02:35.005177+0000","last_became_active":"2023-10-09T05:02:31.008573+0000","last_became_peered":"2023-10-09T05:02:31.008573+0000","last_unstale":"2023-10-09T05:02:35.005177+0000","last_undegraded":"2023-10-09T05:02:35.005177+0000","last_fullsized":"2023-10-09T05:02:35.005177+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:51:34.769919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":222,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.007266+0000","last_change":"2023-10-09T05:01:56.159460+0000","last_active":"2023-10-09T05:02:35.007266+0000","last_peered":"2023-10-09T05:02:35.007266+0000","last_clean":"2023-10-09T05:02:35.007266+0000","last_became_active":"2023-10-09T05:01:56.007027+0000","last_became_peered":"2023-10-09T05:01:56.007027+0000","last_unstale":"2023-10-09T05:02:35.007266+0000","last_undegraded":"2023-10-09T05:02:35.007266+0000","last_fullsized":"2023-10-09T05:02:35.007266+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:17:56.460099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":95,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.005430+0000","last_change":"2023-10-09T05:02:33.078860+0000","last_active":"2023-10-09T05:02:35.005430+0000","last_peered":"2023-10-09T05:02:35.005430+0000","last_clean":"2023-10-09T05:02:35.005430+0000","last_became_active":"2023-10-09T05:02:33.078527+0000","last_became_peered":"2023-10-09T05:02:33.078527+0000","last_unstale":"2023-10-09T05:02:35.005430+0000","last_undegraded":"2023-10-09T05:02:35.005430+0000","last_fullsized":"2023-10-09T05:02:35.005430+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:58.887490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"594'5361","reported_seq":10655,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.014205+0000","last_change":"2023-10-09T05:02:19.884483+0000","last_active":"2023-10-09T05:02:35.014205+0000","last_peered":"2023-10-09T05:02:35.014205+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:02:19.838722+0000","last_became_peered":"2023-10-09T05:02:19.838722+0000","last_unstale":"2023-10-09T05:02:35.014205+0000","last_undegraded":"2023-10-09T05:02:19.827054+0000","last_fullsized":"2023-10-09T05:02:19.743072+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5361,"log_dups_size":0,"ondisk_log_size":5361,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127991808,"num_objects":1953,"num_object_clones":0,"num_object_copies":5859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3055,"num_objects_misplaced":1848,"num_objects_unfound":0,"num_objects_dirty":1953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2862,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1022,"num_bytes_recovered":62488576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[7,6],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":306},{"shards":"3,4,5,6,7","objects":178},{"shards":"3,4,5,7","objects":1},{"shards":"4,5,6,7","objects":1},{"shards":"4,5,7","objects":1359},{"shards":"6,7","objects":108}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":123,"reported_epoch":600,"state":"activating+remapped","last_fresh":"2023-10-09T05:02:21.938176+0000","last_change":"2023-10-09T05:02:20.766503+0000","last_active":"2023-10-09T05:02:18.746957+0000","last_peered":"2023-10-09T05:02:14.615364+0000","last_clean":"2023-10-09T05:02:14.615364+0000","last_became_active":"2023-10-09T05:01:56.002760+0000","last_became_peered":"2023-10-09T05:01:56.002760+0000","last_unstale":"2023-10-09T05:02:21.938176+0000","last_undegraded":"2023-10-09T05:02:21.938176+0000","last_fullsized":"2023-10-09T05:02:21.938176+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:09:19.933697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,3],"avail_no_missing":["4","0","1","3"],"object_location_counts":[{"shards":"0,3,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":633,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006848+0000","last_change":"2023-10-09T04:59:30.861892+0000","last_active":"2023-10-09T05:02:35.006848+0000","last_peered":"2023-10-09T05:02:35.006848+0000","last_clean":"2023-10-09T05:02:35.006848+0000","last_became_active":"2023-10-09T04:59:30.861571+0000","last_became_peered":"2023-10-09T04:59:30.861571+0000","last_unstale":"2023-10-09T05:02:35.006848+0000","last_undegraded":"2023-10-09T05:02:35.006848+0000","last_fullsized":"2023-10-09T05:02:35.006848+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:46:28.783652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"594'5427","reported_seq":7944,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.006807+0000","last_change":"2023-10-09T05:02:27.612963+0000","last_active":"2023-10-09T05:02:35.006807+0000","last_peered":"2023-10-09T05:02:35.006807+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:02:26.958228+0000","last_became_peered":"2023-10-09T05:02:26.958228+0000","last_unstale":"2023-10-09T05:02:35.006807+0000","last_undegraded":"2023-10-09T05:02:23.388536+0000","last_fullsized":"2023-10-09T05:02:22.182418+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5427,"log_dups_size":0,"ondisk_log_size":5427,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129040384,"num_objects":1969,"num_object_clones":0,"num_object_copies":5907,"num_objects_missing_on_primary":1503,"num_objects_missing":1503,"num_objects_degraded":5755,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2895,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":31064064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,3","objects":35},{"shards":"1,2,3,5,6,7","objects":17},{"shards":"2,3,5,6,7","objects":1},{"shards":"2,3,5,7","objects":368},{"shards":"2,5,7","objects":1503},{"shards":"3,5,6","objects":45}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":26,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006130+0000","last_change":"2023-10-09T05:02:19.869921+0000","last_active":"2023-10-09T05:02:35.006130+0000","last_peered":"2023-10-09T05:02:35.006130+0000","last_clean":"2023-10-09T05:02:35.006130+0000","last_became_active":"2023-10-09T05:02:19.753141+0000","last_became_peered":"2023-10-09T05:02:19.753141+0000","last_unstale":"2023-10-09T05:02:35.006130+0000","last_undegraded":"2023-10-09T05:02:35.006130+0000","last_fullsized":"2023-10-09T05:02:35.006130+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:18:12.724558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:32.995757+0000","last_change":"2023-10-09T05:02:19.787734+0000","last_active":"2023-10-09T05:02:32.995757+0000","last_peered":"2023-10-09T05:02:32.995757+0000","last_clean":"2023-10-09T05:02:32.995757+0000","last_became_active":"2023-10-09T05:02:19.787387+0000","last_became_peered":"2023-10-09T05:02:19.787387+0000","last_unstale":"2023-10-09T05:02:32.995757+0000","last_undegraded":"2023-10-09T05:02:32.995757+0000","last_fullsized":"2023-10-09T05:02:32.995757+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:39:01.200294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"594'5446","reported_seq":5826,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.022655+0000","last_change":"2023-10-09T05:01:57.440156+0000","last_active":"2023-10-09T05:02:35.022655+0000","last_peered":"2023-10-09T05:02:35.022655+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.430922+0000","last_became_peered":"2023-10-09T05:01:57.430922+0000","last_unstale":"2023-10-09T05:02:35.022655+0000","last_undegraded":"2023-10-09T05:01:57.022221+0000","last_fullsized":"2023-10-09T05:01:56.935258+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127696896,"num_objects":1948,"num_object_clones":0,"num_object_copies":5844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4787,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2914,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":1998848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,4,5","objects":1883},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":55},{"shards":"5,6","objects":9}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":226,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.019756+0000","last_change":"2023-10-09T04:59:31.025595+0000","last_active":"2023-10-09T05:02:35.019756+0000","last_peered":"2023-10-09T05:02:35.019756+0000","last_clean":"2023-10-09T05:02:35.019756+0000","last_became_active":"2023-10-09T04:59:30.946523+0000","last_became_peered":"2023-10-09T04:59:30.946523+0000","last_unstale":"2023-10-09T05:02:35.019756+0000","last_undegraded":"2023-10-09T05:02:35.019756+0000","last_fullsized":"2023-10-09T05:02:35.019756+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:49:05.945423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.018955+0000","last_change":"2023-10-09T05:02:19.825179+0000","last_active":"2023-10-09T05:02:35.018955+0000","last_peered":"2023-10-09T05:02:35.018955+0000","last_clean":"2023-10-09T05:02:35.018955+0000","last_became_active":"2023-10-09T05:02:19.824360+0000","last_became_peered":"2023-10-09T05:02:19.824360+0000","last_unstale":"2023-10-09T05:02:35.018955+0000","last_undegraded":"2023-10-09T05:02:35.018955+0000","last_fullsized":"2023-10-09T05:02:35.018955+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:36:04.515905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"594'5430","reported_seq":16059,"reported_epoch":607,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:02:35.002632+0000","last_change":"2023-10-09T05:02:20.783960+0000","last_active":"2023-10-09T05:02:35.002632+0000","last_peered":"2023-10-09T05:02:35.002632+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:20.776791+0000","last_became_peered":"2023-10-09T05:02:20.776791+0000","last_unstale":"2023-10-09T05:02:35.002632+0000","last_undegraded":"2023-10-09T05:02:20.740633+0000","last_fullsized":"2023-10-09T05:02:35.002632+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5430,"log_dups_size":0,"ondisk_log_size":5430,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2895,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3303,"num_bytes_recovered":206536704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1964}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":668,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.021864+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:02:35.021864+0000","last_peered":"2023-10-09T05:02:35.021864+0000","last_clean":"2023-10-09T05:02:35.021864+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:02:35.021864+0000","last_undegraded":"2023-10-09T05:02:35.021864+0000","last_fullsized":"2023-10-09T05:02:35.021864+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.987635+0000","last_change":"2023-10-09T05:02:20.548592+0000","last_active":"2023-10-09T05:02:34.987635+0000","last_peered":"2023-10-09T05:02:34.987635+0000","last_clean":"2023-10-09T05:02:34.987635+0000","last_became_active":"2023-10-09T05:02:20.548286+0000","last_became_peered":"2023-10-09T05:02:20.548286+0000","last_unstale":"2023-10-09T05:02:34.987635+0000","last_undegraded":"2023-10-09T05:02:34.987635+0000","last_fullsized":"2023-10-09T05:02:34.987635+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:10:06.022561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1176,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.007695+0000","last_change":"2023-10-09T05:02:22.054178+0000","last_active":"2023-10-09T05:02:35.007695+0000","last_peered":"2023-10-09T05:02:35.007695+0000","last_clean":"2023-10-09T05:02:35.007695+0000","last_became_active":"2023-10-09T05:02:22.053814+0000","last_became_peered":"2023-10-09T05:02:22.053814+0000","last_unstale":"2023-10-09T05:02:35.007695+0000","last_undegraded":"2023-10-09T05:02:35.007695+0000","last_fullsized":"2023-10-09T05:02:35.007695+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"594'5248","reported_seq":7364,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:33.006137+0000","last_change":"2023-10-09T05:02:29.432088+0000","last_active":"2023-10-09T05:02:33.006137+0000","last_peered":"2023-10-09T05:02:33.006137+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:02:27.986460+0000","last_became_peered":"2023-10-09T05:02:27.986460+0000","last_unstale":"2023-10-09T05:02:33.006137+0000","last_undegraded":"2023-10-09T05:02:23.236696+0000","last_fullsized":"2023-10-09T05:02:20.766210+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5248,"log_dups_size":0,"ondisk_log_size":5248,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126124032,"num_objects":1924,"num_object_clones":0,"num_object_copies":5772,"num_objects_missing_on_primary":827,"num_objects_missing":827,"num_objects_degraded":4290,"num_objects_misplaced":1412,"num_objects_unfound":0,"num_objects_dirty":1924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2791,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":434,"num_bytes_recovered":28442624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[4,0],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,4","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":1492}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":195,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006661+0000","last_change":"2023-10-09T05:01:56.161853+0000","last_active":"2023-10-09T05:02:35.006661+0000","last_peered":"2023-10-09T05:02:35.006661+0000","last_clean":"2023-10-09T05:02:35.006661+0000","last_became_active":"2023-10-09T05:01:56.075100+0000","last_became_peered":"2023-10-09T05:01:56.075100+0000","last_unstale":"2023-10-09T05:02:35.006661+0000","last_undegraded":"2023-10-09T05:02:35.006661+0000","last_fullsized":"2023-10-09T05:02:35.006661+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:26:50.128094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":62,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:33.832602+0000","last_change":"2023-10-09T04:58:57.210297+0000","last_active":"2023-10-09T05:02:33.832602+0000","last_peered":"2023-10-09T05:02:33.832602+0000","last_clean":"2023-10-09T05:02:33.832602+0000","last_became_active":"2023-10-09T04:58:57.209978+0000","last_became_peered":"2023-10-09T04:58:57.209978+0000","last_unstale":"2023-10-09T05:02:33.832602+0000","last_undegraded":"2023-10-09T05:02:33.832602+0000","last_fullsized":"2023-10-09T05:02:33.832602+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:48:58.875418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1166,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.007054+0000","last_change":"2023-10-09T05:02:20.800734+0000","last_active":"2023-10-09T05:02:35.007054+0000","last_peered":"2023-10-09T05:02:35.007054+0000","last_clean":"2023-10-09T05:02:14.583184+0000","last_became_active":"2023-10-09T05:02:20.800241+0000","last_became_peered":"2023-10-09T05:02:20.800241+0000","last_unstale":"2023-10-09T05:02:35.007054+0000","last_undegraded":"2023-10-09T05:02:20.759121+0000","last_fullsized":"2023-10-09T05:02:20.758712+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":588,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0],"avail_no_missing":["6","0","2","4","7"],"object_location_counts":[{"shards":"0,2,4,6,7","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"594'5423","reported_seq":14188,"reported_epoch":605,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:33.055996+0000","last_change":"2023-10-09T05:02:26.413854+0000","last_active":"2023-10-09T05:02:18.811242+0000","last_peered":"2023-10-09T05:02:14.602492+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:02:12.434052+0000","last_became_peered":"2023-10-09T05:02:12.434052+0000","last_unstale":"2023-10-09T05:02:33.055996+0000","last_undegraded":"2023-10-09T05:02:26.402283+0000","last_fullsized":"2023-10-09T05:02:21.938064+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5423,"log_dups_size":0,"ondisk_log_size":5423,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131432448,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":93,"num_objects_missing":93,"num_objects_degraded":2903,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2881,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2120,"num_bytes_recovered":138018816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,4,5","objects":1},{"shards":"0,4","objects":3},{"shards":"1,2,4,5","objects":1641},{"shards":"2,4,5","objects":360}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":134,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.018887+0000","last_change":"2023-10-09T05:02:19.826671+0000","last_active":"2023-10-09T05:02:35.018887+0000","last_peered":"2023-10-09T05:02:35.018887+0000","last_clean":"2023-10-09T05:02:35.018887+0000","last_became_active":"2023-10-09T05:02:19.824766+0000","last_became_peered":"2023-10-09T05:02:19.824766+0000","last_unstale":"2023-10-09T05:02:35.018887+0000","last_undegraded":"2023-10-09T05:02:35.018887+0000","last_fullsized":"2023-10-09T05:02:35.018887+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:25.304660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.005554+0000","last_change":"2023-10-09T05:02:19.858873+0000","last_active":"2023-10-09T05:02:35.005554+0000","last_peered":"2023-10-09T05:02:35.005554+0000","last_clean":"2023-10-09T05:02:35.005554+0000","last_became_active":"2023-10-09T05:02:19.858555+0000","last_became_peered":"2023-10-09T05:02:19.858555+0000","last_unstale":"2023-10-09T05:02:35.005554+0000","last_undegraded":"2023-10-09T05:02:35.005554+0000","last_fullsized":"2023-10-09T05:02:35.005554+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:38.232160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.4","version":"593'5433","reported_seq":7665,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.013327+0000","last_change":"2023-10-09T05:01:57.616295+0000","last_active":"2023-10-09T05:02:35.013327+0000","last_peered":"2023-10-09T05:02:35.013327+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:01:57.602806+0000","last_became_peered":"2023-10-09T05:01:57.602806+0000","last_unstale":"2023-10-09T05:02:35.013327+0000","last_undegraded":"2023-10-09T05:01:57.106192+0000","last_fullsized":"2023-10-09T05:01:56.914072+0000","mapping_epoch":586,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5433,"log_dups_size":0,"ondisk_log_size":5433,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128876544,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3532,"num_objects_misplaced":2451,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2893,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":463,"num_bytes_recovered":30375936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[7,3],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,3,6,7","objects":46},{"shards":"2,4,7","objects":1573},{"shards":"3,7","objects":-17}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:32.995608+0000","last_change":"2023-10-09T05:02:31.008778+0000","last_active":"2023-10-09T05:02:32.995608+0000","last_peered":"2023-10-09T05:02:32.995608+0000","last_clean":"2023-10-09T05:02:32.995608+0000","last_became_active":"2023-10-09T05:02:31.008446+0000","last_became_peered":"2023-10-09T05:02:31.008446+0000","last_unstale":"2023-10-09T05:02:32.995608+0000","last_undegraded":"2023-10-09T05:02:32.995608+0000","last_fullsized":"2023-10-09T05:02:32.995608+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:39:42.091985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":91,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:33.000501+0000","last_change":"2023-10-09T05:02:19.883254+0000","last_active":"2023-10-09T05:02:33.000501+0000","last_peered":"2023-10-09T05:02:33.000501+0000","last_clean":"2023-10-09T05:02:33.000501+0000","last_became_active":"2023-10-09T05:02:19.861061+0000","last_became_peered":"2023-10-09T05:02:19.861061+0000","last_unstale":"2023-10-09T05:02:33.000501+0000","last_undegraded":"2023-10-09T05:02:33.000501+0000","last_fullsized":"2023-10-09T05:02:33.000501+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:14:00.954058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1185,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.019112+0000","last_change":"2023-10-09T05:02:22.053863+0000","last_active":"2023-10-09T05:02:35.019112+0000","last_peered":"2023-10-09T05:02:35.019112+0000","last_clean":"2023-10-09T05:02:35.019112+0000","last_became_active":"2023-10-09T05:02:22.053472+0000","last_became_peered":"2023-10-09T05:02:22.053472+0000","last_unstale":"2023-10-09T05:02:35.019112+0000","last_undegraded":"2023-10-09T05:02:35.019112+0000","last_fullsized":"2023-10-09T05:02:35.019112+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"594'5428","reported_seq":14442,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:33.008222+0000","last_change":"2023-10-09T05:02:01.368060+0000","last_active":"2023-10-09T05:02:33.008222+0000","last_peered":"2023-10-09T05:02:33.008222+0000","last_clean":"2023-10-09T04:54:25.113241+0000","last_became_active":"2023-10-09T05:02:00.542229+0000","last_became_peered":"2023-10-09T05:02:00.542229+0000","last_unstale":"2023-10-09T05:02:33.008222+0000","last_undegraded":"2023-10-09T05:01:57.066288+0000","last_fullsized":"2023-10-09T05:01:56.915450+0000","mapping_epoch":586,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129302528,"num_objects":1973,"num_object_clones":0,"num_object_copies":5919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2765,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2888,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2000,"num_bytes_recovered":125960192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,3],"avail_no_missing":["4","5","7"],"object_location_counts":[{"shards":"0,4,5,7","objects":114},{"shards":"3,4","objects":508},{"shards":"3,4,5,7","objects":438},{"shards":"4,5,7","objects":913}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.001126+0000","last_change":"2023-10-09T05:02:19.782073+0000","last_active":"2023-10-09T05:02:35.001126+0000","last_peered":"2023-10-09T05:02:35.001126+0000","last_clean":"2023-10-09T05:02:35.001126+0000","last_became_active":"2023-10-09T05:02:19.781796+0000","last_became_peered":"2023-10-09T05:02:19.781796+0000","last_unstale":"2023-10-09T05:02:35.001126+0000","last_undegraded":"2023-10-09T05:02:35.001126+0000","last_fullsized":"2023-10-09T05:02:35.001126+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:38.154776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":17,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.007046+0000","last_change":"2023-10-09T05:02:32.058090+0000","last_active":"2023-10-09T05:02:35.007046+0000","last_peered":"2023-10-09T05:02:35.007046+0000","last_clean":"2023-10-09T05:02:35.007046+0000","last_became_active":"2023-10-09T05:02:23.725374+0000","last_became_peered":"2023-10-09T05:02:23.725374+0000","last_unstale":"2023-10-09T05:02:35.007046+0000","last_undegraded":"2023-10-09T05:02:35.007046+0000","last_fullsized":"2023-10-09T05:02:35.007046+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:14:21.185079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1191,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.008218+0000","last_change":"2023-10-09T05:02:34.610826+0000","last_active":"2023-10-09T05:02:35.008218+0000","last_peered":"2023-10-09T05:02:35.008218+0000","last_clean":"2023-10-09T05:02:35.008218+0000","last_became_active":"2023-10-09T05:02:34.610472+0000","last_became_peered":"2023-10-09T05:02:34.610472+0000","last_unstale":"2023-10-09T05:02:35.008218+0000","last_undegraded":"2023-10-09T05:02:35.008218+0000","last_fullsized":"2023-10-09T05:02:35.008218+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"594'5578","reported_seq":10290,"reported_epoch":608,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:36.375952+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:02:36.375952+0000","last_peered":"2023-10-09T05:02:36.375952+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:02:36.375952+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":1530,"num_objects_missing":1530,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2969,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1380,"num_bytes_recovered":90439680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,3,5,7","objects":907},{"shards":"0,3","objects":-26},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":738}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.989509+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:02:34.989509+0000","last_peered":"2023-10-09T05:02:34.989509+0000","last_clean":"2023-10-09T05:02:34.989509+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:02:34.989509+0000","last_undegraded":"2023-10-09T05:02:34.989509+0000","last_fullsized":"2023-10-09T05:02:34.989509+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":555,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.004246+0000","last_change":"2023-10-09T05:02:31.187532+0000","last_active":"2023-10-09T05:02:35.004246+0000","last_peered":"2023-10-09T05:02:35.004246+0000","last_clean":"2023-10-09T05:02:35.004246+0000","last_became_active":"2023-10-09T05:02:31.187141+0000","last_became_peered":"2023-10-09T05:02:31.187141+0000","last_unstale":"2023-10-09T05:02:35.004246+0000","last_undegraded":"2023-10-09T05:02:35.004246+0000","last_fullsized":"2023-10-09T05:02:35.004246+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:48:47.219588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.987135+0000","last_change":"2023-10-09T05:02:19.859835+0000","last_active":"2023-10-09T05:02:34.987135+0000","last_peered":"2023-10-09T05:02:34.987135+0000","last_clean":"2023-10-09T05:02:34.987135+0000","last_became_active":"2023-10-09T05:02:19.859407+0000","last_became_peered":"2023-10-09T05:02:19.859407+0000","last_unstale":"2023-10-09T05:02:34.987135+0000","last_undegraded":"2023-10-09T05:02:34.987135+0000","last_fullsized":"2023-10-09T05:02:34.987135+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:39:10.526299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1174,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.007617+0000","last_change":"2023-10-09T05:02:22.053237+0000","last_active":"2023-10-09T05:02:35.007617+0000","last_peered":"2023-10-09T05:02:35.007617+0000","last_clean":"2023-10-09T05:02:35.007617+0000","last_became_active":"2023-10-09T05:02:22.052740+0000","last_became_peered":"2023-10-09T05:02:22.052740+0000","last_unstale":"2023-10-09T05:02:35.007617+0000","last_undegraded":"2023-10-09T05:02:35.007617+0000","last_fullsized":"2023-10-09T05:02:35.007617+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"594'5704","reported_seq":16372,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.014438+0000","last_change":"2023-10-09T05:02:34.204045+0000","last_active":"2023-10-09T05:02:35.014438+0000","last_peered":"2023-10-09T05:02:35.014438+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:02:34.140170+0000","last_became_peered":"2023-10-09T05:02:34.140170+0000","last_unstale":"2023-10-09T05:02:35.014438+0000","last_undegraded":"2023-10-09T05:02:34.126620+0000","last_fullsized":"2023-10-09T05:02:33.982844+0000","mapping_epoch":605,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4023,"log_dups_size":0,"ondisk_log_size":4023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:39.890935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254279680,"num_objects":3880,"num_object_clones":0,"num_object_copies":11640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3880,"num_objects_misplaced":4019,"num_objects_unfound":0,"num_objects_dirty":3880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4796,"num_bytes_recovered":305135616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,3,4,6","objects":170},{"shards":"2,4,6","objects":111},{"shards":"2,4,6,7","objects":2894},{"shards":"2,6","objects":705}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":692,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.021601+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:02:35.021601+0000","last_peered":"2023-10-09T05:02:35.021601+0000","last_clean":"2023-10-09T05:02:35.021601+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:02:35.021601+0000","last_undegraded":"2023-10-09T05:02:35.021601+0000","last_fullsized":"2023-10-09T05:02:35.021601+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T05:02:20.544138+0000","last_change":"2023-10-09T05:01:55.929530+0000","last_active":"2023-10-09T05:02:20.544138+0000","last_peered":"2023-10-09T05:02:20.544138+0000","last_clean":"2023-10-09T05:02:20.544138+0000","last_became_active":"2023-10-09T05:01:55.929193+0000","last_became_peered":"2023-10-09T05:01:55.929193+0000","last_unstale":"2023-10-09T05:02:20.544138+0000","last_undegraded":"2023-10-09T05:02:20.544138+0000","last_fullsized":"2023-10-09T05:02:20.544138+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:22:55.763897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.989438+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:02:34.989438+0000","last_peered":"2023-10-09T05:02:34.989438+0000","last_clean":"2023-10-09T05:02:34.989438+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:02:34.989438+0000","last_undegraded":"2023-10-09T05:02:34.989438+0000","last_fullsized":"2023-10-09T05:02:34.989438+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1185,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.019027+0000","last_change":"2023-10-09T05:02:22.053139+0000","last_active":"2023-10-09T05:02:35.019027+0000","last_peered":"2023-10-09T05:02:35.019027+0000","last_clean":"2023-10-09T05:02:35.019027+0000","last_became_active":"2023-10-09T05:02:22.052703+0000","last_became_peered":"2023-10-09T05:02:22.052703+0000","last_unstale":"2023-10-09T05:02:35.019027+0000","last_undegraded":"2023-10-09T05:02:35.019027+0000","last_fullsized":"2023-10-09T05:02:35.019027+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"594'5717","reported_seq":8595,"reported_epoch":608,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:02:40.374684+0000","last_change":"2023-10-09T05:02:20.072128+0000","last_active":"2023-10-09T05:02:40.374684+0000","last_peered":"2023-10-09T05:02:40.374684+0000","last_clean":"2023-10-09T04:54:44.521216+0000","last_became_active":"2023-10-09T04:59:07.457064+0000","last_became_peered":"2023-10-09T04:59:07.457064+0000","last_unstale":"2023-10-09T05:02:40.374684+0000","last_undegraded":"2023-10-09T05:02:40.374684+0000","last_fullsized":"2023-10-09T04:58:57.981736+0000","mapping_epoch":538,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3424,"log_dups_size":0,"ondisk_log_size":3424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:50:35.050351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935064,"num_objects":3891,"num_object_clones":0,"num_object_copies":11673,"num_objects_missing_on_primary":3358,"num_objects_missing":3358,"num_objects_degraded":0,"num_objects_misplaced":3284,"num_objects_unfound":0,"num_objects_dirty":3891,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5717,"num_write_kb":307393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1323,"num_bytes_recovered":86704128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":3284},{"shards":"2,4,5,6","objects":1},{"shards":"5,6","objects":606}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":101,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.002015+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:02:35.002015+0000","last_peered":"2023-10-09T05:02:35.002015+0000","last_clean":"2023-10-09T05:02:35.002015+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:02:35.002015+0000","last_undegraded":"2023-10-09T05:02:35.002015+0000","last_fullsized":"2023-10-09T05:02:35.002015+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":20,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.520073+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:02:35.520073+0000","last_peered":"2023-10-09T05:02:35.520073+0000","last_clean":"2023-10-09T05:02:35.520073+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:02:35.520073+0000","last_undegraded":"2023-10-09T05:02:35.520073+0000","last_fullsized":"2023-10-09T05:02:35.520073+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":17,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006999+0000","last_change":"2023-10-09T05:02:32.056691+0000","last_active":"2023-10-09T05:02:35.006999+0000","last_peered":"2023-10-09T05:02:35.006999+0000","last_clean":"2023-10-09T05:02:35.006999+0000","last_became_active":"2023-10-09T05:02:23.721411+0000","last_became_peered":"2023-10-09T05:02:23.721411+0000","last_unstale":"2023-10-09T05:02:35.006999+0000","last_undegraded":"2023-10-09T05:02:35.006999+0000","last_fullsized":"2023-10-09T05:02:35.006999+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:49.296331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"594'5651","reported_seq":8983,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.022731+0000","last_change":"2023-10-09T05:02:33.025727+0000","last_active":"2023-10-09T05:02:35.022731+0000","last_peered":"2023-10-09T05:02:35.022731+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:02:32.120012+0000","last_became_peered":"2023-10-09T05:02:32.120012+0000","last_unstale":"2023-10-09T05:02:35.022731+0000","last_undegraded":"2023-10-09T05:02:23.650910+0000","last_fullsized":"2023-10-09T05:02:19.732771+0000","mapping_epoch":596,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3524,"log_dups_size":0,"ondisk_log_size":3524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:28:56.646812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256049152,"num_objects":3907,"num_object_clones":0,"num_object_copies":11721,"num_objects_missing_on_primary":4052,"num_objects_missing":4052,"num_objects_degraded":7579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5651,"num_write_kb":305856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1513,"num_bytes_recovered":99090432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[1,3],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,5,7","objects":3130},{"shards":"3,5,7","objects":776},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":408,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.015570+0000","last_change":"2023-10-09T05:02:31.012455+0000","last_active":"2023-10-09T05:02:35.015570+0000","last_peered":"2023-10-09T05:02:35.015570+0000","last_clean":"2023-10-09T05:02:35.015570+0000","last_became_active":"2023-10-09T05:02:31.012141+0000","last_became_peered":"2023-10-09T05:02:31.012141+0000","last_unstale":"2023-10-09T05:02:35.015570+0000","last_undegraded":"2023-10-09T05:02:35.015570+0000","last_fullsized":"2023-10-09T05:02:35.015570+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:36:07.519541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1191,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006920+0000","last_change":"2023-10-09T05:02:22.053523+0000","last_active":"2023-10-09T05:02:35.006920+0000","last_peered":"2023-10-09T05:02:35.006920+0000","last_clean":"2023-10-09T05:02:35.006920+0000","last_became_active":"2023-10-09T05:02:22.053139+0000","last_became_peered":"2023-10-09T05:02:22.053139+0000","last_unstale":"2023-10-09T05:02:35.006920+0000","last_undegraded":"2023-10-09T05:02:35.006920+0000","last_fullsized":"2023-10-09T05:02:35.006920+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":75,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.003188+0000","last_change":"2023-10-09T04:59:30.856383+0000","last_active":"2023-10-09T05:02:35.003188+0000","last_peered":"2023-10-09T05:02:35.003188+0000","last_clean":"2023-10-09T05:02:35.003188+0000","last_became_active":"2023-10-09T04:59:30.855682+0000","last_became_peered":"2023-10-09T04:59:30.855682+0000","last_unstale":"2023-10-09T05:02:35.003188+0000","last_undegraded":"2023-10-09T05:02:35.003188+0000","last_fullsized":"2023-10-09T05:02:35.003188+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:08:19.088239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":112,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.020016+0000","last_change":"2023-10-09T04:58:56.999352+0000","last_active":"2023-10-09T05:02:35.020016+0000","last_peered":"2023-10-09T05:02:35.020016+0000","last_clean":"2023-10-09T05:02:35.020016+0000","last_became_active":"2023-10-09T04:58:56.999048+0000","last_became_peered":"2023-10-09T04:58:56.999048+0000","last_unstale":"2023-10-09T05:02:35.020016+0000","last_undegraded":"2023-10-09T05:02:35.020016+0000","last_fullsized":"2023-10-09T05:02:35.020016+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:16:53.056597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1164,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.008074+0000","last_change":"2023-10-09T05:02:23.412504+0000","last_active":"2023-10-09T05:02:35.008074+0000","last_peered":"2023-10-09T05:02:35.008074+0000","last_clean":"2023-10-09T05:02:35.008074+0000","last_became_active":"2023-10-09T05:02:23.411959+0000","last_became_peered":"2023-10-09T05:02:23.411959+0000","last_unstale":"2023-10-09T05:02:35.008074+0000","last_undegraded":"2023-10-09T05:02:35.008074+0000","last_fullsized":"2023-10-09T05:02:35.008074+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"583'5794","reported_seq":10957,"reported_epoch":606,"state":"remapped+peering","last_fresh":"2023-10-09T05:01:55.573077+0000","last_change":"2023-10-09T05:01:55.573077+0000","last_active":"2023-10-09T05:01:55.572897+0000","last_peered":"2023-10-09T05:01:53.538479+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T04:58:59.452514+0000","last_became_peered":"2023-10-09T04:58:59.452514+0000","last_unstale":"2023-10-09T05:01:55.573077+0000","last_undegraded":"2023-10-09T05:01:55.573077+0000","last_fullsized":"2023-10-09T05:01:55.573077+0000","mapping_epoch":606,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4031,"log_dups_size":0,"ondisk_log_size":4031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:17:58.482495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259260416,"num_objects":3956,"num_object_clones":0,"num_object_copies":11868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5794,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2263,"num_bytes_recovered":129499136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.520151+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:02:35.520151+0000","last_peered":"2023-10-09T05:02:35.520151+0000","last_clean":"2023-10-09T05:02:35.520151+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:02:35.520151+0000","last_undegraded":"2023-10-09T05:02:35.520151+0000","last_fullsized":"2023-10-09T05:02:35.520151+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":83,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.002098+0000","last_change":"2023-10-09T05:01:55.962359+0000","last_active":"2023-10-09T05:02:35.002098+0000","last_peered":"2023-10-09T05:02:35.002098+0000","last_clean":"2023-10-09T05:02:35.002098+0000","last_became_active":"2023-10-09T05:01:55.961843+0000","last_became_peered":"2023-10-09T05:01:55.961843+0000","last_unstale":"2023-10-09T05:02:35.002098+0000","last_undegraded":"2023-10-09T05:02:35.002098+0000","last_fullsized":"2023-10-09T05:02:35.002098+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:43:45.973049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.025860+0000","last_change":"2023-10-09T05:02:19.794966+0000","last_active":"2023-10-09T05:02:35.025860+0000","last_peered":"2023-10-09T05:02:35.025860+0000","last_clean":"2023-10-09T05:02:35.025860+0000","last_became_active":"2023-10-09T05:02:19.794637+0000","last_became_peered":"2023-10-09T05:02:19.794637+0000","last_unstale":"2023-10-09T05:02:35.025860+0000","last_undegraded":"2023-10-09T05:02:35.025860+0000","last_fullsized":"2023-10-09T05:02:35.025860+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:11:29.452702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1185,"reported_epoch":607,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-09T05:02:35.008431+0000","last_change":"2023-10-09T05:02:20.801449+0000","last_active":"2023-10-09T05:02:35.008431+0000","last_peered":"2023-10-09T05:02:35.008431+0000","last_clean":"2023-10-09T05:02:14.599067+0000","last_became_active":"2023-10-09T05:02:20.801174+0000","last_became_peered":"2023-10-09T05:02:20.801174+0000","last_unstale":"2023-10-09T05:02:35.008431+0000","last_undegraded":"2023-10-09T05:02:35.008431+0000","last_fullsized":"2023-10-09T05:02:35.008431+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,2],"avail_no_missing":["6","0","1","2","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"594'5441","reported_seq":17331,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.022098+0000","last_change":"2023-10-09T05:01:45.610039+0000","last_active":"2023-10-09T05:02:35.022098+0000","last_peered":"2023-10-09T05:02:35.022098+0000","last_clean":"2023-10-09T05:02:35.022098+0000","last_became_active":"2023-10-09T05:01:45.604097+0000","last_became_peered":"2023-10-09T05:01:45.604097+0000","last_unstale":"2023-10-09T05:02:35.022098+0000","last_undegraded":"2023-10-09T05:02:35.022098+0000","last_fullsized":"2023-10-09T05:02:35.022098+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130187264,"num_objects":1986,"num_object_clones":0,"num_object_copies":5958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2902,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2945,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":98,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:32.996716+0000","last_change":"2023-10-09T05:01:55.924505+0000","last_active":"2023-10-09T05:02:32.996716+0000","last_peered":"2023-10-09T05:02:32.996716+0000","last_clean":"2023-10-09T05:02:32.996716+0000","last_became_active":"2023-10-09T05:01:55.924173+0000","last_became_peered":"2023-10-09T05:01:55.924173+0000","last_unstale":"2023-10-09T05:02:32.996716+0000","last_undegraded":"2023-10-09T05:02:32.996716+0000","last_fullsized":"2023-10-09T05:02:32.996716+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:56.093201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.989284+0000","last_change":"2023-10-09T05:02:33.837560+0000","last_active":"2023-10-09T05:02:34.989284+0000","last_peered":"2023-10-09T05:02:34.989284+0000","last_clean":"2023-10-09T05:02:34.989284+0000","last_became_active":"2023-10-09T05:02:33.837357+0000","last_became_peered":"2023-10-09T05:02:33.837357+0000","last_unstale":"2023-10-09T05:02:34.989284+0000","last_undegraded":"2023-10-09T05:02:34.989284+0000","last_fullsized":"2023-10-09T05:02:34.989284+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:18.369260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.987706+0000","last_change":"2023-10-09T05:02:19.770561+0000","last_active":"2023-10-09T05:02:34.987706+0000","last_peered":"2023-10-09T05:02:34.987706+0000","last_clean":"2023-10-09T05:02:34.987706+0000","last_became_active":"2023-10-09T05:02:19.770296+0000","last_became_peered":"2023-10-09T05:02:19.770296+0000","last_unstale":"2023-10-09T05:02:34.987706+0000","last_undegraded":"2023-10-09T05:02:34.987706+0000","last_fullsized":"2023-10-09T05:02:34.987706+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:48:23.611957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1189,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.019942+0000","last_change":"2023-10-09T05:02:22.053490+0000","last_active":"2023-10-09T05:02:35.019942+0000","last_peered":"2023-10-09T05:02:35.019942+0000","last_clean":"2023-10-09T05:02:35.019942+0000","last_became_active":"2023-10-09T05:02:22.053095+0000","last_became_peered":"2023-10-09T05:02:22.053095+0000","last_unstale":"2023-10-09T05:02:35.019942+0000","last_undegraded":"2023-10-09T05:02:35.019942+0000","last_fullsized":"2023-10-09T05:02:35.019942+0000","mapping_epoch":599,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":600,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"594'5630","reported_seq":22916,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.021464+0000","last_change":"2023-10-09T05:02:20.783779+0000","last_active":"2023-10-09T05:02:35.021464+0000","last_peered":"2023-10-09T05:02:35.021464+0000","last_clean":"2023-10-09T05:02:35.021464+0000","last_became_active":"2023-10-09T05:02:20.771939+0000","last_became_peered":"2023-10-09T05:02:20.771939+0000","last_unstale":"2023-10-09T05:02:35.021464+0000","last_undegraded":"2023-10-09T05:02:35.021464+0000","last_fullsized":"2023-10-09T05:02:35.021464+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5230,"log_dups_size":0,"ondisk_log_size":5230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:11:02.708271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256376832,"num_objects":3912,"num_object_clones":0,"num_object_copies":11736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5630,"num_write_kb":305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7908,"num_bytes_recovered":507379712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":226,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.005830+0000","last_change":"2023-10-09T05:01:56.154483+0000","last_active":"2023-10-09T05:02:35.005830+0000","last_peered":"2023-10-09T05:02:35.005830+0000","last_clean":"2023-10-09T05:02:35.005830+0000","last_became_active":"2023-10-09T05:01:56.045082+0000","last_became_peered":"2023-10-09T05:01:56.045082+0000","last_unstale":"2023-10-09T05:02:35.005830+0000","last_undegraded":"2023-10-09T05:02:35.005830+0000","last_fullsized":"2023-10-09T05:02:35.005830+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:00:02.707936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.006055+0000","last_change":"2023-10-09T05:02:19.837313+0000","last_active":"2023-10-09T05:02:35.006055+0000","last_peered":"2023-10-09T05:02:35.006055+0000","last_clean":"2023-10-09T05:02:35.006055+0000","last_became_active":"2023-10-09T05:02:19.830796+0000","last_became_peered":"2023-10-09T05:02:19.830796+0000","last_unstale":"2023-10-09T05:02:35.006055+0000","last_undegraded":"2023-10-09T05:02:35.006055+0000","last_fullsized":"2023-10-09T05:02:35.006055+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:26:10.552707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.001030+0000","last_change":"2023-10-09T05:02:19.765434+0000","last_active":"2023-10-09T05:02:35.001030+0000","last_peered":"2023-10-09T05:02:35.001030+0000","last_clean":"2023-10-09T05:02:35.001030+0000","last_became_active":"2023-10-09T05:02:19.764690+0000","last_became_peered":"2023-10-09T05:02:19.764690+0000","last_unstale":"2023-10-09T05:02:35.001030+0000","last_undegraded":"2023-10-09T05:02:35.001030+0000","last_fullsized":"2023-10-09T05:02:35.001030+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:13:00.604847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1143,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.015382+0000","last_change":"2023-10-09T05:02:33.835658+0000","last_active":"2023-10-09T05:02:35.015382+0000","last_peered":"2023-10-09T05:02:35.015382+0000","last_clean":"2023-10-09T05:02:35.015382+0000","last_became_active":"2023-10-09T05:02:33.835276+0000","last_became_peered":"2023-10-09T05:02:33.835276+0000","last_unstale":"2023-10-09T05:02:35.015382+0000","last_undegraded":"2023-10-09T05:02:35.015382+0000","last_fullsized":"2023-10-09T05:02:35.015382+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1466069,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"574'5902","reported_seq":9308,"reported_epoch":598,"state":"remapped+peering","last_fresh":"2023-10-09T05:02:20.806835+0000","last_change":"2023-10-09T05:02:20.767548+0000","last_active":"2023-10-09T05:01:42.654520+0000","last_peered":"2023-10-09T05:01:42.599792+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T04:59:55.037735+0000","last_became_peered":"2023-10-09T04:59:55.037735+0000","last_unstale":"2023-10-09T05:02:20.806835+0000","last_undegraded":"2023-10-09T05:02:20.806835+0000","last_fullsized":"2023-10-09T05:02:20.806835+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:19:43.613630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265945088,"num_objects":4058,"num_object_clones":0,"num_object_copies":12174,"num_objects_missing_on_primary":1048,"num_objects_missing":1048,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5902,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1489,"num_bytes_recovered":97583104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.008461+0000","last_change":"2023-10-09T05:02:33.078050+0000","last_active":"2023-10-09T05:02:35.008461+0000","last_peered":"2023-10-09T05:02:35.008461+0000","last_clean":"2023-10-09T05:02:35.008461+0000","last_became_active":"2023-10-09T05:02:33.077860+0000","last_became_peered":"2023-10-09T05:02:33.077860+0000","last_unstale":"2023-10-09T05:02:35.008461+0000","last_undegraded":"2023-10-09T05:02:35.008461+0000","last_fullsized":"2023-10-09T05:02:35.008461+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:51.707414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":26,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.005955+0000","last_change":"2023-10-09T05:02:19.890835+0000","last_active":"2023-10-09T05:02:35.005955+0000","last_peered":"2023-10-09T05:02:35.005955+0000","last_clean":"2023-10-09T05:02:35.005955+0000","last_became_active":"2023-10-09T05:02:19.831307+0000","last_became_peered":"2023-10-09T05:02:19.831307+0000","last_unstale":"2023-10-09T05:02:35.005955+0000","last_undegraded":"2023-10-09T05:02:35.005955+0000","last_fullsized":"2023-10-09T05:02:35.005955+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:14:04.353224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:32.995696+0000","last_change":"2023-10-09T05:01:55.929880+0000","last_active":"2023-10-09T05:02:32.995696+0000","last_peered":"2023-10-09T05:02:32.995696+0000","last_clean":"2023-10-09T05:02:32.995696+0000","last_became_active":"2023-10-09T05:01:55.929584+0000","last_became_peered":"2023-10-09T05:01:55.929584+0000","last_unstale":"2023-10-09T05:02:32.995696+0000","last_undegraded":"2023-10-09T05:02:32.995696+0000","last_fullsized":"2023-10-09T05:02:32.995696+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:42:09.226353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"594'5274","reported_seq":7504,"reported_epoch":605,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:33.006722+0000","last_change":"2023-10-09T05:02:32.630440+0000","last_active":"2023-10-09T05:02:33.006722+0000","last_peered":"2023-10-09T05:02:33.006722+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:02:31.252549+0000","last_became_peered":"2023-10-09T05:02:31.252549+0000","last_unstale":"2023-10-09T05:02:33.006722+0000","last_undegraded":"2023-10-09T05:02:26.334279+0000","last_fullsized":"2023-10-09T05:02:23.261195+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5274,"log_dups_size":0,"ondisk_log_size":5274,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124420096,"num_objects":1899,"num_object_clones":0,"num_object_copies":5697,"num_objects_missing_on_primary":843,"num_objects_missing":843,"num_objects_degraded":4394,"num_objects_misplaced":1481,"num_objects_unfound":0,"num_objects_dirty":1899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2818,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":460,"num_bytes_recovered":30146560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[4,0],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,4","objects":-36},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":1548}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":84,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-10-09T05:02:20.544581+0000","last_change":"2023-10-09T05:01:55.925656+0000","last_active":"2023-10-09T05:02:20.544581+0000","last_peered":"2023-10-09T05:02:20.544581+0000","last_clean":"2023-10-09T05:02:20.544581+0000","last_became_active":"2023-10-09T05:01:55.923643+0000","last_became_peered":"2023-10-09T05:01:55.923643+0000","last_unstale":"2023-10-09T05:02:20.544581+0000","last_undegraded":"2023-10-09T05:02:20.544581+0000","last_fullsized":"2023-10-09T05:02:20.544581+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:07:53.361762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:34.989365+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:02:34.989365+0000","last_peered":"2023-10-09T05:02:34.989365+0000","last_clean":"2023-10-09T05:02:34.989365+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:02:34.989365+0000","last_undegraded":"2023-10-09T05:02:34.989365+0000","last_fullsized":"2023-10-09T05:02:34.989365+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:31:45.877723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"594'5387","reported_seq":10168,"reported_epoch":607,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:02:35.002293+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:02:35.002293+0000","last_peered":"2023-10-09T05:02:35.002293+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:02:35.002293+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:02:35.002293+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131530752,"num_objects":2007,"num_object_clones":0,"num_object_copies":6021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2852,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":2007}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":721,"reported_epoch":607,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:02:35.021969+0000","last_change":"2023-10-09T05:02:33.923362+0000","last_active":"2023-10-09T05:02:35.021969+0000","last_peered":"2023-10-09T05:02:35.021969+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:02:33.923078+0000","last_became_peered":"2023-10-09T05:02:33.923078+0000","last_unstale":"2023-10-09T05:02:35.021969+0000","last_undegraded":"2023-10-09T05:02:33.919102+0000","last_fullsized":"2023-10-09T05:02:35.021969+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:08:53.134445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":399,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.015662+0000","last_change":"2023-10-09T05:02:19.776474+0000","last_active":"2023-10-09T05:02:35.015662+0000","last_peered":"2023-10-09T05:02:35.015662+0000","last_clean":"2023-10-09T05:02:35.015662+0000","last_became_active":"2023-10-09T05:02:19.776193+0000","last_became_peered":"2023-10-09T05:02:19.776193+0000","last_unstale":"2023-10-09T05:02:35.015662+0000","last_undegraded":"2023-10-09T05:02:35.015662+0000","last_fullsized":"2023-10-09T05:02:35.015662+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:09.746053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.10","version":"594'5429","reported_seq":6012,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:02:35.024215+0000","last_change":"2023-10-09T05:02:19.767791+0000","last_active":"2023-10-09T05:02:35.024215+0000","last_peered":"2023-10-09T05:02:35.024215+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:02:35.024215+0000","last_undegraded":"2023-10-09T05:02:19.765931+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4891,"log_dups_size":0,"ondisk_log_size":4891,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128811008,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4666,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2898,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":6389760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":1841},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":59}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":672,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.021652+0000","last_change":"2023-10-09T04:59:30.019132+0000","last_active":"2023-10-09T05:02:35.021652+0000","last_peered":"2023-10-09T05:02:35.021652+0000","last_clean":"2023-10-09T05:02:35.021652+0000","last_became_active":"2023-10-09T04:58:56.996966+0000","last_became_peered":"2023-10-09T04:58:56.996966+0000","last_unstale":"2023-10-09T05:02:35.021652+0000","last_undegraded":"2023-10-09T05:02:35.021652+0000","last_fullsized":"2023-10-09T05:02:35.021652+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":545,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:05:43.116063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":236,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.003512+0000","last_change":"2023-10-09T05:01:56.141384+0000","last_active":"2023-10-09T05:02:35.003512+0000","last_peered":"2023-10-09T05:02:35.003512+0000","last_clean":"2023-10-09T05:02:35.003512+0000","last_became_active":"2023-10-09T05:01:56.134812+0000","last_became_peered":"2023-10-09T05:01:56.134812+0000","last_unstale":"2023-10-09T05:02:35.003512+0000","last_undegraded":"2023-10-09T05:02:35.003512+0000","last_fullsized":"2023-10-09T05:02:35.003512+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:10:45.812714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"594'5404","reported_seq":8694,"reported_epoch":608,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:02:41.990665+0000","last_change":"2023-10-09T05:02:30.477225+0000","last_active":"2023-10-09T05:02:41.990665+0000","last_peered":"2023-10-09T05:02:41.990665+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:02:26.964295+0000","last_became_peered":"2023-10-09T05:02:26.964295+0000","last_unstale":"2023-10-09T05:02:41.990665+0000","last_undegraded":"2023-10-09T05:02:41.990665+0000","last_fullsized":"2023-10-09T05:02:20.731812+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130547712,"num_objects":1992,"num_object_clones":0,"num_object_copies":5976,"num_objects_missing_on_primary":1380,"num_objects_missing":1380,"num_objects_degraded":0,"num_objects_misplaced":4932,"num_objects_unfound":0,"num_objects_dirty":1992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2872,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":671,"num_bytes_recovered":43909120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,3","objects":216},{"shards":"2,3,5,7","objects":347},{"shards":"2,5,7","objects":1380},{"shards":"3,5,6","objects":49}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":227,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.996487+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:02:35.996487+0000","last_peered":"2023-10-09T05:02:35.996487+0000","last_clean":"2023-10-09T05:02:35.996487+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:02:35.996487+0000","last_undegraded":"2023-10-09T05:02:35.996487+0000","last_fullsized":"2023-10-09T05:02:35.996487+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":226,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.002888+0000","last_change":"2023-10-09T05:02:19.789771+0000","last_active":"2023-10-09T05:02:35.002888+0000","last_peered":"2023-10-09T05:02:35.002888+0000","last_clean":"2023-10-09T05:02:35.002888+0000","last_became_active":"2023-10-09T05:02:19.789260+0000","last_became_peered":"2023-10-09T05:02:19.789260+0000","last_unstale":"2023-10-09T05:02:35.002888+0000","last_undegraded":"2023-10-09T05:02:35.002888+0000","last_fullsized":"2023-10-09T05:02:35.002888+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:35:41.421645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"594'5329","reported_seq":16878,"reported_epoch":607,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:02:35.001210+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:02:35.001210+0000","last_peered":"2023-10-09T05:02:35.001210+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:02:35.001210+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:02:35.001210+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5329,"log_dups_size":0,"ondisk_log_size":5329,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130088960,"num_objects":1985,"num_object_clones":0,"num_object_copies":5955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2830,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":1985}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":617,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-10-09T05:02:35.008585+0000","last_change":"2023-10-09T05:01:56.147695+0000","last_active":"2023-10-09T05:02:35.008585+0000","last_peered":"2023-10-09T05:02:35.008585+0000","last_clean":"2023-10-09T05:02:35.008585+0000","last_became_active":"2023-10-09T05:01:55.946291+0000","last_became_peered":"2023-10-09T05:01:55.946291+0000","last_unstale":"2023-10-09T05:02:35.008585+0000","last_undegraded":"2023-10-09T05:02:35.008585+0000","last_fullsized":"2023-10-09T05:02:35.008585+0000","mapping_epoch":585,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:11:59.083190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-10-09T05:02:32.995490+0000","last_change":"2023-10-09T05:02:19.788118+0000","last_active":"2023-10-09T05:02:32.995490+0000","last_peered":"2023-10-09T05:02:32.995490+0000","last_clean":"2023-10-09T05:02:32.995490+0000","last_became_active":"2023-10-09T05:02:19.787826+0000","last_became_peered":"2023-10-09T05:02:19.787826+0000","last_unstale":"2023-10-09T05:02:32.995490+0000","last_undegraded":"2023-10-09T05:02:32.995490+0000","last_fullsized":"2023-10-09T05:02:32.995490+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:36:52.237527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":4138205208,"num_objects":63145,"num_object_clones":0,"num_object_copies":189435,"num_objects_missing_on_primary":16812,"num_objects_missing":16812,"num_objects_degraded":64319,"num_objects_misplaced":20587,"num_objects_unfound":0,"num_objects_dirty":63145,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":92073,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48290,"num_bytes_recovered":3073245184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16477929472,"data_stored":16477913184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131350,"ondisk_log_size":131350,"up":78,"acting":58,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":31822,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31051,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":6768058,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234664,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3694016,"kb_used_data":2188856,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":90051136,"statfs":{"total":95995035648,"available":92212363264,"internally_reserved":0,"allocated":2241388544,"data_stored":2239915307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:02:12 2023","interfaces":[{"interface":"back","average":{"1min":3.3210000000000002,"5min":1.0429999999999999,"15min":0.999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":33.402000000000001,"5min":33.402000000000001,"15min":33.402000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":4.2210000000000001,"5min":1.2929999999999999,"15min":1.4239999999999999},"min":{"1min":0.31,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":47.482999999999997,"5min":47.482999999999997,"15min":47.482999999999997},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:02:12 2023","interfaces":[{"interface":"back","average":{"1min":5.1639999999999997,"5min":1.488,"15min":2.669},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.246},"max":{"1min":43.158000000000001,"5min":43.158000000000001,"15min":128.601},"last":0.38},{"interface":"front","average":{"1min":4.1790000000000003,"5min":1.425,"15min":2.6419999999999999},"min":{"1min":0.35999999999999999,"5min":0.29699999999999999,"15min":0.26900000000000002},"max":{"1min":33.167000000000002,"5min":33.167000000000002,"15min":128.381},"last":0.44500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:01:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":3.0139999999999998,"15min":5.0359999999999996},"min":{"1min":0.39600000000000002,"5min":0.34100000000000003,"15min":0.29499999999999998},"max":{"1min":4.2300000000000004,"5min":154.76400000000001,"15min":207.96600000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.0960000000000001,"5min":3.0880000000000001,"15min":5.2229999999999999},"min":{"1min":0.36699999999999999,"5min":0.313,"15min":0.29699999999999999},"max":{"1min":4.2409999999999997,"5min":154.80000000000001,"15min":207.97900000000001},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:02:03 2023","interfaces":[{"interface":"back","average":{"1min":3.7810000000000001,"5min":1.5149999999999999,"15min":1.8280000000000001},"min":{"1min":0.38300000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":43.103000000000002,"5min":43.103000000000002,"15min":74.259},"last":0.52600000000000002},{"interface":"front","average":{"1min":3.448,"5min":1.5389999999999999,"15min":1.962},"min":{"1min":0.434,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":20.832999999999998,"5min":20.832999999999998,"15min":74.100999999999999},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:01:45 2023","interfaces":[{"interface":"back","average":{"1min":3.298,"5min":1.034,"15min":0.65700000000000003},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":57.134999999999998,"5min":57.134999999999998,"15min":57.134999999999998},"last":0.35499999999999998},{"interface":"front","average":{"1min":3.1080000000000001,"5min":1.0740000000000001,"15min":0.73199999999999998},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":53.287999999999997,"5min":53.287999999999997,"15min":53.287999999999997},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:01:45 2023","interfaces":[{"interface":"back","average":{"1min":3.3490000000000002,"5min":1.0580000000000001,"15min":1.853},"min":{"1min":0.33100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":57.064999999999998,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.502},{"interface":"front","average":{"1min":3.1629999999999998,"5min":1.1160000000000001,"15min":2.2879999999999998},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.26000000000000001},"max":{"1min":53.225000000000001,"5min":53.225000000000001,"15min":135.74199999999999},"last":1.6559999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:01:45 2023","interfaces":[{"interface":"back","average":{"1min":3.3820000000000001,"5min":1.085,"15min":1.29},"min":{"1min":0.33200000000000002,"5min":0.315,"15min":0.29399999999999998},"max":{"1min":57.168999999999997,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":3.3809999999999998,"5min":1.1990000000000001,"15min":1.5609999999999999},"min":{"1min":0.34100000000000003,"5min":0.30599999999999999,"15min":0.29999999999999999},"max":{"1min":57.183999999999997,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.42799999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496297,"num_pgs":72,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1748416,"kb_used_data":854328,"kb_used_omap":0,"kb_used_meta":894080,"kb_avail":91996736,"statfs":{"total":95995035648,"available":94204657664,"internally_reserved":0,"allocated":874831872,"data_stored":873358856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915537920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:02:00 2023","interfaces":[{"interface":"back","average":{"1min":19.559000000000001,"5min":4.367,"15min":2.0030000000000001},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.23599999999999999},"max":{"1min":321.346,"5min":321.346,"15min":321.346},"last":0.42099999999999999},{"interface":"front","average":{"1min":10.702999999999999,"5min":2.6509999999999998,"15min":1.645},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":169.119,"5min":169.119,"15min":169.119},"last":0.66600000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:02:00 2023","interfaces":[{"interface":"back","average":{"1min":5.7389999999999999,"5min":3.2589999999999999,"15min":5.702},"min":{"1min":0.33100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":72.840999999999994,"5min":90.278999999999996,"15min":260.738},"last":0.32300000000000001},{"interface":"front","average":{"1min":5.7409999999999997,"5min":3.1840000000000002,"15min":5.4199999999999999},"min":{"1min":0.33500000000000002,"5min":0.251,"15min":0.251},"max":{"1min":72.832999999999998,"5min":90.414000000000001,"15min":261.26100000000002},"last":0.47299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:02:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":1.2490000000000001,"15min":2.298},"min":{"1min":0.36599999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":23.943000000000001,"5min":23.943000000000001,"15min":133.535},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.3079999999999998,"5min":1.292,"15min":2.3109999999999999},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":25.978999999999999,"5min":25.978999999999999,"15min":133.51900000000001},"last":0.40899999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:02:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":2.105,"15min":2.3069999999999999},"min":{"1min":0.375,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":14.919,"5min":91.260999999999996,"15min":147.43600000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":2.1469999999999998,"15min":2.46},"min":{"1min":0.38700000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.2949999999999999,"5min":92.087000000000003,"15min":147.41999999999999},"last":0.70199999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:01:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45800000000000002,"15min":0.46600000000000003},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.7290000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.51700000000000002,"15min":0.501},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.9729999999999999,"5min":2.9729999999999999,"15min":2.9729999999999999},"last":0.40400000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:02:00 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":0.89100000000000001,"15min":1.651},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":25.934000000000001,"5min":25.934000000000001,"15min":66.995999999999995},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.54100000000000004,"15min":1.6120000000000001},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.254},"max":{"1min":1.246,"5min":2.3879999999999999,"15min":67.189999999999998},"last":0.50900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.65900000000000003,"15min":1.1990000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.30399999999999999},"max":{"1min":5.835,"5min":5.835,"15min":36.204999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.1070000000000002,"5min":0.89500000000000002,"15min":1.2450000000000001},"min":{"1min":0.36299999999999999,"5min":0.32600000000000001,"15min":0.32500000000000001},"max":{"1min":20.041,"5min":20.041,"15min":20.041},"last":0.84599999999999997}]}]},{"osd":1,"up_from":118,"seq":506806141366,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2263064,"kb_used_data":1031760,"kb_used_omap":4,"kb_used_meta":1231291,"kb_avail":91482088,"statfs":{"total":95995035648,"available":93677658112,"internally_reserved":0,"allocated":1056522240,"data_stored":1053384608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1260842894},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.331,"15min":1.0369999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.20799999999999999},"max":{"1min":4.8959999999999999,"5min":9.6999999999999993,"15min":11.763999999999999},"last":0.23499999999999999},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.103,"15min":1.01},"min":{"1min":0.252,"5min":0.252,"15min":0.214},"max":{"1min":3.6869999999999998,"5min":7.032,"15min":8.8190000000000008},"last":0.32700000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.873,"15min":3.641},"min":{"1min":0.28100000000000003,"5min":0.25,"15min":0.21099999999999999},"max":{"1min":5.6299999999999999,"5min":63.143999999999998,"15min":263.089},"last":0.313},{"interface":"front","average":{"1min":2.0880000000000001,"5min":1.9199999999999999,"15min":3.6299999999999999},"min":{"1min":0.28299999999999997,"5min":0.26000000000000001,"15min":0.222},"max":{"1min":13.763999999999999,"5min":66.198999999999998,"15min":254.50899999999999},"last":0.38100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:02:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.841,"15min":1.599},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.26500000000000001},"max":{"1min":6.7480000000000002,"5min":39.289000000000001,"15min":77.819999999999993},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.76000000000000001,"5min":1.1870000000000001,"15min":1.484},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.248},"max":{"1min":3.7989999999999999,"5min":14.212,"15min":77.796000000000006},"last":0.26900000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:01:44 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.4710000000000001,"15min":1.738},"min":{"1min":0.34999999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.1239999999999997,"5min":19.634,"15min":19.634},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.2150000000000001,"15min":1.284},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.702,"5min":13.045,"15min":13.045},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":1.0760000000000001,"15min":1.411},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":3.7629999999999999,"5min":7.79,"15min":9.782},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.8149999999999999,"5min":1.5109999999999999,"15min":1.407},"min":{"1min":0.45000000000000001,"5min":0.379,"15min":0.315},"max":{"1min":7.7089999999999996,"5min":19.597999999999999,"15min":19.597999999999999},"last":0.63800000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.2390000000000001,"15min":1.3360000000000001},"min":{"1min":0.39800000000000002,"5min":0.38400000000000001,"15min":0.34399999999999997},"max":{"1min":6.2389999999999999,"5min":7.4370000000000003,"15min":9.7840000000000007},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.25,"5min":1.5549999999999999,"15min":1.446},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.32400000000000001},"max":{"1min":6.2549999999999999,"5min":19.548999999999999,"15min":19.548999999999999},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.6579999999999999,"15min":2.0009999999999999},"min":{"1min":0.436,"5min":0.373,"15min":0.35899999999999999},"max":{"1min":5.9800000000000004,"5min":19.664999999999999,"15min":59.972000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.3540000000000001,"15min":1.885},"min":{"1min":0.437,"5min":0.40500000000000003,"15min":0.378},"max":{"1min":5.2649999999999997,"5min":9.8710000000000004,"15min":60.011000000000003},"last":0.56599999999999995}]}]},{"osd":0,"up_from":91,"seq":390842024450,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309860,"kb_used_data":328860,"kb_used_omap":5,"kb_used_meta":980986,"kb_avail":92435292,"statfs":{"total":95995035648,"available":94653739008,"internally_reserved":0,"allocated":336752640,"data_stored":334189821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1004530298},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:02:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":3.3140000000000001,"15min":4.282},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.19600000000000001},"max":{"1min":30.007999999999999,"5min":155.411,"15min":247.68899999999999},"last":0.31},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.9630000000000001,"15min":4.4279999999999999},"min":{"1min":0.27000000000000002,"5min":0.24099999999999999,"15min":0.214},"max":{"1min":29.940000000000001,"5min":154.75,"15min":246.38},"last":0.32300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.7230000000000001,"15min":2.839},"min":{"1min":0.32500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":14.207000000000001,"5min":26.556999999999999,"15min":151.06399999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.454,"15min":2.867},"min":{"1min":0.28199999999999997,"5min":0.27300000000000002,"15min":0.24199999999999999},"max":{"1min":3.355,"5min":29.032,"15min":151.22399999999999},"last":0.35499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:01:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":4.9569999999999999,"15min":2.3919999999999999},"min":{"1min":0.27700000000000002,"5min":0.26300000000000001,"15min":0.252},"max":{"1min":13.997,"5min":153.91,"15min":153.91},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.405,"5min":4.6680000000000001,"15min":2.4830000000000001},"min":{"1min":0.33000000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.1840000000000002,"5min":153.88300000000001,"15min":153.88300000000001},"last":0.27500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.83499999999999996,"15min":0.64100000000000001},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.1269999999999998,"5min":8.4000000000000004,"15min":8.4000000000000004},"last":0.432},{"interface":"front","average":{"1min":1.0660000000000001,"5min":0.98599999999999999,"15min":0.81599999999999995},"min":{"1min":0.40300000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.9300000000000002,"5min":12.048999999999999,"15min":12.048999999999999},"last":5.0289999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:02:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.0089999999999999,"15min":1.2849999999999999},"min":{"1min":0.48499999999999999,"5min":0.36399999999999999,"15min":0.33500000000000002},"max":{"1min":5.4429999999999996,"5min":8.3599999999999994,"15min":20.434999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.7669999999999999,"5min":1.448,"15min":1.236},"min":{"1min":0.38,"5min":0.38,"15min":0.32500000000000001},"max":{"1min":14.036,"5min":14.036,"15min":14.036},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:02:14 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.595,"15min":1.4419999999999999},"min":{"1min":0.46700000000000003,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":20.129999999999999,"5min":39.594999999999999,"15min":39.594999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":5.2149999999999999,"5min":2.3239999999999998,"15min":1.6339999999999999},"min":{"1min":0.41799999999999998,"5min":0.378,"15min":0.35099999999999998},"max":{"1min":47.241,"5min":47.241,"15min":47.241},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:02:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4769999999999999,"5min":1.579,"15min":1.579},"min":{"1min":0.49099999999999999,"5min":0.42999999999999999,"15min":0.38100000000000001},"max":{"1min":21.658000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":3.7149999999999999,"5min":1.5229999999999999,"15min":1.468},"min":{"1min":0.45900000000000002,"5min":0.432,"15min":0.38200000000000001},"max":{"1min":21.68,"5min":21.68,"15min":21.68},"last":0.61199999999999999}]}]},{"osd":2,"up_from":0,"seq":949187772726,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356421,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2354952,"kb_used_data":1506176,"kb_used_omap":2,"kb_used_meta":848765,"kb_avail":91390200,"statfs":{"total":95995035648,"available":93583564800,"internally_reserved":0,"allocated":1542324224,"data_stored":1539727245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":869135413},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.86899999999999999,"15min":0.78500000000000003},"min":{"1min":0.23799999999999999,"5min":0.23200000000000001,"15min":0.20699999999999999},"max":{"1min":5.1459999999999999,"5min":10.167,"15min":11.817},"last":0.38},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.095,"15min":0.93000000000000005},"min":{"1min":0.28299999999999997,"5min":0.28000000000000003,"15min":0.23499999999999999},"max":{"1min":6.0529999999999999,"5min":6.0529999999999999,"15min":8.6769999999999996},"last":0.29399999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.2649999999999999,"15min":2.6179999999999999},"min":{"1min":0.249,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":6.5039999999999996,"5min":10.145,"15min":125.233},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.3300000000000001,"15min":2.484},"min":{"1min":0.30199999999999999,"5min":0.255,"15min":0.21299999999999999},"max":{"1min":6.7789999999999999,"5min":26.692,"15min":126.102},"last":0.34999999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":2.9590000000000001,"15min":2.4049999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23499999999999999},"max":{"1min":6.976,"5min":212.065,"15min":212.065},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.93799999999999994,"5min":2.915,"15min":2.4460000000000002},"min":{"1min":0.26000000000000001,"5min":0.255,"15min":0.22600000000000001},"max":{"1min":6.149,"5min":211.98400000000001,"15min":211.98400000000001},"last":0.253}]},{"osd":4,"last update":"Mon Oct 9 05:01:48 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.0760000000000001,"15min":1.002},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.9960000000000004,"5min":5.9960000000000004,"15min":5.9960000000000004},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.4990000000000001,"5min":1.2470000000000001,"15min":1.268},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.4649999999999999,"5min":8.5779999999999994,"15min":8.5779999999999994},"last":2.7890000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.79300000000000004,"15min":1.296},"min":{"1min":0.315,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":2.4399999999999999,"5min":11.368,"15min":28.302},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0820000000000001,"15min":1.5},"min":{"1min":0.46100000000000002,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":6.0869999999999997,"5min":7.3040000000000003,"15min":28.280000000000001},"last":0.45400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.077,"15min":1.1040000000000001},"min":{"1min":0.41799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.1189999999999998,"5min":6.1189999999999998,"15min":8.7859999999999996},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.228,"15min":1.466},"min":{"1min":0.435,"5min":0.40400000000000003,"15min":0.36199999999999999},"max":{"1min":5.992,"5min":8.6069999999999993,"15min":24.916},"last":0.505}]},{"osd":7,"last update":"Mon Oct 9 05:02:04 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.2050000000000001,"15min":1.6930000000000001},"min":{"1min":0.50900000000000001,"5min":0.39800000000000002,"15min":0.375},"max":{"1min":6.0339999999999998,"5min":8.6370000000000005,"15min":25.004000000000001},"last":0.755},{"interface":"front","average":{"1min":1.2390000000000001,"5min":0.89000000000000001,"15min":1.159},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":6.0179999999999998,"5min":7.2409999999999997,"15min":10.092000000000001},"last":0.74099999999999999}]}]},{"osd":4,"up_from":507,"seq":2177548419126,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3454512,"kb_used_data":3045416,"kb_used_omap":11,"kb_used_meta":409076,"kb_avail":90290640,"statfs":{"total":95995035648,"available":92457615360,"internally_reserved":0,"allocated":3118505984,"data_stored":3117068569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":418894728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,7],"upper_bound":131072},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.81000000000000005,"15min":0.81599999999999995},"min":{"1min":0.29099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.1080000000000001,"5min":7.9020000000000001,"15min":7.9020000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.75600000000000001,"15min":0.70899999999999996},"min":{"1min":0.33800000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.7639999999999998,"5min":5.4770000000000003,"15min":5.4770000000000003},"last":0.48799999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":12.382999999999999,"5min":6.4130000000000003,"15min":7.774},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":76.640000000000001,"5min":147.49600000000001,"15min":147.49600000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":12.214,"5min":6.4699999999999998,"15min":7.9699999999999998},"min":{"1min":0.35999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":76.603999999999999,"5min":147.51300000000001,"15min":147.51300000000001},"last":0.67600000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1.077,"15min":0.97499999999999998},"min":{"1min":0.41099999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.9039999999999999,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.145,"15min":1.3280000000000001},"min":{"1min":0.33900000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.1040000000000001,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.55800000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.5980000000000001,"15min":1.236},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":13.478999999999999,"5min":24.379000000000001,"15min":24.379000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.724,"5min":2.1219999999999999,"15min":2.343},"min":{"1min":0.42699999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":13.542,"5min":24.23,"15min":24.23},"last":0.41699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.497,"15min":0.48699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.68000000000000005,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.57699999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.85399999999999998,"15min":0.748},"min":{"1min":0.27100000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.5179999999999998,"5min":22.350999999999999,"15min":22.350999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.438,"5min":0.75600000000000001,"15min":0.60399999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67600000000000005,"5min":22.433,"15min":22.433},"last":0.50900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54100000000000004,"15min":0.54900000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.008,"5min":1.9179999999999999,"15min":1.9179999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.59199999999999997,"15min":0.69899999999999995},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.74199999999999999,"5min":4.5949999999999998,"15min":4.5949999999999998},"last":0.52000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757941,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5123052,"kb_used_data":3633828,"kb_used_omap":0,"kb_used_meta":1489216,"kb_avail":88622100,"statfs":{"total":95995035648,"available":90749030400,"internally_reserved":0,"allocated":3721039872,"data_stored":3719581757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1524957184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:02:17 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.66100000000000003,"15min":1.4550000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.970000000000001,"5min":11.970000000000001,"15min":106.465},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.115,"5min":0.83999999999999997,"15min":1.47},"min":{"1min":0.314,"5min":0.28199999999999997,"15min":0.23799999999999999},"max":{"1min":12.206,"5min":12.206,"15min":106.54600000000001},"last":0.27400000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:02:17 2023","interfaces":[{"interface":"back","average":{"1min":14.234,"5min":3.5510000000000002,"15min":3.9670000000000001},"min":{"1min":0.309,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":262.26900000000001,"5min":262.26900000000001,"15min":262.26900000000001},"last":1.3029999999999999},{"interface":"front","average":{"1min":14.122999999999999,"5min":3.6000000000000001,"15min":4.2030000000000003},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.255},"max":{"1min":262.291,"5min":262.291,"15min":262.291},"last":0.34599999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:02:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1819999999999999,"5min":2.8650000000000002,"15min":2.6930000000000001},"min":{"1min":0.377,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.949999999999999,"5min":106.238,"15min":106.238},"last":1.266},{"interface":"front","average":{"1min":1.6970000000000001,"5min":2.6629999999999998,"15min":2.9249999999999998},"min":{"1min":0.371,"5min":0.29999999999999999,"15min":0.29699999999999999},"max":{"1min":11.912000000000001,"5min":104.31699999999999,"15min":104.31699999999999},"last":0.371}]},{"osd":3,"last update":"Mon Oct 9 05:01:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":1.1180000000000001,"15min":1.7450000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":12.365,"5min":12.365,"15min":105.879},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.677,"5min":1.1639999999999999,"15min":1.6970000000000001},"min":{"1min":0.47199999999999998,"5min":0.316,"15min":0.316},"max":{"1min":12.297000000000001,"5min":12.297000000000001,"15min":105.625},"last":1.335}]},{"osd":4,"last update":"Mon Oct 9 05:01:41 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.501,"15min":0.55300000000000005},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.65100000000000002,"5min":2.2360000000000002,"15min":2.2360000000000002},"last":1.2350000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.46800000000000003,"15min":0.47999999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.74099999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.39700000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:02:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":0.76100000000000001,"15min":0.92700000000000005},"min":{"1min":0.34399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.093,"5min":11.093,"15min":11.093},"last":1.1919999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.54500000000000004,"15min":1.032},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":2.698,"5min":2.9609999999999999,"15min":13.083},"last":0.36099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:01:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.54600000000000004,"15min":2.3010000000000002},"min":{"1min":0.33700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57199999999999995,"5min":2.8540000000000001,"15min":110.521},"last":0.496},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.57199999999999995,"15min":2.3809999999999998},"min":{"1min":0.35099999999999998,"5min":0.34599999999999997,"15min":0.29499999999999998},"max":{"1min":0.71499999999999997,"5min":2.266,"15min":110.746},"last":1.496}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11384,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":333774848,"data_stored":333774848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1048379392,"data_stored":1048379392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3626897408,"data_stored":3626893336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1539309568,"data_stored":1539309568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3111849984,"data_stored":3111845912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3714846720,"data_stored":3714842648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":868290560,"data_stored":868286488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2234580992,"data_stored":2234580992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:02:46.302 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-09T05:02:46.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_9 pg_num 36 2023-10-09T05:02:46.304 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:46.331 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:46.367 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:46.418 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:46.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:46.526 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:46.565 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:46.590 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:02:46.684 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:46.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:46.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:46.776 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:02:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:46 smithi177 ceph-mon[195797]: failed to encode map e609 with expected crc 2023-10-09T05:02:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:46 smithi177 ceph-mon[195797]: osdmap e609: 8 total, 8 up, 7 in 2023-10-09T05:02:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:46 smithi177 ceph-mon[195797]: failed to encode map e609 with expected crc 2023-10-09T05:02:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:46 smithi177 ceph-mon[195797]: pgmap v1993: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20512/189507 objects misplaced (10.824%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:02:46.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4130133589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:46.788 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:46.806 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:02:46.824 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:46.827 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:02:46.849 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:02:46.878 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:46.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[216903]: failed to encode map e609 with expected crc 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[216903]: osdmap e609: 8 total, 8 up, 7 in 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[216903]: failed to encode map e609 with expected crc 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[216903]: pgmap v1993: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20512/189507 objects misplaced (10.824%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4130133589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[223858]: failed to encode map e609 with expected crc 2023-10-09T05:02:46.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[223858]: osdmap e609: 8 total, 8 up, 7 in 2023-10-09T05:02:46.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[223858]: failed to encode map e609 with expected crc 2023-10-09T05:02:46.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[223858]: pgmap v1993: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20512/189507 objects misplaced (10.824%); 5.3 MiB/s, 84 objects/s recovering 2023-10-09T05:02:46.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4130133589' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:02:46.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:47.016 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:47.072 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:47.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:47.178 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:47.206 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4130133589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: osdmap e610: 8 total, 8 up, 7 in 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/657248809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4130133589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: osdmap e610: 8 total, 8 up, 7 in 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:47.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/657248809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4130133589' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: osdmap e610: 8 total, 8 up, 7 in 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/657248809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:48.106 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:48.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:48.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:48.114 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:02:48.137 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:02:48.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:48.244 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:48.300 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:48.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:48.322 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:02:48.373 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.374 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:02:48.406 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:48.437 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:48.493 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:48.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:48.600 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:48.635 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:48.688 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:48.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:48.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:02:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:48 smithi177 ceph-mon[195797]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:48 smithi177 ceph-mon[195797]: pgmap v1995: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20150/189507 objects misplaced (10.633%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:02:48.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:48.822 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:48 smithi029 ceph-mon[216903]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.823 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:48 smithi029 ceph-mon[216903]: pgmap v1995: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20150/189507 objects misplaced (10.633%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:02:48.823 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:48 smithi029 ceph-mon[223858]: failed to encode map e610 with expected crc 2023-10-09T05:02:48.823 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:48 smithi029 ceph-mon[223858]: pgmap v1995: 109 pgs: 1 activating+remapped, 2 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 remapped+peering, 4 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64321/189507 objects degraded (33.941%); 20150/189507 objects misplaced (10.633%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:02:48.826 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:49.683 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:49.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:49.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:49.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:49.831 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:49.865 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:49.927 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:49.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:49.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:49 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T05:02:49.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:49 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80239/189507 objects degraded (42.341%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:49.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:49 smithi029 ceph-mon[216903]: Health check failed: 4 slow ops, oldest one blocked for 38 sec, osd.4 has slow ops (SLOW_OPS) 2023-10-09T05:02:49.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:49 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T05:02:49.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:49 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80239/189507 objects degraded (42.341%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:49.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:49 smithi029 ceph-mon[223858]: Health check failed: 4 slow ops, oldest one blocked for 38 sec, osd.4 has slow ops (SLOW_OPS) 2023-10-09T05:02:50.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:49 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-10-09T05:02:50.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:49 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80239/189507 objects degraded (42.341%), 23 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:50.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:49 smithi177 ceph-mon[195797]: Health check failed: 4 slow ops, oldest one blocked for 38 sec, osd.4 has slow ops (SLOW_OPS) 2023-10-09T05:02:50.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:50.065 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:50.171 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:50.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:50.245 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:02:50.246 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:02:50.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:50.310 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:50.368 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:50.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:50.474 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:50.489 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:02:50.507 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:50.773 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:50 smithi177 ceph-mon[195797]: pgmap v1996: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80239/189507 objects degraded (42.341%); 25860/189507 objects misplaced (13.646%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:02:50.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:50 smithi029 ceph-mon[216903]: pgmap v1996: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80239/189507 objects degraded (42.341%); 25860/189507 objects misplaced (13.646%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:02:50.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:50 smithi029 ceph-mon[223858]: pgmap v1996: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80239/189507 objects degraded (42.341%); 25860/189507 objects misplaced (13.646%); 4.6 MiB/s, 73 objects/s recovering 2023-10-09T05:02:51.788 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:02:51.915 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:51.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:52.019 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:52.034 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:02:52.056 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:52.131 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:52.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:52.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:52.235 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:52.262 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:52.292 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:52.369 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:52.390 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:52.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:52.498 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:52.527 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:52.597 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:52.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:52.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:52.740 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:52.789 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:52.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:52.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:52.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:52.927 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:53.011 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:53.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:53.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:53.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:52 smithi029 ceph-mon[216903]: pgmap v1997: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80237/189507 objects degraded (42.340%); 24616/189507 objects misplaced (12.989%) 2023-10-09T05:02:53.125 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:52 smithi029 ceph-mon[223858]: pgmap v1997: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80237/189507 objects degraded (42.340%); 24616/189507 objects misplaced (12.989%) 2023-10-09T05:02:53.126 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:02:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:02:52] "GET /metrics HTTP/1.1" 200 36728 "" "Prometheus/2.43.0" 2023-10-09T05:02:53.163 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:52 smithi177 ceph-mon[195797]: pgmap v1997: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80237/189507 objects degraded (42.340%); 24616/189507 objects misplaced (12.989%) 2023-10-09T05:02:53.927 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:53.930 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 14 pg_num to 36 2023-10-09T05:02:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:53.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:54.043 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:54.076 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:54.131 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:54.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4208328652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:02:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:53 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:02:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4208328652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:02:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:53 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:02:54.237 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:54.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4208328652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:02:54.264 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:53 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]: dispatch 2023-10-09T05:02:54.267 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:54.323 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:54.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:54.433 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:54.466 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: pgmap v1998: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80237/189507 objects degraded (42.340%); 24247/189507 objects misplaced (12.795%) 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[216903]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: pgmap v1998: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80237/189507 objects degraded (42.340%); 24247/189507 objects misplaced (12.795%) 2023-10-09T05:02:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:02:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T05:02:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:54 smithi029 ceph-mon[223858]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: pgmap v1998: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 80237/189507 objects degraded (42.340%); 24247/189507 objects misplaced (12.795%) 2023-10-09T05:02:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2023-10-09T05:02:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: osdmap e611: 8 total, 8 up, 7 in 2023-10-09T05:02:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:54 smithi177 ceph-mon[195797]: failed to encode map e611 with expected crc 2023-10-09T05:02:55.523 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:55.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:55.628 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:55.656 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:55.710 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:55.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:55.716 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:02:55.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:55.852 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:55.925 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:55.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:56.034 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:56.063 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:56.112 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:56.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:56.150 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:02:56.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:55 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 80237/189507 objects degraded (42.340%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:56.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:55 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 80237/189507 objects degraded (42.340%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:56.176 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:02:56.201 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:02:56.218 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:02:56.227 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:02:56.251 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:56.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:55 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 80237/189507 objects degraded (42.340%), 22 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-09T05:02:56.559 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:02:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:56.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:02:56.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:56.693 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:56.794 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:56.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:56.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:56.927 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:56.937 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:02:56.960 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[216903]: pgmap v2000: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 80213/189372 objects degraded (42.357%); 24158/189372 objects misplaced (12.757%) 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[216903]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4072804776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2108052350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[223858]: pgmap v2000: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 80213/189372 objects degraded (42.357%); 24158/189372 objects misplaced (12.757%) 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[223858]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4072804776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:56.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2108052350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:56.975 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:56.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:57.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:57.108 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:57.161 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:57.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:57.265 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:02:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:56 smithi177 ceph-mon[195797]: pgmap v2000: 109 pgs: 1 activating+degraded, 3 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 4 op/s; 80213/189372 objects degraded (42.357%); 24158/189372 objects misplaced (12.757%) 2023-10-09T05:02:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:56 smithi177 ceph-mon[195797]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-10-09T05:02:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4072804776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:02:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2108052350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:02:57.292 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:57.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:02:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:57.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:02:57.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:57.477 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:57.578 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:57.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:57.622 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:02:57.641 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:02:57.662 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:02:57.682 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:02:57.682 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.685 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.686 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:02:57.712 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:58.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:58.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[216903]: failed to encode map e612 with expected crc 2023-10-09T05:02:58.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2108052350' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:02:58.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[216903]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T05:02:58.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[216903]: failed to encode map e612 with expected crc 2023-10-09T05:02:58.238 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:02:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[223858]: failed to encode map e612 with expected crc 2023-10-09T05:02:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2108052350' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:02:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[223858]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T05:02:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[223858]: failed to encode map e612 with expected crc 2023-10-09T05:02:58.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:57 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:02:58.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:57 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:02:58.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:57 smithi177 ceph-mon[195797]: failed to encode map e612 with expected crc 2023-10-09T05:02:58.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2108052350' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:02:58.280 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:57 smithi177 ceph-mon[195797]: osdmap e612: 8 total, 8 up, 7 in 2023-10-09T05:02:58.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:57 smithi177 ceph-mon[195797]: failed to encode map e612 with expected crc 2023-10-09T05:02:58.281 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:57 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-10-09T05:02:58.679 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:02:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:58.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:02:58.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:02:58.814 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:58.925 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:02:58.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:58.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:02:59.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:02:59.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: pgmap v2002: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 81442/189108 objects degraded (43.066%); 21816/189108 objects misplaced (11.536%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T05:02:59.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-10-09T05:02:59.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: Health check cleared: SLOW_OPS (was: 4 slow ops, oldest one blocked for 38 sec, osd.4 has slow ops) 2023-10-09T05:02:59.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:02:59.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T05:02:59.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: failed to encode map e613 with expected crc 2023-10-09T05:02:59.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: failed to encode map e613 with expected crc 2023-10-09T05:02:59.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: pgmap v2002: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 81442/189108 objects degraded (43.066%); 21816/189108 objects misplaced (11.536%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T05:02:59.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-10-09T05:02:59.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: Health check cleared: SLOW_OPS (was: 4 slow ops, oldest one blocked for 38 sec, osd.4 has slow ops) 2023-10-09T05:02:59.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:02:59.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T05:02:59.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: failed to encode map e613 with expected crc 2023-10-09T05:02:59.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: failed to encode map e613 with expected crc 2023-10-09T05:02:59.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.073 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: pgmap v2002: 109 pgs: 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 17 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 81442/189108 objects degraded (43.066%); 21816/189108 objects misplaced (11.536%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-10-09T05:02:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-10-09T05:02:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: Health check cleared: SLOW_OPS (was: 4 slow ops, oldest one blocked for 38 sec, osd.4 has slow ops) 2023-10-09T05:02:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2023-10-09T05:02:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: osdmap e613: 8 total, 8 up, 7 in 2023-10-09T05:02:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: failed to encode map e613 with expected crc 2023-10-09T05:02:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: failed to encode map e613 with expected crc 2023-10-09T05:02:59.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:02:59.302 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:02:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:59.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:02:59.407 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:02:59.438 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:59.545 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:02:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:59.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:02:59.651 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:02:59.679 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:59.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:02:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:59.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:02:59.747 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:02:59.747 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:02:59.747 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:02:59.747 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:02:59.747 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:02:59.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:02:59.863 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:02:59.907 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:02:59.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:02:59.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:00.017 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:00.082 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:00.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:00.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:00.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: osdmap e614: 8 total, 8 up, 7 in 2023-10-09T05:03:00.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[216903]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: osdmap e614: 8 total, 8 up, 7 in 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:02:59 smithi029 ceph-mon[223858]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.268 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: osdmap e614: 8 total, 8 up, 7 in 2023-10-09T05:03:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:02:59 smithi177 ceph-mon[195797]: failed to encode map e614 with expected crc 2023-10-09T05:03:00.298 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:00.353 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:00.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:00.458 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:00.495 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:00.557 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:00.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:00.663 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:00.711 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:00.745 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 3, 6, 4, 0, 1] out_osds: [2] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:03:00.745 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:03:00.745 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 3, 5, 6, 7, 2, 4] 2023-10-09T05:03:00.745 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T05:03:00.745 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T05:03:00.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:00.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:00.888 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:00.922 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:00.998 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:01.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:01.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:01.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:03:01.087 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:03:01.106 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:01.154 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:00 smithi177 systemd[1]: Stopping Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:03:01.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: pgmap v2005: 119 pgs: 1 remapped+peering, 10 unknown, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.1 MiB/s wr, 47 op/s; 73848/189282 objects degraded (39.015%); 21816/189282 objects misplaced (11.526%); 8.5 MiB/s, 1 keys/s, 136 objects/s recovering 2023-10-09T05:03:01.154 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: osdmap e615: 8 total, 8 up, 7 in 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: 14.e continuing backfill to osd.0 from (445'1681,612'5719] MIN to 612'5719 2023-10-09T05:03:01.155 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:00 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 73848/189282 objects degraded (39.015%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:01.158 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:01.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: pgmap v2005: 119 pgs: 1 remapped+peering, 10 unknown, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.1 MiB/s wr, 47 op/s; 73848/189282 objects degraded (39.015%); 21816/189282 objects misplaced (11.526%); 8.5 MiB/s, 1 keys/s, 136 objects/s recovering 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: osdmap e615: 8 total, 8 up, 7 in 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: 14.e continuing backfill to osd.0 from (445'1681,612'5719] MIN to 612'5719 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 73848/189282 objects degraded (39.015%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: pgmap v2005: 119 pgs: 1 remapped+peering, 10 unknown, 2 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.1 MiB/s wr, 47 op/s; 73848/189282 objects degraded (39.015%); 21816/189282 objects misplaced (11.526%); 8.5 MiB/s, 1 keys/s, 136 objects/s recovering 2023-10-09T05:03:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: osdmap e615: 8 total, 8 up, 7 in 2023-10-09T05:03:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: failed to encode map e615 with expected crc 2023-10-09T05:03:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: 14.e continuing backfill to osd.0 from (445'1681,612'5719] MIN to 612'5719 2023-10-09T05:03:01.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:00 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 73848/189282 objects degraded (39.015%), 21 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:01.221 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:01.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:01.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:01.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:01.363 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:01.411 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:03:01.428 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:01.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:01.529 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:01.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:01.588 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:01.593 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:01.753 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:01.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:01.858 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:01.883 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:01 smithi177 conmon[372243]: 2023-10-09T05:03:01.563+0000 7f0817c62700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:03:01.883 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:01 smithi177 conmon[372243]: 2023-10-09T05:03:01.563+0000 7f0817c62700 -1 osd.4 616 *** Got signal Terminated *** 2023-10-09T05:03:01.883 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:01 smithi177 conmon[372243]: 2023-10-09T05:03:01.563+0000 7f0817c62700 -1 osd.4 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:03:01.886 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:01.938 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:01.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:02.042 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: osdmap e616: 8 total, 8 up, 7 in 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: 14.1e continuing backfill to osd.0 from (445'1681,612'5719] MIN to 612'5719 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[216903]: osd.4 marked itself down and dead 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: osdmap e616: 8 total, 8 up, 7 in 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: 14.1e continuing backfill to osd.0 from (445'1681,612'5719] MIN to 612'5719 2023-10-09T05:03:02.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:01 smithi029 ceph-mon[223858]: osd.4 marked itself down and dead 2023-10-09T05:03:02.074 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: osdmap e616: 8 total, 8 up, 7 in 2023-10-09T05:03:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: failed to encode map e616 with expected crc 2023-10-09T05:03:02.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: 14.1e continuing backfill to osd.0 from (445'1681,612'5719] MIN to 612'5719 2023-10-09T05:03:02.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:01 smithi177 ceph-mon[195797]: osd.4 marked itself down and dead 2023-10-09T05:03:02.580 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:02.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:02.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:02.685 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:02.717 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:02.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:03:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:03:02] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2023-10-09T05:03:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: pgmap v2008: 119 pgs: 1 activating+undersized+remapped, 1 peering, 3 remapped+peering, 5 unknown, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 84 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.6 MiB/s wr, 42 op/s; 72928/189204 objects degraded (38.545%); 14943/189204 objects misplaced (7.898%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:03:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:03:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: osdmap e617: 8 total, 7 up, 7 in 2023-10-09T05:03:03.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:02 smithi177 ceph-mon[195797]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: pgmap v2008: 119 pgs: 1 activating+undersized+remapped, 1 peering, 3 remapped+peering, 5 unknown, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 84 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.6 MiB/s wr, 42 op/s; 72928/189204 objects degraded (38.545%); 14943/189204 objects misplaced (7.898%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: osdmap e617: 8 total, 7 up, 7 in 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[216903]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: pgmap v2008: 119 pgs: 1 activating+undersized+remapped, 1 peering, 3 remapped+peering, 5 unknown, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+degraded, 84 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.6 MiB/s wr, 42 op/s; 72928/189204 objects degraded (38.545%); 14943/189204 objects misplaced (7.898%); 1.2 MiB/s, 19 objects/s recovering 2023-10-09T05:03:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-09T05:03:03.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: osdmap e617: 8 total, 7 up, 7 in 2023-10-09T05:03:03.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: failed to encode map e617 with expected crc 2023-10-09T05:03:03.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:02 smithi029 ceph-mon[223858]: failed to encode map e617 with expected crc 2023-10-09T05:03:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: osdmap e618: 8 total, 7 up, 7 in 2023-10-09T05:03:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:03:04.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[216903]: osdmap e619: 8 total, 7 up, 7 in 2023-10-09T05:03:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: osdmap e618: 8 total, 7 up, 7 in 2023-10-09T05:03:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:03:04.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:04 smithi029 ceph-mon[223858]: osdmap e619: 8 total, 7 up, 7 in 2023-10-09T05:03:04.332 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:04.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:04.438 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:04.482 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: osdmap e618: 8 total, 7 up, 7 in 2023-10-09T05:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: failed to encode map e618 with expected crc 2023-10-09T05:03:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:03:04.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:04 smithi177 ceph-mon[195797]: osdmap e619: 8 total, 7 up, 7 in 2023-10-09T05:03:04.567 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:04.632 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:04.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:04.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:04.796 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:04.858 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:04.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:04.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:05.029 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:05.088 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:05.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:05.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:05.206 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[216903]: pgmap v2011: 119 pgs: 1 active+undersized+degraded, 4 active+undersized, 66 active+clean, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 unknown, 1 stale+active+remapped+backfill_wait, 6 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 1 activating+undersized+remapped, 1 active+remapped+backfill_wait, 11 stale+active+clean, 1 stale+active+undersized+degraded+remapped; 3.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67308/183141 objects degraded (36.752%); 11174/183141 objects misplaced (6.101%) 2023-10-09T05:03:05.206 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[216903]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.206 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[216903]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.206 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[216903]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.206 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[216903]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[216903]: osdmap e620: 8 total, 7 up, 7 in 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[223858]: pgmap v2011: 119 pgs: 1 active+undersized+degraded, 4 active+undersized, 66 active+clean, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 unknown, 1 stale+active+remapped+backfill_wait, 6 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 1 activating+undersized+remapped, 1 active+remapped+backfill_wait, 11 stale+active+clean, 1 stale+active+undersized+degraded+remapped; 3.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67308/183141 objects degraded (36.752%); 11174/183141 objects misplaced (6.101%) 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[223858]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[223858]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[223858]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[223858]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.207 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:05 smithi029 ceph-mon[223858]: osdmap e620: 8 total, 7 up, 7 in 2023-10-09T05:03:05.231 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:05.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:04 smithi177 bash[409036]: faaf1f5dae86182e625adaeb73753b0c5c0b3d539c17e98ddf25945033dbbe6f 2023-10-09T05:03:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:05 smithi177 ceph-mon[195797]: pgmap v2011: 119 pgs: 1 active+undersized+degraded, 4 active+undersized, 66 active+clean, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 3 unknown, 1 stale+active+remapped+backfill_wait, 6 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 2 peering, 1 activating+undersized+remapped, 1 active+remapped+backfill_wait, 11 stale+active+clean, 1 stale+active+undersized+degraded+remapped; 3.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67308/183141 objects degraded (36.752%); 11174/183141 objects misplaced (6.101%) 2023-10-09T05:03:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:05 smithi177 ceph-mon[195797]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:05 smithi177 ceph-mon[195797]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:05 smithi177 ceph-mon[195797]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:05 smithi177 ceph-mon[195797]: failed to encode map e619 with expected crc 2023-10-09T05:03:05.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:05 smithi177 ceph-mon[195797]: osdmap e620: 8 total, 7 up, 7 in 2023-10-09T05:03:05.475 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:05.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:05.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:05.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:05.613 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:05.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:05.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:05.782 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:05.815 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:05.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:05.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:05.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:05.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:06.072 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:06.159 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:06.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[216903]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[216903]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[216903]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[216903]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 67308/183141 objects degraded (36.752%), 24 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:06.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[223858]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[223858]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[223858]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[223858]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 67308/183141 objects degraded (36.752%), 24 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:06.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:06 smithi177 ceph-mon[195797]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:06 smithi177 ceph-mon[195797]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:06 smithi177 ceph-mon[195797]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:06 smithi177 ceph-mon[195797]: failed to encode map e620 with expected crc 2023-10-09T05:03:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 67308/183141 objects degraded (36.752%), 24 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:06.296 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:06.346 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:06.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:06.450 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:06.488 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:06.574 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:06.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:06.678 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:06.709 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:06.762 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:06.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:06.866 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:06.909 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:07.129 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:03:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:07 smithi029 ceph-mon[216903]: pgmap v2014: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 active+undersized+degraded, 8 active+undersized, 58 active+clean, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 unknown, 1 stale+active+remapped+backfill_wait, 4 stale+active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 10 stale+active+clean, 1 stale+active+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 73161/188985 objects degraded (38.713%); 12479/188985 objects misplaced (6.603%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:03:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3816981910' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:07.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:07 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:07.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:07 smithi029 ceph-mon[223858]: pgmap v2014: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 active+undersized+degraded, 8 active+undersized, 58 active+clean, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 unknown, 1 stale+active+remapped+backfill_wait, 4 stale+active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 10 stale+active+clean, 1 stale+active+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 73161/188985 objects degraded (38.713%); 12479/188985 objects misplaced (6.603%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:03:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3816981910' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:07.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:07 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:07.279 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:03:07.279 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:03:07.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79979 14.8 14:13238338:::benchmark_data_smithi029_550670_object2368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:57.399715+0000", 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 369.86875020899998, 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 32.366814924000003, 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.280 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79979 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399715+0000", 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399715+0000", 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399713+0000", 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399723+0000", 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0036e-05 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399739+0000", 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5603999999999998e-05 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.399747+0000", 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4919999999999993e-06 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.356872+0000", 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 31.957124335 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.376280+0000", 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019408621000000001 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.442399+0000", 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.066119066000000004 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.461617+0000", 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019218057 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.559776+0000", 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.098158714999999994 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.577639+0000", 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017863029999999998 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,3", 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.614223+0000", 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036583362000000001 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.658662+0000", 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044439182000000001 2023-10-09T05:03:07.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.719412+0000", 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.060750266999999997 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.723436+0000", 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040238560000000001 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.730398+0000", 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069622659999999999 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:57:29.766530+0000", 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036131933999999997 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80507 14.1 14:8936b98f:::benchmark_data_smithi029_550670_object2896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:05.499214+0000", 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 301.76925158199998, 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 30.860218122999999, 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.289 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80507 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.499214+0000", 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.499214+0000", 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.499212+0000", 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.499221+0000", 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1139999999999995e-06 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.499237+0000", 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6426000000000002e-05 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.499248+0000", 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0397999999999999e-05 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.674791+0000", 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17554329099999999 2023-10-09T05:03:07.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.691739+0000", 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016947650000000002 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.827328+0000", 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13558912100000001 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.850974+0000", 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023646476 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.816753+0000", 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.96577903200000004 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.833556+0000", 2023-10-09T05:03:07.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016802536999999999 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,5", 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.866977+0000", 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033421278999999998 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022044+0000", 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15506640899999999 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.295 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022062+0000", 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7978999999999998e-05 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833109+0000", 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81104734000000001 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833123+0000", 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4479e-05 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.116567+0000", 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.283443739999999 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for readable", 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.146819+0000", 2023-10-09T05:03:07.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030251931999999999 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.216272+0000", 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.069452881999999994 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.241510+0000", 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025238171 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.353380+0000", 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.111869726 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.359432+0000", 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006052071 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80512 14.1 14:899076e7:::benchmark_data_smithi029_550670_object2901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:05.586266+0000", 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 301.68219975800002, 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 31.072087216, 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80512 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.300 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.586266+0000", 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.586266+0000", 2023-10-09T05:03:07.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.586264+0000", 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.586272+0000", 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0949999999999996e-06 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.586285+0000", 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2957e-05 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.586294+0000", 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6640000000000004e-06 2023-10-09T05:03:07.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.853050+0000", 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26675618600000001 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.876702+0000", 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023651838000000001 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.061734+0000", 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.18503239799999999 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.079929+0000", 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018194707000000001 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022203+0000", 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94227429500000004 2023-10-09T05:03:07.306 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022221+0000", 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8077999999999999e-05 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833236+0000", 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81101484499999998 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833245+0000", 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0270000000000001e-06 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.490888+0000", 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.657642590000002 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.512768+0000", 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021880687999999999 2023-10-09T05:03:07.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.536355+0000", 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023586780000000002 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.608750+0000", 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.072394927999999997 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.610487+0000", 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001737169 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.616330+0000", 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058426509999999999 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.624481+0000", 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081510209999999996 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.658353+0000", 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033871971000000001 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.311 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80513 14.1 14:8884ed32:::benchmark_data_smithi029_550670_object2902:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:05.610250+0000", 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 301.65821560500001, 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 31.247123149, 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80513 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.610250+0000", 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.610250+0000", 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.610248+0000", 2023-10-09T05:03:07.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.610261+0000", 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3217000000000001e-05 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.610281+0000", 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9908e-05 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.610293+0000", 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2468e-05 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.878159+0000", 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26786597699999998 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:05.896923+0000", 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018763598999999999 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.135449+0000", 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23852663099999999 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.153658+0000", 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018208361999999999 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022264+0000", 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.86860612299999995 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022273+0000", 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4790000000000002e-06 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833275+0000", 2023-10-09T05:03:07.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.811001109 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833283+0000", 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7620000000000001e-06 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.660280+0000", 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.826996512000001 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.687294+0000", 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027014512000000001 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.711560+0000", 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024265403000000001 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.806442+0000", 2023-10-09T05:03:07.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.094882741000000007 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.808253+0000", 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001810718 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.815539+0000", 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007285896 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.823491+0000", 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079523839999999998 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.857373+0000", 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033881454999999998 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80529 14.1 14:8f300ffc:::benchmark_data_smithi029_550670_object2918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.016496+0000", 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 301.25196972800001, 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 30.586740634999998, 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80529 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.322 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.016496+0000", 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.016496+0000", 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.016494+0000", 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.016502+0000", 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2539999999999992e-06 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.016516+0000", 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4051e-05 2023-10-09T05:03:07.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.016524+0000", 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4649999999999997e-06 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.215510+0000", 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19898660600000001 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.236443+0000", 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020932916999999999 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.408216+0000", 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.171772906 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.426935+0000", 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018719044000000001 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022141+0000", 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.59520552299999996 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022150+0000", 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5880000000000004e-06 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833198+0000", 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81104821500000002 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833208+0000", 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5019999999999998e-06 2023-10-09T05:03:07.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.441263+0000", 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.60805465 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.455207+0000", 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013944796000000001 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.473615+0000", 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01840729 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.556659+0000", 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.083044733999999995 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.557960+0000", 2023-10-09T05:03:07.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013009320000000001 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.562364+0000", 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044039409999999998 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.569296+0000", 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069318549999999998 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.603236+0000", 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.033940235999999999 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80537 14.1 14:81101edd:::benchmark_data_smithi029_550670_object2926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.449119+0000", 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 300.81934675100001, 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 30.472305052999999, 2023-10-09T05:03:07.332 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80537 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.449119+0000", 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.449119+0000", 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.449117+0000", 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.449125+0000", 2023-10-09T05:03:07.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.464e-06 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.449137+0000", 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.224e-05 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.449145+0000", 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4749999999999993e-06 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.499066+0000", 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.049920900999999997 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.526084+0000", 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027017330999999999 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.698639+0000", 2023-10-09T05:03:07.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.172555446 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.719725+0000", 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021085680999999998 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022349+0000", 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30262440299999999 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022357+0000", 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2009999999999999e-06 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833311+0000", 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.810953594 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833320+0000", 2023-10-09T05:03:07.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0299999999999999e-06 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.731718+0000", 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.898398279999999 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.757616+0000", 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025898074 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.339 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.783844+0000", 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026228199000000001 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.809709+0000", 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025864608000000001 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.862363+0000", 2023-10-09T05:03:07.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.052653563 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.867721+0000", 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053588109999999998 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.875993+0000", 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082716300000000003 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.921424+0000", 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.045430656 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80551 14.1 14:8518cc6e:::benchmark_data_smithi029_550670_object2940:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.752989+0000", 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 300.51547623499999, 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 31.102100502999999, 2023-10-09T05:03:07.342 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80551 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.752989+0000", 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.343 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.752989+0000", 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.752987+0000", 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.344 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.753001+0000", 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3763e-05 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.753028+0000", 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7487999999999998e-05 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.753041+0000", 2023-10-09T05:03:07.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.292e-05 2023-10-09T05:03:07.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.895488+0000", 2023-10-09T05:03:07.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14244686400000001 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.918843+0000", 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023355343000000001 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022100+0000", 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10325679 2023-10-09T05:03:07.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022109+0000", 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6770000000000001e-06 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833156+0000", 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.811047725 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833166+0000", 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5689999999999994e-06 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.276238+0000", 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.44307199 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.287194+0000", 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010956379 2023-10-09T05:03:07.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.421742+0000", 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13454733299999999 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.436817+0000", 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015075600999999999 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.587413+0000", 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.150595982 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.596005+0000", 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085919280000000004 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.610602+0000", 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014596427 2023-10-09T05:03:07.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.759116+0000", 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14851415100000001 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.812388+0000", 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053272594999999999 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.352 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.818282+0000", 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058938039999999999 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.826111+0000", 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078286559999999998 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.855090+0000", 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028978791 2023-10-09T05:03:07.353 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80554 14.1 14:88d54b09:::benchmark_data_smithi029_550670_object2943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.826139+0000", 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 300.44232599899999, 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 31.088094795, 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.354 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80554 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.826139+0000", 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.826139+0000", 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.826138+0000", 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.826146+0000", 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1130000000000001e-06 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.826160+0000", 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4409999999999999e-05 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.826173+0000", 2023-10-09T05:03:07.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2286999999999999e-05 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022382+0000", 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19620926699999999 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.022388+0000", 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6390000000000003e-06 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833345+0000", 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.810956169 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.833352+0000", 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8839999999999994e-06 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.978573+0000", 2023-10-09T05:03:07.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 29.145220642000002 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:36.999514+0000", 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020941272 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.200104+0000", 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20058942399999999 2023-10-09T05:03:07.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.218378+0000", 2023-10-09T05:03:07.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018274540999999998 2023-10-09T05:03:07.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.362 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.362 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.676437+0000", 2023-10-09T05:03:07.362 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.45805869599999999 2023-10-09T05:03:07.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.363 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.687258+0000", 2023-10-09T05:03:07.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010820778 2023-10-09T05:03:07.363 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.363 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.704169+0000", 2023-10-09T05:03:07.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016910773 2023-10-09T05:03:07.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.813493+0000", 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.109324132 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.860904+0000", 2023-10-09T05:03:07.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.047411163999999999 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.867778+0000", 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068745660000000004 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.876828+0000", 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0090494890000000008 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.914234+0000", 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037406254 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:80574 14.9 e515/491 14:96f986b2:::benchmark_data_smithi029_550670_object2963:head v 515'5062, mlcod=444'139)", 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:09.224386+0000", 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 298.04407961700002, 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.383176670999999, 2023-10-09T05:03:07.367 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224386+0000", 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224386+0000", 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224385+0000", 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224399+0000", 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4201e-05 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224411+0000", 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2653000000000001e-05 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224420+0000", 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3149999999999992e-06 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.595932+0000", 2023-10-09T05:03:07.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.371512599000001 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.595954+0000", 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2086000000000002e-05 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.607510+0000", 2023-10-09T05:03:07.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011555107 2023-10-09T05:03:07.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.607562+0000", 2023-10-09T05:03:07.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2859000000000003e-05 2023-10-09T05:03:07.372 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.373 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.373 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.373 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:80575 14.9 e515/491 14:99a4fb36:::benchmark_data_smithi029_550670_object2964:head v 515'5063, mlcod=444'139)", 2023-10-09T05:03:07.373 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:09.233300+0000", 2023-10-09T05:03:07.374 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 298.03516588500003, 2023-10-09T05:03:07.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.374491719, 2023-10-09T05:03:07.374 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.374 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.374 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.233300+0000", 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.233300+0000", 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.233298+0000", 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:03:07.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.233307+0000", 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4470000000000006e-06 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.233322+0000", 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5401999999999999e-05 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.233335+0000", 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2279000000000001e-05 2023-10-09T05:03:07.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.601600+0000", 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.368265886 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.601630+0000", 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9699000000000001e-05 2023-10-09T05:03:07.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.607762+0000", 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006131432 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:19.607791+0000", 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9652e-05 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.380 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80580 14.1 14:889dc462:::benchmark_data_smithi029_550670_object2969:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:09.277037+0000", 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 297.99142839799998, 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 28.687254853999999, 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.381 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80580 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.277037+0000", 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.277037+0000", 2023-10-09T05:03:07.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.277035+0000", 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.277045+0000", 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0074e-05 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.277062+0000", 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7317e-05 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.277075+0000", 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3222e-05 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.001410+0000", 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 27.724334823 2023-10-09T05:03:07.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.115979+0000", 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11456833900000001 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.275505+0000", 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.15952666400000001 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.296446+0000", 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020940345999999999 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.760506+0000", 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.46406033299999999 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.772945+0000", 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012439220000000001 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.793890+0000", 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020944600000000001 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.862366+0000", 2023-10-09T05:03:07.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.068476420999999996 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.920139+0000", 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057773111000000002 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.924830+0000", 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046911289999999996 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.933118+0000", 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0082874179999999995 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.964292+0000", 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031173960000000001 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.389 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80582 14.1 14:8b64ee67:::benchmark_data_smithi029_550670_object2971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e515)", 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:09.367611+0000", 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 297.90085461899997, 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 29.121693274999998, 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.390 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80582 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.367611+0000", 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.367611+0000", 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.367609+0000", 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.367619+0000", 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0370999999999999e-05 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.392 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.367637+0000", 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7935999999999999e-05 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.367649+0000", 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.201e-05 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.118043+0000", 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 27.750393580000001 2023-10-09T05:03:07.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.134519+0000", 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016476074 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.345848+0000", 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21132904199999999 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:37.360942+0000", 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015094386 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.143931+0000", 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.78298905399999996 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.158159+0000", 2023-10-09T05:03:07.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014227841999999999 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.179904+0000", 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021744531000000001 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.398884+0000", 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.21898046400000001 2023-10-09T05:03:07.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.408799+0000", 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099152159999999993 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.414795+0000", 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00599535 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.424590+0000", 2023-10-09T05:03:07.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097957340000000004 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:38.489304+0000", 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.064713654999999995 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91912 14.b 14:d0cac1ad:::benchmark_data_smithi029_550670_object14301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:41.732359+0000", 2023-10-09T05:03:07.399 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.536106509000007, 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.863427890000001, 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91912 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732359+0000", 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732359+0000", 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732358+0000", 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732364+0000", 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7499999999999997e-06 2023-10-09T05:03:07.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732376+0000", 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1871999999999999e-05 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732384+0000", 2023-10-09T05:03:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.289e-06 2023-10-09T05:03:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.769573+0000", 2023-10-09T05:03:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037189978999999998 2023-10-09T05:03:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.780750+0000", 2023-10-09T05:03:07.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011176317 2023-10-09T05:03:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.932536+0000", 2023-10-09T05:03:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.151786217 2023-10-09T05:03:07.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.936798+0000", 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042621489999999998 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.413913+0000", 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47711435200000002 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.541782+0000", 2023-10-09T05:03:07.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.127869381 2023-10-09T05:03:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:03:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.563926+0000", 2023-10-09T05:03:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022143889999999999 2023-10-09T05:03:07.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.655023+0000", 2023-10-09T05:03:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.091097062000000006 2023-10-09T05:03:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.655040+0000", 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6758000000000001e-05 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421648+0000", 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76660828000000003 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421661+0000", 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2758999999999999e-05 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.859547+0000", 2023-10-09T05:03:07.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.437885865999998 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.859564+0000", 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7229999999999999e-05 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595770+0000", 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73620618999999998 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595787+0000", 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6833999999999998e-05 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91922 14.b 14:dc20e122:::benchmark_data_smithi029_550670_object14311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:41.955425+0000", 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.313040752000006, 2023-10-09T05:03:07.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.640405913999999, 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91922 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955425+0000", 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955425+0000", 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955423+0000", 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955433+0000", 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0509000000000001e-05 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955450+0000", 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7051999999999999e-05 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955462+0000", 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1929e-05 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.971473+0000", 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016010832999999999 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.982740+0000", 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011266905000000001 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.037740+0000", 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054999795999999997 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.043790+0000", 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060503049999999997 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656039+0000", 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.61224903200000003 2023-10-09T05:03:07.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656054+0000", 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4387e-05 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421695+0000", 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76564138299999995 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421704+0000", 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6920000000000008e-06 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865416+0000", 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.44371245 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865428+0000", 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1542999999999999e-05 2023-10-09T05:03:07.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595820+0000", 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73039237700000004 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595831+0000", 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0611999999999999e-05 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.422 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91924 14.b 14:d1ba9b3e:::benchmark_data_smithi029_550670_object14313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:41.980177+0000", 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.288288011999995, 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.615808210000001, 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.423 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91924 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980177+0000", 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980177+0000", 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980175+0000", 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980184+0000", 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3799999999999994e-06 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.982798+0000", 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026139610000000001 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.982823+0000", 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5154999999999999e-05 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.175554+0000", 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19273112000000001 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.181103+0000", 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055490849999999996 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.247397+0000", 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.066294343000000006 2023-10-09T05:03:07.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.253507+0000", 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061097640000000002 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656152+0000", 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40264439899999999 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656156+0000", 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6589999999999998e-06 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421835+0000", 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76567873099999995 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421848+0000", 2023-10-09T05:03:07.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3365e-05 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865794+0000", 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443945398 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865800+0000", 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4579999999999998e-06 2023-10-09T05:03:07.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595976+0000", 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.730175558 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595986+0000", 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8530000000000004e-06 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91937 14.b 14:d29ea32b:::benchmark_data_smithi029_550670_object14326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.105162+0000", 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.163302920000007, 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.490785804000001, 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91937 2023-10-09T05:03:07.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105162+0000", 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105162+0000", 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.433 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105161+0000", 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105168+0000", 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8450000000000003e-06 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105183+0000", 2023-10-09T05:03:07.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4808e-05 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105192+0000", 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9930000000000001e-06 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.182056+0000", 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076864049000000004 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.190862+0000", 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088061900000000002 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334875+0000", 2023-10-09T05:03:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.144012217 2023-10-09T05:03:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.342436+0000", 2023-10-09T05:03:07.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075609090000000002 2023-10-09T05:03:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656130+0000", 2023-10-09T05:03:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.313694588 2023-10-09T05:03:07.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656137+0000", 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.319e-06 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421791+0000", 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76565495100000003 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421799+0000", 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5320000000000001e-06 2023-10-09T05:03:07.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865761+0000", 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443961875999999 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865770+0000", 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9390000000000003e-06 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595938+0000", 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73016817700000003 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595948+0000", 2023-10-09T05:03:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0283e-05 2023-10-09T05:03:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.443 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91941 14.b 14:d4fdc961:::benchmark_data_smithi029_550670_object14330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.136326+0000", 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 85.132139488000007, 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.459581669999999, 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.444 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91941 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136326+0000", 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136326+0000", 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136324+0000", 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136334+0000", 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.268e-06 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136352+0000", 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8000999999999999e-05 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136362+0000", 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0259e-05 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.211276+0000", 2023-10-09T05:03:07.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.074914003000000007 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.216717+0000", 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054411390000000002 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.292465+0000", 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.075747729999999999 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.297780+0000", 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053155199999999998 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656110+0000", 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.358329862 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656116+0000", 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6370000000000004e-06 2023-10-09T05:03:07.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421759+0000", 2023-10-09T05:03:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76564343599999996 2023-10-09T05:03:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421767+0000", 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5839999999999997e-06 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865731+0000", 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443964449999999 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865738+0000", 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7830000000000001e-06 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595897+0000", 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73015950100000004 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595908+0000", 2023-10-09T05:03:07.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0091999999999999e-05 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91961 14.b 14:db9a68b4:::benchmark_data_smithi029_550670_object14350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.334243+0000", 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 84.934222860999995, 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.261626548999999, 2023-10-09T05:03:07.453 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91961 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334243+0000", 2023-10-09T05:03:07.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334243+0000", 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334241+0000", 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334249+0000", 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1519999999999992e-06 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334264+0000", 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5427999999999999e-05 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334273+0000", 2023-10-09T05:03:07.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3599999999999996e-06 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.351183+0000", 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016909699 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.360166+0000", 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089832769999999996 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.620803+0000", 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.260637008 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.624479+0000", 2023-10-09T05:03:07.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003676548 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656082+0000", 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031603054999999998 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656092+0000", 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7399999999999999e-06 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421728+0000", 2023-10-09T05:03:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76563604299999999 2023-10-09T05:03:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421736+0000", 2023-10-09T05:03:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9410000000000007e-06 2023-10-09T05:03:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865697+0000", 2023-10-09T05:03:07.462 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443961297000001 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865705+0000", 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8460000000000008e-06 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.463 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595859+0000", 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73015364100000002 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.464 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595869+0000", 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0155e-05 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.465 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91966 14.16 14:696ae628:::benchmark_data_smithi029_550670_object14355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.368550+0000", 2023-10-09T05:03:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 84.899915870000001, 2023-10-09T05:03:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.744465222000001, 2023-10-09T05:03:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.466 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91966 2023-10-09T05:03:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368550+0000", 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368550+0000", 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368548+0000", 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368556+0000", 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8619999999999992e-06 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368573+0000", 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6878999999999999e-05 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368583+0000", 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7140000000000005e-06 2023-10-09T05:03:07.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.369429+0000", 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00084586399999999999 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.373983+0000", 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045541640000000003 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.389294+0000", 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015310486 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.391535+0000", 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022415410000000001 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.557489+0000", 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.165954305 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.560884+0000", 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033946269999999999 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:03:07.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.586880+0000", 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025995541 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.613945+0000", 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027065378000000001 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.613962+0000", 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7087000000000001e-05 2023-10-09T05:03:07.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.419798+0000", 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.80583640899999998 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.419816+0000", 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7589999999999999e-05 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:52.682897+0000", 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2630812010000003 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:52.688813+0000", 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059156670000000003 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:53.110634+0000", 2023-10-09T05:03:07.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.42182064200000002 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:53.113015+0000", 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023811959999999999 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.477 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91974 14.b 14.b620bdab (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:51.596975+0000", 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 75.671490160000005, 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.000508494, 2023-10-09T05:03:07.478 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:07.479 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91974 2023-10-09T05:03:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.481 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596975+0000", 2023-10-09T05:03:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596975+0000", 2023-10-09T05:03:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596974+0000", 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596983+0000", 2023-10-09T05:03:07.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7499999999999998e-06 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.597000+0000", 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6419e-05 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.597028+0000", 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8773999999999999e-05 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865819+0000", 2023-10-09T05:03:07.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2687909620000006 2023-10-09T05:03:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865830+0000", 2023-10-09T05:03:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0088999999999999e-05 2023-10-09T05:03:07.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.597476+0000", 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73164620300000005 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:07.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.597484+0000", 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0169999999999996e-06 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:07.489 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:03:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:07.491 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:07.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:07.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:07 smithi177 ceph-mon[195797]: pgmap v2014: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 active+undersized+degraded, 8 active+undersized, 58 active+clean, 1 active+degraded, 1 active+recovery_wait+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 unknown, 1 stale+active+remapped+backfill_wait, 4 stale+active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 10 stale+active+clean, 1 stale+active+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 73161/188985 objects degraded (38.713%); 12479/188985 objects misplaced (6.603%); 1.8 MiB/s, 28 objects/s recovering 2023-10-09T05:03:07.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3816981910' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:07.519 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:07 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:07.526 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T05:03:07.570 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:03:07.570 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T05:03:07.591 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:07.646 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:08.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:03:08.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:08.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[216903]: osdmap e621: 8 total, 7 up, 7 in 2023-10-09T05:03:08.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[216903]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[216903]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[216903]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:03:08.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:08.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[223858]: osdmap e621: 8 total, 7 up, 7 in 2023-10-09T05:03:08.377 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[223858]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.377 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[223858]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.377 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:08 smithi029 ceph-mon[223858]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:08 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:03:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:08 smithi177 ceph-mon[195797]: osdmap e621: 8 total, 7 up, 7 in 2023-10-09T05:03:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:08 smithi177 ceph-mon[195797]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:08 smithi177 ceph-mon[195797]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:08 smithi177 ceph-mon[195797]: failed to encode map e621 with expected crc 2023-10-09T05:03:08.791 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:08.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:08.896 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:08.924 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:08.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:08.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:09.062 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:09 smithi029 ceph-mon[216903]: pgmap v2016: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 6 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 79086/188970 objects degraded (41.851%); 17889/188970 objects misplaced (9.467%); 1541/62990 objects unfound (2.446%); 1.6 MiB/s, 22 objects/s recovering 2023-10-09T05:03:09.063 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:09 smithi029 ceph-mon[216903]: Health check failed: 1541/62990 objects unfound (2.446%) (OBJECT_UNFOUND) 2023-10-09T05:03:09.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:09.116 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:09.274 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:09 smithi177 ceph-mon[195797]: pgmap v2016: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 6 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 79086/188970 objects degraded (41.851%); 17889/188970 objects misplaced (9.467%); 1541/62990 objects unfound (2.446%); 1.6 MiB/s, 22 objects/s recovering 2023-10-09T05:03:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:09 smithi177 ceph-mon[195797]: Health check failed: 1541/62990 objects unfound (2.446%) (OBJECT_UNFOUND) 2023-10-09T05:03:09.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:09.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:09.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:09 smithi029 ceph-mon[223858]: pgmap v2016: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 6 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 79086/188970 objects degraded (41.851%); 17889/188970 objects misplaced (9.467%); 1541/62990 objects unfound (2.446%); 1.6 MiB/s, 22 objects/s recovering 2023-10-09T05:03:09.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:09 smithi029 ceph-mon[223858]: Health check failed: 1541/62990 objects unfound (2.446%) (OBJECT_UNFOUND) 2023-10-09T05:03:09.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:03:09.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:09.442 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:09.629 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:03:09.633 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:09.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:09.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:09.770 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:09.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:09.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:09.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:09.924 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:09.974 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:09.992 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:10.068 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:10.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:10.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:10.202 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:10.248 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:10.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:10.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:10.325 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2975531316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:10.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2975531316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:10.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:10.412 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2975531316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:10.740 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:10.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:10.846 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:10.879 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:10.947 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:10.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:10.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:11.056 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:11.089 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:11.137 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:11.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:11.241 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:11.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:11 smithi029 ceph-mon[216903]: pgmap v2017: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 6 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 25 MiB/s wr, 472 op/s; 79086/188970 objects degraded (41.851%); 17889/188970 objects misplaced (9.467%); 1541/62990 objects unfound (2.446%); 9.1 MiB/s, 142 objects/s recovering 2023-10-09T05:03:11.266 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 79086/188970 objects degraded (41.851%), 37 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:11.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:11 smithi029 ceph-mon[223858]: pgmap v2017: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 6 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 25 MiB/s wr, 472 op/s; 79086/188970 objects degraded (41.851%); 17889/188970 objects misplaced (9.467%); 1541/62990 objects unfound (2.446%); 9.1 MiB/s, 142 objects/s recovering 2023-10-09T05:03:11.267 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 79086/188970 objects degraded (41.851%), 37 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:11.269 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:11.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:03:11.323 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:11.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:11.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:11.427 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:11.456 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:11 smithi177 ceph-mon[195797]: pgmap v2017: 119 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+undersized+remapped, 6 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 peering, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 25 MiB/s wr, 472 op/s; 79086/188970 objects degraded (41.851%); 17889/188970 objects misplaced (9.467%); 1541/62990 objects unfound (2.446%); 9.1 MiB/s, 142 objects/s recovering 2023-10-09T05:03:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 79086/188970 objects degraded (41.851%), 37 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:11.739 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:11.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:11.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:11.884 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:11.892 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:11.952 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:11.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:12.057 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:12.086 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:12.133 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:12.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:12.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:12.280 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:12.365 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:03:12.366 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:03:12.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.366 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92263 14.1f 14:fdfcca44:::benchmark_data_smithi029_550670_object14652:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e613)", 2023-10-09T05:03:12.366 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:58.282065+0000", 2023-10-09T05:03:12.366 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 14.082316895, 2023-10-09T05:03:12.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 14.082342679, 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "waiting for sub ops", 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92263 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.367 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282065+0000", 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282065+0000", 2023-10-09T05:03:12.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282063+0000", 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282069+0000", 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.207e-06 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282080+0000", 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.118e-05 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282088+0000", 2023-10-09T05:03:12.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7249999999999997e-06 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282375+0000", 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028714000000000002 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.282388+0000", 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2762000000000001e-05 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:59.955374+0000", 2023-10-09T05:03:12.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.672985537 2023-10-09T05:03:12.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:12.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:59.955388+0000", 2023-10-09T05:03:12.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4214e-05 2023-10-09T05:03:12.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.737751+0000", 2023-10-09T05:03:12.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7823628779999998 2023-10-09T05:03:12.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.741299+0000", 2023-10-09T05:03:12.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035482640000000002 2023-10-09T05:03:12.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.377 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.377 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.783267+0000", 2023-10-09T05:03:12.377 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.041968182 2023-10-09T05:03:12.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.378 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:12.378 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.788534+0000", 2023-10-09T05:03:12.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052663340000000001 2023-10-09T05:03:12.378 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.378 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.808953+0000", 2023-10-09T05:03:12.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020419113999999999 2023-10-09T05:03:12.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.812671+0000", 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037182309999999998 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5", 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.819453+0000", 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067820179999999999 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.845465+0000", 2023-10-09T05:03:12.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026012128999999998 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.845869+0000", 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00040375600000000002 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92291 14.1f 14:fc8ac7c7:::benchmark_data_smithi029_550670_object14680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:12.381 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:09.825432+0000", 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.5389496629999999, 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.539067572, 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92291 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.825432+0000", 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.825432+0000", 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.825430+0000", 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.825438+0000", 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5399999999999996e-06 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.825460+0000", 2023-10-09T05:03:12.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1172999999999999e-05 2023-10-09T05:03:12.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.825471+0000", 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1192000000000001e-05 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.826496+0000", 2023-10-09T05:03:12.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010250999999999999 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.832268+0000", 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057719390000000002 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.848215+0000", 2023-10-09T05:03:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015946881999999999 2023-10-09T05:03:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.850632+0000", 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024172830000000001 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92268 14.1f 14:fca182c9:::benchmark_data_smithi029_550670_object14657:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e615)", 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:00.908823+0000", 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 11.455558727, 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 11.455741891000001, 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.389 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92268 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908823+0000", 2023-10-09T05:03:12.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.391 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908823+0000", 2023-10-09T05:03:12.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908820+0000", 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908829+0000", 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.098e-06 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908847+0000", 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7799999999999999e-05 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908857+0000", 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6549999999999999e-06 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908916+0000", 2023-10-09T05:03:12.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9106e-05 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.908925+0000", 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7830000000000004e-06 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.790988+0000", 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8820629310000001 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.796397+0000", 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054091 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.847089+0000", 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.050692741999999999 2023-10-09T05:03:12.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:05.849665+0000", 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025753569999999999 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.398 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92288 14.1f 14:fb2ee6f4:::benchmark_data_smithi029_550670_object14677:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:09.567953+0000", 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.7964286710000001, 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7966706659999998, 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92288 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.399 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.567953+0000", 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.567953+0000", 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.567950+0000", 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.567962+0000", 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1649e-05 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.567979+0000", 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6426000000000002e-05 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.567992+0000", 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3185e-05 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.569006+0000", 2023-10-09T05:03:12.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001013845 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.577662+0000", 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.008656202 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.594507+0000", 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016845068000000001 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.597085+0000", 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025779380000000001 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 621/597 [PushOp(14:cb95c603:::benchmark_data_smithi029_550670_object40507:head, version: 445'2562, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:cb95c603:::benchmark_data_smithi029_550670_object40507:head@445'2562, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:12.362770+0000", 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 0.0016116489999999999, 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00191556, 2023-10-09T05:03:12.404 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.362770+0000", 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.362770+0000", 2023-10-09T05:03:12.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.362769+0000", 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.362830+0000", 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0588000000000003e-05 2023-10-09T05:03:12.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.362845+0000", 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5639999999999999e-05 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.362857+0000", 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1151e-05 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.363096+0000", 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023976499999999999 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:12.363269+0000", 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017239100000000001 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92289 14.1f 14:fc5cced3:::benchmark_data_smithi029_550670_object14678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:09.584336+0000", 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.7800456369999997, 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7803895399999998, 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92289 2023-10-09T05:03:12.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.584336+0000", 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.584336+0000", 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.584334+0000", 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.584345+0000", 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1522000000000001e-05 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.584362+0000", 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7305000000000001e-05 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.584377+0000", 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4199000000000001e-05 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.598786+0000", 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014409325000000001 2023-10-09T05:03:12.412 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.602243+0000", 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034567479999999999 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.620208+0000", 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017965750999999999 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.622042+0000", 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00183366 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92306 14.1f 14:fc44c71a:::benchmark_data_smithi029_550670_object14695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:09.934455+0000", 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.429926488, 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4303215200000001, 2023-10-09T05:03:12.414 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92306 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.934455+0000", 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.934455+0000", 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.934453+0000", 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.934461+0000", 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0260000000000007e-06 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.934477+0000", 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6169999999999999e-05 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.934487+0000", 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7200000000000001e-06 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.935473+0000", 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098632499999999992 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.941574+0000", 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061009660000000002 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.963254+0000", 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021679530999999998 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:09.967372+0000", 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041180050000000001 2023-10-09T05:03:12.419 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92323 14.1f 14:fa15fab6:::benchmark_data_smithi029_550670_object14712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:10.194203+0000", 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 2.1701788579999999, 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1706236900000002, 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.420 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92323 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.194203+0000", 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.194203+0000", 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.194201+0000", 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.194207+0000", 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9569999999999999e-06 2023-10-09T05:03:12.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.194220+0000", 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2411e-05 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.194226+0000", 2023-10-09T05:03:12.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2820000000000002e-06 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.195127+0000", 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00090094399999999999 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.199252+0000", 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041254799999999999 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.215809+0000", 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016556810000000002 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.217807+0000", 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001997979 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.425 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92350 14.1f 14:ff9542fb:::benchmark_data_smithi029_550670_object14739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:10.516726+0000", 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 1.847655788, 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8481498059999999, 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "delayed", 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:12.426 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92350 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.516726+0000", 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.516726+0000", 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.516723+0000", 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.516736+0000", 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2964e-05 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:12.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.516759+0000", 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2376000000000001e-05 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.516772+0000", 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3687e-05 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.517720+0000", 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00094708599999999998 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.525060+0000", 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073407869999999997 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.537856+0000", 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012795825 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:12.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:10.539924+0000", 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020678519999999998 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: ], 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 9 2023-10-09T05:03:12.431 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:03:12.570 INFO:tasks.thrashosds.thrasher:in_osds: [5, 7, 3, 6, 4, 0, 1] out_osds: [2] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-10-09T05:03:12.571 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-10-09T05:03:12.571 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-10-09T05:03:12.571 DEBUG:teuthology.orchestra.run.smithi177:osd.4> sudo journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-09T05:03:12.574 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl start ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T05:03:12.599 INFO:journalctl@ceph.osd.4.smithi177.stdout:-- Logs begin at Mon 2023-10-09 03:43:11 UTC. -- 2023-10-09T05:03:12.649 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:12.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:12.753 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:12.784 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:12.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:03:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:03:12] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2023-10-09T05:03:13.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:12 smithi177 systemd[1]: Starting Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:03:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:13 smithi029 ceph-mon[216903]: pgmap v2018: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 4 activating+undersized+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 19 MiB/s wr, 368 op/s; 83776/188943 objects degraded (44.339%); 21059/188943 objects misplaced (11.146%); 1541/62981 objects unfound (2.447%); 11 MiB/s, 218 objects/s recovering 2023-10-09T05:03:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:13 smithi029 ceph-mon[223858]: pgmap v2018: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 4 activating+undersized+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 19 MiB/s wr, 368 op/s; 83776/188943 objects degraded (44.339%); 21059/188943 objects misplaced (11.146%); 1541/62981 objects unfound (2.447%); 11 MiB/s, 218 objects/s recovering 2023-10-09T05:03:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:13 smithi177 ceph-mon[195797]: pgmap v2018: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 4 activating+undersized+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 19 MiB/s wr, 368 op/s; 83776/188943 objects degraded (44.339%); 21059/188943 objects misplaced (11.146%); 1541/62981 objects unfound (2.447%); 11 MiB/s, 218 objects/s recovering 2023-10-09T05:03:14.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-09T05:03:14.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T05:03:14.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2023-10-09T05:03:14.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2023-10-09T05:03:14.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2023-10-09T05:03:14.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-09T05:03:14.028 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-09T05:03:14.028 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:13 smithi177 bash[410230]: --> ceph-volume lvm activate successful for osd ID: 4 2023-10-09T05:03:14.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:03:14.292 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:14.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:14.396 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: osdmap e622: 8 total, 7 up, 7 in 2023-10-09T05:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: osdmap e622: 8 total, 7 up, 7 in 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:14.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: osdmap e622: 8 total, 7 up, 7 in 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: failed to encode map e622 with expected crc 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:14.423 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:14.427 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:14.478 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:14.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:14.503 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:03:14.529 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:03:14.569 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:03:14.583 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:14.604 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:03:14.625 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:14.629 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:14.641 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.672 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.673 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:03:14.898 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:14.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:15.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:15.039 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:15.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: pgmap v2020: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 83694/188757 objects degraded (44.340%); 21032/188757 objects misplaced (11.142%); 4.3 MiB/s, 107 objects/s recovering 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: osdmap e623: 8 total, 7 up, 7 in 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[223858]: Health check cleared: OBJECT_UNFOUND (was: 1541/62981 objects unfound (2.447%)) 2023-10-09T05:03:15.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: pgmap v2020: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 83694/188757 objects degraded (44.340%); 21032/188757 objects misplaced (11.142%); 4.3 MiB/s, 107 objects/s recovering 2023-10-09T05:03:15.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: osdmap e623: 8 total, 7 up, 7 in 2023-10-09T05:03:15.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:15 smithi029 ceph-mon[216903]: Health check cleared: OBJECT_UNFOUND (was: 1541/62981 objects unfound (2.447%)) 2023-10-09T05:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: pgmap v2020: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 stale+active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 9 op/s; 83694/188757 objects degraded (44.340%); 21032/188757 objects misplaced (11.142%); 4.3 MiB/s, 107 objects/s recovering 2023-10-09T05:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: osdmap e623: 8 total, 7 up, 7 in 2023-10-09T05:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: failed to encode map e623 with expected crc 2023-10-09T05:03:15.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:15 smithi177 ceph-mon[195797]: Health check cleared: OBJECT_UNFOUND (was: 1541/62981 objects unfound (2.447%)) 2023-10-09T05:03:16.000 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:16.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:16.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:16.122 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-09T05:03:16.123 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-09T05:03:16.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:16.157 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:16.210 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:16.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:16.315 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:16.349 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:16.400 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:16.404 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 83694/188757 objects degraded (44.340%), 39 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:16.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:16.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:16.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 83694/188757 objects degraded (44.340%), 39 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:16.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 83694/188757 objects degraded (44.340%), 39 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:16.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:16.656 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:16.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:03:16.776 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:16.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:16.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:16.954 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:17.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:16 smithi177 ceph-osd[410870]: -- 172.21.15.177:0/1191169221 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560c29444b60 con 0x560c2940ac00 2023-10-09T05:03:17.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:16 smithi177 bash[410765]: c60e52eb4c6c3745edfb13038b08fb6256d1b6865674e57326f534793cbb4af2 2023-10-09T05:03:17.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:16 smithi177 systemd[1]: Started Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:03:17.029 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:17.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:17.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:17.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:17.165 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:17.215 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:03:17.219 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:17.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:17 smithi177 ceph-mon[195797]: pgmap v2022: 119 pgs: 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 100524/188610 objects degraded (53.297%); 20605/188610 objects misplaced (10.925%); 5.6 MiB/s, 129 objects/s recovering 2023-10-09T05:03:17.475 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:17 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/179684847' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:03:17.476 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:17 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:03:17.476 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:17 smithi177 conmon[410843]: 2023-10-09T05:03:17.255+0000 7ff983eac540 -1 Falling back to public interface 2023-10-09T05:03:17.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:17 smithi029 ceph-mon[223858]: pgmap v2022: 119 pgs: 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 100524/188610 objects degraded (53.297%); 20605/188610 objects misplaced (10.925%); 5.6 MiB/s, 129 objects/s recovering 2023-10-09T05:03:17.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:17 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/179684847' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:03:17.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:17 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:03:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:17 smithi029 ceph-mon[216903]: pgmap v2022: 119 pgs: 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 100524/188610 objects degraded (53.297%); 20605/188610 objects misplaced (10.925%); 5.6 MiB/s, 129 objects/s recovering 2023-10-09T05:03:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:17 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/179684847' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:03:17.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:17 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:03:17.777 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:17.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:17.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:17.890 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:17.923 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:17.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:17.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:18.074 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:18.100 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:18.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:03:18.143 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:18.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:18.247 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:18.399 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:18.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:18.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:03:18.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: osdmap e624: 8 total, 7 up, 7 in 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[223858]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: osdmap e624: 8 total, 7 up, 7 in 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:18 smithi029 ceph-mon[216903]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:03:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: osdmap e624: 8 total, 7 up, 7 in 2023-10-09T05:03:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:18 smithi177 ceph-mon[195797]: failed to encode map e624 with expected crc 2023-10-09T05:03:18.723 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:18.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:18.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:18.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:18.992 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:19.125 INFO:teuthology.orchestra.run.smithi177.stderr:no valid command found; 10 closest matches: 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:0 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:1 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:2 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:abort 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:assert 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:config diff 2023-10-09T05:03:19.126 INFO:teuthology.orchestra.run.smithi177.stderr:config diff get 2023-10-09T05:03:19.127 INFO:teuthology.orchestra.run.smithi177.stderr:config get 2023-10-09T05:03:19.127 INFO:teuthology.orchestra.run.smithi177.stderr:config help [] 2023-10-09T05:03:19.127 INFO:teuthology.orchestra.run.smithi177.stderr:config set ... 2023-10-09T05:03:19.127 INFO:teuthology.orchestra.run.smithi177.stderr:admin_socket: invalid command 2023-10-09T05:03:19.271 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:19.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:19.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:19.467 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:03:19.467 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:03:19.484 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:19.507 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:19 smithi029 ceph-mon[216903]: pgmap v2024: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 3 active+clean+remapped, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 98451/188283 objects degraded (52.289%); 21149/188283 objects misplaced (11.233%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T05:03:19.508 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:19 smithi029 ceph-mon[223858]: pgmap v2024: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 3 active+clean+remapped, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 98451/188283 objects degraded (52.289%); 21149/188283 objects misplaced (11.233%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T05:03:19.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:19 smithi177 ceph-mon[195797]: pgmap v2024: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+degraded+remapped, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 3 active+clean+remapped, 5 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 36 op/s; 98451/188283 objects degraded (52.289%); 21149/188283 objects misplaced (11.233%); 8.7 MiB/s, 140 objects/s recovering 2023-10-09T05:03:19.601 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:19.723 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:19.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:19.832 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:19.868 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-09T05:03:19.969 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-10-09T05:03:19.973 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:19.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:20.081 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:20.113 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:20.159 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:20.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:20.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:20.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:20.310 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:20.374 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:20.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:20.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:20.509 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:20.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:20.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:20.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:20.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:03:20.766 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:20.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:03:21.007 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:21.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:21.118 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:21.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:21 smithi029 ceph-mon[216903]: pgmap v2025: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 41 op/s; 98157/187938 objects degraded (52.228%); 21149/187938 objects misplaced (11.253%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T05:03:21.261 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 98157/187938 objects degraded (52.228%), 40 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:21.261 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:21 smithi029 ceph-mon[223858]: pgmap v2025: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 41 op/s; 98157/187938 objects degraded (52.228%); 21149/187938 objects misplaced (11.253%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T05:03:21.262 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 98157/187938 objects degraded (52.228%), 40 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:21.291 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:21 smithi177 ceph-mon[195797]: pgmap v2025: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 41 op/s; 98157/187938 objects degraded (52.228%); 21149/187938 objects misplaced (11.253%); 8.7 MiB/s, 138 objects/s recovering 2023-10-09T05:03:21.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 98157/187938 objects degraded (52.228%), 40 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:21.638 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:21.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:21.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:21.800 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:21.932 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:21.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:21.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:22.040 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:22.069 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:22.135 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:22.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:22.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:22.252 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:22.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:21 smithi177 conmon[410843]: 2023-10-09T05:03:21.895+0000 7ff983eac540 -1 osd.4 616 log_to_monitors true 2023-10-09T05:03:22.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:22 smithi029 ceph-mon[216903]: from='osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:03:22.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:22 smithi029 ceph-mon[216903]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:03:22.411 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:22.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:22 smithi029 ceph-mon[223858]: from='osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:03:22.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:22 smithi029 ceph-mon[223858]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:03:22.682 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:22.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:22.777 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:03:22 smithi177 conmon[410843]: 2023-10-09T05:03:22.326+0000 7ff973c89700 -1 osd.4 616 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-09T05:03:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:22 smithi177 ceph-mon[195797]: from='osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:03:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:22 smithi177 ceph-mon[195797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-09T05:03:22.796 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:22.817 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:22.856 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:22.925 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:03:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:03:22] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2023-10-09T05:03:22.927 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:23.166 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:23.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:23.276 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:23.307 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:23.424 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:23.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:23.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:23.539 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:23.650 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:03:23.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: pgmap v2026: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 97986/187251 objects degraded (52.329%); 20930/187251 objects misplaced (11.178%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:03:23.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:03:23.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: osdmap e625: 8 total, 7 up, 7 in 2023-10-09T05:03:23.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: from='osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[216903]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: pgmap v2026: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 97986/187251 objects degraded (52.329%); 20930/187251 objects misplaced (11.178%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: osdmap e625: 8 total, 7 up, 7 in 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: from='osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T05:03:23.658 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.659 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.659 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.659 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:23 smithi029 ceph-mon[223858]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T05:03:23.693 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:03:23.725 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:23.730 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:03:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: pgmap v2026: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 97986/187251 objects degraded (52.329%); 20930/187251 objects misplaced (11.178%); 8.1 MiB/s, 130 objects/s recovering 2023-10-09T05:03:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-09T05:03:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: osdmap e625: 8 total, 7 up, 7 in 2023-10-09T05:03:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: from='osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T05:03:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: failed to encode map e625 with expected crc 2023-10-09T05:03:23.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:23 smithi177 ceph-mon[195797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2023-10-09T05:03:23.791 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:03:23.839 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:03:23.862 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:03:23.898 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:03:23.934 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:03:23.937 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:03:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:03:24.052 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:24.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:24.158 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:24.203 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:24.355 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:24.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:24.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:24.517 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:03:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329] boot 2023-10-09T05:03:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: osdmap e626: 8 total, 8 up, 7 in 2023-10-09T05:03:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:03:24.518 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1209706280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:24.519 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[216903]: osdmap e627: 8 total, 8 up, 7 in 2023-10-09T05:03:24.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:03:24.519 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329] boot 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: osdmap e626: 8 total, 8 up, 7 in 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1209706280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:24.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:24 smithi029 ceph-mon[223858]: osdmap e627: 8 total, 8 up, 7 in 2023-10-09T05:03:24.521 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-09T05:03:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: osd.4 [v2:172.21.15.177:6800/1622844329,v1:172.21.15.177:6801/1622844329] boot 2023-10-09T05:03:24.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: osdmap e626: 8 total, 8 up, 7 in 2023-10-09T05:03:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-09T05:03:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: failed to encode map e626 with expected crc 2023-10-09T05:03:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1209706280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:24.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:24 smithi177 ceph-mon[195797]: osdmap e627: 8 total, 8 up, 7 in 2023-10-09T05:03:24.970 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:03:25.312 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:25.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[223858]: failed to encode map e617 with expected crc 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[223858]: failed to encode map e626 with expected crc 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[223858]: pgmap v2029: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 97667/186516 objects degraded (52.364%); 20506/186516 objects misplaced (10.994%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[223858]: failed to encode map e627 with expected crc 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[223858]: osdmap e628: 8 total, 8 up, 7 in 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[216903]: failed to encode map e617 with expected crc 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[216903]: failed to encode map e626 with expected crc 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[216903]: pgmap v2029: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 97667/186516 objects degraded (52.364%); 20506/186516 objects misplaced (10.994%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:03:25.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[216903]: failed to encode map e627 with expected crc 2023-10-09T05:03:25.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:25 smithi029 ceph-mon[216903]: osdmap e628: 8 total, 8 up, 7 in 2023-10-09T05:03:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:25 smithi177 ceph-mon[195797]: failed to encode map e617 with expected crc 2023-10-09T05:03:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:25 smithi177 ceph-mon[195797]: failed to encode map e626 with expected crc 2023-10-09T05:03:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:25 smithi177 ceph-mon[195797]: pgmap v2029: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+clean+remapped, 7 active+recovery_wait+degraded+remapped, 9 active+undersized+degraded, 19 active+undersized, 54 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 94 op/s; 97667/186516 objects degraded (52.364%); 20506/186516 objects misplaced (10.994%); 7.2 MiB/s, 114 objects/s recovering 2023-10-09T05:03:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:25 smithi177 ceph-mon[195797]: failed to encode map e627 with expected crc 2023-10-09T05:03:25.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:25 smithi177 ceph-mon[195797]: osdmap e628: 8 total, 8 up, 7 in 2023-10-09T05:03:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:26 smithi177 ceph-mon[195797]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:26 smithi177 ceph-mon[195797]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:26 smithi177 ceph-mon[195797]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:03:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:26 smithi177 ceph-mon[195797]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 97667/186516 objects degraded (52.364%), 40 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:26 smithi177 ceph-mon[195797]: osdmap e629: 8 total, 8 up, 7 in 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[216903]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[216903]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[216903]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[216903]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 97667/186516 objects degraded (52.364%), 40 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[216903]: osdmap e629: 8 total, 8 up, 7 in 2023-10-09T05:03:26.812 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[223858]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[223858]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[223858]: 3.f continuing backfill to osd.1 from (0'0,68'1] MIN to 68'1 2023-10-09T05:03:26.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[223858]: failed to encode map e628 with expected crc 2023-10-09T05:03:26.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 97667/186516 objects degraded (52.364%), 40 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:26.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:26 smithi029 ceph-mon[223858]: osdmap e629: 8 total, 8 up, 7 in 2023-10-09T05:03:26.813 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:26.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:26.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:26.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:27.067 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:27 smithi177 ceph-mon[195797]: pgmap v2032: 119 pgs: 5 remapped+peering, 3 peering, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 remapped, 1 active+clean+remapped, 6 active+recovery_wait+degraded+remapped, 3 active+undersized+degraded, 15 active+undersized, 64 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 59662/185850 objects degraded (32.102%); 20382/185850 objects misplaced (10.967%); 9.8 MiB/s, 174 objects/s recovering 2023-10-09T05:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:27 smithi177 ceph-mon[195797]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:27 smithi177 ceph-mon[195797]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:27 smithi177 ceph-mon[195797]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:27 smithi177 ceph-mon[195797]: 14.e continuing backfill to osd.0 from (445'1681,625'5753] MIN to 625'5753 2023-10-09T05:03:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:27 smithi177 ceph-mon[195797]: 14.1e continuing backfill to osd.0 from (445'1681,625'5762] MIN to 625'5762 2023-10-09T05:03:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[216903]: pgmap v2032: 119 pgs: 5 remapped+peering, 3 peering, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 remapped, 1 active+clean+remapped, 6 active+recovery_wait+degraded+remapped, 3 active+undersized+degraded, 15 active+undersized, 64 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 59662/185850 objects degraded (32.102%); 20382/185850 objects misplaced (10.967%); 9.8 MiB/s, 174 objects/s recovering 2023-10-09T05:03:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[216903]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[216903]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[216903]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[216903]: 14.e continuing backfill to osd.0 from (445'1681,625'5753] MIN to 625'5753 2023-10-09T05:03:27.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[216903]: 14.1e continuing backfill to osd.0 from (445'1681,625'5762] MIN to 625'5762 2023-10-09T05:03:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[223858]: pgmap v2032: 119 pgs: 5 remapped+peering, 3 peering, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 remapped, 1 active+clean+remapped, 6 active+recovery_wait+degraded+remapped, 3 active+undersized+degraded, 15 active+undersized, 64 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 59662/185850 objects degraded (32.102%); 20382/185850 objects misplaced (10.967%); 9.8 MiB/s, 174 objects/s recovering 2023-10-09T05:03:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[223858]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[223858]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[223858]: failed to encode map e629 with expected crc 2023-10-09T05:03:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[223858]: 14.e continuing backfill to osd.0 from (445'1681,625'5753] MIN to 625'5753 2023-10-09T05:03:27.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:27 smithi029 ceph-mon[223858]: 14.1e continuing backfill to osd.0 from (445'1681,625'5762] MIN to 625'5762 2023-10-09T05:03:27.979 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:27.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:28.051 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:03:28.051 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93400 14.1c 14.cdc803bc (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.6692214349999999, 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6692521149999999, 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for peered", 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:28.052 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93400 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381706+0000", 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8140000000000001e-06 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381711+0000", 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.575e-06 2023-10-09T05:03:28.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381715+0000", 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8240000000000001e-06 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381812+0000", 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6555e-05 2023-10-09T05:03:28.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381822+0000", 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0302000000000001e-05 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93407 14.5 14:a209006e:::benchmark_data_smithi029_550670_object15796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.668817448, 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.668943096, 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for missing object", 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:28.058 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93407 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:03:28.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382114+0000", 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2089999999999999e-06 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382124+0000", 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0268000000000001e-05 2023-10-09T05:03:28.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382134+0000", 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382191+0000", 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6889999999999999e-05 2023-10-09T05:03:28.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382201+0000", 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0359e-05 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.374861+0000", 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9926602410000003 2023-10-09T05:03:28.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.375679+0000", 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00081763400000000003 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.064 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93418 14.1c 14.6fb62d3c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.6687746460000001, 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6689906560000001, 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for peered", 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93418 2023-10-09T05:03:28.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382153+0000", 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0680000000000001e-06 2023-10-09T05:03:28.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382160+0000", 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9639999999999997e-06 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382165+0000", 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4569999999999996e-06 2023-10-09T05:03:28.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382198+0000", 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2703999999999997e-05 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382205+0000", 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.069 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93419 14.15 14.823c8c15 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 3.6457789699999998, 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6460682520000001, 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "waiting for peered", 2023-10-09T05:03:28.070 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93419 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:03:28.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405145+0000", 2023-10-09T05:03:28.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405149+0000", 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6789999999999996e-06 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405159+0000", 2023-10-09T05:03:28.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8779999999999993e-06 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405163+0000", 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1359999999999999e-06 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:28.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405203+0000", 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9718999999999997e-05 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405212+0000", 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2439999999999999e-06 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.075 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:28.076 INFO:teuthology.orchestra.run.smithi177.stdout: ], 2023-10-09T05:03:28.076 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 4 2023-10-09T05:03:28.076 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:03:28.086 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:28.122 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:28.325 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:28.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:28.430 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:28.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:03:28.505 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:28.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:28 smithi029 ceph-mon[216903]: pgmap v2034: 119 pgs: 2 activating+degraded+remapped, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 remapped, 4 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded, 12 active+undersized, 68 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 56 op/s; 59753/185775 objects degraded (32.164%); 18460/185775 objects misplaced (9.937%); 2.3 MiB/s, 53 objects/s recovering 2023-10-09T05:03:28.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:28 smithi029 ceph-mon[223858]: pgmap v2034: 119 pgs: 2 activating+degraded+remapped, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 remapped, 4 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded, 12 active+undersized, 68 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 56 op/s; 59753/185775 objects degraded (32.164%); 18460/185775 objects misplaced (9.937%); 2.3 MiB/s, 53 objects/s recovering 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75785 14.1 14:8b563722:::benchmark_data_smithi029_550670_object75784:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T05:03:28.644 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581965+0000", 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 523.05925355500005, 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 12.481576762, 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75785 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581965+0000", 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581965+0000", 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581964+0000", 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581992+0000", 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7560999999999999e-05 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582001+0000", 2023-10-09T05:03:28.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2410000000000001e-06 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582014+0000", 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2964e-05 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582129+0000", 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000115244 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582138+0000", 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0499999999999997e-06 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:57.982937+0000", 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 12.400798625 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.649 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:57.990054+0000", 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0071166620000000002 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.008517+0000", 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018463080999999999 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.021868+0000", 2023-10-09T05:03:28.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01335124 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.042048+0000", 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020179806000000002 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.044195+0000", 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021467069999999999 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.047476+0000", 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032809219999999999 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.063542+0000", 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016066225999999999 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75806 14.9 e454/449)", 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.525967+0000", 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.11525131999997, 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.546416377, 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525967+0000", 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525967+0000", 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.525966+0000", 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.654 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526032+0000", 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6188999999999999e-05 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526047+0000", 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4789e-05 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.526056+0000", 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6300000000000004e-06 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.042096+0000", 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.516040319 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.042131+0000", 2023-10-09T05:03:28.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4876000000000003e-05 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072334+0000", 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030203741999999999 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072384+0000", 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9141999999999998e-05 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.657 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75808 14.9 e454/449)", 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.529907+0000", 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.11131137300004, 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.547706969, 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529907+0000", 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529907+0000", 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529906+0000", 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:28.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529941+0000", 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4762999999999999e-05 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529954+0000", 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3297e-05 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.529963+0000", 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3389999999999999e-06 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.063586+0000", 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.533623241000001 2023-10-09T05:03:28.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.063618+0000", 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1579999999999999e-05 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077574+0000", 2023-10-09T05:03:28.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013955778 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077614+0000", 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0300999999999999e-05 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75811 14.9 e454/449)", 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.544466+0000", 2023-10-09T05:03:28.664 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.09675276899998, 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.533182881, 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544466+0000", 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544466+0000", 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544465+0000", 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544531+0000", 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5989999999999997e-05 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544543+0000", 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1668000000000001e-05 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.544550+0000", 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8689999999999998e-06 2023-10-09T05:03:28.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.068317+0000", 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.523767645 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.068341+0000", 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3470999999999999e-05 2023-10-09T05:03:28.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077621+0000", 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092804050000000003 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077649+0000", 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7525000000000002e-05 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.669 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75815 14.1 14:83dd8a20:::benchmark_data_smithi029_550670_object75814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.548948+0000", 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.09227049399999, 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.711953803, 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.670 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75815 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548948+0000", 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548948+0000", 2023-10-09T05:03:28.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548946+0000", 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548972+0000", 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5953999999999999e-05 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.673 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548991+0000", 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8706e-05 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.549004+0000", 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2743e-05 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.101680+0000", 2023-10-09T05:03:28.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.552676115000001 2023-10-09T05:03:28.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.114195+0000", 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012514759 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.134726+0000", 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020531269000000001 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.216945+0000", 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.082219406999999994 2023-10-09T05:03:28.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.221124+0000", 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041786879999999998 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.222701+0000", 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015774960000000001 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.230571+0000", 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078695970000000007 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.260902+0000", 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030330778999999999 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.678 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75824 14.1 14:86e81355:::benchmark_data_smithi029_550670_object75823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.566240+0000", 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.07497877900005, 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.729266398, 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.679 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75824 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566240+0000", 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566240+0000", 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566238+0000", 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566260+0000", 2023-10-09T05:03:28.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1810999999999998e-05 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566270+0000", 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.6870000000000006e-06 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.566277+0000", 2023-10-09T05:03:28.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.481e-06 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.147946+0000", 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.581669004 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.169373+0000", 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021426062999999999 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.199762+0000", 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030389360000000001 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.262198+0000", 2023-10-09T05:03:28.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062436396999999998 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.266913+0000", 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047142499999999997 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.271230+0000", 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043171829999999996 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.277067+0000", 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005837582 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.295506+0000", 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01843887 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.686 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75829 14.1 14:8b2b9eaf:::benchmark_data_smithi029_550670_object75828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.571726+0000", 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.06949219000001, 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.818256821, 2023-10-09T05:03:28.687 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.688 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.688 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.688 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.688 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.688 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75829 2023-10-09T05:03:28.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.689 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571726+0000", 2023-10-09T05:03:28.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571726+0000", 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571724+0000", 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571762+0000", 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8056000000000003e-05 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571776+0000", 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.349e-05 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.571786+0000", 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8719999999999997e-06 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.296115+0000", 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.724328866 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.304023+0000", 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0079088049999999997 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.315603+0000", 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011579145000000001 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.358599+0000", 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042996389000000003 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.360933+0000", 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023335909999999999 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.363428+0000", 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024950300000000001 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.366629+0000", 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032013940000000002 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.389983+0000", 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.02335419 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.701 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75832 14.9 e454/449)", 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.582541+0000", 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.05867774900003, 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.518063569000001, 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582541+0000", 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582541+0000", 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582540+0000", 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582578+0000", 2023-10-09T05:03:28.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8028000000000003e-05 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582594+0000", 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6560999999999999e-05 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.582603+0000", 2023-10-09T05:03:28.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3489999999999992e-06 2023-10-09T05:03:28.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072417+0000", 2023-10-09T05:03:28.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.489814439 2023-10-09T05:03:28.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.072445+0000", 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7645999999999999e-05 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.100571+0000", 2023-10-09T05:03:28.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028126284000000001 2023-10-09T05:03:28.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.100604+0000", 2023-10-09T05:03:28.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3399000000000003e-05 2023-10-09T05:03:28.709 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.710 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.710 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.711 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75834 14.9 e454/449)", 2023-10-09T05:03:28.711 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.585970+0000", 2023-10-09T05:03:28.711 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.05524896199995, 2023-10-09T05:03:28.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.628231904, 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585970+0000", 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585970+0000", 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.585969+0000", 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586023+0000", 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4001e-05 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586033+0000", 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0575e-05 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.586040+0000", 2023-10-09T05:03:28.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0829999999999998e-06 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077690+0000", 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.491649827 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.077716+0000", 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5324999999999999e-05 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214158+0000", 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.136442024 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214201+0000", 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3862000000000001e-05 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75837 14.1 14:8c05572a:::benchmark_data_smithi029_550670_object75836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e454)", 2023-10-09T05:03:28.718 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.601333+0000", 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.039885411, 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.873180149, 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75837 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.719 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601333+0000", 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601333+0000", 2023-10-09T05:03:28.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601332+0000", 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601361+0000", 2023-10-09T05:03:28.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9600000000000001e-05 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601376+0000", 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4403000000000001e-05 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.601386+0000", 2023-10-09T05:03:28.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0416e-05 2023-10-09T05:03:28.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.323704+0000", 2023-10-09T05:03:28.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.72231845 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.332083+0000", 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0083783480000000007 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 2,6", 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.346710+0000", 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014627693000000001 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.362819+0000", 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016108062999999999 2023-10-09T05:03:28.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.392608+0000", 2023-10-09T05:03:28.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029789645999999999 2023-10-09T05:03:28.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.445466+0000", 2023-10-09T05:03:28.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.052858166999999998 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.451631+0000", 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061645199999999997 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.474513+0000", 2023-10-09T05:03:28.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022882422999999999 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:75840 14.9 e454/449)", 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:47.614574+0000", 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 521.02664472499998, 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.599658032000001, 2023-10-09T05:03:28.729 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614574+0000", 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.730 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614574+0000", 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614573+0000", 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614607+0000", 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3590999999999997e-05 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614616+0000", 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.081e-06 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.614624+0000", 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4540000000000007e-06 2023-10-09T05:03:28.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.083376+0000", 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.468752077 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.083408+0000", 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2087e-05 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214204+0000", 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13079591800000001 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:58.214232+0000", 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7529e-05 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.734 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPushReply(14.3 467/464 [PushReplyOp(14:cffce7f4:::benchmark_data_smithi029_550670_object77452:head)])", 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:36.986257+0000", 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 471.65496200600001, 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9163757380000002, 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.735 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986257+0000", 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986257+0000", 2023-10-09T05:03:28.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986255+0000", 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986265+0000", 2023-10-09T05:03:28.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9890000000000004e-06 2023-10-09T05:03:28.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986281+0000", 2023-10-09T05:03:28.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5446000000000001e-05 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:36.986291+0000", 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8539999999999992e-06 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:43.883633+0000", 2023-10-09T05:03:28.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8973423409999999 2023-10-09T05:03:28.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:43.902632+0000", 2023-10-09T05:03:28.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018999194 2023-10-09T05:03:28.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.741 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.742 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.742 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.742 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.742 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79398 14.3 14:c5eeef8d:::benchmark_data_smithi029_550670_object1787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:03:28.742 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:10.796523+0000", 2023-10-09T05:03:28.742 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 437.84469601400002, 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6241380140000001, 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.743 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79398 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796523+0000", 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.744 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796523+0000", 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.745 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796520+0000", 2023-10-09T05:03:28.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.746 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.746 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.746 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.746 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796533+0000", 2023-10-09T05:03:28.746 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2126e-05 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796548+0000", 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5633000000000001e-05 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.747 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:10.796564+0000", 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5344000000000001e-05 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.219053+0000", 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.422489797 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.235375+0000", 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016321806000000001 2023-10-09T05:03:28.748 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.331532+0000", 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.096156778999999998 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.337123+0000", 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055913389999999999 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.749 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.158303+0000", 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8211796690000002 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.168693+0000", 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010389667999999999 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:03:28.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.180089+0000", 2023-10-09T05:03:28.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011396702999999999 2023-10-09T05:03:28.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.256103+0000", 2023-10-09T05:03:28.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076013678000000001 2023-10-09T05:03:28.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.395645+0000", 2023-10-09T05:03:28.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13954192900000001 2023-10-09T05:03:28.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.397350+0000", 2023-10-09T05:03:28.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001704672 2023-10-09T05:03:28.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.401058+0000", 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037084610000000001 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.420661+0000", 2023-10-09T05:03:28.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019602482000000001 2023-10-09T05:03:28.755 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.755 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.755 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.756 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79458 14.3 14:c243e4b1:::benchmark_data_smithi029_550670_object1847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:03:28.756 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:11.471770+0000", 2023-10-09T05:03:28.756 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 437.16944869700001, 2023-10-09T05:03:28.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5716450159999997, 2023-10-09T05:03:28.756 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.757 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.757 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.757 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.757 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.757 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79458 2023-10-09T05:03:28.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.758 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471770+0000", 2023-10-09T05:03:28.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471770+0000", 2023-10-09T05:03:28.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.760 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.760 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.760 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471768+0000", 2023-10-09T05:03:28.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.760 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.760 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471775+0000", 2023-10-09T05:03:28.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2960000000000001e-06 2023-10-09T05:03:28.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.762 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471790+0000", 2023-10-09T05:03:28.762 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4711e-05 2023-10-09T05:03:28.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.762 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.471799+0000", 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8880000000000003e-06 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.722387+0000", 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25058769800000003 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.736403+0000", 2023-10-09T05:03:28.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014016631 2023-10-09T05:03:28.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.835566+0000", 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099163167999999996 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.839398+0000", 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038317640000000001 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.899006+0000", 2023-10-09T05:03:28.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0596074309999999 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.906843+0000", 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0078377759999999994 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.917496+0000", 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010652557999999999 2023-10-09T05:03:28.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.945373+0000", 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027876995000000002 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.768 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.021672+0000", 2023-10-09T05:03:28.768 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076299464999999997 2023-10-09T05:03:28.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.768 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.768 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.024686+0000", 2023-10-09T05:03:28.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030134469999999998 2023-10-09T05:03:28.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.769 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.769 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.028127+0000", 2023-10-09T05:03:28.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034407800000000001 2023-10-09T05:03:28.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.043415+0000", 2023-10-09T05:03:28.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015288339999999999 2023-10-09T05:03:28.770 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.771 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.771 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.771 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79468 14.3 14:cf2d94b1:::benchmark_data_smithi029_550670_object1857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:03:28.771 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:11.676314+0000", 2023-10-09T05:03:28.772 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 436.96490408300002, 2023-10-09T05:03:28.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3917630189999999, 2023-10-09T05:03:28.772 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.772 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.772 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.772 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.773 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.773 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79468 2023-10-09T05:03:28.773 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.773 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.773 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.774 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676314+0000", 2023-10-09T05:03:28.774 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.774 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.774 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.774 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.775 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676314+0000", 2023-10-09T05:03:28.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.775 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.775 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.775 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676312+0000", 2023-10-09T05:03:28.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:28.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676321+0000", 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.166e-06 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676337+0000", 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5852e-05 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.676346+0000", 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9590000000000001e-06 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.774451+0000", 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.09810518 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.786991+0000", 2023-10-09T05:03:28.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012539385 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.898062+0000", 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.111071581 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:11.909583+0000", 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011521103 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.991195+0000", 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0816113939999998 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:14.996599+0000", 2023-10-09T05:03:28.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054041219999999999 2023-10-09T05:03:28.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:03:28.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.006568+0000", 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099691250000000006 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.022256+0000", 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015688437999999999 2023-10-09T05:03:28.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.045384+0000", 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023128057 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.047287+0000", 2023-10-09T05:03:28.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001902513 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.050149+0000", 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002862065 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.068077+0000", 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017928436999999998 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79489 14.3 14:c00d8f05:::benchmark_data_smithi029_550670_object1878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:12.219889+0000", 2023-10-09T05:03:28.785 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 436.42132934400001, 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4242194270000001, 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79489 2023-10-09T05:03:28.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219889+0000", 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219889+0000", 2023-10-09T05:03:28.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219887+0000", 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219896+0000", 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8710000000000003e-06 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219915+0000", 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9021e-05 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.219928+0000", 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2598e-05 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.273993+0000", 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054065509999999997 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.293072+0000", 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019078660000000001 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.529355+0000", 2023-10-09T05:03:28.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23628290199999999 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.537965+0000", 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086095359999999992 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.443954+0000", 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9059896649999999 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.452576+0000", 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0086220489999999997 2023-10-09T05:03:28.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.463464+0000", 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010887732000000001 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.528955+0000", 2023-10-09T05:03:28.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.065491266000000006 2023-10-09T05:03:28.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.605316+0000", 2023-10-09T05:03:28.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076360474999999997 2023-10-09T05:03:28.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.608931+0000", 2023-10-09T05:03:28.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036152419999999998 2023-10-09T05:03:28.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.796 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.796 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.612435+0000", 2023-10-09T05:03:28.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035043959999999999 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.644109+0000", 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031673289 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.797 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79493 14.3 14:c64f621f:::benchmark_data_smithi029_550670_object1882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:12.269365+0000", 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 436.371853192, 2023-10-09T05:03:28.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4045552749999999, 2023-10-09T05:03:28.799 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.799 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.799 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.799 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.799 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.800 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79493 2023-10-09T05:03:28.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.800 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269365+0000", 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269365+0000", 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269364+0000", 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269371+0000", 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8739999999999997e-06 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269385+0000", 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4453999999999999e-05 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.269393+0000", 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3969999999999997e-06 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.609689+0000", 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34029656000000003 2023-10-09T05:03:28.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.619319+0000", 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0096294910000000004 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.718361+0000", 2023-10-09T05:03:28.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.099042179999999994 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.722461+0000", 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041005870000000002 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.573673+0000", 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8512119760000001 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.581260+0000", 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075868860000000002 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.596270+0000", 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015009972 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.605958+0000", 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.009687279 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.649286+0000", 2023-10-09T05:03:28.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043328558000000003 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.652757+0000", 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034707100000000001 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.808 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.657267+0000", 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045104250000000002 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.673921+0000", 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016653411 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79501 14.3 14:cfa44866:::benchmark_data_smithi029_550670_object1890:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e477)", 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:12.343757+0000", 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 436.29746118700001, 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5847279009999999, 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.810 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79501 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343757+0000", 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343757+0000", 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343755+0000", 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343763+0000", 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3429999999999998e-06 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343776+0000", 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3511e-05 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.343785+0000", 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6260000000000001e-06 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.620575+0000", 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27679018100000002 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.640756+0000", 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020180974000000001 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.756643+0000", 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.115886903 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:12.760712+0000", 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040691850000000003 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.796144+0000", 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0354321290000001 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.801378+0000", 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052342739999999997 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:03:28.816 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.810629+0000", 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0092507800000000001 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.865118+0000", 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054488254999999999 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.896139+0000", 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031021271999999999 2023-10-09T05:03:28.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.898233+0000", 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020940500000000001 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.901551+0000", 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033177020000000001 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:15.928485+0000", 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026934738 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80535 14.3 14:cc98a142:::benchmark_data_smithi029_550670_object2924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 322.39128881699997, 2023-10-09T05:03:28.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 40.148030116999998, 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80535 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.820 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249927+0000", 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249939+0000", 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1640999999999999e-05 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249963+0000", 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3714999999999999e-05 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249976+0000", 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3699e-05 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.632019+0000", 2023-10-09T05:03:28.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38204312600000001 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.653132+0000", 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021113060999999999 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.779084+0000", 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12595120000000001 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.797955+0000", 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018871354 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103793+0000", 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30583778099999998 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103806+0000", 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3494e-05 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844372+0000", 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.74056582000000004 2023-10-09T05:03:28.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.829 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.829 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844384+0000", 2023-10-09T05:03:28.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2394e-05 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.144509+0000", 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 38.300124437999997 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.170256+0000", 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025747190999999999 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.299425+0000", 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12916846400000001 2023-10-09T05:03:28.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.304976+0000", 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055519480000000001 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.314789+0000", 2023-10-09T05:03:28.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098120340000000007 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.351252+0000", 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036463151999999999 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.353456+0000", 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022039910000000002 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.356260+0000", 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028044680000000001 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.358811+0000", 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025511090000000002 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.365432+0000", 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066203989999999999 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.397960+0000", 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032528230999999998 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80543 14.3 14:cf2f7cf9:::benchmark_data_smithi029_550670_object2932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 322.11194885499998, 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 39.820873380999998, 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:03:28.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80543 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:03:28.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529268+0000", 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529279+0000", 2023-10-09T05:03:28.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1258e-05 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529301+0000", 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1912e-05 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529314+0000", 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3030000000000001e-05 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.655681+0000", 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12636664 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.679149+0000", 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023468648000000002 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.887190+0000", 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20804069 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.911521+0000", 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0243314 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103710+0000", 2023-10-09T05:03:28.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19218885399999999 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103727+0000", 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6439999999999998e-05 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844324+0000", 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.740597536 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844341+0000", 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7054999999999998e-05 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.011168+0000", 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 38.166826931000003 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for readable", 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.028691+0000", 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017522685 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.108149+0000", 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.079458184000000001 2023-10-09T05:03:28.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.127919+0000", 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019770056000000001 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.216507+0000", 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.088587677000000004 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.223312+0000", 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068052829999999996 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.233101+0000", 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097893839999999999 2023-10-09T05:03:28.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.265633+0000", 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032531428000000001 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.323546+0000", 2023-10-09T05:03:28.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057912784000000002 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.325561+0000", 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020157949999999999 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.327379+0000", 2023-10-09T05:03:28.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001817639 2023-10-09T05:03:28.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:03:28.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.330515+0000", 2023-10-09T05:03:28.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031356869999999998 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.350143+0000", 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019628362 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.850 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.851 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:03:28.851 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:03:28.851 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:03:28.852 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:28 smithi177 ceph-mon[195797]: pgmap v2034: 119 pgs: 2 activating+degraded+remapped, 1 activating+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 1 active+undersized+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 remapped, 4 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded, 12 active+undersized, 68 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 56 op/s; 59753/185775 objects degraded (32.164%); 18460/185775 objects misplaced (9.937%); 2.3 MiB/s, 53 objects/s recovering 2023-10-09T05:03:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:28.853 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:28.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:28.954 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:29.003 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:29.025 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:03:29.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:29.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:30.006 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:30.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:30.110 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:30.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:03:30.153 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:30.281 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:30.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:30.300 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:30.339 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:30.386 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:30.426 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:30.473 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:30.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:30 smithi029 ceph-mon[216903]: pgmap v2035: 119 pgs: 1 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 5 activating+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 1 active+recovering+undersized+remapped, 3 remapped, 2 active+recovery_wait+degraded+remapped, 82 active+clean, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 49 op/s; 59355/185625 objects degraded (31.976%); 9428/185625 objects misplaced (5.079%); 2.7 MiB/s, 55 objects/s recovering 2023-10-09T05:03:30.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1984694844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:30.578 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:30.630 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:03:30.703 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:30.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:30.816 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:30.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:30 smithi029 ceph-mon[223858]: pgmap v2035: 119 pgs: 1 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 5 activating+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 1 active+recovering+undersized+remapped, 3 remapped, 2 active+recovery_wait+degraded+remapped, 82 active+clean, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 49 op/s; 59355/185625 objects degraded (31.976%); 9428/185625 objects misplaced (5.079%); 2.7 MiB/s, 55 objects/s recovering 2023-10-09T05:03:30.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1984694844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:30.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:30.842 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:30 smithi177 ceph-mon[195797]: pgmap v2035: 119 pgs: 1 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 5 activating+undersized+degraded+remapped, 5 remapped+peering, 3 peering, 1 active+recovering+undersized+remapped, 3 remapped, 2 active+recovery_wait+degraded+remapped, 82 active+clean, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 49 op/s; 59355/185625 objects degraded (31.976%); 9428/185625 objects misplaced (5.079%); 2.7 MiB/s, 55 objects/s recovering 2023-10-09T05:03:30.843 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1984694844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:30.843 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:30.845 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:31.092 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:31.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:31.195 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:31.223 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:31.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:31.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:31.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:31.396 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:31.549 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:31.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:31.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:31.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:31 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:31.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:31 smithi177 ceph-mon[195797]: failed to encode map e630 with expected crc 2023-10-09T05:03:31.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:31.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:31 smithi177 ceph-mon[195797]: osdmap e630: 8 total, 8 up, 7 in 2023-10-09T05:03:31.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:31 smithi177 ceph-mon[195797]: failed to encode map e630 with expected crc 2023-10-09T05:03:31.678 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 59355/185625 objects degraded (31.976%), 23 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:31.680 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:31.758 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:31.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:31.861 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:31.897 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[216903]: failed to encode map e630 with expected crc 2023-10-09T05:03:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[216903]: osdmap e630: 8 total, 8 up, 7 in 2023-10-09T05:03:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[216903]: failed to encode map e630 with expected crc 2023-10-09T05:03:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 59355/185625 objects degraded (31.976%), 23 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[223858]: failed to encode map e630 with expected crc 2023-10-09T05:03:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[223858]: osdmap e630: 8 total, 8 up, 7 in 2023-10-09T05:03:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[223858]: failed to encode map e630 with expected crc 2023-10-09T05:03:31.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 59355/185625 objects degraded (31.976%), 23 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:32.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:32.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:32.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:32.103 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:03:32.104 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:03:32.104 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:03:32.104 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:03:32.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:32.180 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:32.327 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:32.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:32.431 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:32.458 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:32.501 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:32.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:32.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:32.604 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:32.631 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:32.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:32 smithi029 ceph-mon[216903]: pgmap v2037: 119 pgs: 1 activating+degraded, 1 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 5 activating+undersized+degraded+remapped, 3 remapped+peering, 1 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 68743/185595 objects degraded (37.039%); 9291/185595 objects misplaced (5.006%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:03:32.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:32 smithi029 ceph-mon[223858]: pgmap v2037: 119 pgs: 1 activating+degraded, 1 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 5 activating+undersized+degraded+remapped, 3 remapped+peering, 1 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 68743/185595 objects degraded (37.039%); 9291/185595 objects misplaced (5.006%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:03:32.845 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:32.847 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:32 smithi177 ceph-mon[195797]: pgmap v2037: 119 pgs: 1 activating+degraded, 1 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 5 activating+undersized+degraded+remapped, 3 remapped+peering, 1 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 13 op/s; 68743/185595 objects degraded (37.039%); 9291/185595 objects misplaced (5.006%); 2.4 MiB/s, 38 objects/s recovering 2023-10-09T05:03:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:32.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:32.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:03:32.949 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:32.986 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:33.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:03:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:03:32] "GET /metrics HTTP/1.1" 200 36730 "" "Prometheus/2.43.0" 2023-10-09T05:03:33.262 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:33.656 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:33.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:33.763 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:33.816 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:34.131 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:34.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:34.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:34.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:34.269 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:34.450 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:34.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:34.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:34.586 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:34.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:34.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:34.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:34.842 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:34.921 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:34.937 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:03:34.937 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:03:34.937 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:03:34.937 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:03:34.937 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:03:34.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:34 smithi177 ceph-mon[195797]: pgmap v2038: 119 pgs: 1 activating+degraded, 2 activating+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 68221/185505 objects degraded (36.776%); 9651/185505 objects misplaced (5.203%); 2.0 MiB/s, 78 objects/s recovering 2023-10-09T05:03:34.964 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2931817132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:35.062 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:35.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:35.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:35.071 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:34 smithi029 ceph-mon[216903]: pgmap v2038: 119 pgs: 1 activating+degraded, 2 activating+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 68221/185505 objects degraded (36.776%); 9651/185505 objects misplaced (5.203%); 2.0 MiB/s, 78 objects/s recovering 2023-10-09T05:03:35.072 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2931817132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:35.072 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:34 smithi029 ceph-mon[223858]: pgmap v2038: 119 pgs: 1 activating+degraded, 2 activating+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 68221/185505 objects degraded (36.776%); 9651/185505 objects misplaced (5.203%); 2.0 MiB/s, 78 objects/s recovering 2023-10-09T05:03:35.072 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2931817132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:35.173 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:35.207 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:35.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:35.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:35.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:35.366 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:35.412 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:35.534 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:35.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:35.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:35.642 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:35.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:03:35.670 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:35.831 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:03:35.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:35.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:35.961 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:35.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1062634930' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:03:35.965 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:35.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:03:36.010 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:36.067 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:36.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:36.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:36.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1062634930' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:03:36.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1062634930' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:03:36.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:03:36.176 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:36.214 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:36.323 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:36.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:36.428 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:36.454 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:36.551 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:36.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:36.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:36.571 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:36.651 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:03:36.656 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:36.684 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:36.686 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:03:36.719 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:03:36.734 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:36.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:36.744 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:03:36.766 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:03:36.800 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:03:36.820 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:03:36.838 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:36.841 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.843 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:03:36.864 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:36.983 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:36.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: pgmap v2039: 119 pgs: 2 activating+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 68202/185370 objects degraded (36.792%); 9358/185370 objects misplaced (5.048%); 2.4 MiB/s, 77 objects/s recovering 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: failed to encode map e631 with expected crc 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: failed to encode map e631 with expected crc 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1062634930' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: osdmap e631: 8 total, 8 up, 7 in 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: failed to encode map e631 with expected crc 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 68202/185370 objects degraded (36.792%), 27 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: osdmap e632: 8 total, 8 up, 7 in 2023-10-09T05:03:36.989 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:36.990 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:36 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:37.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:37.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: pgmap v2039: 119 pgs: 2 activating+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 68202/185370 objects degraded (36.792%); 9358/185370 objects misplaced (5.048%); 2.4 MiB/s, 77 objects/s recovering 2023-10-09T05:03:37.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: failed to encode map e631 with expected crc 2023-10-09T05:03:37.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: failed to encode map e631 with expected crc 2023-10-09T05:03:37.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1062634930' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:03:37.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: osdmap e631: 8 total, 8 up, 7 in 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: failed to encode map e631 with expected crc 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 68202/185370 objects degraded (36.792%), 27 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: osdmap e632: 8 total, 8 up, 7 in 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: pgmap v2039: 119 pgs: 2 activating+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+remapped, 1 remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 16 op/s; 68202/185370 objects degraded (36.792%); 9358/185370 objects misplaced (5.048%); 2.4 MiB/s, 77 objects/s recovering 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: failed to encode map e631 with expected crc 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: failed to encode map e631 with expected crc 2023-10-09T05:03:37.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1062634930' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:03:37.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: osdmap e631: 8 total, 8 up, 7 in 2023-10-09T05:03:37.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: failed to encode map e631 with expected crc 2023-10-09T05:03:37.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 68202/185370 objects degraded (36.792%), 27 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:37.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: osdmap e632: 8 total, 8 up, 7 in 2023-10-09T05:03:37.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:37.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:36 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:37.118 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: osdmap e633: 8 total, 8 up, 7 in 2023-10-09T05:03:38.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[223858]: failed to encode map e633 with expected crc 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: osdmap e633: 8 total, 8 up, 7 in 2023-10-09T05:03:38.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:37 smithi029 ceph-mon[216903]: failed to encode map e633 with expected crc 2023-10-09T05:03:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: failed to encode map e632 with expected crc 2023-10-09T05:03:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: osdmap e633: 8 total, 8 up, 7 in 2023-10-09T05:03:38.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:37 smithi177 ceph-mon[195797]: failed to encode map e633 with expected crc 2023-10-09T05:03:38.923 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:03:38.923 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:03:38.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: failed to encode map e633 with expected crc 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: failed to encode map e633 with expected crc 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: failed to encode map e633 with expected crc 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: pgmap v2043: 119 pgs: 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 1 remapped+peering, 2 active+recovering+undersized+remapped, 84 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 21 op/s; 85216/185199 objects degraded (46.013%); 6941/185199 objects misplaced (3.748%); 1.7 MiB/s, 90 objects/s recovering 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: osdmap e634: 8 total, 8 up, 7 in 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:38.947 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:38 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: failed to encode map e633 with expected crc 2023-10-09T05:03:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: failed to encode map e633 with expected crc 2023-10-09T05:03:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: failed to encode map e633 with expected crc 2023-10-09T05:03:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: pgmap v2043: 119 pgs: 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 1 remapped+peering, 2 active+recovering+undersized+remapped, 84 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 21 op/s; 85216/185199 objects degraded (46.013%); 6941/185199 objects misplaced (3.748%); 1.7 MiB/s, 90 objects/s recovering 2023-10-09T05:03:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: osdmap e634: 8 total, 8 up, 7 in 2023-10-09T05:03:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: failed to encode map e633 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: failed to encode map e633 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: failed to encode map e633 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: pgmap v2043: 119 pgs: 1 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 2 activating+degraded+remapped, 1 remapped+peering, 2 active+recovering+undersized+remapped, 84 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 8 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 21 op/s; 85216/185199 objects degraded (46.013%); 6941/185199 objects misplaced (3.748%); 1.7 MiB/s, 90 objects/s recovering 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: osdmap e634: 8 total, 8 up, 7 in 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:38 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: failed to encode map e634 with expected crc 2023-10-09T05:03:39.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: failed to encode map e635 with expected crc 2023-10-09T05:03:39.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: osdmap e635: 8 total, 8 up, 7 in 2023-10-09T05:03:39.946 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:39 smithi177 ceph-mon[195797]: failed to encode map e635 with expected crc 2023-10-09T05:03:39.976 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:03:40.016 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:03:40.037 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.037 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: failed to encode map e635 with expected crc 2023-10-09T05:03:40.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: osdmap e635: 8 total, 8 up, 7 in 2023-10-09T05:03:40.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[216903]: failed to encode map e635 with expected crc 2023-10-09T05:03:40.039 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:40.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: failed to encode map e634 with expected crc 2023-10-09T05:03:40.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: failed to encode map e635 with expected crc 2023-10-09T05:03:40.039 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: osdmap e635: 8 total, 8 up, 7 in 2023-10-09T05:03:40.040 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:39 smithi029 ceph-mon[223858]: failed to encode map e635 with expected crc 2023-10-09T05:03:40.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:40.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:40.130 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:40.148 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:40.185 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:40.253 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:40.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:40.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:40.399 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:40.468 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:40.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:40.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:40.573 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:40.612 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:40.612 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:40.681 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:40.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:40.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:40.785 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:40.820 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:40.884 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:40 smithi177 ceph-mon[195797]: failed to encode map e635 with expected crc 2023-10-09T05:03:40.884 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:40 smithi177 ceph-mon[195797]: pgmap v2046: 119 pgs: 1 peering, 2 active+recovery_wait+degraded+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 83 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 91100/185049 objects degraded (49.230%); 6128/185049 objects misplaced (3.312%); 14 MiB/s, 260 objects/s recovering 2023-10-09T05:03:41.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:40 smithi029 ceph-mon[216903]: failed to encode map e635 with expected crc 2023-10-09T05:03:41.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:40 smithi029 ceph-mon[216903]: pgmap v2046: 119 pgs: 1 peering, 2 active+recovery_wait+degraded+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 83 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 91100/185049 objects degraded (49.230%); 6128/185049 objects misplaced (3.312%); 14 MiB/s, 260 objects/s recovering 2023-10-09T05:03:41.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:40 smithi029 ceph-mon[223858]: failed to encode map e635 with expected crc 2023-10-09T05:03:41.098 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:40 smithi029 ceph-mon[223858]: pgmap v2046: 119 pgs: 1 peering, 2 active+recovery_wait+degraded+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 83 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 91100/185049 objects degraded (49.230%); 6128/185049 objects misplaced (3.312%); 14 MiB/s, 260 objects/s recovering 2023-10-09T05:03:41.834 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:41.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:41.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:41.942 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:41.983 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:42.069 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:03:42.100 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 91100/185049 objects degraded (49.230%), 32 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:42.100 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 91100/185049 objects degraded (49.230%), 32 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:42.204 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:42.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:42.274 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 91100/185049 objects degraded (49.230%), 32 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:42.310 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:42.351 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:42.502 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:03:42.503 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:03:42.622 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:03:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:42.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:03:42.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:42.762 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:42.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:42.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:42.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:42.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:03:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:03:42] "GET /metrics HTTP/1.1" 200 36730 "" "Prometheus/2.43.0" 2023-10-09T05:03:42.941 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:42.988 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:42.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:43.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:43.129 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:43.132 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:03:43.379 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:43 smithi177 ceph-mon[195797]: pgmap v2047: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 90748/184806 objects degraded (49.104%); 5535/184806 objects misplaced (2.995%); 12 MiB/s, 221 objects/s recovering 2023-10-09T05:03:43.379 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:43.379 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:43.379 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3604817945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:03:43.379 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:43 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:03:43.380 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:03:43.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[216903]: pgmap v2047: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 90748/184806 objects degraded (49.104%); 5535/184806 objects misplaced (2.995%); 12 MiB/s, 221 objects/s recovering 2023-10-09T05:03:43.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:43.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3604817945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[223858]: pgmap v2047: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 32 op/s; 90748/184806 objects degraded (49.104%); 5535/184806 objects misplaced (2.995%); 12 MiB/s, 221 objects/s recovering 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3604817945' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:03:43.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:43 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:03:43.745 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:44.134 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:44.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:44.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:44.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: osdmap e636: 8 total, 8 up, 7 in 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:03:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: osdmap e636: 8 total, 8 up, 7 in 2023-10-09T05:03:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:44.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:44.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:03:44.265 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:03:44.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: osdmap e636: 8 total, 8 up, 7 in 2023-10-09T05:03:44.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: failed to encode map e636 with expected crc 2023-10-09T05:03:44.266 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:44.268 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:44.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:44.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:44.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:44.594 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:44.638 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:44.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:44.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:44.771 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:44.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:44.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:44.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:45.090 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:45.125 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:45.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:45 smithi177 ceph-mon[195797]: pgmap v2049: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 21 op/s; 90748/184806 objects degraded (49.104%); 5535/184806 objects misplaced (2.995%); 11 MiB/s, 203 objects/s recovering 2023-10-09T05:03:45.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:45 smithi029 ceph-mon[216903]: pgmap v2049: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 21 op/s; 90748/184806 objects degraded (49.104%); 5535/184806 objects misplaced (2.995%); 11 MiB/s, 203 objects/s recovering 2023-10-09T05:03:45.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:45 smithi029 ceph-mon[223858]: pgmap v2049: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded+remapped, 1 activating+undersized+degraded+remapped, 80 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 21 op/s; 90748/184806 objects degraded (49.104%); 5535/184806 objects misplaced (2.995%); 11 MiB/s, 203 objects/s recovering 2023-10-09T05:03:45.496 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:03:45.496 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:03:45.574 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:45.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:45.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:45.718 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:45.735 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:45.816 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:45.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:45.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:45.946 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:46.245 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:03:46.266 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:46.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:46.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:46.401 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 90748/184806 objects degraded (49.104%), 33 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:46.401 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3331809431' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:03:46.401 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:46 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:03:46.402 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 90748/184806 objects degraded (49.104%), 33 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:46.402 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3331809431' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:03:46.402 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:46 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:03:46.406 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 90748/184806 objects degraded (49.104%), 33 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3331809431' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:03:46.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:46 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:03:46.635 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:46.766 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:46.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:46.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:46.902 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:46.974 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:03:47.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:47 smithi029 ceph-mon[216903]: pgmap v2050: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean, 20 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 17 op/s; 93150/184797 objects degraded (50.407%); 4737/184797 objects misplaced (2.563%); 11 MiB/s, 192 objects/s recovering 2023-10-09T05:03:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:03:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:47 smithi029 ceph-mon[216903]: osdmap e637: 8 total, 8 up, 7 in 2023-10-09T05:03:47.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:47 smithi029 ceph-mon[223858]: pgmap v2050: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean, 20 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 17 op/s; 93150/184797 objects degraded (50.407%); 4737/184797 objects misplaced (2.563%); 11 MiB/s, 192 objects/s recovering 2023-10-09T05:03:47.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:03:47.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:47 smithi029 ceph-mon[223858]: osdmap e637: 8 total, 8 up, 7 in 2023-10-09T05:03:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:47 smithi177 ceph-mon[195797]: pgmap v2050: 119 pgs: 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 80 active+clean, 20 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 17 op/s; 93150/184797 objects degraded (50.407%); 4737/184797 objects misplaced (2.563%); 11 MiB/s, 192 objects/s recovering 2023-10-09T05:03:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:03:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:47 smithi177 ceph-mon[195797]: osdmap e637: 8 total, 8 up, 7 in 2023-10-09T05:03:47.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:03:48.137 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.138 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:48 smithi177 ceph-mon[195797]: osdmap e638: 8 total, 8 up, 7 in 2023-10-09T05:03:48.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[216903]: osdmap e638: 8 total, 8 up, 7 in 2023-10-09T05:03:48.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: failed to encode map e637 with expected crc 2023-10-09T05:03:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:48 smithi029 ceph-mon[223858]: osdmap e638: 8 total, 8 up, 7 in 2023-10-09T05:03:48.449 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:03:48.449 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: pgmap v2052: 119 pgs: 1 active+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean, 21 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 93147/184794 objects degraded (50.406%); 4737/184794 objects misplaced (2.563%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[216903]: osdmap e639: 8 total, 8 up, 7 in 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: pgmap v2052: 119 pgs: 1 active+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean, 21 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 93147/184794 objects degraded (50.406%); 4737/184794 objects misplaced (2.563%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:49 smithi029 ceph-mon[223858]: osdmap e639: 8 total, 8 up, 7 in 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: pgmap v2052: 119 pgs: 1 active+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 77 active+clean, 21 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 2 active+remapped+backfill_wait; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 93147/184794 objects degraded (50.406%); 4737/184794 objects misplaced (2.563%); 4.1 MiB/s, 65 objects/s recovering 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: failed to encode map e638 with expected crc 2023-10-09T05:03:49.465 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:49 smithi177 ceph-mon[195797]: osdmap e639: 8 total, 8 up, 7 in 2023-10-09T05:03:49.492 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:03:49.916 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:50.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[223858]: osdmap e640: 8 total, 8 up, 7 in 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:50.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:50.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:50 smithi029 ceph-mon[216903]: osdmap e640: 8 total, 8 up, 7 in 2023-10-09T05:03:50.474 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:50.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:50.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: failed to encode map e639 with expected crc 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:50.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:50 smithi177 ceph-mon[195797]: osdmap e640: 8 total, 8 up, 7 in 2023-10-09T05:03:50.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:50.630 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:50.686 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:50.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:50.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:50.790 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:50.875 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:51.054 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:51.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:51.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:51.127 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:03:51.154 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:03:51.187 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:03:51.201 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:51.235 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:03:51.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:03:51.305 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:51.325 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:03:51.366 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: pgmap v2055: 119 pgs: 2 activating+undersized+degraded+remapped, 10 remapped+peering, 1 active+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 85066/184794 objects degraded (46.033%); 8335/184794 objects misplaced (4.510%); 3.5 MiB/s, 63 objects/s recovering 2023-10-09T05:03:51.366 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 85066/184794 objects degraded (46.033%), 34 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: osdmap e641: 8 total, 8 up, 7 in 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1652479071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: pgmap v2055: 119 pgs: 2 activating+undersized+degraded+remapped, 10 remapped+peering, 1 active+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 85066/184794 objects degraded (46.033%); 8335/184794 objects misplaced (4.510%); 3.5 MiB/s, 63 objects/s recovering 2023-10-09T05:03:51.367 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 85066/184794 objects degraded (46.033%), 34 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: osdmap e641: 8 total, 8 up, 7 in 2023-10-09T05:03:51.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1652479071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:51.385 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:03:51.423 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:03:51.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.430 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.431 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.431 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.431 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.431 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.431 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:03:51.464 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:51.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:51.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:51.586 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: pgmap v2055: 119 pgs: 2 activating+undersized+degraded+remapped, 10 remapped+peering, 1 active+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 69 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 0 op/s; 85066/184794 objects degraded (46.033%); 8335/184794 objects misplaced (4.510%); 3.5 MiB/s, 63 objects/s recovering 2023-10-09T05:03:51.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.587 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: failed to encode map e640 with expected crc 2023-10-09T05:03:51.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:03:51.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:03:51.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:51.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 85066/184794 objects degraded (46.033%), 34 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:51.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: osdmap e641: 8 total, 8 up, 7 in 2023-10-09T05:03:51.588 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1652479071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:03:51.605 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:51.698 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:51.714 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:51.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:51.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:03:51.850 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:51.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:03:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:51.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:03:52.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:52.067 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:52.118 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:03:52.118 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:03:52.118 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:03:52.119 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:03:52.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[216903]: osdmap e642: 8 total, 8 up, 7 in 2023-10-09T05:03:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:52 smithi177 ceph-mon[195797]: osdmap e642: 8 total, 8 up, 7 in 2023-10-09T05:03:52.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: failed to encode map e641 with expected crc 2023-10-09T05:03:52.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:52 smithi029 ceph-mon[223858]: osdmap e642: 8 total, 8 up, 7 in 2023-10-09T05:03:53.119 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:03:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:03:52] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.43.0" 2023-10-09T05:03:53.410 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:03:53.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: pgmap v2058: 119 pgs: 2 activating+undersized+degraded+remapped, 13 remapped+peering, 1 active+undersized+degraded+remapped, 5 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 68 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69109/185622 objects degraded (37.231%); 9624/185622 objects misplaced (5.185%); 5.9 MiB/s, 198 objects/s recovering 2023-10-09T05:03:53.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[216903]: osdmap e643: 8 total, 8 up, 7 in 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: pgmap v2058: 119 pgs: 2 activating+undersized+degraded+remapped, 13 remapped+peering, 1 active+undersized+degraded+remapped, 5 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 68 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69109/185622 objects degraded (37.231%); 9624/185622 objects misplaced (5.185%); 5.9 MiB/s, 198 objects/s recovering 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:03:53.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:53 smithi029 ceph-mon[223858]: osdmap e643: 8 total, 8 up, 7 in 2023-10-09T05:03:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: pgmap v2058: 119 pgs: 2 activating+undersized+degraded+remapped, 13 remapped+peering, 1 active+undersized+degraded+remapped, 5 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 68 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69109/185622 objects degraded (37.231%); 9624/185622 objects misplaced (5.185%); 5.9 MiB/s, 198 objects/s recovering 2023-10-09T05:03:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: failed to encode map e642 with expected crc 2023-10-09T05:03:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:03:53.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:53 smithi177 ceph-mon[195797]: osdmap e643: 8 total, 8 up, 7 in 2023-10-09T05:03:53.782 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:54.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:54.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:54.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:54.202 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:54.234 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:54.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:54.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:54.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:54 smithi029 ceph-mon[216903]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:54 smithi029 ceph-mon[216903]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:54 smithi029 ceph-mon[216903]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:54 smithi029 ceph-mon[223858]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:54 smithi029 ceph-mon[223858]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:54 smithi029 ceph-mon[223858]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.484 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:54.512 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:54.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:54.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:54.661 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:54.685 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:54 smithi177 ceph-mon[195797]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.685 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:54 smithi177 ceph-mon[195797]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.686 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:54 smithi177 ceph-mon[195797]: failed to encode map e643 with expected crc 2023-10-09T05:03:54.688 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:54.733 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:54.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:54.836 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:54.864 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:54.909 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:54.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:54.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:55.013 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:03:55.061 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:55.320 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:55 smithi029 ceph-mon[223858]: pgmap v2061: 119 pgs: 2 activating+undersized+degraded+remapped, 13 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 71 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69103/185493 objects degraded (37.254%); 9624/185493 objects misplaced (5.188%); 5.0 MiB/s, 170 objects/s recovering 2023-10-09T05:03:55.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:55 smithi029 ceph-mon[216903]: pgmap v2061: 119 pgs: 2 activating+undersized+degraded+remapped, 13 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 71 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69103/185493 objects degraded (37.254%); 9624/185493 objects misplaced (5.188%); 5.0 MiB/s, 170 objects/s recovering 2023-10-09T05:03:55.713 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:55 smithi177 ceph-mon[195797]: pgmap v2061: 119 pgs: 2 activating+undersized+degraded+remapped, 13 remapped+peering, 2 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 71 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 69103/185493 objects degraded (37.254%); 9624/185493 objects misplaced (5.188%); 5.0 MiB/s, 170 objects/s recovering 2023-10-09T05:03:55.982 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:03:55.982 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:03:55.982 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:03:55.982 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:03:55.982 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:03:56.086 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:03:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:56.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:03:56.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:03:56.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:56.224 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:56.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:56.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:56.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:56.406 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:56.456 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:56.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 69103/185493 objects degraded (37.254%), 31 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:56 smithi177 ceph-mon[195797]: osdmap e644: 8 total, 8 up, 7 in 2023-10-09T05:03:56.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:56.586 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:56.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:56.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:56.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 69103/185493 objects degraded (37.254%), 31 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:56.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:56 smithi029 ceph-mon[216903]: osdmap e644: 8 total, 8 up, 7 in 2023-10-09T05:03:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 69103/185493 objects degraded (37.254%), 31 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:03:56.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:56 smithi029 ceph-mon[223858]: osdmap e644: 8 total, 8 up, 7 in 2023-10-09T05:03:56.735 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:56.761 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:56.765 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:03:56.892 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:56.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:56.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:56.971 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:03:56.978 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:03:56.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:03:57.039 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:57.392 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:03:57.603 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: pgmap v2062: 119 pgs: 3 remapped+peering, 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 82 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 71764/185403 objects degraded (38.707%); 6108/185403 objects misplaced (3.294%); 6.4 MiB/s, 130 objects/s recovering 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/373799457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/373799457' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: osdmap e645: 8 total, 8 up, 7 in 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4003063680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:03:57.604 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:57 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: pgmap v2062: 119 pgs: 3 remapped+peering, 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 82 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 71764/185403 objects degraded (38.707%); 6108/185403 objects misplaced (3.294%); 6.4 MiB/s, 130 objects/s recovering 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/373799457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:03:57.616 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/373799457' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: osdmap e645: 8 total, 8 up, 7 in 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4003063680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: pgmap v2062: 119 pgs: 3 remapped+peering, 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 82 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 13 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 71764/185403 objects degraded (38.707%); 6108/185403 objects misplaced (3.294%); 6.4 MiB/s, 130 objects/s recovering 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: failed to encode map e644 with expected crc 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/373799457' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:03:57.617 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/373799457' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:03:57.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: osdmap e645: 8 total, 8 up, 7 in 2023-10-09T05:03:57.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4003063680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:03:57.618 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:57 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:03:57.661 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:03:57.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:57.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:03:57.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:57.805 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:58.008 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:03:58.162 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:58.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:58.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:58.266 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:03:58.296 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:58.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:03:58.567 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:58 smithi177 ceph-mon[195797]: failed to encode map e645 with expected crc 2023-10-09T05:03:58.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:03:58.568 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:58 smithi177 ceph-mon[195797]: osdmap e646: 8 total, 8 up, 7 in 2023-10-09T05:03:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:58.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:03:58.670 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:03:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:58 smithi029 ceph-mon[216903]: failed to encode map e645 with expected crc 2023-10-09T05:03:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:03:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:58 smithi029 ceph-mon[216903]: osdmap e646: 8 total, 8 up, 7 in 2023-10-09T05:03:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:58 smithi029 ceph-mon[223858]: failed to encode map e645 with expected crc 2023-10-09T05:03:58.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:03:58.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:58 smithi029 ceph-mon[223858]: osdmap e646: 8 total, 8 up, 7 in 2023-10-09T05:03:58.701 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:58.856 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:03:58.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:58.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:03:58.965 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:03:58.999 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:59.062 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:03:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:59.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:03:59.130 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:03:59.130 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:03:59.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:03:59.130 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:03:59.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:78164 14.5 14:ab2c19dc:::benchmark_data_smithi029_550670_object553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:55:53.873914+0000", 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 485.25242698800002, 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.48484511200000002, 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 78164 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.131 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873914+0000", 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873914+0000", 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873913+0000", 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873918+0000", 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5060000000000003e-06 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873928+0000", 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5289999999999997e-06 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873934+0000", 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9819999999999996e-06 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873985+0000", 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1035999999999997e-05 2023-10-09T05:03:59.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:53.873993+0000", 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.738e-06 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.281511+0000", 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40751733000000001 2023-10-09T05:03:59.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.282315+0000", 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00080389999999999997 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.341124+0000", 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.058808968000000003 2023-10-09T05:03:59.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.342375+0000", 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012515429999999999 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.342484+0000", 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108472 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.345377+0000", 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028932519999999998 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.349321+0000", 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039443680000000002 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.355687+0000", 2023-10-09T05:03:59.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063658669999999999 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.355730+0000", 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2814e-05 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:55:54.358759+0000", 2023-10-09T05:03:59.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003029505 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:79931 14.5 14:aa6ec0f4:::benchmark_data_smithi029_550670_object2320:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:56:55.803323+0000", 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 423.323018295, 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.185262104, 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.140 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 79931 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803323+0000", 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803323+0000", 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803322+0000", 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803329+0000", 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.238e-06 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803343+0000", 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3878e-05 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803363+0000", 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0746000000000001e-05 2023-10-09T05:03:59.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803437+0000", 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3337000000000006e-05 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.803447+0000", 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9240000000000002e-06 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.814483+0000", 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011036403 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:55.814495+0000", 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1610999999999999e-05 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.972831+0000", 2023-10-09T05:03:59.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.158336517 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.973026+0000", 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000194429 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.973208+0000", 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000182401 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.980540+0000", 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073322370000000001 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.980587+0000", 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7122e-05 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.986228+0000", 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056403210000000002 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.986283+0000", 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5247000000000001e-05 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.988585+0000", 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023021299999999999 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:79958 14.5 14:a053d0c1:::benchmark_data_smithi029_550670_object2347:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e496)", 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:56:56.079965+0000", 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 423.04637680000002, 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.91067540300000005, 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.149 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 79958 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.079965+0000", 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.079965+0000", 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.079962+0000", 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:59.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.079976+0000", 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3971000000000001e-05 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.080002+0000", 2023-10-09T05:03:59.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6111000000000001e-05 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.080012+0000", 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0059e-05 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.080074+0000", 2023-10-09T05:03:59.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1187999999999998e-05 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.080084+0000", 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0871e-05 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.978072+0000", 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.89798709099999996 2023-10-09T05:03:59.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.978260+0000", 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018879500000000001 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.978445+0000", 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000184489 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.980742+0000", 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002297402 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.981710+0000", 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00096747300000000005 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.988639+0000", 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0069291520000000001 2023-10-09T05:03:59.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.988682+0000", 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3547000000000002e-05 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:56:56.990640+0000", 2023-10-09T05:03:59.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019575040000000001 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:80558 14.4 14:28ce501b:::benchmark_data_smithi029_550670_object2947:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T05:03:59.159 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:58:07.141255+0000", 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 351.98508593299999, 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.97944381700000005, 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 80558 2023-10-09T05:03:59.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141255+0000", 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141255+0000", 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141254+0000", 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141263+0000", 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0229999999999998e-06 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141282+0000", 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9267e-05 2023-10-09T05:03:59.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141291+0000", 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8629999999999997e-06 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141358+0000", 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6608999999999999e-05 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.141368+0000", 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.008e-05 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.086919+0000", 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94555103100000004 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.087739+0000", 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00082011300000000003 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.099849+0000", 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012109725999999999 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.099930+0000", 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0887e-05 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.100025+0000", 2023-10-09T05:03:59.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5088999999999997e-05 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.111506+0000", 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011481798 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.111559+0000", 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2154999999999998e-05 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.117554+0000", 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059952590000000002 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.169 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.117623+0000", 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9224e-05 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.120699+0000", 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030761629999999998 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.170 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:80561 14.4 14:2803bdfb:::benchmark_data_smithi029_550670_object2950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:58:07.404102+0000", 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 351.72223955300001, 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71871560199999995, 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 80561 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.171 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404102+0000", 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404102+0000", 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.172 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404100+0000", 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404110+0000", 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0411000000000001e-05 2023-10-09T05:03:59.173 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404134+0000", 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4179e-05 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404143+0000", 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9649999999999997e-06 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.174 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404217+0000", 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3256000000000005e-05 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:07.404231+0000", 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4187e-05 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.087791+0000", 2023-10-09T05:03:59.175 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.683560108 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.088513+0000", 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00072212199999999995 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.107567+0000", 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019053810000000001 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.176 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.107657+0000", 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0606999999999995e-05 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.107764+0000", 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000106913 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.111569+0000", 2023-10-09T05:03:59.177 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038048209999999999 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.111894+0000", 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032508299999999999 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.120756+0000", 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0088616370000000003 2023-10-09T05:03:59.178 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.120821+0000", 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4681e-05 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:58:08.122817+0000", 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019968170000000001 2023-10-09T05:03:59.179 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82836 14.4 14:267e5921:::benchmark_data_smithi029_550670_object5225:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e546)", 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:30.847604+0000", 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 268.27873763399998, 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.281529715, 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.180 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82836 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847604+0000", 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.181 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847604+0000", 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847602+0000", 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.182 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847611+0000", 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3780000000000007e-06 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847633+0000", 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1658000000000002e-05 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847641+0000", 2023-10-09T05:03:59.183 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5420000000000005e-06 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847712+0000", 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0710999999999994e-05 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.847720+0000", 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1259999999999998e-06 2023-10-09T05:03:59.184 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.874442+0000", 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026721327999999999 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.874451+0000", 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7310000000000005e-06 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.185 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.062992+0000", 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1885407560000001 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.063862+0000", 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00086985999999999997 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.186 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.099550+0000", 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.035687925000000002 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.099652+0000", 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102246 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.099748+0000", 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5533999999999997e-05 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.187 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.101902+0000", 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021546780000000001 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.104793+0000", 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028909970000000002 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.123451+0000", 2023-10-09T05:03:59.188 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018657776000000001 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.127769+0000", 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043183229999999998 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.127814+0000", 2023-10-09T05:03:59.189 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4467e-05 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:32.129133+0000", 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001319492 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.190 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:91970 14.4 14:21827fbb:::benchmark_data_smithi029_550670_object14359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:42.408999+0000", 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 136.71734192299999, 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0219353019999999, 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 91970 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.191 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.408999+0000", 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.408999+0000", 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.192 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.408998+0000", 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409006+0000", 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4769999999999997e-06 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409022+0000", 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6721000000000001e-05 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.193 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409029+0000", 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9090000000000003e-06 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409089+0000", 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9553999999999999e-05 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.194 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.410296+0000", 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001207414 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.416006+0000", 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057100390000000001 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.416141+0000", 2023-10-09T05:03:59.195 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013511600000000001 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.416305+0000", 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000163812 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.422708+0000", 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064030770000000001 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.196 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.461435+0000", 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.038726869999999997 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.461448+0000", 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3370000000000001e-05 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.197 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.430921+0000", 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96947220599999995 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.430935+0000", 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4063000000000001e-05 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.198 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:91985 14.2 14:410dd76c:::benchmark_data_smithi029_550670_object14374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:01.825669+0000", 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 117.300672273, 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1116379600000004, 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.199 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 91985 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825669+0000", 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.200 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825669+0000", 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825666+0000", 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999971 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.201 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825683+0000", 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6543999999999999e-05 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825706+0000", 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2974999999999999e-05 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.202 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825717+0000", 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1087000000000001e-05 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.801677+0000", 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9759603759999997 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.822265+0000", 2023-10-09T05:03:59.203 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020587215999999998 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.882023+0000", 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.059758517999999997 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.887238+0000", 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052146980000000002 2023-10-09T05:03:59.204 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.896986+0000", 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097482279999999994 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.909274+0000", 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012287836999999999 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.205 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.911026+0000", 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017521800000000001 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.912977+0000", 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001950818 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.206 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.915190+0000", 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022132549999999999 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.918685+0000", 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00349512 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.937307+0000", 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018621746000000002 2023-10-09T05:03:59.207 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 593/586 [PushReplyOp(14:4529595e:::benchmark_data_smithi029_550670_object74277:head)])", 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:06.566413+0000", 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 112.55992836, 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.060305104999999998, 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.208 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566413+0000", 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566413+0000", 2023-10-09T05:03:59.209 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566411+0000", 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566424+0000", 2023-10-09T05:03:59.210 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2454e-05 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566441+0000", 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6787e-05 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566451+0000", 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0146000000000001e-05 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.211 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.581924+0000", 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015473169 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.626718+0000", 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.044794187999999999 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.212 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:443b08e2:::benchmark_data_smithi029_550670_object74358:head)])", 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:06.754907+0000", 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 112.371434072, 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.064498344999999999, 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754907+0000", 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.213 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754907+0000", 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754906+0000", 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.214 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754918+0000", 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2072000000000001e-05 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754936+0000", 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7836999999999999e-05 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754946+0000", 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0248000000000001e-05 2023-10-09T05:03:59.215 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.797183+0000", 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.042237441000000001 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.819406+0000", 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022222301999999999 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.216 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:44043727:::benchmark_data_smithi029_550670_object74467:head)])", 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:07.009229+0000", 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 112.117112581, 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.067423306000000002, 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.217 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009229+0000", 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009229+0000", 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009228+0000", 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.218 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009237+0000", 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2399999999999996e-06 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009253+0000", 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6350000000000001e-05 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.219 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009261+0000", 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4080000000000001e-06 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.033341+0000", 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024080958 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.220 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.076652+0000", 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.043310596999999999 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:46efb29e:::benchmark_data_smithi029_550670_object74990:head)])", 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:08.028376+0000", 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 111.09796502899999, 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.069276355999999997, 2023-10-09T05:03:59.221 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028376+0000", 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028376+0000", 2023-10-09T05:03:59.222 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028375+0000", 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028382+0000", 2023-10-09T05:03:59.223 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8260000000000004e-06 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028398+0000", 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5622999999999999e-05 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028404+0000", 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4189999999999999e-06 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.224 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.053474+0000", 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025070012999999999 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.097653+0000", 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.044178336999999998 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.225 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92031 14.2 14:400f9103:::benchmark_data_smithi029_550670_object14420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:10.530240+0000", 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 108.59610141100001, 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14341484199999999, 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.226 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92031 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530240+0000", 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.227 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530240+0000", 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530238+0000", 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.228 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530248+0000", 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0087e-05 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530267+0000", 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9137999999999999e-05 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.229 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530275+0000", 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0879999999999995e-06 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.535696+0000", 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054208019999999997 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.230 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.552710+0000", 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017013654 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.609958+0000", 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.057248649999999998 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.612646+0000", 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026877989999999998 2023-10-09T05:03:59.231 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.617288+0000", 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046419479999999999 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.653851+0000", 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.036563307000000003 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.655902+0000", 2023-10-09T05:03:59.232 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020504949999999998 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.656580+0000", 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00067814399999999999 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.658429+0000", 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001848992 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.233 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.660996+0000", 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025669460000000001 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.673655+0000", 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012658849 2023-10-09T05:03:59.234 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:4589460d:::benchmark_data_smithi029_550670_object76508:head)])", 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:10.570016+0000", 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 108.556325661, 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081763137, 2023-10-09T05:03:59.235 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570016+0000", 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570016+0000", 2023-10-09T05:03:59.236 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570014+0000", 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570025+0000", 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1127e-05 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.237 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570044+0000", 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8454e-05 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570053+0000", 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8780000000000004e-06 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.238 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.622346+0000", 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.052293555999999998 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.651779+0000", 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029432494 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.239 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.14 607/586 [PushReplyOp(14:2811b6fd:::benchmark_data_smithi029_550670_object48381:head)])", 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:35.992499+0000", 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 83.133842709999996, 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017789461999999999, 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:35.992499+0000", 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.240 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:35.992499+0000", 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:35.992497+0000", 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.241 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:35.992505+0000", 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7489999999999998e-06 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:35.992522+0000", 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6430999999999999e-05 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:35.992530+0000", 2023-10-09T05:03:59.242 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1359999999999997e-06 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:36.008790+0000", 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016260368000000001 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:36.010288+0000", 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001498026 2023-10-09T05:03:59.243 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92277 14.4 14:2748781f:::benchmark_data_smithi029_550670_object14666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:09.402221+0000", 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 49.724120050000003, 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.070626093000000001, 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.244 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92277 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402221+0000", 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.245 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402221+0000", 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402219+0000", 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.246 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402235+0000", 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5835999999999999e-05 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402258+0000", 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3258e-05 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.247 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402267+0000", 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4150000000000005e-06 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402335+0000", 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7275999999999998e-05 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.403203+0000", 2023-10-09T05:03:59.248 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00086857000000000004 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.454287+0000", 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.051083739000000003 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.454400+0000", 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113029 2023-10-09T05:03:59.249 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.454574+0000", 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017359400000000001 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.456547+0000", 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019731409999999999 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.250 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.462441+0000", 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058945339999999999 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.463065+0000", 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062352499999999999 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.251 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.471014+0000", 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079486799999999996 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.471090+0000", 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5967e-05 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.252 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.472847+0000", 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001757855 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93464 14.4 14:267b05b2:::benchmark_data_smithi029_550670_object15853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:31.775033+0000", 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 27.351308522, 2023-10-09T05:03:59.253 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.086900612000000002, 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93464 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775033+0000", 2023-10-09T05:03:59.254 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775033+0000", 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775032+0000", 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.255 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775038+0000", 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2709999999999998e-06 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775053+0000", 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4953000000000001e-05 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.256 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775059+0000", 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.789e-06 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775129+0000", 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0958999999999996e-05 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.257 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.776109+0000", 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097991599999999995 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.855584+0000", 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.079474808999999993 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.258 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.855678+0000", 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4109000000000003e-05 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.855787+0000", 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010880200000000001 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.858366+0000", 2023-10-09T05:03:59.259 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025790800000000001 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.859247+0000", 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000880618 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.859494+0000", 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024704600000000001 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.260 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.860324+0000", 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083051700000000002 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.860366+0000", 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1770999999999997e-05 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.261 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.861933+0000", 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015672850000000001 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93501 14.4 14:21f386db:::benchmark_data_smithi029_550670_object15890:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:32.142236+0000", 2023-10-09T05:03:59.262 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 26.984105340999999, 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016609658999999999, 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93501 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.263 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142236+0000", 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142236+0000", 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.264 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142235+0000", 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142242+0000", 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8560000000000007e-06 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142262+0000", 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9721999999999999e-05 2023-10-09T05:03:59.265 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142269+0000", 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.477e-06 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.142320+0000", 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0952999999999998e-05 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.266 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.143119+0000", 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00079970300000000002 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.151157+0000", 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080373140000000003 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.267 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.151248+0000", 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1766000000000002e-05 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.151359+0000", 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000110967 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.154008+0000", 2023-10-09T05:03:59.268 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026487260000000001 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.154616+0000", 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00060828900000000003 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.156754+0000", 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00213773 2023-10-09T05:03:59.269 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.157047+0000", 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029260000000000001 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.157312+0000", 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026542100000000001 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.270 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:32.158846+0000", 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001533487 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93532 14.4 14:20bd476b:::benchmark_data_smithi029_550670_object15921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:34.770178+0000", 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 24.356163037000002, 2023-10-09T05:03:59.271 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017043624, 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93532 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.272 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770178+0000", 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770178+0000", 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770177+0000", 2023-10-09T05:03:59.273 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770185+0000", 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.2309999999999996e-06 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770204+0000", 2023-10-09T05:03:59.274 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8759000000000001e-05 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770211+0000", 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0099999999999998e-06 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.770275+0000", 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4138000000000002e-05 2023-10-09T05:03:59.275 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.771261+0000", 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00098623000000000005 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.776790+0000", 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055290499999999998 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.276 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.776902+0000", 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011179999999999999 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.777071+0000", 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016920999999999999 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.779357+0000", 2023-10-09T05:03:59.277 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022859289999999999 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.779681+0000", 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032415800000000001 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.278 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.781014+0000", 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001332934 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.786005+0000", 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049910809999999996 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.786042+0000", 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6767000000000002e-05 2023-10-09T05:03:59.279 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:34.787222+0000", 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001179863 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93769 14.1f 14.e529d53f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:03:59.280 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:51.428253+0000", 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 7.6980880190000001, 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.582461071, 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93769 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:03:59.281 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428253+0000", 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428253+0000", 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:03:59.282 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428252+0000", 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428259+0000", 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0709999999999997e-06 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428273+0000", 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3718e-05 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.283 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428280+0000", 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3209999999999999e-06 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428351+0000", 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1095999999999998e-05 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:03:59.284 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428364+0000", 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2873e-05 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:52.010701+0000", 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.58233696199999996 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:52.010714+0000", 2023-10-09T05:03:59.285 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3111e-05 2023-10-09T05:03:59.286 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.286 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.286 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.286 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:03:59.286 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:03:59.286 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:03:59.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:03:59.315 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:03:59.441 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:03:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:03:59.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:03:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: pgmap v2065: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 79997/184404 objects degraded (43.381%); 7285/184404 objects misplaced (3.951%); 13 MiB/s, 243 objects/s recovering 2023-10-09T05:03:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:03:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:03:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:59.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:03:59.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: pgmap v2065: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 79997/184404 objects degraded (43.381%); 7285/184404 objects misplaced (3.951%); 13 MiB/s, 243 objects/s recovering 2023-10-09T05:03:59.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.583 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: pgmap v2065: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 70 op/s; 79997/184404 objects degraded (43.381%); 7285/184404 objects misplaced (3.951%); 13 MiB/s, 243 objects/s recovering 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: failed to encode map e646 with expected crc 2023-10-09T05:03:59.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-09T05:03:59.585 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:03:59.585 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:03:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:03:59.586 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:00.177 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:04:00.481 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:04:00.486 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:00.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:00.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:00.618 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:00.646 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:00.693 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:00.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:00.798 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:00.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:04:00.967 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:01.175 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:01.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:01.280 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:01.425 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:01.581 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:01.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:01 smithi029 ceph-mon[216903]: pgmap v2067: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 77849/184098 objects degraded (42.287%); 8416/184098 objects misplaced (4.571%); 14 MiB/s, 350 objects/s recovering 2023-10-09T05:04:01.593 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 77849/184098 objects degraded (42.287%), 32 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:01.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:01 smithi029 ceph-mon[223858]: pgmap v2067: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 77849/184098 objects degraded (42.287%); 8416/184098 objects misplaced (4.571%); 14 MiB/s, 350 objects/s recovering 2023-10-09T05:04:01.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 77849/184098 objects degraded (42.287%), 32 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:01.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:01.645 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:04:01.651 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:01.697 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:01.715 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:04:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:01 smithi177 ceph-mon[195797]: pgmap v2067: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 77 op/s; 77849/184098 objects degraded (42.287%); 8416/184098 objects misplaced (4.571%); 14 MiB/s, 350 objects/s recovering 2023-10-09T05:04:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 77849/184098 objects degraded (42.287%), 32 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:01.783 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:04:01.799 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:01.859 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:04:01.861 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:01.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:04:01.983 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:04:02.017 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:04:02.048 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.050 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:04:02.090 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:02.384 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:02.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:02.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:04.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3648051975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:04.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3648051975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:04.190 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:04:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:04:02] "GET /metrics HTTP/1.1" 200 36708 "" "Prometheus/2.43.0" 2023-10-09T05:04:04.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:03 smithi029 ceph-mon[216903]: pgmap v2068: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 77703/183924 objects degraded (42.247%); 8416/183924 objects misplaced (4.576%); 8.0 MiB/s, 221 objects/s recovering 2023-10-09T05:04:04.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:03 smithi029 ceph-mon[223858]: pgmap v2068: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 77703/183924 objects degraded (42.247%); 8416/183924 objects misplaced (4.576%); 8.0 MiB/s, 221 objects/s recovering 2023-10-09T05:04:04.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3648051975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:04.194 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:03 smithi177 ceph-mon[195797]: pgmap v2068: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 81 op/s; 77703/183924 objects degraded (42.247%); 8416/183924 objects misplaced (4.576%); 8.0 MiB/s, 221 objects/s recovering 2023-10-09T05:04:04.310 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:04.887 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:04.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:05.072 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:05.104 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:05.162 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:05.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:05.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:05.303 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:05.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:05.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:05.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:05.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:05.545 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:05 smithi177 ceph-mon[195797]: pgmap v2069: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 77323/182349 objects degraded (42.404%); 8146/182349 objects misplaced (4.467%); 8.2 MiB/s, 215 objects/s recovering 2023-10-09T05:04:05.808 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:05.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:05 smithi029 ceph-mon[223858]: pgmap v2069: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 77323/182349 objects degraded (42.404%); 8146/182349 objects misplaced (4.467%); 8.2 MiB/s, 215 objects/s recovering 2023-10-09T05:04:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:05.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:05 smithi029 ceph-mon[216903]: pgmap v2069: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 77323/182349 objects degraded (42.404%); 8146/182349 objects misplaced (4.467%); 8.2 MiB/s, 215 objects/s recovering 2023-10-09T05:04:05.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:05.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:05.978 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:06.141 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:06.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:06.195 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:04:06.248 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:06.338 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 77323/182349 objects degraded (42.404%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:06.912 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:06.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:06.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:06.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 77323/182349 objects degraded (42.404%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:06.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 77323/182349 objects degraded (42.404%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:06.987 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:04:06.987 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:04:07.018 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:07.046 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:07.101 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:07.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:07.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:07.211 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:07.272 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:07.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:07 smithi029 ceph-mon[223858]: pgmap v2070: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 112 op/s; 77223/181473 objects degraded (42.553%); 7708/181473 objects misplaced (4.247%); 6.9 MiB/s, 184 objects/s recovering 2023-10-09T05:04:07.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:07 smithi029 ceph-mon[216903]: pgmap v2070: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 112 op/s; 77223/181473 objects degraded (42.553%); 7708/181473 objects misplaced (4.247%); 6.9 MiB/s, 184 objects/s recovering 2023-10-09T05:04:07.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:07 smithi177 ceph-mon[195797]: pgmap v2070: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 112 op/s; 77223/181473 objects degraded (42.553%); 7708/181473 objects misplaced (4.247%); 6.9 MiB/s, 184 objects/s recovering 2023-10-09T05:04:08.345 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:04:08.365 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:08.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:08.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:08.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:08.690 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:08.855 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:08.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:08.977 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:09.101 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:09.130 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:09.276 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:09.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:09.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:09.478 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:09.524 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:04:09.552 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:09 smithi177 ceph-mon[195797]: pgmap v2071: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 76532/179550 objects degraded (42.624%); 7656/179550 objects misplaced (4.264%); 8.0 MiB/s, 197 objects/s recovering 2023-10-09T05:04:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/485078081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:04:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:04:09.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:09 smithi029 ceph-mon[216903]: pgmap v2071: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 76532/179550 objects degraded (42.624%); 7656/179550 objects misplaced (4.264%); 8.0 MiB/s, 197 objects/s recovering 2023-10-09T05:04:09.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/485078081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:04:09.791 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:04:09.791 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:09 smithi029 ceph-mon[223858]: pgmap v2071: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 76532/179550 objects degraded (42.624%); 7656/179550 objects misplaced (4.264%); 8.0 MiB/s, 197 objects/s recovering 2023-10-09T05:04:09.791 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/485078081' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:04:09.791 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:04:09.815 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:09.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:09.923 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:09.955 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:10.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:10.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:10.128 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:10.199 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:10.536 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:04:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: failed to encode map e647 with expected crc 2023-10-09T05:04:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:04:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: osdmap e647: 8 total, 8 up, 7 in 2023-10-09T05:04:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: failed to encode map e647 with expected crc 2023-10-09T05:04:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: failed to encode map e647 with expected crc 2023-10-09T05:04:10.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: pgmap v2073: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 76435/178824 objects degraded (42.743%); 7114/178824 objects misplaced (3.978%); 10 MiB/s, 182 objects/s recovering 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1729245799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: failed to encode map e647 with expected crc 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: osdmap e647: 8 total, 8 up, 7 in 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: failed to encode map e647 with expected crc 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: failed to encode map e647 with expected crc 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: pgmap v2073: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 76435/178824 objects degraded (42.743%); 7114/178824 objects misplaced (3.978%); 10 MiB/s, 182 objects/s recovering 2023-10-09T05:04:10.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1729245799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:04:10.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:04:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: failed to encode map e647 with expected crc 2023-10-09T05:04:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:04:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: osdmap e647: 8 total, 8 up, 7 in 2023-10-09T05:04:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: failed to encode map e647 with expected crc 2023-10-09T05:04:11.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: failed to encode map e647 with expected crc 2023-10-09T05:04:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: pgmap v2073: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 76435/178824 objects degraded (42.743%); 7114/178824 objects misplaced (3.978%); 10 MiB/s, 182 objects/s recovering 2023-10-09T05:04:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1729245799' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:04:11.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:04:11.309 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:11.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:11.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:11.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:11.508 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:11.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:11.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: failed to encode map e648 with expected crc 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: failed to encode map e648 with expected crc 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: osdmap e648: 8 total, 8 up, 7 in 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: failed to encode map e648 with expected crc 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 76435/178824 objects degraded (42.743%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: failed to encode map e648 with expected crc 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: failed to encode map e648 with expected crc 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:04:11.576 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: osdmap e648: 8 total, 8 up, 7 in 2023-10-09T05:04:11.577 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: failed to encode map e648 with expected crc 2023-10-09T05:04:11.577 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 76435/178824 objects degraded (42.743%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: failed to encode map e648 with expected crc 2023-10-09T05:04:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: failed to encode map e648 with expected crc 2023-10-09T05:04:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:04:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: osdmap e648: 8 total, 8 up, 7 in 2023-10-09T05:04:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: failed to encode map e648 with expected crc 2023-10-09T05:04:12.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 76435/178824 objects degraded (42.743%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:12.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:04:12.555 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:12.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:12.675 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:12.704 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:12.756 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:12.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:12.823 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:12 smithi029 ceph-mon[216903]: pgmap v2075: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 76284/178638 objects degraded (42.703%); 7114/178638 objects misplaced (3.982%); 11 MiB/s, 187 objects/s recovering 2023-10-09T05:04:12.824 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:04:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:04:12] "GET /metrics HTTP/1.1" 200 36708 "" "Prometheus/2.43.0" 2023-10-09T05:04:12.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:12 smithi029 ceph-mon[223858]: pgmap v2075: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 76284/178638 objects degraded (42.703%); 7114/178638 objects misplaced (3.982%); 11 MiB/s, 187 objects/s recovering 2023-10-09T05:04:12.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:13.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:12 smithi177 ceph-mon[195797]: pgmap v2075: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 76284/178638 objects degraded (42.703%); 7114/178638 objects misplaced (3.982%); 11 MiB/s, 187 objects/s recovering 2023-10-09T05:04:13.040 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:13.317 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:13.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:13.436 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:13.464 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:13.934 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:13.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:14.062 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:14.251 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:14.510 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:14.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:14.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:14.572 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:14 smithi029 ceph-mon[223858]: pgmap v2076: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 75745/176997 objects degraded (42.795%); 7031/176997 objects misplaced (3.972%); 9.3 MiB/s, 174 objects/s recovering 2023-10-09T05:04:14.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:14.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:14.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:14 smithi029 ceph-mon[216903]: pgmap v2076: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 75745/176997 objects degraded (42.795%); 7031/176997 objects misplaced (3.972%); 9.3 MiB/s, 174 objects/s recovering 2023-10-09T05:04:14.658 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:14.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:14.726 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:04:14.765 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:04:14.783 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:14.811 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:14.818 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:04:14.856 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:04:14.858 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:14.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:14.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:14.864 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:14 smithi177 ceph-mon[195797]: pgmap v2076: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 75745/176997 objects degraded (42.795%); 7031/176997 objects misplaced (3.972%); 9.3 MiB/s, 174 objects/s recovering 2023-10-09T05:04:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:14.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:14.914 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:04:14.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:04:14.962 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:04:14.967 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:14.997 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:15.006 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:04:15.045 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:04:15.050 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:15.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:15.069 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:04:15.069 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:04:15.069 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:04:15.069 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:04:15.070 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:04:15.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.084 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.085 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.085 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:04:15.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:15.185 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:15.235 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:15.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:15.341 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:15.431 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:15.594 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3088929010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:15.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3088929010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3088929010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:16.616 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:16.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:16.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:16.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:16.760 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:16.809 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:16.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:16.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:16.917 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:16 smithi029 ceph-mon[216903]: pgmap v2077: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 75622/176145 objects degraded (42.932%); 6621/176145 objects misplaced (3.759%); 10 MiB/s, 198 objects/s recovering 2023-10-09T05:04:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 75745/176997 objects degraded (42.795%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:16 smithi029 ceph-mon[216903]: osdmap e649: 8 total, 8 up, 7 in 2023-10-09T05:04:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:16 smithi029 ceph-mon[223858]: pgmap v2077: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 75622/176145 objects degraded (42.932%); 6621/176145 objects misplaced (3.759%); 10 MiB/s, 198 objects/s recovering 2023-10-09T05:04:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 75745/176997 objects degraded (42.795%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:16 smithi029 ceph-mon[223858]: osdmap e649: 8 total, 8 up, 7 in 2023-10-09T05:04:16.978 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:16 smithi177 ceph-mon[195797]: pgmap v2077: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 17 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.6 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 75622/176145 objects degraded (42.932%); 6621/176145 objects misplaced (3.759%); 10 MiB/s, 198 objects/s recovering 2023-10-09T05:04:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 75745/176997 objects degraded (42.795%), 32 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:16 smithi177 ceph-mon[195797]: osdmap e649: 8 total, 8 up, 7 in 2023-10-09T05:04:17.412 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:17.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:17.522 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:17.549 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:17.598 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:17.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:17.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:17.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:17.735 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:17.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:17.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:17.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:17.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[216903]: failed to encode map e649 with expected crc 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[216903]: failed to encode map e649 with expected crc 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[216903]: osdmap e650: 8 total, 8 up, 7 in 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[216903]: failed to encode map e650 with expected crc 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[223858]: failed to encode map e649 with expected crc 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[223858]: failed to encode map e649 with expected crc 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[223858]: osdmap e650: 8 total, 8 up, 7 in 2023-10-09T05:04:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:17 smithi029 ceph-mon[223858]: failed to encode map e650 with expected crc 2023-10-09T05:04:17.974 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:17 smithi177 ceph-mon[195797]: failed to encode map e649 with expected crc 2023-10-09T05:04:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:17 smithi177 ceph-mon[195797]: failed to encode map e649 with expected crc 2023-10-09T05:04:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:17 smithi177 ceph-mon[195797]: osdmap e650: 8 total, 8 up, 7 in 2023-10-09T05:04:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:17 smithi177 ceph-mon[195797]: failed to encode map e650 with expected crc 2023-10-09T05:04:18.094 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:04:18.474 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:18.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:18.590 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:18.663 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:18.705 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:18.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:18 smithi029 ceph-mon[216903]: failed to encode map e650 with expected crc 2023-10-09T05:04:18.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:18 smithi029 ceph-mon[216903]: pgmap v2080: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 70429/174336 objects degraded (40.398%); 9626/174336 objects misplaced (5.522%); 13 MiB/s, 251 objects/s recovering 2023-10-09T05:04:18.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:18 smithi029 ceph-mon[223858]: failed to encode map e650 with expected crc 2023-10-09T05:04:18.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:18 smithi029 ceph-mon[223858]: pgmap v2080: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 70429/174336 objects degraded (40.398%); 9626/174336 objects misplaced (5.522%); 13 MiB/s, 251 objects/s recovering 2023-10-09T05:04:18.937 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:18.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:18 smithi177 ceph-mon[195797]: failed to encode map e650 with expected crc 2023-10-09T05:04:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:18 smithi177 ceph-mon[195797]: pgmap v2080: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 70429/174336 objects degraded (40.398%); 9626/174336 objects misplaced (5.522%); 13 MiB/s, 251 objects/s recovering 2023-10-09T05:04:19.061 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:19.090 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:19.147 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:19.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:19.254 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:19.282 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:19.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:19.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:19.384 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:04:19.442 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:19.567 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:19.790 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:19.844 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:19.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:19.925 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:04:19.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:20.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2956607193' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:04:20.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2956607193' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:04:20.061 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:20.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2956607193' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:04:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[216903]: pgmap v2081: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 70306/173463 objects degraded (40.531%); 9556/173463 objects misplaced (5.509%); 12 MiB/s, 253 objects/s recovering 2023-10-09T05:04:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[216903]: failed to encode map e651 with expected crc 2023-10-09T05:04:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2956607193' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:04:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[216903]: osdmap e651: 8 total, 8 up, 7 in 2023-10-09T05:04:21.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[216903]: osdmap e652: 8 total, 8 up, 7 in 2023-10-09T05:04:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[223858]: pgmap v2081: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 70306/173463 objects degraded (40.531%); 9556/173463 objects misplaced (5.509%); 12 MiB/s, 253 objects/s recovering 2023-10-09T05:04:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[223858]: failed to encode map e651 with expected crc 2023-10-09T05:04:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2956607193' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:04:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[223858]: osdmap e651: 8 total, 8 up, 7 in 2023-10-09T05:04:21.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:20 smithi029 ceph-mon[223858]: osdmap e652: 8 total, 8 up, 7 in 2023-10-09T05:04:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:20 smithi177 ceph-mon[195797]: pgmap v2081: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 70306/173463 objects degraded (40.531%); 9556/173463 objects misplaced (5.509%); 12 MiB/s, 253 objects/s recovering 2023-10-09T05:04:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:20 smithi177 ceph-mon[195797]: failed to encode map e651 with expected crc 2023-10-09T05:04:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2956607193' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:04:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:20 smithi177 ceph-mon[195797]: osdmap e651: 8 total, 8 up, 7 in 2023-10-09T05:04:21.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:20 smithi177 ceph-mon[195797]: osdmap e652: 8 total, 8 up, 7 in 2023-10-09T05:04:22.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 70306/173463 objects degraded (40.531%), 30 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: failed to encode map e653 with expected crc 2023-10-09T05:04:22.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[223858]: osdmap e653: 8 total, 8 up, 7 in 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 70306/173463 objects degraded (40.531%), 30 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: failed to encode map e653 with expected crc 2023-10-09T05:04:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:21 smithi029 ceph-mon[216903]: osdmap e653: 8 total, 8 up, 7 in 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: failed to encode map e652 with expected crc 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 70306/173463 objects degraded (40.531%), 30 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: failed to encode map e653 with expected crc 2023-10-09T05:04:22.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:21 smithi177 ceph-mon[195797]: osdmap e653: 8 total, 8 up, 7 in 2023-10-09T05:04:22.468 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:22.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:22.596 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:22.723 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:22.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:04:22.984 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:04:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:04:22] "GET /metrics HTTP/1.1" 200 36707 "" "Prometheus/2.43.0" 2023-10-09T05:04:22.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[223858]: pgmap v2084: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 70185/173313 objects degraded (40.496%); 9556/173313 objects misplaced (5.514%); 9.1 MiB/s, 194 objects/s recovering 2023-10-09T05:04:22.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[223858]: failed to encode map e653 with expected crc 2023-10-09T05:04:22.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[223858]: failed to encode map e653 with expected crc 2023-10-09T05:04:22.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[223858]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.016 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:23.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:23.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:23.136 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:23.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[216903]: pgmap v2084: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 70185/173313 objects degraded (40.496%); 9556/173313 objects misplaced (5.514%); 9.1 MiB/s, 194 objects/s recovering 2023-10-09T05:04:23.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[216903]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[216903]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.242 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:22 smithi029 ceph-mon[216903]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.259 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:23.272 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:22 smithi177 ceph-mon[195797]: pgmap v2084: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 70185/173313 objects degraded (40.496%); 9556/173313 objects misplaced (5.514%); 9.1 MiB/s, 194 objects/s recovering 2023-10-09T05:04:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:22 smithi177 ceph-mon[195797]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:22 smithi177 ceph-mon[195797]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:22 smithi177 ceph-mon[195797]: failed to encode map e653 with expected crc 2023-10-09T05:04:23.550 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:23.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:23.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:23.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:23.780 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:24.065 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:04:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:23 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1731084850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:24.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:23 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:24.394 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:24.400 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:23 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1731084850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:24.400 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:23 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:24.400 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:23 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1731084850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:24.400 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:23 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:24.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:24.502 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:24.531 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:24.584 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:24.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:24.690 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:24.718 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:24.765 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:24.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:24.870 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:24.900 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:24 smithi177 ceph-mon[195797]: pgmap v2086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 69705/172056 objects degraded (40.513%); 8265/172056 objects misplaced (4.804%); 12 MiB/s, 223 objects/s recovering 2023-10-09T05:04:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:24 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:24 smithi177 ceph-mon[195797]: failed to encode map e654 with expected crc 2023-10-09T05:04:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:04:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:24 smithi177 ceph-mon[195797]: osdmap e654: 8 total, 8 up, 7 in 2023-10-09T05:04:25.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[216903]: pgmap v2086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 69705/172056 objects degraded (40.513%); 8265/172056 objects misplaced (4.804%); 12 MiB/s, 223 objects/s recovering 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[216903]: failed to encode map e654 with expected crc 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[216903]: osdmap e654: 8 total, 8 up, 7 in 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[223858]: pgmap v2086: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 87 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 11 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 69705/172056 objects degraded (40.513%); 8265/172056 objects misplaced (4.804%); 12 MiB/s, 223 objects/s recovering 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[223858]: failed to encode map e654 with expected crc 2023-10-09T05:04:25.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:04:25.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:24 smithi029 ceph-mon[223858]: osdmap e654: 8 total, 8 up, 7 in 2023-10-09T05:04:25.709 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:25.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:25.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:25.818 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:25.846 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:25.895 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:25.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:25.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:26.001 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:26.134 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:26.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 69705/172056 objects degraded (40.513%), 30 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:26.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 69705/172056 objects degraded (40.513%), 30 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:26.309 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 69705/172056 objects degraded (40.513%), 30 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:27.038 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:04:27.038 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:04:27.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:04:27.038 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:04:27.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75767 14.6 14:67eeec07:::benchmark_data_smithi029_550670_object75766:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587164+0000", 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.37258865299998, 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8369943580000001, 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75767 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.039 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587164+0000", 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587164+0000", 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587162+0000", 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587206+0000", 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3853000000000002e-05 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587228+0000", 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1642999999999999e-05 2023-10-09T05:04:27.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587239+0000", 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1773999999999999e-05 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587432+0000", 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000193165 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587621+0000", 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000188137 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587810+0000", 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000189638 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.595548+0000", 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077380579999999999 2023-10-09T05:04:27.043 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.595558+0000", 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3070000000000008e-06 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.626520+0000", 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030962798 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.044 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.626536+0000", 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5619000000000001e-05 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578599+0000", 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95206309499999997 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578612+0000", 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2500000000000001e-05 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.424024+0000", 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.845412107 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.424159+0000", 2023-10-09T05:04:27.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013491100000000001 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75768 14.b 14:df4bb7ab:::benchmark_data_smithi029_550670_object75767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587258+0000", 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.37249463499995, 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9327158849999999, 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.047 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75768 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587258+0000", 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587258+0000", 2023-10-09T05:04:27.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587258+0000", 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587297+0000", 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8970000000000001e-05 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587305+0000", 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5720000000000003e-06 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587311+0000", 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8479999999999997e-06 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.591805+0000", 2023-10-09T05:04:27.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044936780000000001 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.591955+0000", 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000150617 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.592115+0000", 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000159679 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.051 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619278+0000", 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027162781 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619295+0000", 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6892999999999999e-05 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577449+0000", 2023-10-09T05:04:27.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95815421199999995 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577461+0000", 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2148e-05 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.519849+0000", 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.942388059 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.519974+0000", 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012499899999999999 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75769 14.6 14:640c7ebb:::benchmark_data_smithi029_550670_object75768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587318+0000", 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.37243459399997, 2023-10-09T05:04:27.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8637364299999999, 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75769 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587318+0000", 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587318+0000", 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587318+0000", 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587337+0000", 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9017000000000001e-05 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587344+0000", 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6760000000000001e-06 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587349+0000", 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9899999999999997e-06 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.590673+0000", 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003324151 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.590837+0000", 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000164186 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,5", 2023-10-09T05:04:27.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.591004+0000", 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000166897 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.627208+0000", 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036204436 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.627221+0000", 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2875e-05 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578647+0000", 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95142564900000004 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578657+0000", 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0594999999999999e-05 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.426758+0000", 2023-10-09T05:04:27.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8481005879999999 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.427461+0000", 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00070262299999999998 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.450926+0000", 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023465465000000001 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.061 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.451055+0000", 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012875300000000001 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75770 14.b 14:dc4bda7c:::benchmark_data_smithi029_550670_object75769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.587353+0000", 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.372399632, 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.961170659, 2023-10-09T05:04:27.062 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75770 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587353+0000", 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587353+0000", 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587353+0000", 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587375+0000", 2023-10-09T05:04:27.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1727000000000001e-05 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587381+0000", 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6769999999999998e-06 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.587386+0000", 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9930000000000003e-06 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.600078+0000", 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012692050999999999 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.600255+0000", 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000176885 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,7", 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.600426+0000", 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017075099999999999 2023-10-09T05:04:27.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619779+0000", 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019352582 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.619791+0000", 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2045000000000001e-05 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577498+0000", 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95770707099999997 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.577508+0000", 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8859999999999999e-06 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523040+0000", 2023-10-09T05:04:27.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9455329180000001 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523516+0000", 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00047530000000000001 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548407+0000", 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024890834000000001 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548524+0000", 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117376 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75771 14.0 14:090f761e:::benchmark_data_smithi029_550670_object75770:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e451)", 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.599830+0000", 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.35992324799997, 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.927342222, 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.070 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75771 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599830+0000", 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.071 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599830+0000", 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599827+0000", 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599868+0000", 2023-10-09T05:04:27.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0200999999999997e-05 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599883+0000", 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5826000000000001e-05 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.599894+0000", 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0865e-05 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.073 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.623358+0000", 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023463588 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.623366+0000", 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9079999999999995e-06 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576263+0000", 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.95289678799999999 2023-10-09T05:04:27.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576275+0000", 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2527999999999999e-05 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.510946+0000", 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9346704300000002 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.511127+0000", 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018167300000000001 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.511327+0000", 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000199676 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523885+0000", 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012557972000000001 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.524081+0000", 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000196119 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.524092+0000", 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1073999999999999e-05 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.524147+0000", 2023-10-09T05:04:27.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4941000000000001e-05 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.527172+0000", 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030249029999999998 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75778 14.b 14:db8e7e52:::benchmark_data_smithi029_550670_object75777:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T05:04:27.078 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.632090+0000", 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.32766329799995, 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.920090541, 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75778 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632090+0000", 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632090+0000", 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632088+0000", 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:27.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632115+0000", 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6480000000000001e-05 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632133+0000", 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8165999999999999e-05 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632141+0000", 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8639999999999996e-06 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632222+0000", 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1005000000000001e-05 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.632232+0000", 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0789e-05 2023-10-09T05:04:27.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581266+0000", 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94903347299999996 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581273+0000", 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2880000000000004e-06 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523554+0000", 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9422803749999999 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523699+0000", 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014507999999999999 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.523877+0000", 2023-10-09T05:04:27.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017794499999999999 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.538285+0000", 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014408287000000001 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.541528+0000", 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032427770000000001 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548696+0000", 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007167919 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.548754+0000", 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8739999999999997e-05 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.552180+0000", 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034258280000000001 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75780 14.0 14:0dbd6af5:::benchmark_data_smithi029_550670_object75779:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.637479+0000", 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.32227390599996, 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9000195930000001, 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.087 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75780 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637479+0000", 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637479+0000", 2023-10-09T05:04:27.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637479+0000", 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637500+0000", 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1426000000000001e-05 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637511+0000", 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0839000000000001e-05 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.637517+0000", 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4470000000000003e-06 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.639412+0000", 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018943899999999999 2023-10-09T05:04:27.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.639418+0000", 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7129999999999999e-06 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576299+0000", 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93688061199999995 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.576307+0000", 2023-10-09T05:04:27.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5490000000000001e-06 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520128+0000", 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.943821799 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520331+0000", 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020220200000000001 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 5,6", 2023-10-09T05:04:27.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.520519+0000", 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000188445 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.527279+0000", 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067602900000000004 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.527328+0000", 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8647999999999999e-05 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.534681+0000", 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073529279999999999 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.534756+0000", 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4950000000000006e-05 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.537499+0000", 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027427860000000001 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75782 14.6 14:66f59f82:::benchmark_data_smithi029_550670_object75781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e452)", 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:44.638790+0000", 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 582.32096288100001, 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8162426200000001, 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.095 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75782 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638790+0000", 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638790+0000", 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638789+0000", 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638819+0000", 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0349e-05 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638837+0000", 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8090000000000001e-05 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638847+0000", 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9690000000000006e-06 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638900+0000", 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3232000000000001e-05 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:44.638910+0000", 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.9480000000000003e-06 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578683+0000", 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.93977262699999997 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.578692+0000", 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8759999999999994e-06 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.427504+0000", 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.848811832 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.427820+0000", 2023-10-09T05:04:27.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031670199999999998 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.428026+0000", 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000205756 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.430801+0000", 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027751910000000002 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.439921+0000", 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091198389999999994 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.451125+0000", 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01120434 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.451194+0000", 2023-10-09T05:04:27.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8336000000000004e-05 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.455033+0000", 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003838957 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75763 14.d 14:bffa6910:::benchmark_data_smithi029_550670_object75762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581881+0000", 2023-10-09T05:04:27.103 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 581.37787222199995, 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.884113382, 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75763 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581881+0000", 2023-10-09T05:04:27.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581881+0000", 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581879+0000", 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.105 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581922+0000", 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2283999999999997e-05 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581937+0000", 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.552e-05 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581947+0000", 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0441999999999999e-05 2023-10-09T05:04:27.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582166+0000", 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021813 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582175+0000", 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4239999999999999e-06 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.464358+0000", 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.882182969 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.465994+0000", 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001636071 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.108 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75766 14.d 14:b17bdc11:::benchmark_data_smithi029_550670_object75765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.581967+0000", 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 581.37778600299998, 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8861905270000001, 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75766 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581967+0000", 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581967+0000", 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.581966+0000", 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582002+0000", 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5385000000000001e-05 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582031+0000", 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9578999999999999e-05 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582039+0000", 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4399999999999999e-06 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582199+0000", 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016029400000000001 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582207+0000", 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7670000000000003e-06 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.468032+0000", 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.885825222 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.468157+0000", 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012543 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75774 14.d 14:bdf921c9:::benchmark_data_smithi029_550670_object75773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.582046+0000", 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 581.37770717900003, 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9239243049999999, 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75774 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.114 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582046+0000", 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582046+0000", 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582045+0000", 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582067+0000", 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2084999999999999e-05 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582073+0000", 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1639999999999998e-06 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582078+0000", 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1139999999999999e-06 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582232+0000", 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015455800000000001 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582239+0000", 2023-10-09T05:04:27.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9959999999999996e-06 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.470621+0000", 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.888381734 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.471199+0000", 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00057843300000000005 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.471326+0000", 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012646000000000001 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.476776+0000", 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005449756 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.478331+0000", 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001555802 2023-10-09T05:04:27.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.478980+0000", 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064857800000000002 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.503195+0000", 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024215161999999998 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.503259+0000", 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.3878000000000001e-05 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.505970+0000", 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027110989999999998 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:75775 14.d 14:b648c4c3:::benchmark_data_smithi029_550670_object75774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e453)", 2023-10-09T05:04:27.121 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:54:45.582081+0000", 2023-10-09T05:04:27.122 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 581.37767163900003, 2023-10-09T05:04:27.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9269593029999998, 2023-10-09T05:04:27.122 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.122 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 75775 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582081+0000", 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582081+0000", 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582081+0000", 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:27.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582104+0000", 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2773999999999999e-05 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582113+0000", 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0110000000000006e-06 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582119+0000", 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7860000000000002e-06 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582262+0000", 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014312 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:45.582268+0000", 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8600000000000004e-06 2023-10-09T05:04:27.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.473499+0000", 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.891230465 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.473690+0000", 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019085700000000001 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,5,6", 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.473900+0000", 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000209731 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.476972+0000", 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030722330000000002 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.478460+0000", 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001488367 2023-10-09T05:04:27.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.479284+0000", 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00082400300000000002 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.506072+0000", 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.026788055000000002 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.506156+0000", 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3523000000000007e-05 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:54:47.509041+0000", 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002884869 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.130 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:78125 14.c 14:3c481121:::benchmark_data_smithi029_550670_object514:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.270549+0000", 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 514.68920402399999, 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.060580334, 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 78125 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270549+0000", 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270549+0000", 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270548+0000", 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:27.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270553+0000", 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7250000000000002e-06 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270563+0000", 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5680000000000005e-06 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.133 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.270569+0000", 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9610000000000002e-06 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.316048+0000", 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.045478969000000001 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.316056+0000", 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7819999999999994e-06 2023-10-09T05:04:27.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138551+0000", 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.82249487200000004 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138560+0000", 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4860000000000003e-06 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.299702+0000", 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1611424079999999 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.300295+0000", 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059211799999999998 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.311832+0000", 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011537016000000001 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.313194+0000", 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013622269999999999 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.313294+0000", 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100395 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327670+0000", 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014375489 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327710+0000", 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0241999999999999e-05 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327745+0000", 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5070000000000001e-05 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.327792+0000", 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7407999999999997e-05 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.331129+0000", 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033368450000000002 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:78130 14.c 14:3226741c:::benchmark_data_smithi029_550670_object519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.321301+0000", 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 514.63845235899998, 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0139341900000001, 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 78130 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.140 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321301+0000", 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321301+0000", 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321298+0000", 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321306+0000", 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0960000000000001e-06 2023-10-09T05:04:27.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321324+0000", 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7490999999999999e-05 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.321334+0000", 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8709999999999992e-06 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.324175+0000", 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028410649999999998 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.324186+0000", 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0808999999999999e-05 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138579+0000", 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81439388400000001 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138585+0000", 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6629999999999998e-06 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.303198+0000", 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1646125330000001 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.303820+0000", 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00062219 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.324676+0000", 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020855728 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.324801+0000", 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012591999999999999 2023-10-09T05:04:27.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.324956+0000", 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000154742 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332201+0000", 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072445599999999997 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332255+0000", 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4011999999999997e-05 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332328+0000", 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.3379000000000003e-05 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.332363+0000", 2023-10-09T05:04:27.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5015999999999998e-05 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.335235+0000", 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028716240000000001 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:78138 14.c 14:36c9f53e:::benchmark_data_smithi029_550670_object527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e473)", 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:55:52.497048+0000", 2023-10-09T05:04:27.150 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 514.46270445699997, 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.88800638, 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 78138 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.151 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497048+0000", 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497048+0000", 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497047+0000", 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497057+0000", 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0536e-05 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497073+0000", 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5494000000000001e-05 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497085+0000", 2023-10-09T05:04:27.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2255999999999999e-05 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497132+0000", 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7321000000000003e-05 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:52.497141+0000", 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3399999999999998e-06 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138603+0000", 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.64146234800000002 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:53.138609+0000", 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8409999999999996e-06 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.331198+0000", 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1925896629999999 2023-10-09T05:04:27.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.331703+0000", 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00050408300000000001 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.352160+0000", 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020457389999999999 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.352290+0000", 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000130576 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.352446+0000", 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015550699999999999 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.368379+0000", 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015933460999999999 2023-10-09T05:04:27.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.378110+0000", 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097306489999999992 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.381781+0000", 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036712279999999999 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.381822+0000", 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0844999999999999e-05 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:55:54.385055+0000", 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032326949999999998 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.159 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79944 14.b 14:dc480776:::benchmark_data_smithi029_550670_object2333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:53.873492+0000", 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 453.08626140199999, 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.313680212, 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79944 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873492+0000", 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873492+0000", 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873490+0000", 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:04:27.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873500+0000", 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0532e-05 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873520+0000", 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0228e-05 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873529+0000", 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9409999999999996e-06 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873596+0000", 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6738999999999999e-05 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.873608+0000", 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.184e-05 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.899229+0000", 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025621357000000001 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:53.899240+0000", 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0434999999999999e-05 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.068466+0000", 2023-10-09T05:04:27.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1692258600000001 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.069075+0000", 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00060947900000000001 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.176690+0000", 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10761480599999999 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.176829+0000", 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000139042 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.177029+0000", 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00020046300000000001 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.181269+0000", 2023-10-09T05:04:27.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042392059999999997 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.183001+0000", 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017322769999999999 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.184173+0000", 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001172551 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.184246+0000", 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2984999999999993e-05 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.187172+0000", 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029253909999999998 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.168 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:79950 14.6 14:6b7b279e:::benchmark_data_smithi029_550670_object2339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e495)", 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:56:55.235020+0000", 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 451.724732743, 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0513258090000002, 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 79950 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235020+0000", 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235020+0000", 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235019+0000", 2023-10-09T05:04:27.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235025+0000", 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4529999999999999e-06 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235038+0000", 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2843000000000001e-05 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235045+0000", 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7139999999999996e-06 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235087+0000", 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1946000000000001e-05 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.235097+0000", 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0424e-05 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.806392+0000", 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.57129448100000002 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:55.806403+0000", 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1620000000000001e-05 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.939987+0000", 2023-10-09T05:04:27.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1335838650000001 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:56.940515+0000", 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00052770499999999995 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.276476+0000", 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.33596088499999999 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.276610+0000", 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013452700000000001 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.276798+0000", 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000187529 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283471+0000", 2023-10-09T05:04:27.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066732329999999998 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283506+0000", 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5008000000000001e-05 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283526+0000", 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9721999999999999e-05 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.283592+0000", 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5778999999999995e-05 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:56:57.286346+0000", 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002754458 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.177 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80555 14.b 14:db16aa4d:::benchmark_data_smithi029_550670_object2944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.903074+0000", 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 380.05667877299999, 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3293849390000001, 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80555 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903074+0000", 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903074+0000", 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903072+0000", 2023-10-09T05:04:27.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903084+0000", 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1606999999999999e-05 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903111+0000", 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7042999999999998e-05 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.903124+0000", 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3463e-05 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.905222+0000", 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020976089999999998 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.905239+0000", 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6869999999999999e-05 2023-10-09T05:04:27.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846128+0000", 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.94088980499999997 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846141+0000", 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2454e-05 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.182 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.216737+0000", 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.370596132 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.216952+0000", 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021524599999999999 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.217187+0000", 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00023498700000000001 2023-10-09T05:04:27.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228059+0000", 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010872122 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228118+0000", 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9052000000000003e-05 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228177+0000", 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8903999999999997e-05 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.228253+0000", 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5881000000000001e-05 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.232459+0000", 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042059209999999996 2023-10-09T05:04:27.185 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80560 14.b 14:d972c932:::benchmark_data_smithi029_550670_object2949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:07.374946+0000", 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.58480668599998, 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8664385380000001, 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.186 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80560 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374946+0000", 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374946+0000", 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374944+0000", 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374955+0000", 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0813e-05 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374974+0000", 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9607999999999999e-05 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.374988+0000", 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3927999999999999e-05 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.375053+0000", 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4863000000000006e-05 2023-10-09T05:04:27.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.375065+0000", 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2286000000000001e-05 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846161+0000", 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47109602 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.190 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.846168+0000", 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1029999999999996e-06 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224329+0000", 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.378160863 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224524+0000", 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019509599999999999 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.224719+0000", 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00019503399999999999 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.232728+0000", 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080089949999999997 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.232876+0000", 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014811100000000001 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.237189+0000", 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043121560000000001 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.237263+0000", 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4667999999999997e-05 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.241385+0000", 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041214670000000002 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80562 14.e 14:732ec9d2:::benchmark_data_smithi029_550670_object2951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:07.495374+0000", 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 379.46437941900001, 2023-10-09T05:04:27.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6830923520000001, 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80562 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495374+0000", 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495374+0000", 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495372+0000", 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495382+0000", 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0210000000000001e-05 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495408+0000", 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5658999999999999e-05 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495420+0000", 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2395000000000001e-05 2023-10-09T05:04:27.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495481+0000", 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1062000000000006e-05 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.495490+0000", 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3810000000000005e-06 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.842050+0000", 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34655991600000002 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.842066+0000", 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6103000000000002e-05 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:08.980071+0000", 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1380047360000001 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:08.980571+0000", 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00049947900000000005 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.116782+0000", 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13621124900000001 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.116901+0000", 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000119215 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.117081+0000", 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018009100000000001 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174495+0000", 2023-10-09T05:04:27.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057413829 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174546+0000", 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0671999999999999e-05 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174592+0000", 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6103000000000002e-05 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.174660+0000", 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8343999999999995e-05 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:09.178466+0000", 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003805775 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:04:27.203 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:04:27.204 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:04:27.204 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:27.338 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:27.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:27 smithi029 ceph-mon[216903]: pgmap v2088: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 113 op/s; 70697/171462 objects degraded (41.232%); 4580/171462 objects misplaced (2.671%); 12 MiB/s, 191 objects/s recovering 2023-10-09T05:04:27.355 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:27 smithi029 ceph-mon[223858]: pgmap v2088: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 113 op/s; 70697/171462 objects degraded (41.232%); 4580/171462 objects misplaced (2.671%); 12 MiB/s, 191 objects/s recovering 2023-10-09T05:04:27.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:27.456 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:27.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:27 smithi177 ceph-mon[195797]: pgmap v2088: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 113 op/s; 70697/171462 objects degraded (41.232%); 4580/171462 objects misplaced (2.671%); 12 MiB/s, 191 objects/s recovering 2023-10-09T05:04:27.612 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:27.869 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:04:28.436 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:28.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:28.543 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:28.586 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:04:28.682 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:28.973 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:28.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:28.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:29.044 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:04:29.083 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:29.123 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:29.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:29.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:29.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:04:29.237 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:29 smithi029 ceph-mon[216903]: pgmap v2089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 69942/169635 objects degraded (41.231%); 3617/169635 objects misplaced (2.132%); 19 MiB/s, 300 objects/s recovering 2023-10-09T05:04:29.237 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:29.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:29 smithi029 ceph-mon[223858]: pgmap v2089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 69942/169635 objects degraded (41.231%); 3617/169635 objects misplaced (2.132%); 19 MiB/s, 300 objects/s recovering 2023-10-09T05:04:29.237 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:29.286 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:04:29.300 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:29.326 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:04:29.330 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:29.350 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:29 smithi177 ceph-mon[195797]: pgmap v2089: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 69942/169635 objects degraded (41.231%); 3617/169635 objects misplaced (2.132%); 19 MiB/s, 300 objects/s recovering 2023-10-09T05:04:29.350 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:29.363 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:04:29.395 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:29.399 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:04:29.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:29.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:29.435 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:04:29.470 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:04:29.499 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:04:29.500 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:29.502 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.502 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.502 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.502 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.502 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.503 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.503 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.503 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:04:29.531 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:29.788 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:29.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:29.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:29.893 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:29.957 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:30.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/687917561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:30.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/687917561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:30.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/687917561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:30.709 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:30.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:30.824 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:30.868 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:31.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:31 smithi029 ceph-mon[223858]: pgmap v2090: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 69789/168768 objects degraded (41.352%); 3617/168768 objects misplaced (2.143%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:04:31.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 69789/168768 objects degraded (41.352%), 31 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:31.090 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:31 smithi029 ceph-mon[216903]: pgmap v2090: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 69789/168768 objects degraded (41.352%); 3617/168768 objects misplaced (2.143%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:04:31.090 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 69789/168768 objects degraded (41.352%), 31 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:31 smithi177 ceph-mon[195797]: pgmap v2090: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 69789/168768 objects degraded (41.352%); 3617/168768 objects misplaced (2.143%); 17 MiB/s, 264 objects/s recovering 2023-10-09T05:04:31.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 69789/168768 objects degraded (41.352%), 31 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:31.622 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:31.673 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:31.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:31.796 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:31.826 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:31.879 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:31.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:31.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:31.984 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:32.011 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:32.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:32.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:32.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:32.159 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:32.194 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:32.249 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:32.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:32.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:32.354 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:32.383 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:32.430 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:32.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:32.534 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:32.624 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:33.100 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:04:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:04:32] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.43.0" 2023-10-09T05:04:33.104 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:33.104 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:33 smithi029 ceph-mon[216903]: pgmap v2091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 69648/168594 objects degraded (41.311%); 3617/168594 objects misplaced (2.145%); 14 MiB/s, 220 objects/s recovering 2023-10-09T05:04:33.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:33.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:33.217 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:33.282 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:33.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:33 smithi029 ceph-mon[223858]: pgmap v2091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 69648/168594 objects degraded (41.311%); 3617/168594 objects misplaced (2.145%); 14 MiB/s, 220 objects/s recovering 2023-10-09T05:04:33.430 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:33.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:33.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:33.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:33 smithi177 ceph-mon[195797]: pgmap v2091: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 69648/168594 objects degraded (41.311%); 3617/168594 objects misplaced (2.145%); 14 MiB/s, 220 objects/s recovering 2023-10-09T05:04:33.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:33.636 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:33.640 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:04:33.917 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:33.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:34.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:34.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3525777899' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:04:34.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:34 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:04:34.165 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:34.191 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:04:34.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3525777899' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:04:34.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:34 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:04:34.493 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:34.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:34.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3525777899' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:04:34.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:34 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:04:34.609 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:34.710 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:34.858 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:34.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:34.962 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:34.992 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:35.044 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:35.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:35.147 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:35.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[216903]: pgmap v2092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 68997/166995 objects degraded (41.317%); 2558/166995 objects misplaced (1.532%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:04:35.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[216903]: failed to encode map e655 with expected crc 2023-10-09T05:04:35.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:04:35.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[216903]: osdmap e655: 8 total, 8 up, 7 in 2023-10-09T05:04:35.213 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:35.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:35 smithi177 ceph-mon[195797]: pgmap v2092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 68997/166995 objects degraded (41.317%); 2558/166995 objects misplaced (1.532%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:04:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:35 smithi177 ceph-mon[195797]: failed to encode map e655 with expected crc 2023-10-09T05:04:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:35 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:04:35.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:35 smithi177 ceph-mon[195797]: osdmap e655: 8 total, 8 up, 7 in 2023-10-09T05:04:35.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[223858]: pgmap v2092: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 68997/166995 objects degraded (41.317%); 2558/166995 objects misplaced (1.532%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:04:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[223858]: failed to encode map e655 with expected crc 2023-10-09T05:04:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:04:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:35 smithi029 ceph-mon[223858]: osdmap e655: 8 total, 8 up, 7 in 2023-10-09T05:04:35.656 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:35.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:04:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:35.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:35.787 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:35.816 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:35.837 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:35.868 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:35.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:35.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:36.148 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:36.375 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:36.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:36.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:36.481 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:36.509 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:36.556 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:36.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:36.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:36.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:36.746 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:36.851 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:36.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:36.960 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:37.058 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:04:37.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:36 smithi029 ceph-mon[216903]: pgmap v2094: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 68829/166170 objects degraded (41.421%); 2558/166170 objects misplaced (1.539%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:04:37.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 68997/166995 objects degraded (41.317%), 31 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:37.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2727606629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:04:37.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:36 smithi029 ceph-mon[223858]: pgmap v2094: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 68829/166170 objects degraded (41.421%); 2558/166170 objects misplaced (1.539%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:04:37.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 68997/166995 objects degraded (41.317%), 31 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:37.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2727606629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:04:37.074 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:37.186 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:37.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:37.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:36 smithi177 ceph-mon[195797]: pgmap v2094: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 12 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 68829/166170 objects degraded (41.421%); 2558/166170 objects misplaced (1.539%); 14 MiB/s, 219 objects/s recovering 2023-10-09T05:04:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 68997/166995 objects degraded (41.317%), 31 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:37.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2727606629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:04:37.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:37.369 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:37.429 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:37.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:37.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:37.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:37.617 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:38.018 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:38.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:38.108 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:04:38.108 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:04:38.124 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:38.131 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:38 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:38.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:38 smithi177 ceph-mon[195797]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:38 smithi177 ceph-mon[195797]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:38 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2727606629' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:04:38.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:38 smithi177 ceph-mon[195797]: osdmap e656: 8 total, 8 up, 7 in 2023-10-09T05:04:38.132 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:38 smithi177 ceph-mon[195797]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.153 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:38.354 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:38.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:38.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:37 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:38.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:37 smithi029 ceph-mon[216903]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:37 smithi029 ceph-mon[216903]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2727606629' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:37 smithi029 ceph-mon[216903]: osdmap e656: 8 total, 8 up, 7 in 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:37 smithi029 ceph-mon[216903]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:38 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:38 smithi029 ceph-mon[223858]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:38 smithi029 ceph-mon[223858]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:38 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2727606629' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:04:38.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:38 smithi029 ceph-mon[223858]: osdmap e656: 8 total, 8 up, 7 in 2023-10-09T05:04:38.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:38 smithi029 ceph-mon[223858]: failed to encode map e656 with expected crc 2023-10-09T05:04:38.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:38.600 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:38.763 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:38.886 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:38.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:39.008 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:39.039 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:39.090 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:39.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[223858]: pgmap v2096: 119 pgs: 1 peering, 1 active+recovering, 1 active+recovering+undersized+remapped, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 68247/164799 objects degraded (41.412%); 2107/164799 objects misplaced (1.279%); 15 MiB/s, 266 objects/s recovering 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[223858]: osdmap e657: 8 total, 8 up, 7 in 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[223858]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[223858]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[223858]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[223858]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[216903]: pgmap v2096: 119 pgs: 1 peering, 1 active+recovering, 1 active+recovering+undersized+remapped, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 68247/164799 objects degraded (41.412%); 2107/164799 objects misplaced (1.279%); 15 MiB/s, 266 objects/s recovering 2023-10-09T05:04:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[216903]: osdmap e657: 8 total, 8 up, 7 in 2023-10-09T05:04:39.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[216903]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[216903]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[216903]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:39 smithi029 ceph-mon[216903]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.194 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:39.221 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:39.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:39.466 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:39 smithi177 ceph-mon[195797]: pgmap v2096: 119 pgs: 1 peering, 1 active+recovering, 1 active+recovering+undersized+remapped, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 68247/164799 objects degraded (41.412%); 2107/164799 objects misplaced (1.279%); 15 MiB/s, 266 objects/s recovering 2023-10-09T05:04:39.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:39 smithi177 ceph-mon[195797]: osdmap e657: 8 total, 8 up, 7 in 2023-10-09T05:04:39.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:39 smithi177 ceph-mon[195797]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:39 smithi177 ceph-mon[195797]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:39 smithi177 ceph-mon[195797]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.467 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:39 smithi177 ceph-mon[195797]: failed to encode map e657 with expected crc 2023-10-09T05:04:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:39.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:39.525 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:04:39.569 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:39.615 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:39.670 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:39.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:39.776 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:39.815 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:39.872 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:39.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:39.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:39.920 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:04:39.951 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:04:39.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:39.979 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:04:39.988 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:04:40.026 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:04:40.050 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:40.062 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:04:40.114 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:04:40.149 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:04:40.153 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:40.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:40.178 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:04:40.199 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.199 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.200 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.200 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.200 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.200 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.200 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.200 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:04:40.236 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:40 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3186989691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:40.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:40.309 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:40.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:40 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3186989691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:40.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:40 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3186989691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:40.615 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:40.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:40.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:40.720 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:40.798 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:40.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:04:41.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:41 smithi029 ceph-mon[216903]: pgmap v2098: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 67677/163347 objects degraded (41.431%); 1992/163347 objects misplaced (1.219%); 13 MiB/s, 229 objects/s recovering 2023-10-09T05:04:41.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 67677/163347 objects degraded (41.431%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:41.045 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:41 smithi029 ceph-mon[223858]: pgmap v2098: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 67677/163347 objects degraded (41.431%); 1992/163347 objects misplaced (1.219%); 13 MiB/s, 229 objects/s recovering 2023-10-09T05:04:41.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 67677/163347 objects degraded (41.431%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:41 smithi177 ceph-mon[195797]: pgmap v2098: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 67677/163347 objects degraded (41.431%); 1992/163347 objects misplaced (1.219%); 13 MiB/s, 229 objects/s recovering 2023-10-09T05:04:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 67677/163347 objects degraded (41.431%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:42.023 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:42.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:42.135 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:42.151 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:04:42.151 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:04:42.151 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:04:42.151 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:04:42.151 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:04:42.212 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:42.301 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:42.369 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:42.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:42.486 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:42.578 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:42.703 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:42.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:42.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:42.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:42.832 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:04:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:04:42] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.43.0" 2023-10-09T05:04:42.840 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:43.161 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:43.163 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:43 smithi177 ceph-mon[195797]: pgmap v2099: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 67677/163347 objects degraded (41.431%); 1992/163347 objects misplaced (1.219%); 9.0 MiB/s, 166 objects/s recovering 2023-10-09T05:04:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:43.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:43.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:43 smithi029 ceph-mon[216903]: pgmap v2099: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 67677/163347 objects degraded (41.431%); 1992/163347 objects misplaced (1.219%); 9.0 MiB/s, 166 objects/s recovering 2023-10-09T05:04:43.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:43 smithi029 ceph-mon[223858]: pgmap v2099: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovering, 87 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 67677/163347 objects degraded (41.431%); 1992/163347 objects misplaced (1.219%); 9.0 MiB/s, 166 objects/s recovering 2023-10-09T05:04:43.247 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:04:43.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:43.365 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:43.602 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:43.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:43.685 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:04:43.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:43.757 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:43.807 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:43.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:43.911 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:43.946 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:44.008 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:44.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:44.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:44.039 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:04:44.101 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:04:44.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:44.141 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:44.169 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:44 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1680806889' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:04:44.169 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:44.183 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:44.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:44.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:44.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:44.347 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:44 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1680806889' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:04:44.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:44.348 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:44 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1680806889' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:04:44.348 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:44.352 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:45.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[216903]: pgmap v2100: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 66784/160650 objects degraded (41.571%); 993/160650 objects misplaced (0.618%); 15 MiB/s, 267 objects/s recovering 2023-10-09T05:04:45.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[216903]: failed to encode map e658 with expected crc 2023-10-09T05:04:45.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[216903]: failed to encode map e658 with expected crc 2023-10-09T05:04:45.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1680806889' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:04:45.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[216903]: osdmap e658: 8 total, 8 up, 7 in 2023-10-09T05:04:45.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[223858]: pgmap v2100: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 66784/160650 objects degraded (41.571%); 993/160650 objects misplaced (0.618%); 15 MiB/s, 267 objects/s recovering 2023-10-09T05:04:45.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[223858]: failed to encode map e658 with expected crc 2023-10-09T05:04:45.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[223858]: failed to encode map e658 with expected crc 2023-10-09T05:04:45.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1680806889' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:04:45.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:45 smithi029 ceph-mon[223858]: osdmap e658: 8 total, 8 up, 7 in 2023-10-09T05:04:45.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:45 smithi177 ceph-mon[195797]: pgmap v2100: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 229 op/s; 66784/160650 objects degraded (41.571%); 993/160650 objects misplaced (0.618%); 15 MiB/s, 267 objects/s recovering 2023-10-09T05:04:45.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:45 smithi177 ceph-mon[195797]: failed to encode map e658 with expected crc 2023-10-09T05:04:45.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:45 smithi177 ceph-mon[195797]: failed to encode map e658 with expected crc 2023-10-09T05:04:45.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1680806889' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:04:45.495 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:45 smithi177 ceph-mon[195797]: osdmap e658: 8 total, 8 up, 7 in 2023-10-09T05:04:45.629 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:45.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:45.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:45.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:45.779 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:45.793 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:04:45.793 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93380 14.7 14.dfd5a7a7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.409111838000001, 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94592668400000002, 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.794 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93380 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.795 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381649+0000", 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381660+0000", 2023-10-09T05:04:45.796 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1054e-05 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381674+0000", 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.385e-05 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381683+0000", 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7490000000000004e-06 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.797 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381756+0000", 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3750000000000004e-05 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381767+0000", 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0208e-05 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.798 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327567+0000", 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94579980799999996 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327577+0000", 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.08e-05 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.799 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93400 14.1c 14:3dc013b3:::benchmark_data_smithi029_550670_object15789:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.409059158999995, 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3919103770000003, 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.800 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93400 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.801 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.802 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381706+0000", 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8140000000000001e-06 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381711+0000", 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.575e-06 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381715+0000", 2023-10-09T05:04:45.803 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8240000000000001e-06 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381812+0000", 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6555e-05 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381822+0000", 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0302000000000001e-05 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.804 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.015600+0000", 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6337784260000001 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.017575+0000", 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019744390000000001 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.771430+0000", 2023-10-09T05:04:45.805 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.75385499199999995 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.773614+0000", 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021838650000000001 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.806 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93401 14.7 14.decc63e7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.409023684000005, 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94601877700000003, 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93401 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.807 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.808 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381738+0000", 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381741+0000", 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6749999999999998e-06 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381747+0000", 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5799999999999999e-06 2023-10-09T05:04:45.809 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381751+0000", 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6689999999999997e-06 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381798+0000", 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6303e-05 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.810 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381805+0000", 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0190000000000001e-06 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327746+0000", 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94594101799999997 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.811 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327758+0000", 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1878999999999999e-05 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93405 14.17 14.5711f157 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.409004135999993, 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94466796099999994, 2023-10-09T05:04:45.812 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93405 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.813 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.814 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381761+0000", 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5280000000000002e-06 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381766+0000", 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9130000000000002e-06 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381769+0000", 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.788e-06 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.815 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382138+0000", 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036829199999999999 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382146+0000", 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8200000000000003e-06 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.816 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.326413+0000", 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94426679400000002 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.326426+0000", 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3084999999999999e-05 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.817 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93407 14.5 14:a209006e:::benchmark_data_smithi029_550670_object15796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.408655171999996, 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2579248559999998, 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93407 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.818 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.819 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382114+0000", 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2089999999999999e-06 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.820 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382124+0000", 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0268000000000001e-05 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382134+0000", 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382191+0000", 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6889999999999999e-05 2023-10-09T05:04:45.821 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.822 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.822 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.822 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382201+0000", 2023-10-09T05:04:45.822 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0359e-05 2023-10-09T05:04:45.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.823 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.823 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.823 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.374861+0000", 2023-10-09T05:04:45.823 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9926602410000003 2023-10-09T05:04:45.823 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.375679+0000", 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00081763400000000003 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.639295+0000", 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.263616246 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.824 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.640032+0000", 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00073696500000000002 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93418 14.1c 14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.40861237, 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3997008879999999, 2023-10-09T05:04:45.825 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93418 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.826 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.827 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382153+0000", 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0680000000000001e-06 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.828 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382160+0000", 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9639999999999997e-06 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382165+0000", 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4569999999999996e-06 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.829 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382198+0000", 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2703999999999997e-05 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382205+0000", 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.017835+0000", 2023-10-09T05:04:45.830 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6356300770000001 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.025741+0000", 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079060310000000009 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.751118+0000", 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72537742100000002 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.831 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.753397+0000", 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022792559999999999 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.757192+0000", 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037947889999999998 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.832 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.760499+0000", 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033070700000000001 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.774353+0000", 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013853202 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.775194+0000", 2023-10-09T05:04:45.833 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00084157500000000001 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.776410+0000", 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012155670000000001 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.781851+0000", 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054413259999999998 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.834 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93419 14.15 14:a8313c41:::benchmark_data_smithi029_550670_object15808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 81.385616694000007, 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2483047789999997, 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.835 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93419 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:04:45.836 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405145+0000", 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405149+0000", 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6789999999999996e-06 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.837 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405159+0000", 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8779999999999993e-06 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405163+0000", 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1359999999999999e-06 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.838 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405203+0000", 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9718999999999997e-05 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405212+0000", 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2439999999999999e-06 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.625967+0000", 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2207548089999998 2023-10-09T05:04:45.839 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.629287+0000", 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003320042 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.633107+0000", 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038200420000000001 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.840 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.641198+0000", 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080906099999999998 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.646808+0000", 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056098529999999997 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.647586+0000", 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000778562 2023-10-09T05:04:45.841 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.648854+0000", 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001267617 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.653451+0000", 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045966180000000002 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.842 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93432 14.c 14:374a58ab:::benchmark_data_smithi029_550670_object15821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 77.065902769999994, 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1048139219999999, 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93432 2023-10-09T05:04:45.843 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.844 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724859+0000", 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724864+0000", 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1170000000000003e-06 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.845 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724876+0000", 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1911e-05 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724881+0000", 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0379999999999999e-06 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.774671+0000", 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0497900250000001 2023-10-09T05:04:45.846 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.783271+0000", 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008600056 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.806956+0000", 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023684614999999999 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.847 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.809403+0000", 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002447467 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.813396+0000", 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039930909999999998 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.848 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.816971+0000", 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00357516 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.822865+0000", 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058936639999999998 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.823152+0000", 2023-10-09T05:04:45.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028665299999999999 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.824381+0000", 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012293689999999999 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.829674+0000", 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00529255 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.850 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93434 14.1c 14:38f7b3c2:::benchmark_data_smithi029_550670_object15823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 77.049591749000001, 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0950609299999998, 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.851 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93434 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741170+0000", 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741175+0000", 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1479999999999997e-06 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741186+0000", 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1286e-05 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741191+0000", 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5639999999999998e-06 2023-10-09T05:04:45.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.782108+0000", 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0409173940000001 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.790417+0000", 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083089150000000001 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.810827+0000", 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020409265999999999 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.812966+0000", 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002139448 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.816862+0000", 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038954900000000002 2023-10-09T05:04:45.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.822572+0000", 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057100429999999997 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.823625+0000", 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010534870000000001 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.857 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.830012+0000", 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063868370000000002 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.831216+0000", 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012044099999999999 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.836232+0000", 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005015345 2023-10-09T05:04:45.858 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.1c 629/627 [PushReplyOp(14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head)])", 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.763670665999996, 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71326082199999996, 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.859 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027091+0000", 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027097+0000", 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5639999999999996e-06 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027107+0000", 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.082e-05 2023-10-09T05:04:45.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027113+0000", 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8239999999999996e-06 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.735833+0000", 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70872009899999999 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.740353+0000", 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045193639999999997 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.1c 629/627 [PushReplyOp(14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head)])", 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 75.763468544999995, 2023-10-09T05:04:45.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72351207900000003, 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:04:45.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027293+0000", 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027298+0000", 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9649999999999999e-06 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027310+0000", 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1353e-05 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027316+0000", 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2890000000000003e-06 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.740750+0000", 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71343399900000004 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.750806+0000", 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010056034 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93889 14.c 14:34db2e8f:::benchmark_data_smithi029_550670_object16278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:04:45.867 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 48.083608546000001, 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.485937375, 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93889 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707152+0000", 2023-10-09T05:04:45.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707158+0000", 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7729999999999996e-06 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707171+0000", 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2500000000000001e-05 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707176+0000", 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2159999999999997e-06 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806421+0000", 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.099245228000000005 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.814839+0000", 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0084180560000000001 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.035337+0000", 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.220497577 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.038518+0000", 2023-10-09T05:04:45.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031815480000000002 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.044091+0000", 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055728160000000004 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.181816+0000", 2023-10-09T05:04:45.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.137724604 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.182864+0000", 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010481100000000001 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.183975+0000", 2023-10-09T05:04:45.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011108789999999999 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.185725+0000", 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00175004 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.193091+0000", 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073666629999999999 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.875 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93893 14.1c 14:3ca6af9b:::benchmark_data_smithi029_550670_object16282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 48.051648962000002, 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.309352242, 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.876 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93893 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739112+0000", 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739121+0000", 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2280000000000004e-06 2023-10-09T05:04:45.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739139+0000", 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7754e-05 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739149+0000", 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0462000000000001e-05 2023-10-09T05:04:45.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806087+0000", 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.066937742999999994 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806189+0000", 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102457 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806269+0000", 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9692999999999997e-05 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.814900+0000", 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086310050000000006 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.830030+0000", 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015129956 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.047451+0000", 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.217420735 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.047488+0000", 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7416999999999998e-05 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.048466+0000", 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097746399999999994 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93904 14.c 14:33d8ac2a:::benchmark_data_smithi029_550670_object16293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 47.987824623000002, 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39957331299999999, 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.883 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93904 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802936+0000", 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802945+0000", 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4160000000000004e-06 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802956+0000", 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.117e-05 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802962+0000", 2023-10-09T05:04:45.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0020000000000003e-06 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.815216+0000", 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012254335999999999 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.823445+0000", 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082289100000000007 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.063574+0000", 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.240128587 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.066504+0000", 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029302120000000002 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.177873+0000", 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.111368644 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.193885+0000", 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016012229999999999 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.194152+0000", 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026738700000000001 2023-10-09T05:04:45.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.195130+0000", 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097748699999999997 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.196716+0000", 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001586212 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.202511+0000", 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057953019999999996 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97141 14.15 14:ae3b35de:::benchmark_data_smithi029_550670_object19530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 29.699441942, 2023-10-09T05:04:45.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94747084699999995, 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97141 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091319+0000", 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091327+0000", 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9740000000000002e-06 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091342+0000", 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4747e-05 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091351+0000", 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2089999999999994e-06 2023-10-09T05:04:45.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091416+0000", 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4838999999999994e-05 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091423+0000", 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8240000000000002e-06 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.018840+0000", 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92741732099999996 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.018983+0000", 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014266600000000001 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.019109+0000", 2023-10-09T05:04:45.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012617699999999999 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022437+0000", 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033283459999999998 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022749+0000", 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031135000000000001 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038718+0000", 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015969552000000001 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038764+0000", 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5419e-05 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038791+0000", 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7798e-05 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97180 14.15 14:ad3b1b35:::benchmark_data_smithi029_550670_object19569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 29.453643104000001, 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.705525913, 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.899 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97180 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:04:45.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337118+0000", 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337124+0000", 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1090000000000003e-06 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337138+0000", 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4467000000000001e-05 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337145+0000", 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0099999999999998e-06 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337214+0000", 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8980999999999996e-05 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337222+0000", 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7780000000000007e-06 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020444+0000", 2023-10-09T05:04:45.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68322159100000002 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020544+0000", 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100602 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020650+0000", 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105921 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.904 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022505+0000", 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018547959999999999 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022791+0000", 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028577600000000001 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042584+0000", 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019793139000000001 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.905 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042615+0000", 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1220000000000003e-05 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042645+0000", 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0244999999999999e-05 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97245 14.15 14:ae966d39:::benchmark_data_smithi029_550670_object19634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 29.148471156999999, 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40766333999999999, 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97245 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.907 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.908 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642290+0000", 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642299+0000", 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9490000000000008e-06 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642312+0000", 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2159e-05 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642319+0000", 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3329999999999999e-06 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642389+0000", 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0358000000000006e-05 2023-10-09T05:04:45.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642399+0000", 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2269999999999999e-06 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021495+0000", 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37909622700000001 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.911 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021614+0000", 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011913600000000001 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021734+0000", 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119611 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022758+0000", 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010246599999999999 2023-10-09T05:04:45.912 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.023321+0000", 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00056311399999999998 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049876+0000", 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026554853999999999 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.913 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049924+0000", 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7633999999999998e-05 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049955+0000", 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0854e-05 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.914 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:101455 14.c 14:314cb78e:::benchmark_data_smithi029_550670_object23844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e657)", 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:40.969066+0000", 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.8216962560000001, 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.24501521400000001, 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101455 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.915 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969066+0000", 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969066+0000", 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.916 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969065+0000", 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969070+0000", 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6009999999999997e-06 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969078+0000", 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5529999999999993e-06 2023-10-09T05:04:45.917 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969082+0000", 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6710000000000001e-06 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.969665+0000", 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00058305200000000003 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.977156+0000", 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0074911659999999996 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.154930+0000", 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.177773493 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.919 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.156961+0000", 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00203178 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.160463+0000", 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035013650000000002 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.920 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.197011+0000", 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.036547914000000001 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.206669+0000", 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096579979999999992 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.207564+0000", 2023-10-09T05:04:45.921 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00089536100000000001 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.208698+0000", 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011344390000000001 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.214081+0000", 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053829469999999999 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.922 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:101461 14.c 14:31ef2839:::benchmark_data_smithi029_550670_object23850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e657)", 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:40.993460+0000", 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 4.7973023709999998, 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.22866044599999999, 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.923 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 101461 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993460+0000", 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993460+0000", 2023-10-09T05:04:45.924 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993453+0000", 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999928 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993480+0000", 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6825000000000002e-05 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993490+0000", 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0475999999999999e-05 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993495+0000", 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0350000000000001e-06 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:40.993800+0000", 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000304522 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.001120+0000", 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073197389999999996 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.173035+0000", 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.171915074 2023-10-09T05:04:45.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.175047+0000", 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020119719999999999 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.179004+0000", 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039571900000000002 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.206887+0000", 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.027883512999999999 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.214774+0000", 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0078862770000000006 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.215678+0000", 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00090387400000000004 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.216759+0000", 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010817279999999999 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:41.222121+0000", 2023-10-09T05:04:45.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053612549999999997 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:102064 14.c 14:358318a3:::benchmark_data_smithi029_550670_object24453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e657)", 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:44.049185+0000", 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 1.741577782, 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.21477426899999999, 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:04:45.931 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 102064 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049185+0000", 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:04:45.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049185+0000", 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049184+0000", 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049188+0000", 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.155e-06 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049197+0000", 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3160000000000003e-06 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049202+0000", 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3669999999999999e-06 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.049564+0000", 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036267 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.057110+0000", 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.007546082 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.077547+0000", 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020436388 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.935 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.079721+0000", 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021741830000000001 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.084085+0000", 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043645380000000003 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.102189+0000", 2023-10-09T05:04:45.936 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018103474000000001 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.103299+0000", 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011096649999999999 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.257700+0000", 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.15440130599999999 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.259003+0000", 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013032390000000001 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:44.263959+0000", 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0049558029999999999 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:04:45.938 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:04:45.939 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:04:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:45.940 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:04:45.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:04:45.941 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:45.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:45.948 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:46.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:46.075 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:46.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:46.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:46.375 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 66784/160650 objects degraded (41.571%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:46.379 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 66784/160650 objects degraded (41.571%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:46.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:46.436 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 66784/160650 objects degraded (41.571%), 29 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:46.440 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:46.672 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:46.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:46.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:46.726 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:04:46.776 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:46.931 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:47.165 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:47 smithi029 ceph-mon[223858]: pgmap v2102: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 66628/160239 objects degraded (41.580%); 993/160239 objects misplaced (0.620%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T05:04:47.165 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1757671523' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:47.165 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:47.173 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:47.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:47.231 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:04:47.280 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:47.311 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:47.356 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:47.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:47 smithi029 ceph-mon[216903]: pgmap v2102: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 66628/160239 objects degraded (41.580%); 993/160239 objects misplaced (0.620%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T05:04:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1757671523' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:47.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:47.515 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:47 smithi177 ceph-mon[195797]: pgmap v2102: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 189 op/s; 66628/160239 objects degraded (41.580%); 993/160239 objects misplaced (0.620%); 9.8 MiB/s, 156 objects/s recovering 2023-10-09T05:04:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1757671523' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:04:48.221 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:48.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:48.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:48.353 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:48 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:04:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:48 smithi177 ceph-mon[195797]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:04:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:48 smithi177 ceph-mon[195797]: osdmap e659: 8 total, 8 up, 7 in 2023-10-09T05:04:48.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:48 smithi177 ceph-mon[195797]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:48 smithi177 ceph-mon[195797]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[216903]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[216903]: osdmap e659: 8 total, 8 up, 7 in 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[216903]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[216903]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:04:48.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[223858]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:04:48.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[223858]: osdmap e659: 8 total, 8 up, 7 in 2023-10-09T05:04:48.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[223858]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:48 smithi029 ceph-mon[223858]: failed to encode map e659 with expected crc 2023-10-09T05:04:48.900 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:48.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:48.953 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:49.006 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:49.089 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:49.272 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:49.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:49.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:49.460 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:49 smithi029 ceph-mon[216903]: pgmap v2104: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+remapped, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 66221/158757 objects degraded (41.712%); 255/158757 objects misplaced (0.161%); 12 MiB/s, 186 objects/s recovering 2023-10-09T05:04:49.461 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:49 smithi029 ceph-mon[223858]: pgmap v2104: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+remapped, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 66221/158757 objects degraded (41.712%); 255/158757 objects misplaced (0.161%); 12 MiB/s, 186 objects/s recovering 2023-10-09T05:04:49.487 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:49.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:49 smithi177 ceph-mon[195797]: pgmap v2104: 119 pgs: 1 active+recovering, 1 active+recovering+undersized+remapped, 88 active+clean, 19 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 66221/158757 objects degraded (41.712%); 255/158757 objects misplaced (0.161%); 12 MiB/s, 186 objects/s recovering 2023-10-09T05:04:49.644 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:49.990 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:49.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:50.096 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:50.124 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:50.172 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:50.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:50.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:50.278 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:04:50.306 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:50.358 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:04:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:50.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:04:50.464 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:04:50.494 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:50.542 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:04:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:50.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:04:50.650 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:04:50.669 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.670 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:04:50.742 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:51.017 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:04:51.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[223858]: pgmap v2105: 119 pgs: 1 active+recovering, 2 active+recovering+undersized+remapped, 88 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 61554/157446 objects degraded (39.095%); 3464/157446 objects misplaced (2.200%); 14 MiB/s, 223 objects/s recovering 2023-10-09T05:04:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/693478364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:04:51.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 61554/157446 objects degraded (39.095%), 28 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:51.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[216903]: pgmap v2105: 119 pgs: 1 active+recovering, 2 active+recovering+undersized+remapped, 88 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 61554/157446 objects degraded (39.095%); 3464/157446 objects misplaced (2.200%); 14 MiB/s, 223 objects/s recovering 2023-10-09T05:04:51.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/693478364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:51.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:04:51.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 61554/157446 objects degraded (39.095%), 28 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:51.480 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:51.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:51 smithi177 ceph-mon[195797]: pgmap v2105: 119 pgs: 1 active+recovering, 2 active+recovering+undersized+remapped, 88 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 61554/157446 objects degraded (39.095%); 3464/157446 objects misplaced (2.200%); 14 MiB/s, 223 objects/s recovering 2023-10-09T05:04:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/693478364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:04:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:51 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:04:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 61554/157446 objects degraded (39.095%), 28 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:51.592 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:51.670 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:52.451 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:52.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:52.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:52.562 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:04:52.620 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:52.771 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:04:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:52.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:04:52.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:52.917 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:53.007 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:53.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:53.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:53.139 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:04:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:04:52] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.43.0" 2023-10-09T05:04:53.141 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:53.216 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:53.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:53.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:53.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:53.357 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:53.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:53.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:53.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:53 smithi029 ceph-mon[216903]: pgmap v2106: 119 pgs: 1 active+recovering, 2 active+recovering+undersized+remapped, 88 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 61554/157446 objects degraded (39.095%); 3464/157446 objects misplaced (2.200%); 7.0 MiB/s, 114 objects/s recovering 2023-10-09T05:04:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:53 smithi029 ceph-mon[223858]: pgmap v2106: 119 pgs: 1 active+recovering, 2 active+recovering+undersized+remapped, 88 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 61554/157446 objects degraded (39.095%); 3464/157446 objects misplaced (2.200%); 7.0 MiB/s, 114 objects/s recovering 2023-10-09T05:04:53.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:53.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:53 smithi177 ceph-mon[195797]: pgmap v2106: 119 pgs: 1 active+recovering, 2 active+recovering+undersized+remapped, 88 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 61554/157446 objects degraded (39.095%); 3464/157446 objects misplaced (2.200%); 7.0 MiB/s, 114 objects/s recovering 2023-10-09T05:04:53.561 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:53.679 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:53.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:53.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:53.939 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:54.083 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:54.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:54.110 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:54.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:54.217 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:54.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:54.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:54.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:54.385 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:04:54.417 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:54.470 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:04:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:54.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:04:54.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:04:54.577 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:54.659 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:04:54.706 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:04:54.728 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:54.947 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:04:54.947 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:04:55.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:55 smithi029 ceph-mon[216903]: pgmap v2107: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 60645/154872 objects degraded (39.158%); 3290/154872 objects misplaced (2.124%); 8.8 MiB/s, 168 objects/s recovering 2023-10-09T05:04:55.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:55.029 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:55 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:55.032 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:55 smithi029 ceph-mon[223858]: pgmap v2107: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 60645/154872 objects degraded (39.158%); 3290/154872 objects misplaced (2.124%); 8.8 MiB/s, 168 objects/s recovering 2023-10-09T05:04:55.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:55.033 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:55 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:55 smithi177 ceph-mon[195797]: pgmap v2107: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 60645/154872 objects degraded (39.158%); 3290/154872 objects misplaced (2.124%); 8.8 MiB/s, 168 objects/s recovering 2023-10-09T05:04:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:55 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:56.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:56 smithi029 ceph-mon[216903]: from='client.47501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:04:56.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 60645/154872 objects degraded (39.158%), 28 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:56.374 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:56 smithi029 ceph-mon[223858]: from='client.47501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:04:56.374 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 60645/154872 objects degraded (39.158%), 28 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:56 smithi177 ceph-mon[195797]: from='client.47501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:04:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 60645/154872 objects degraded (39.158%), 28 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:04:56.852 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:56.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:56.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:56.963 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:57.071 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:57.158 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:57 smithi029 ceph-mon[216903]: pgmap v2108: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 60583/154485 objects degraded (39.216%); 2696/154485 objects misplaced (1.745%); 10 MiB/s, 189 objects/s recovering 2023-10-09T05:04:57.159 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:57 smithi029 ceph-mon[223858]: pgmap v2108: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 60583/154485 objects degraded (39.216%); 2696/154485 objects misplaced (1.745%); 10 MiB/s, 189 objects/s recovering 2023-10-09T05:04:57.241 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2107,"stamp":"2023-10-09T05:04:53.746819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3382187960,"num_objects":51624,"num_object_clones":0,"num_object_copies":154872,"num_objects_missing_on_primary":18308,"num_objects_missing":18308,"num_objects_degraded":60645,"num_objects_misplaced":3290,"num_objects_unfound":0,"num_objects_dirty":51624,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":103918,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67659,"num_bytes_recovered":4127558286,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":189223,"ondisk_log_size":189223,"up":357,"acting":337,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":486,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20212216,"kb_used_data":12332800,"kb_used_omap":34,"kb_used_meta":7879261,"kb_avail":636003848,"statfs":{"total":671965249536,"available":651267940352,"internally_reserved":0,"allocated":12628787200,"data_stored":12612892680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8068363658},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,3,0,1,3,0,11,9,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-126222336,"num_objects":-1926,"num_object_clones":0,"num_object_copies":-5778,"num_objects_missing_on_primary":-693,"num_objects_missing":-693,"num_objects_degraded":-6139,"num_objects_misplaced":2297,"num_objects_unfound":0,"num_objects_dirty":-1926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1926,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1626,"num_bytes_recovered":89522176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.673519"},"pg_stats":[{"pgid":"14.23","version":"659'5637","reported_seq":11836,"reported_epoch":659,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:04:53.328717+0000","last_change":"2023-10-09T05:03:47.287202+0000","last_active":"2023-10-09T05:04:53.328717+0000","last_peered":"2023-10-09T05:04:53.328717+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:01.873192+0000","last_became_peered":"2023-10-09T05:03:01.873192+0000","last_unstale":"2023-10-09T05:04:53.328717+0000","last_undegraded":"2023-10-09T05:04:53.328717+0000","last_fullsized":"2023-10-09T05:03:00.955106+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5637,"log_dups_size":0,"ondisk_log_size":5637,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51052544,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":205,"num_objects_missing":205,"num_objects_degraded":0,"num_objects_misplaced":9,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1653,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1196,"num_bytes_recovered":78282752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"0,1","objects":774},{"shards":"0,3,5,6","objects":1},{"shards":"3,5,6","objects":4}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.22","version":"659'5538","reported_seq":11368,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:51.877666+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:04:51.877666+0000","last_peered":"2023-10-09T05:04:51.877666+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:04:51.877666+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52723712,"num_objects":805,"num_object_clones":0,"num_object_copies":2415,"num_objects_missing_on_primary":779,"num_objects_missing":779,"num_objects_degraded":2180,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1603,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":678,"num_bytes_recovered":42188800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"4,5,7","objects":621}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"659'5605","reported_seq":16294,"reported_epoch":659,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:51.697207+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:04:51.697207+0000","last_peered":"2023-10-09T05:04:51.697207+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:04:51.697207+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:04:51.697207+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5605,"log_dups_size":0,"ondisk_log_size":5605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52953088,"num_objects":808,"num_object_clones":0,"num_object_copies":2424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1622,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":808}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"659'5621","reported_seq":6598,"reported_epoch":659,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:51.394393+0000","last_change":"2023-10-09T05:03:26.464077+0000","last_active":"2023-10-09T05:04:51.394393+0000","last_peered":"2023-10-09T05:04:51.394393+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:04:51.394393+0000","last_undegraded":"2023-10-09T05:03:26.367100+0000","last_fullsized":"2023-10-09T05:04:51.394393+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52641792,"num_objects":803,"num_object_clones":0,"num_object_copies":2409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2026,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1628,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":12206080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":754},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"659'5791","reported_seq":19344,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:52.115557+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:04:52.115557+0000","last_peered":"2023-10-09T05:04:52.115557+0000","last_clean":"2023-10-09T05:04:52.115557+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:04:52.115557+0000","last_undegraded":"2023-10-09T05:04:52.115557+0000","last_fullsized":"2023-10-09T05:04:52.115557+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107315200,"num_objects":1638,"num_object_clones":0,"num_object_copies":4914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3250,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":161,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085720+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:04:44.085720+0000","last_peered":"2023-10-09T05:04:44.085720+0000","last_clean":"2023-10-09T05:04:44.085720+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:04:44.085720+0000","last_undegraded":"2023-10-09T05:04:44.085720+0000","last_fullsized":"2023-10-09T05:04:44.085720+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":306,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:47.197837+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:04:47.197837+0000","last_peered":"2023-10-09T05:04:47.197837+0000","last_clean":"2023-10-09T05:04:47.197837+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:04:47.197837+0000","last_undegraded":"2023-10-09T05:04:47.197837+0000","last_fullsized":"2023-10-09T05:04:47.197837+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"659'5800","reported_seq":9019,"reported_epoch":659,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:53.556550+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:04:53.556550+0000","last_peered":"2023-10-09T05:04:53.556550+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:04:53.556550+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:04:53.556550+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5800,"log_dups_size":0,"ondisk_log_size":5800,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104824832,"num_objects":1600,"num_object_clones":0,"num_object_copies":4800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3261,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":1600}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":83,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:47.157960+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:04:47.157960+0000","last_peered":"2023-10-09T05:04:47.157960+0000","last_clean":"2023-10-09T05:04:47.157960+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:04:47.157960+0000","last_undegraded":"2023-10-09T05:04:47.157960+0000","last_fullsized":"2023-10-09T05:04:47.157960+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.090078+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:04:44.090078+0000","last_peered":"2023-10-09T05:04:44.090078+0000","last_clean":"2023-10-09T05:04:44.090078+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:04:44.090078+0000","last_undegraded":"2023-10-09T05:04:44.090078+0000","last_fullsized":"2023-10-09T05:04:44.090078+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"659'5752","reported_seq":17402,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:48.628615+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:04:48.628615+0000","last_peered":"2023-10-09T05:04:48.628615+0000","last_clean":"2023-10-09T05:04:48.628615+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:04:48.628615+0000","last_undegraded":"2023-10-09T05:04:48.628615+0000","last_fullsized":"2023-10-09T05:04:48.628615+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108068864,"num_objects":1649,"num_object_clones":0,"num_object_copies":4947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3212,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":288,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086257+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:04:44.086257+0000","last_peered":"2023-10-09T05:04:44.086257+0000","last_clean":"2023-10-09T05:04:44.086257+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:04:44.086257+0000","last_undegraded":"2023-10-09T05:04:44.086257+0000","last_fullsized":"2023-10-09T05:04:44.086257+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":185,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086128+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:04:44.086128+0000","last_peered":"2023-10-09T05:04:44.086128+0000","last_clean":"2023-10-09T05:04:44.086128+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:04:44.086128+0000","last_undegraded":"2023-10-09T05:04:44.086128+0000","last_fullsized":"2023-10-09T05:04:44.086128+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"659'6002","reported_seq":23349,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:52.081798+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:04:52.081798+0000","last_peered":"2023-10-09T05:04:52.081798+0000","last_clean":"2023-10-09T05:04:52.081798+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:04:52.081798+0000","last_undegraded":"2023-10-09T05:04:52.081798+0000","last_fullsized":"2023-10-09T05:04:52.081798+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104202240,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3181,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":33,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088684+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:04:44.088684+0000","last_peered":"2023-10-09T05:04:44.088684+0000","last_clean":"2023-10-09T05:04:44.088684+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:04:44.088684+0000","last_undegraded":"2023-10-09T05:04:44.088684+0000","last_fullsized":"2023-10-09T05:04:44.088684+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":279,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086177+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:04:44.086177+0000","last_peered":"2023-10-09T05:04:44.086177+0000","last_clean":"2023-10-09T05:04:44.086177+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:04:44.086177+0000","last_undegraded":"2023-10-09T05:04:44.086177+0000","last_fullsized":"2023-10-09T05:04:44.086177+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"659'6287","reported_seq":11286,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.275205+0000","last_change":"2023-10-09T05:03:31.501660+0000","last_active":"2023-10-09T05:04:53.275205+0000","last_peered":"2023-10-09T05:04:53.275205+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:04:53.275205+0000","last_undegraded":"2023-10-09T05:03:27.167651+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4862,"log_dups_size":0,"ondisk_log_size":4862,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108331008,"num_objects":1653,"num_object_clones":0,"num_object_copies":4959,"num_objects_missing_on_primary":2035,"num_objects_missing":2035,"num_objects_degraded":5443,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3327,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1119,"num_bytes_recovered":73269248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":2},{"shards":"0,2,4,7","objects":1},{"shards":"2,4,7","objects":1595},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":47,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088745+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:04:44.088745+0000","last_peered":"2023-10-09T05:04:44.088745+0000","last_clean":"2023-10-09T05:04:44.088745+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:04:44.088745+0000","last_undegraded":"2023-10-09T05:04:44.088745+0000","last_fullsized":"2023-10-09T05:04:44.088745+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":269,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085129+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:04:44.085129+0000","last_peered":"2023-10-09T05:04:44.085129+0000","last_clean":"2023-10-09T05:04:44.085129+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:04:44.085129+0000","last_undegraded":"2023-10-09T05:04:44.085129+0000","last_fullsized":"2023-10-09T05:04:44.085129+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"659'5937","reported_seq":9442,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:52.005682+0000","last_change":"2023-10-09T05:02:36.475072+0000","last_active":"2023-10-09T05:04:52.005682+0000","last_peered":"2023-10-09T05:04:52.005682+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:04:52.005682+0000","last_undegraded":"2023-10-09T05:02:32.875126+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4401,"log_dups_size":0,"ondisk_log_size":4401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":1640,"num_objects_missing":1640,"num_objects_degraded":4951,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3328,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":833,"num_bytes_recovered":54591488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,3,5,7","objects":1},{"shards":"1,2,5,7","objects":1},{"shards":"1,3","objects":26},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1207}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":332,"reported_epoch":658,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:44.089388+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:04:44.089388+0000","last_peered":"2023-10-09T05:04:44.089388+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:04:44.089388+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:04:44.089388+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086842+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:04:44.086842+0000","last_peered":"2023-10-09T05:04:44.086842+0000","last_clean":"2023-10-09T05:04:44.086842+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:04:44.086842+0000","last_undegraded":"2023-10-09T05:04:44.086842+0000","last_fullsized":"2023-10-09T05:04:44.086842+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"659'5801","reported_seq":17866,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:52.119874+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:04:52.119874+0000","last_peered":"2023-10-09T05:04:52.119874+0000","last_clean":"2023-10-09T05:04:52.119874+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:04:52.119874+0000","last_undegraded":"2023-10-09T05:04:52.119874+0000","last_fullsized":"2023-10-09T05:04:52.119874+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5801,"log_dups_size":0,"ondisk_log_size":5801,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106594304,"num_objects":1627,"num_object_clones":0,"num_object_copies":4881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3263,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":114,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.091088+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:04:44.091088+0000","last_peered":"2023-10-09T05:04:44.091088+0000","last_clean":"2023-10-09T05:04:44.091088+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:04:44.091088+0000","last_undegraded":"2023-10-09T05:04:44.091088+0000","last_fullsized":"2023-10-09T05:04:44.091088+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":129,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085855+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:04:44.085855+0000","last_peered":"2023-10-09T05:04:44.085855+0000","last_clean":"2023-10-09T05:04:44.085855+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:04:44.085855+0000","last_undegraded":"2023-10-09T05:04:44.085855+0000","last_fullsized":"2023-10-09T05:04:44.085855+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"659'6062","reported_seq":17501,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.689602+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:04:53.689602+0000","last_peered":"2023-10-09T05:04:53.689602+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:04:53.689602+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4381,"log_dups_size":0,"ondisk_log_size":4381,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104169472,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2114,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3203,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2741,"num_bytes_recovered":175046656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":1086}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":135,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085866+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:04:44.085866+0000","last_peered":"2023-10-09T05:04:44.085866+0000","last_clean":"2023-10-09T05:04:44.085866+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:04:44.085866+0000","last_undegraded":"2023-10-09T05:04:44.085866+0000","last_fullsized":"2023-10-09T05:04:44.085866+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.097830+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:04:44.097830+0000","last_peered":"2023-10-09T05:04:44.097830+0000","last_clean":"2023-10-09T05:04:44.097830+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:04:44.097830+0000","last_undegraded":"2023-10-09T05:04:44.097830+0000","last_fullsized":"2023-10-09T05:04:44.097830+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"659'6060","reported_seq":10671,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:52.062800+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:04:52.062800+0000","last_peered":"2023-10-09T05:04:52.062800+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:04:52.062800+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3933,"log_dups_size":0,"ondisk_log_size":3933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101449728,"num_objects":1548,"num_object_clones":0,"num_object_copies":4644,"num_objects_missing_on_primary":1651,"num_objects_missing":1651,"num_objects_degraded":3279,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3231,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1162,"num_bytes_recovered":76120064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,5,7","objects":2},{"shards":"1,7","objects":-39},{"shards":"2,5,7","objects":1210},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":162,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.093379+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:04:44.093379+0000","last_peered":"2023-10-09T05:04:44.093379+0000","last_clean":"2023-10-09T05:04:44.093379+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:04:44.093379+0000","last_undegraded":"2023-10-09T05:04:44.093379+0000","last_fullsized":"2023-10-09T05:04:44.093379+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":37,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088497+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:04:44.088497+0000","last_peered":"2023-10-09T05:04:44.088497+0000","last_clean":"2023-10-09T05:04:44.088497+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:04:44.088497+0000","last_undegraded":"2023-10-09T05:04:44.088497+0000","last_fullsized":"2023-10-09T05:04:44.088497+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"659'6139","reported_seq":12335,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:49.046949+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:04:49.046949+0000","last_peered":"2023-10-09T05:04:49.046949+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:04:49.046949+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4376,"log_dups_size":0,"ondisk_log_size":4376,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107151360,"num_objects":1635,"num_object_clones":0,"num_object_copies":4905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3854,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3240,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1529,"num_bytes_recovered":87228416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5","objects":101},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":1452},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.084089+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:04:44.084089+0000","last_peered":"2023-10-09T05:04:44.084089+0000","last_clean":"2023-10-09T05:04:44.084089+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:04:44.084089+0000","last_undegraded":"2023-10-09T05:04:44.084089+0000","last_fullsized":"2023-10-09T05:04:44.084089+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":296,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.090460+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:04:44.090460+0000","last_peered":"2023-10-09T05:04:44.090460+0000","last_clean":"2023-10-09T05:04:44.090460+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:04:44.090460+0000","last_undegraded":"2023-10-09T05:04:44.090460+0000","last_fullsized":"2023-10-09T05:04:44.090460+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"659'6070","reported_seq":12941,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:53.576886+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:04:53.576886+0000","last_peered":"2023-10-09T05:04:53.576886+0000","last_clean":"2023-10-09T05:04:53.576886+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:04:53.576886+0000","last_undegraded":"2023-10-09T05:04:53.576886+0000","last_fullsized":"2023-10-09T05:04:53.576886+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3777,"log_dups_size":0,"ondisk_log_size":3777,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104562700,"num_objects":1596,"num_object_clones":0,"num_object_copies":4788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1596,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3208,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":293,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082560+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:04:44.082560+0000","last_peered":"2023-10-09T05:04:44.082560+0000","last_clean":"2023-10-09T05:04:44.082560+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:04:44.082560+0000","last_undegraded":"2023-10-09T05:04:44.082560+0000","last_fullsized":"2023-10-09T05:04:44.082560+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":154,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.098549+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:04:44.098549+0000","last_peered":"2023-10-09T05:04:44.098549+0000","last_clean":"2023-10-09T05:04:44.098549+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:04:44.098549+0000","last_undegraded":"2023-10-09T05:04:44.098549+0000","last_fullsized":"2023-10-09T05:04:44.098549+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"659'5568","reported_seq":11506,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:52.019624+0000","last_change":"2023-10-09T05:03:28.636503+0000","last_active":"2023-10-09T05:04:52.019624+0000","last_peered":"2023-10-09T05:04:52.019624+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.593716+0000","last_became_peered":"2023-10-09T05:03:28.593716+0000","last_unstale":"2023-10-09T05:04:52.019624+0000","last_undegraded":"2023-10-09T05:03:26.697990+0000","last_fullsized":"2023-10-09T05:03:26.339481+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50757632,"num_objects":774,"num_object_clones":0,"num_object_copies":2322,"num_objects_missing_on_primary":741,"num_objects_missing":741,"num_objects_degraded":2040,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1633,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":44154880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":39},{"shards":"1,3","objects":65},{"shards":"3,4,5,6,7","objects":90},{"shards":"3,4,5,7","objects":1},{"shards":"4,5,7","objects":579}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":188,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.084639+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:04:44.084639+0000","last_peered":"2023-10-09T05:04:44.084639+0000","last_clean":"2023-10-09T05:04:44.084639+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:04:44.084639+0000","last_undegraded":"2023-10-09T05:04:44.084639+0000","last_fullsized":"2023-10-09T05:04:44.084639+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":47,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.087644+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:04:44.087644+0000","last_peered":"2023-10-09T05:04:44.087644+0000","last_clean":"2023-10-09T05:04:44.087644+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:04:44.087644+0000","last_undegraded":"2023-10-09T05:04:44.087644+0000","last_fullsized":"2023-10-09T05:04:44.087644+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"659'5614","reported_seq":8906,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.200766+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:04:53.200766+0000","last_peered":"2023-10-09T05:04:53.200766+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:04:53.200766+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5614,"log_dups_size":0,"ondisk_log_size":5614,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":980,"num_objects_missing":980,"num_objects_degraded":2358,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1630,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":27492352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":582},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":16,"reported_epoch":658,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:44.084759+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:04:44.084759+0000","last_peered":"2023-10-09T05:04:44.084759+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:04:44.084759+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:04:44.084759+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":145,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083959+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:04:44.083959+0000","last_peered":"2023-10-09T05:04:44.083959+0000","last_clean":"2023-10-09T05:04:44.083959+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:04:44.083959+0000","last_undegraded":"2023-10-09T05:04:44.083959+0000","last_fullsized":"2023-10-09T05:04:44.083959+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"659'5637","reported_seq":9749,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:51.873325+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:04:51.873325+0000","last_peered":"2023-10-09T05:04:51.873325+0000","last_clean":"2023-10-09T05:04:51.873325+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:04:51.873325+0000","last_undegraded":"2023-10-09T05:04:51.873325+0000","last_fullsized":"2023-10-09T05:04:51.873325+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5099,"log_dups_size":0,"ondisk_log_size":5099,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51593216,"num_objects":787,"num_object_clones":0,"num_object_copies":2361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1644,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":41,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088425+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:04:44.088425+0000","last_peered":"2023-10-09T05:04:44.088425+0000","last_clean":"2023-10-09T05:04:44.088425+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:04:44.088425+0000","last_undegraded":"2023-10-09T05:04:44.088425+0000","last_fullsized":"2023-10-09T05:04:44.088425+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":205,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082736+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:04:44.082736+0000","last_peered":"2023-10-09T05:04:44.082736+0000","last_clean":"2023-10-09T05:04:44.082736+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:04:44.082736+0000","last_undegraded":"2023-10-09T05:04:44.082736+0000","last_fullsized":"2023-10-09T05:04:44.082736+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"659'5623","reported_seq":16312,"reported_epoch":659,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:51.957978+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:04:51.957978+0000","last_peered":"2023-10-09T05:04:51.957978+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:04:51.957978+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:04:51.957978+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5623,"log_dups_size":0,"ondisk_log_size":5623,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51773440,"num_objects":790,"num_object_clones":0,"num_object_copies":2370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1639,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":790}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":719,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.089117+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:04:44.089117+0000","last_peered":"2023-10-09T05:04:44.089117+0000","last_clean":"2023-10-09T05:04:44.089117+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:04:44.089117+0000","last_undegraded":"2023-10-09T05:04:44.089117+0000","last_fullsized":"2023-10-09T05:04:44.089117+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":94,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.077680+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:04:44.077680+0000","last_peered":"2023-10-09T05:04:44.077680+0000","last_clean":"2023-10-09T05:04:44.077680+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:04:44.077680+0000","last_undegraded":"2023-10-09T05:04:44.077680+0000","last_fullsized":"2023-10-09T05:04:44.077680+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1247,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082660+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:04:44.082660+0000","last_peered":"2023-10-09T05:04:44.082660+0000","last_clean":"2023-10-09T05:04:44.082660+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:04:44.082660+0000","last_undegraded":"2023-10-09T05:04:44.082660+0000","last_fullsized":"2023-10-09T05:04:44.082660+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"659'5597","reported_seq":9172,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.368578+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:04:53.368578+0000","last_peered":"2023-10-09T05:04:53.368578+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:04:53.368578+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5597,"log_dups_size":0,"ondisk_log_size":5597,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103251968,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":1890,"num_objects_missing":1890,"num_objects_degraded":4655,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3140,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":783,"num_bytes_recovered":51314688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":1143}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":262,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.087517+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:04:44.087517+0000","last_peered":"2023-10-09T05:04:44.087517+0000","last_clean":"2023-10-09T05:04:44.087517+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:04:44.087517+0000","last_undegraded":"2023-10-09T05:04:44.087517+0000","last_fullsized":"2023-10-09T05:04:44.087517+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.090997+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:04:44.090997+0000","last_peered":"2023-10-09T05:04:44.090997+0000","last_clean":"2023-10-09T05:04:44.090997+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:04:44.090997+0000","last_undegraded":"2023-10-09T05:04:44.090997+0000","last_fullsized":"2023-10-09T05:04:44.090997+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1236,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082485+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:04:44.082485+0000","last_peered":"2023-10-09T05:04:44.082485+0000","last_clean":"2023-10-09T05:04:44.082485+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:04:44.082485+0000","last_undegraded":"2023-10-09T05:04:44.082485+0000","last_fullsized":"2023-10-09T05:04:44.082485+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"659'5806","reported_seq":14178,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:51.876434+0000","last_change":"2023-10-09T05:03:46.353847+0000","last_active":"2023-10-09T05:04:51.876434+0000","last_peered":"2023-10-09T05:04:51.876434+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:03:45.583447+0000","last_became_peered":"2023-10-09T05:03:45.583447+0000","last_unstale":"2023-10-09T05:04:51.876434+0000","last_undegraded":"2023-10-09T05:03:41.179518+0000","last_fullsized":"2023-10-09T05:03:38.964620+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106332160,"num_objects":1622,"num_object_clones":0,"num_object_copies":4866,"num_objects_missing_on_primary":328,"num_objects_missing":328,"num_objects_degraded":2781,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3264,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2158,"num_bytes_recovered":140443648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,2,4,5","objects":1258},{"shards":"2,4,5","objects":360},{"shards":"4,5","objects":-342}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":202,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082691+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:04:44.082691+0000","last_peered":"2023-10-09T05:04:44.082691+0000","last_clean":"2023-10-09T05:04:44.082691+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:04:44.082691+0000","last_undegraded":"2023-10-09T05:04:44.082691+0000","last_fullsized":"2023-10-09T05:04:44.082691+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083667+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:04:44.083667+0000","last_peered":"2023-10-09T05:04:44.083667+0000","last_clean":"2023-10-09T05:04:44.083667+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:04:44.083667+0000","last_undegraded":"2023-10-09T05:04:44.083667+0000","last_fullsized":"2023-10-09T05:04:44.083667+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"659'5780","reported_seq":9156,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:52.315147+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:04:52.315147+0000","last_peered":"2023-10-09T05:04:52.315147+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:04:52.315147+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5780,"log_dups_size":0,"ondisk_log_size":5780,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106135552,"num_objects":1619,"num_object_clones":0,"num_object_copies":4857,"num_objects_missing_on_primary":1226,"num_objects_missing":1226,"num_objects_degraded":2839,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3240,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":810,"num_bytes_recovered":53116928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,4,7","objects":1226},{"shards":"3,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":131,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.084024+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:04:44.084024+0000","last_peered":"2023-10-09T05:04:44.084024+0000","last_clean":"2023-10-09T05:04:44.084024+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:04:44.084024+0000","last_undegraded":"2023-10-09T05:04:44.084024+0000","last_fullsized":"2023-10-09T05:04:44.084024+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":132,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085044+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:04:44.085044+0000","last_peered":"2023-10-09T05:04:44.085044+0000","last_clean":"2023-10-09T05:04:44.085044+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:04:44.085044+0000","last_undegraded":"2023-10-09T05:04:44.085044+0000","last_fullsized":"2023-10-09T05:04:44.085044+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1256,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082883+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:04:44.082883+0000","last_peered":"2023-10-09T05:04:44.082883+0000","last_clean":"2023-10-09T05:04:44.082883+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:04:44.082883+0000","last_undegraded":"2023-10-09T05:04:44.082883+0000","last_fullsized":"2023-10-09T05:04:44.082883+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"659'5787","reported_seq":19594,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:48.599948+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:04:48.599948+0000","last_peered":"2023-10-09T05:04:48.599948+0000","last_clean":"2023-10-09T05:04:48.599948+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:04:48.599948+0000","last_undegraded":"2023-10-09T05:04:48.599948+0000","last_fullsized":"2023-10-09T05:04:48.599948+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105775104,"num_objects":1614,"num_object_clones":0,"num_object_copies":4842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3247,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":177,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085527+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:04:44.085527+0000","last_peered":"2023-10-09T05:04:44.085527+0000","last_clean":"2023-10-09T05:04:44.085527+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:04:44.085527+0000","last_undegraded":"2023-10-09T05:04:44.085527+0000","last_fullsized":"2023-10-09T05:04:44.085527+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":86,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.107893+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:04:44.107893+0000","last_peered":"2023-10-09T05:04:44.107893+0000","last_clean":"2023-10-09T05:04:44.107893+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:04:44.107893+0000","last_undegraded":"2023-10-09T05:04:44.107893+0000","last_fullsized":"2023-10-09T05:04:44.107893+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1262,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086137+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:04:44.086137+0000","last_peered":"2023-10-09T05:04:44.086137+0000","last_clean":"2023-10-09T05:04:44.086137+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:04:44.086137+0000","last_undegraded":"2023-10-09T05:04:44.086137+0000","last_fullsized":"2023-10-09T05:04:44.086137+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"659'5950","reported_seq":10714,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.213111+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:04:53.213111+0000","last_peered":"2023-10-09T05:04:53.213111+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:04:53.213111+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5950,"log_dups_size":0,"ondisk_log_size":5950,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108331008,"num_objects":1653,"num_object_clones":0,"num_object_copies":4959,"num_objects_missing_on_primary":1530,"num_objects_missing":1530,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3341,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1380,"num_bytes_recovered":90439680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,3,5,7","objects":535},{"shards":"0,3","objects":-398},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":738}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":74,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.077441+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:04:44.077441+0000","last_peered":"2023-10-09T05:04:44.077441+0000","last_clean":"2023-10-09T05:04:44.077441+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:04:44.077441+0000","last_undegraded":"2023-10-09T05:04:44.077441+0000","last_fullsized":"2023-10-09T05:04:44.077441+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":37,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.091556+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:04:44.091556+0000","last_peered":"2023-10-09T05:04:44.091556+0000","last_clean":"2023-10-09T05:04:44.091556+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:04:44.091556+0000","last_undegraded":"2023-10-09T05:04:44.091556+0000","last_fullsized":"2023-10-09T05:04:44.091556+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":92,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086770+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:04:44.086770+0000","last_peered":"2023-10-09T05:04:44.086770+0000","last_clean":"2023-10-09T05:04:44.086770+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:04:44.086770+0000","last_undegraded":"2023-10-09T05:04:44.086770+0000","last_fullsized":"2023-10-09T05:04:44.086770+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1245,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082551+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:04:44.082551+0000","last_peered":"2023-10-09T05:04:44.082551+0000","last_clean":"2023-10-09T05:04:44.082551+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:04:44.082551+0000","last_undegraded":"2023-10-09T05:04:44.082551+0000","last_fullsized":"2023-10-09T05:04:44.082551+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"659'6103","reported_seq":17625,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.351562+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:04:53.351562+0000","last_peered":"2023-10-09T05:04:53.351562+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:04:53.351562+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101482496,"num_objects":1548,"num_object_clones":0,"num_object_copies":4644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3243,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2782,"num_bytes_recovered":177733632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":1066}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":743,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.089265+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:04:44.089265+0000","last_peered":"2023-10-09T05:04:44.089265+0000","last_clean":"2023-10-09T05:04:44.089265+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:04:44.089265+0000","last_undegraded":"2023-10-09T05:04:44.089265+0000","last_fullsized":"2023-10-09T05:04:44.089265+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085681+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:04:44.085681+0000","last_peered":"2023-10-09T05:04:44.085681+0000","last_clean":"2023-10-09T05:04:44.085681+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:04:44.085681+0000","last_undegraded":"2023-10-09T05:04:44.085681+0000","last_fullsized":"2023-10-09T05:04:44.085681+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":74,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.077396+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:04:44.077396+0000","last_peered":"2023-10-09T05:04:44.077396+0000","last_clean":"2023-10-09T05:04:44.077396+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:04:44.077396+0000","last_undegraded":"2023-10-09T05:04:44.077396+0000","last_fullsized":"2023-10-09T05:04:44.077396+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1256,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082791+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:04:44.082791+0000","last_peered":"2023-10-09T05:04:44.082791+0000","last_clean":"2023-10-09T05:04:44.082791+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:04:44.082791+0000","last_undegraded":"2023-10-09T05:04:44.082791+0000","last_fullsized":"2023-10-09T05:04:44.082791+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"659'6092","reported_seq":11864,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:53.628072+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:04:53.628072+0000","last_peered":"2023-10-09T05:04:53.628072+0000","last_clean":"2023-10-09T05:04:53.628072+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:04:53.628072+0000","last_undegraded":"2023-10-09T05:04:53.628072+0000","last_fullsized":"2023-10-09T05:04:53.628072+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3799,"log_dups_size":0,"ondisk_log_size":3799,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103120908,"num_objects":1574,"num_object_clones":0,"num_object_copies":4722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3230,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":153,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.084433+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:04:44.084433+0000","last_peered":"2023-10-09T05:04:44.084433+0000","last_clean":"2023-10-09T05:04:44.084433+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:04:44.084433+0000","last_undegraded":"2023-10-09T05:04:44.084433+0000","last_fullsized":"2023-10-09T05:04:44.084433+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.119229+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:04:44.119229+0000","last_peered":"2023-10-09T05:04:44.119229+0000","last_clean":"2023-10-09T05:04:44.119229+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:04:44.119229+0000","last_undegraded":"2023-10-09T05:04:44.119229+0000","last_fullsized":"2023-10-09T05:04:44.119229+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.107824+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:04:44.107824+0000","last_peered":"2023-10-09T05:04:44.107824+0000","last_clean":"2023-10-09T05:04:44.107824+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:04:44.107824+0000","last_undegraded":"2023-10-09T05:04:44.107824+0000","last_fullsized":"2023-10-09T05:04:44.107824+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"659'5966","reported_seq":10267,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:51.909692+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:04:51.909692+0000","last_peered":"2023-10-09T05:04:51.909692+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:04:51.909692+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3839,"log_dups_size":0,"ondisk_log_size":3839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107610112,"num_objects":1642,"num_object_clones":0,"num_object_copies":4926,"num_objects_missing_on_primary":1292,"num_objects_missing":1292,"num_objects_degraded":3238,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3137,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1056,"num_bytes_recovered":69173248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,7","objects":1201},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":47,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088002+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:04:44.088002+0000","last_peered":"2023-10-09T05:04:44.088002+0000","last_clean":"2023-10-09T05:04:44.088002+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:04:44.088002+0000","last_undegraded":"2023-10-09T05:04:44.088002+0000","last_fullsized":"2023-10-09T05:04:44.088002+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1262,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.082415+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:04:44.082415+0000","last_peered":"2023-10-09T05:04:44.082415+0000","last_clean":"2023-10-09T05:04:44.082415+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:04:44.082415+0000","last_undegraded":"2023-10-09T05:04:44.082415+0000","last_fullsized":"2023-10-09T05:04:44.082415+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":151,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.093476+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:04:44.093476+0000","last_peered":"2023-10-09T05:04:44.093476+0000","last_clean":"2023-10-09T05:04:44.093476+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:04:44.093476+0000","last_undegraded":"2023-10-09T05:04:44.093476+0000","last_fullsized":"2023-10-09T05:04:44.093476+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":179,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083586+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:04:44.083586+0000","last_peered":"2023-10-09T05:04:44.083586+0000","last_clean":"2023-10-09T05:04:44.083586+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:04:44.083586+0000","last_undegraded":"2023-10-09T05:04:44.083586+0000","last_fullsized":"2023-10-09T05:04:44.083586+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1235,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083149+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:04:44.083149+0000","last_peered":"2023-10-09T05:04:44.083149+0000","last_clean":"2023-10-09T05:04:44.083149+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:04:44.083149+0000","last_undegraded":"2023-10-09T05:04:44.083149+0000","last_fullsized":"2023-10-09T05:04:44.083149+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"659'6144","reported_seq":12316,"reported_epoch":659,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:04:49.086105+0000","last_change":"2023-10-09T05:04:48.881602+0000","last_active":"2023-10-09T05:04:49.086105+0000","last_peered":"2023-10-09T05:04:49.086105+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:04:49.086105+0000","last_undegraded":"2023-10-09T05:04:49.086105+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4381,"log_dups_size":0,"ondisk_log_size":4381,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106823680,"num_objects":1630,"num_object_clones":0,"num_object_copies":4890,"num_objects_missing_on_primary":23,"num_objects_missing":23,"num_objects_degraded":0,"num_objects_misplaced":3281,"num_objects_unfound":0,"num_objects_dirty":1630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3245,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1501,"num_bytes_recovered":87556096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":-34},{"shards":"1,2,4,5","objects":126},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":1491}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":72,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.119315+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:04:44.119315+0000","last_peered":"2023-10-09T05:04:44.119315+0000","last_clean":"2023-10-09T05:04:44.119315+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:04:44.119315+0000","last_undegraded":"2023-10-09T05:04:44.119315+0000","last_fullsized":"2023-10-09T05:04:44.119315+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":17,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083727+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:04:44.083727+0000","last_peered":"2023-10-09T05:04:44.083727+0000","last_clean":"2023-10-09T05:04:44.083727+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:04:44.083727+0000","last_undegraded":"2023-10-09T05:04:44.083727+0000","last_fullsized":"2023-10-09T05:04:44.083727+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":110,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.249198+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:04:44.249198+0000","last_peered":"2023-10-09T05:04:44.249198+0000","last_clean":"2023-10-09T05:04:44.249198+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:04:44.249198+0000","last_undegraded":"2023-10-09T05:04:44.249198+0000","last_fullsized":"2023-10-09T05:04:44.249198+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1256,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083637+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:04:44.083637+0000","last_peered":"2023-10-09T05:04:44.083637+0000","last_clean":"2023-10-09T05:04:44.083637+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:04:44.083637+0000","last_undegraded":"2023-10-09T05:04:44.083637+0000","last_fullsized":"2023-10-09T05:04:44.083637+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"659'5800","reported_seq":17827,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:51.944745+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:04:51.944745+0000","last_peered":"2023-10-09T05:04:51.944745+0000","last_clean":"2023-10-09T05:04:51.944745+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:04:51.944745+0000","last_undegraded":"2023-10-09T05:04:51.944745+0000","last_fullsized":"2023-10-09T05:04:51.944745+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5800,"log_dups_size":0,"ondisk_log_size":5800,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106659840,"num_objects":1627,"num_object_clones":0,"num_object_copies":4881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3261,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":124,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.084156+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:04:44.084156+0000","last_peered":"2023-10-09T05:04:44.084156+0000","last_clean":"2023-10-09T05:04:44.084156+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:04:44.084156+0000","last_undegraded":"2023-10-09T05:04:44.084156+0000","last_fullsized":"2023-10-09T05:04:44.084156+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.077310+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:04:44.077310+0000","last_peered":"2023-10-09T05:04:44.077310+0000","last_clean":"2023-10-09T05:04:44.077310+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:04:44.077310+0000","last_undegraded":"2023-10-09T05:04:44.077310+0000","last_fullsized":"2023-10-09T05:04:44.077310+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088085+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:04:44.088085+0000","last_peered":"2023-10-09T05:04:44.088085+0000","last_clean":"2023-10-09T05:04:44.088085+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:04:44.088085+0000","last_undegraded":"2023-10-09T05:04:44.088085+0000","last_fullsized":"2023-10-09T05:04:44.088085+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1259,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083505+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:04:44.083505+0000","last_peered":"2023-10-09T05:04:44.083505+0000","last_clean":"2023-10-09T05:04:44.083505+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:04:44.083505+0000","last_undegraded":"2023-10-09T05:04:44.083505+0000","last_fullsized":"2023-10-09T05:04:44.083505+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"659'5960","reported_seq":23308,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:52.138772+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:04:52.138772+0000","last_peered":"2023-10-09T05:04:52.138772+0000","last_clean":"2023-10-09T05:04:52.138772+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:04:52.138772+0000","last_undegraded":"2023-10-09T05:04:52.138772+0000","last_fullsized":"2023-10-09T05:04:52.138772+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5560,"log_dups_size":0,"ondisk_log_size":5560,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3139,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":282,"reported_epoch":658,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:44.100321+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:04:44.100321+0000","last_peered":"2023-10-09T05:04:44.100321+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:04:44.100321+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:04:44.100321+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.078156+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:04:44.078156+0000","last_peered":"2023-10-09T05:04:44.078156+0000","last_clean":"2023-10-09T05:04:44.078156+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:04:44.078156+0000","last_undegraded":"2023-10-09T05:04:44.078156+0000","last_fullsized":"2023-10-09T05:04:44.078156+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":162,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085448+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:04:44.085448+0000","last_peered":"2023-10-09T05:04:44.085448+0000","last_clean":"2023-10-09T05:04:44.085448+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:04:44.085448+0000","last_undegraded":"2023-10-09T05:04:44.085448+0000","last_fullsized":"2023-10-09T05:04:44.085448+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1201,"reported_epoch":658,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:44.083121+0000","last_change":"2023-10-09T05:03:48.986735+0000","last_active":"2023-10-09T05:04:44.083121+0000","last_peered":"2023-10-09T05:04:44.083121+0000","last_clean":"2023-10-09T05:03:43.187034+0000","last_became_active":"2023-10-09T05:03:48.981834+0000","last_became_peered":"2023-10-09T05:03:48.981834+0000","last_unstale":"2023-10-09T05:04:44.083121+0000","last_undegraded":"2023-10-09T05:03:48.960205+0000","last_fullsized":"2023-10-09T05:03:48.959273+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1466069,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,0],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"659'6312","reported_seq":11413,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.335531+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:04:53.335531+0000","last_peered":"2023-10-09T05:04:53.335531+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:04:53.335531+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4887,"log_dups_size":0,"ondisk_log_size":4887,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106692608,"num_objects":1628,"num_object_clones":0,"num_object_copies":4884,"num_objects_missing_on_primary":2020,"num_objects_missing":2020,"num_objects_degraded":5362,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3352,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1144,"num_bytes_recovered":74973184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":1558},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":175,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.086284+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:04:44.086284+0000","last_peered":"2023-10-09T05:04:44.086284+0000","last_clean":"2023-10-09T05:04:44.086284+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:04:44.086284+0000","last_undegraded":"2023-10-09T05:04:44.086284+0000","last_fullsized":"2023-10-09T05:04:44.086284+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":93,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.078040+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:04:44.078040+0000","last_peered":"2023-10-09T05:04:44.078040+0000","last_clean":"2023-10-09T05:04:44.078040+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:04:44.078040+0000","last_undegraded":"2023-10-09T05:04:44.078040+0000","last_fullsized":"2023-10-09T05:04:44.078040+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083896+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:04:44.083896+0000","last_peered":"2023-10-09T05:04:44.083896+0000","last_clean":"2023-10-09T05:04:44.083896+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:04:44.083896+0000","last_undegraded":"2023-10-09T05:04:44.083896+0000","last_fullsized":"2023-10-09T05:04:44.083896+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"659'5630","reported_seq":9352,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:53.270384+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:04:53.270384+0000","last_peered":"2023-10-09T05:04:53.270384+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:04:53.270384+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101089280,"num_objects":1543,"num_object_clones":0,"num_object_copies":4629,"num_objects_missing_on_primary":1967,"num_objects_missing":1967,"num_objects_degraded":4804,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3174,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":817,"num_bytes_recovered":53542912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-35},{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":1191}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":124,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085777+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:04:44.085777+0000","last_peered":"2023-10-09T05:04:44.085777+0000","last_clean":"2023-10-09T05:04:44.085777+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:04:44.085777+0000","last_undegraded":"2023-10-09T05:04:44.085777+0000","last_fullsized":"2023-10-09T05:04:44.085777+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":74,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:47.158404+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:04:47.158404+0000","last_peered":"2023-10-09T05:04:47.158404+0000","last_clean":"2023-10-09T05:04:47.158404+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:04:47.158404+0000","last_undegraded":"2023-10-09T05:04:47.158404+0000","last_fullsized":"2023-10-09T05:04:47.158404+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"659'5716","reported_seq":10549,"reported_epoch":659,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:52.176695+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:04:52.176695+0000","last_peered":"2023-10-09T05:04:52.176695+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:04:52.176695+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:04:52.176695+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3181,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1678}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":794,"reported_epoch":658,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:44.089215+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:04:44.089215+0000","last_peered":"2023-10-09T05:04:44.089215+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:04:44.089215+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:04:44.089215+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.084184+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:04:44.084184+0000","last_peered":"2023-10-09T05:04:44.084184+0000","last_clean":"2023-10-09T05:04:44.084184+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:04:44.084184+0000","last_undegraded":"2023-10-09T05:04:44.084184+0000","last_fullsized":"2023-10-09T05:04:44.084184+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"659'5772","reported_seq":10189,"reported_epoch":659,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:04:52.050952+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:04:52.050952+0000","last_peered":"2023-10-09T05:04:52.050952+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:04:52.050952+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106332160,"num_objects":1623,"num_object_clones":0,"num_object_copies":4869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3241,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":976}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":740,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088946+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:04:44.088946+0000","last_peered":"2023-10-09T05:04:44.088946+0000","last_clean":"2023-10-09T05:04:44.088946+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:04:44.088946+0000","last_undegraded":"2023-10-09T05:04:44.088946+0000","last_fullsized":"2023-10-09T05:04:44.088946+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083537+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:04:44.083537+0000","last_peered":"2023-10-09T05:04:44.083537+0000","last_clean":"2023-10-09T05:04:44.083537+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:04:44.083537+0000","last_undegraded":"2023-10-09T05:04:44.083537+0000","last_fullsized":"2023-10-09T05:04:44.083537+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"659'5761","reported_seq":16559,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-10-09T05:04:52.211493+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:04:52.211493+0000","last_peered":"2023-10-09T05:04:52.211493+0000","last_clean":"2023-10-09T05:04:52.211493+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:04:52.211493+0000","last_undegraded":"2023-10-09T05:04:52.211493+0000","last_fullsized":"2023-10-09T05:04:52.211493+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5761,"log_dups_size":0,"ondisk_log_size":5761,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107151360,"num_objects":1635,"num_object_clones":0,"num_object_copies":4905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3229,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":279,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.090390+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:04:44.090390+0000","last_peered":"2023-10-09T05:04:44.090390+0000","last_clean":"2023-10-09T05:04:44.090390+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:04:44.090390+0000","last_undegraded":"2023-10-09T05:04:44.090390+0000","last_fullsized":"2023-10-09T05:04:44.090390+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":287,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.085210+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:04:44.085210+0000","last_peered":"2023-10-09T05:04:44.085210+0000","last_clean":"2023-10-09T05:04:44.085210+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:04:44.085210+0000","last_undegraded":"2023-10-09T05:04:44.085210+0000","last_fullsized":"2023-10-09T05:04:44.085210+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"659'5697","reported_seq":17297,"reported_epoch":659,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:04:53.618152+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:04:53.618152+0000","last_peered":"2023-10-09T05:04:53.618152+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:04:53.618152+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:04:53.618152+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105971712,"num_objects":1617,"num_object_clones":0,"num_object_copies":4851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3198,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":1617}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":39,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.088769+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:04:44.088769+0000","last_peered":"2023-10-09T05:04:44.088769+0000","last_clean":"2023-10-09T05:04:44.088769+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:04:44.088769+0000","last_undegraded":"2023-10-09T05:04:44.088769+0000","last_fullsized":"2023-10-09T05:04:44.088769+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":135,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-10-09T05:04:44.083504+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:04:44.083504+0000","last_peered":"2023-10-09T05:04:44.083504+0000","last_clean":"2023-10-09T05:04:44.083504+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:04:44.083504+0000","last_undegraded":"2023-10-09T05:04:44.083504+0000","last_fullsized":"2023-10-09T05:04:44.083504+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":3381592088,"num_objects":51600,"num_object_clones":0,"num_object_copies":154800,"num_objects_missing_on_primary":18307,"num_objects_missing":18307,"num_objects_degraded":60639,"num_objects_misplaced":3290,"num_objects_unfound":0,"num_objects_dirty":51600,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103618,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67491,"num_bytes_recovered":4120772608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15437737984,"data_stored":15437725768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":187397,"ondisk_log_size":187397,"up":108,"acting":89,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":6768058,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2314240,"data_stored":2296480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234690,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3432552,"kb_used_data":1915104,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":90312600,"statfs":{"total":95995035648,"available":92480102400,"internally_reserved":0,"allocated":1961066496,"data_stored":1959404088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:14 2023","interfaces":[{"interface":"back","average":{"1min":11.875999999999999,"5min":3.4199999999999999,"15min":1.633},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":123.04000000000001,"5min":123.04000000000001,"15min":123.04000000000001},"last":3.7999999999999998},{"interface":"front","average":{"1min":11.739000000000001,"5min":3.5819999999999999,"15min":1.9039999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":123.059,"5min":123.059,"15min":123.059},"last":0.34000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:04:14 2023","interfaces":[{"interface":"back","average":{"1min":4.3799999999999999,"5min":2.3450000000000002,"15min":2.29},"min":{"1min":0.29999999999999999,"5min":0.28899999999999998,"15min":0.246},"max":{"1min":40.651000000000003,"5min":43.158000000000001,"15min":122.56},"last":3.0070000000000001},{"interface":"front","average":{"1min":3.5350000000000001,"5min":1.976,"15min":2.214},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":43.331000000000003,"5min":43.331000000000003,"15min":122.666},"last":3.4300000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.4569999999999999,"15min":3.8889999999999998},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.30299999999999999},"max":{"1min":10.545,"5min":154.76400000000001,"15min":207.96600000000001},"last":6.7009999999999996},{"interface":"front","average":{"1min":2.8029999999999999,"5min":4.4340000000000002,"15min":4.0110000000000001},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.29699999999999999},"max":{"1min":10.532,"5min":154.80000000000001,"15min":207.97900000000001},"last":6.6799999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":2.3730000000000002,"15min":1.8959999999999999},"min":{"1min":0.36299999999999999,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":8.5950000000000006,"5min":43.103000000000002,"15min":74.259},"last":2.698},{"interface":"front","average":{"1min":1.319,"5min":2.2810000000000001,"15min":1.9159999999999999},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":5.593,"5min":32.804000000000002,"15min":74.100999999999999},"last":2.6269999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.38500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.6579999999999999,"15min":1.7929999999999999},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":5.5970000000000004,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.8169999999999999,"15min":2.149},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":0.57899999999999996,"5min":53.225000000000001,"15min":135.74199999999999},"last":0.41899999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":1.4730000000000001,"15min":1.23},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.57299999999999995,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":1.736,"15min":1.4339999999999999},"min":{"1min":0.34899999999999998,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":0.56100000000000005,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.45900000000000002}]}]},{"osd":6,"up_from":46,"seq":197568496323,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2212136,"kb_used_data":1017376,"kb_used_omap":0,"kb_used_meta":1194752,"kb_avail":91533016,"statfs":{"total":95995035648,"available":93729808384,"internally_reserved":0,"allocated":1041793024,"data_stored":1040130849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1223426048},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5489999999999999,"5min":5.0910000000000002,"15min":2.1720000000000002},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":65.721999999999994,"5min":321.346,"15min":321.346},"last":2.8079999999999998},{"interface":"front","average":{"1min":4.3700000000000001,"5min":3.5390000000000001,"15min":1.855},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":65.585999999999999,"5min":169.119,"15min":169.119},"last":3.5670000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:04:01 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":2.427,"15min":4.6070000000000002},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.3289999999999997,"5min":72.840999999999994,"15min":260.738},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.3839999999999999,"5min":2.5619999999999998,"15min":4.5170000000000003},"min":{"1min":0.25600000000000001,"5min":0.251,"15min":0.251},"max":{"1min":7.0979999999999999,"5min":72.832999999999998,"15min":261.26100000000002},"last":3.6890000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.4399999999999999,"15min":1.994},"min":{"1min":0.35299999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.032999999999999,"5min":23.943000000000001,"15min":133.535},"last":3.8799999999999999},{"interface":"front","average":{"1min":1.591,"5min":1.421,"15min":1.9390000000000001},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.30599999999999999},"max":{"1min":8.5920000000000005,"5min":25.978999999999999,"15min":133.51900000000001},"last":3.8929999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:04:01 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.4219999999999999,"15min":2.3090000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.1790000000000003,"5min":14.919,"15min":147.43600000000001},"last":1.399},{"interface":"front","average":{"1min":1.603,"5min":1.3069999999999999,"15min":2.4049999999999998},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":11.066000000000001,"5min":11.066000000000001,"15min":147.41999999999999},"last":2.5379999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1710000000000003,"5min":7.1710000000000003,"15min":7.1710000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":11.102,"5min":11.102,"15min":11.102},"last":0.29899999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:04:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.0629999999999999,"15min":1.3260000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4510000000000001,"5min":25.934000000000001,"15min":66.995999999999995},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79200000000000004,"15min":1.302},"min":{"1min":0.33900000000000002,"5min":0.32200000000000001,"15min":0.254},"max":{"1min":0.97099999999999997,"5min":16.789000000000001,"15min":67.189999999999998},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.66900000000000004,"15min":1.1180000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":0.59299999999999997,"5min":5.835,"15min":36.204999999999998},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.002,"15min":1.1259999999999999},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":2.8069999999999999,"5min":20.041,"15min":20.041},"last":0.36599999999999999}]}]},{"osd":1,"up_from":118,"seq":506806141390,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2319736,"kb_used_data":1052272,"kb_used_omap":4,"kb_used_meta":1267387,"kb_avail":91425416,"statfs":{"total":95995035648,"available":93619625984,"internally_reserved":0,"allocated":1077526528,"data_stored":1074030989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1297805198},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8949999999999996,"5min":2.431,"15min":1.361},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":85.753,"5min":85.753,"15min":85.753},"last":3.2469999999999999},{"interface":"front","average":{"1min":5.9269999999999996,"5min":2.4049999999999998,"15min":1.355},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.214},"max":{"1min":85.227999999999994,"5min":85.227999999999994,"15min":85.227999999999994},"last":10.497999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1639999999999999,"15min":2.4140000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":6.3129999999999997,"5min":12.460000000000001,"15min":263.089},"last":2.726},{"interface":"front","average":{"1min":2.2509999999999999,"5min":1.4870000000000001,"15min":2.4689999999999999},"min":{"1min":0.254,"5min":0.252,"15min":0.222},"max":{"1min":8.5250000000000004,"5min":13.933999999999999,"15min":254.50899999999999},"last":10.385}]},{"osd":3,"last update":"Mon Oct 9 05:04:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":1.6040000000000001,"15min":1.6379999999999999},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":28.797000000000001,"5min":28.797000000000001,"15min":77.819999999999993},"last":2.3969999999999998},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.0189999999999999,"15min":1.331},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.249},"max":{"1min":5.8319999999999999,"5min":7.0860000000000003,"15min":77.796000000000006},"last":10.534000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:04:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.7800000000000002,"5min":5.7800000000000002,"15min":5.7800000000000002},"last":10.25},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":2.492}]},{"osd":5,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.03,"15min":1.1779999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":5.8109999999999999,"5min":7.79,"15min":7.79},"last":10.353},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.4510000000000001,"15min":1.2350000000000001},"min":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":10.554,"5min":10.554,"15min":19.597999999999999},"last":2.552}]},{"osd":6,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.125,"15min":1.157},"min":{"1min":0.41099999999999998,"5min":0.39700000000000002,"15min":0.35699999999999998},"max":{"1min":5.7489999999999997,"5min":7.4370000000000003,"15min":7.4370000000000003},"last":10.311999999999999},{"interface":"front","average":{"1min":1.4430000000000001,"5min":1.3440000000000001,"15min":1.252},"min":{"1min":0.39500000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.472,"5min":10.472,"15min":19.548999999999999},"last":2.714}]},{"osd":7,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.484,"15min":1.722},"min":{"1min":0.42399999999999999,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":10.583,"5min":10.583,"15min":59.972000000000001},"last":2.4500000000000002},{"interface":"front","average":{"1min":0.745,"5min":1.2150000000000001,"15min":1.6279999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":3.5670000000000002,"5min":11.903,"15min":60.011000000000003},"last":2.3530000000000002}]}]},{"osd":0,"up_from":91,"seq":390842024474,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1876520,"kb_used_data":786208,"kb_used_omap":5,"kb_used_meta":1090298,"kb_avail":91868632,"statfs":{"total":95995035648,"available":94073479168,"internally_reserved":0,"allocated":805076992,"data_stored":802242014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1116465786},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,10,9,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:04:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":2.2400000000000002,"15min":4.407},"min":{"1min":0.222,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":16.731000000000002,"5min":30.007999999999999,"15min":247.68899999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.77,"5min":1.921,"15min":4.3559999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":16.823,"5min":29.940000000000001,"15min":246.38},"last":1.6870000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.9399999999999999,"15min":2.2930000000000001},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.9340000000000002,"5min":26.556999999999999,"15min":151.06399999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":4.4400000000000004,"5min":2.2400000000000002,"15min":2.4820000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":20.239000000000001,"5min":29.032,"15min":151.22399999999999},"last":1.754}]},{"osd":3,"last update":"Mon Oct 9 05:03:54 2023","interfaces":[{"interface":"back","average":{"1min":17.937999999999999,"5min":6.4749999999999996,"15min":3.4129999999999998},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":290.89100000000002,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.54300000000000004},{"interface":"front","average":{"1min":18.800999999999998,"5min":6.3259999999999996,"15min":3.5129999999999999},"min":{"1min":0.30599999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":290.73899999999998,"5min":290.73899999999998,"15min":290.73899999999998},"last":1.726}]},{"osd":4,"last update":"Mon Oct 9 05:04:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":1.8520000000000001,"15min":1.8520000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.9320000000000004,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.5349999999999999,"5min":1.5349999999999999,"15min":1.5349999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.9249999999999998,"5min":6.9249999999999998,"15min":6.9249999999999998},"last":0.45200000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:04:17 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.3939999999999999,"15min":1.1830000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33500000000000002},"max":{"1min":16.888000000000002,"5min":16.888000000000002,"15min":16.888000000000002},"last":2.02},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.659,"15min":1.1619999999999999},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":7.9909999999999997,"5min":14.036,"15min":14.036},"last":0.52600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:04:17 2023","interfaces":[{"interface":"back","average":{"1min":2.609,"5min":1.6160000000000001,"15min":1.3120000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.34799999999999998},"max":{"1min":16.917000000000002,"5min":20.129999999999999,"15min":39.594999999999999},"last":1.839},{"interface":"front","average":{"1min":2.1629999999999998,"5min":2.1989999999999998,"15min":1.4850000000000001},"min":{"1min":0.436,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":8.0619999999999994,"5min":47.241,"15min":47.241},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:04:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.909,"15min":1.381},"min":{"1min":0.47499999999999998,"5min":0.41899999999999998,"15min":0.38100000000000001},"max":{"1min":8.0269999999999992,"5min":21.658000000000001,"15min":21.658000000000001},"last":0.621},{"interface":"front","average":{"1min":1.702,"5min":1.736,"15min":1.383},"min":{"1min":0.52800000000000002,"5min":0.41999999999999998,"15min":0.38200000000000001},"max":{"1min":6.8730000000000002,"5min":21.68,"15min":21.68},"last":0.50700000000000001}]}]},{"osd":2,"up_from":0,"seq":949187772752,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356446,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2361600,"kb_used_data":1438392,"kb_used_omap":2,"kb_used_meta":923197,"kb_avail":91383552,"statfs":{"total":95995035648,"available":93576757248,"internally_reserved":0,"allocated":1472913408,"data_stored":1469994770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":945353781},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,2,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":6.5350000000000001,"5min":2.077,"15min":1.0669999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":108.631,"5min":108.631,"15min":108.631},"last":3.415},{"interface":"front","average":{"1min":6.5250000000000004,"5min":2.226,"15min":1.1859999999999999},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.23499999999999999},"max":{"1min":110.012,"5min":110.012,"15min":110.012},"last":2.0030000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.347,"15min":2.427},"min":{"1min":0.223,"5min":0.221,"15min":0.221},"max":{"1min":15.218,"5min":15.218,"15min":125.233},"last":0.71399999999999997},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.391,"15min":2.4689999999999999},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.21299999999999999},"max":{"1min":17.544,"5min":26.692,"15min":126.102},"last":3.4849999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:36 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.3009999999999999,"15min":2.1389999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":6.4160000000000004,"5min":15.606999999999999,"15min":212.065},"last":4.0330000000000004},{"interface":"front","average":{"1min":2.0800000000000001,"5min":1.3899999999999999,"15min":2.1789999999999998},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":12.987,"5min":12.987,"15min":211.98400000000001},"last":2.133}]},{"osd":4,"last update":"Mon Oct 9 05:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.72,"15min":1.72},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.193,"5min":11.193,"15min":11.193},"last":2.04},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.6030000000000002,"5min":3.6030000000000002,"15min":3.6030000000000002},"last":0.623}]},{"osd":5,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":0.90600000000000003,"15min":1.161},"min":{"1min":0.36499999999999999,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":7.6779999999999999,"5min":7.6779999999999999,"15min":28.302},"last":3.456},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.2070000000000001,"15min":1.3440000000000001},"min":{"1min":0.40300000000000002,"5min":0.372,"15min":0.33600000000000002},"max":{"1min":3.7869999999999999,"5min":7.3040000000000003,"15min":28.280000000000001},"last":2.0979999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.2789999999999999,"15min":1.0229999999999999},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.8149999999999999,"5min":6.383,"15min":6.383},"last":2.073},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.2529999999999999,"15min":1.2470000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.731,"5min":14.731,"15min":24.916},"last":0.65800000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.353,"15min":1.349},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.375},"max":{"1min":14.784000000000001,"5min":14.784000000000001,"15min":25.004000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.165,"15min":1.081},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.758,"5min":7.758,"15min":7.9009999999999998},"last":3.5089999999999999}]}]},{"osd":4,"up_from":626,"seq":2688649527314,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3142000,"kb_used_data":2745000,"kb_used_omap":22,"kb_used_meta":396969,"kb_avail":90603152,"statfs":{"total":95995035648,"available":92777627648,"internally_reserved":0,"allocated":2810880000,"data_stored":2809209998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":406497101},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":2.2909999999999999,"15min":2.2909999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":21.292000000000002,"5min":21.292000000000002,"15min":21.292000000000002},"last":0.81599999999999995},{"interface":"front","average":{"1min":3.1709999999999998,"5min":3.1709999999999998,"15min":3.1709999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":21.391999999999999,"5min":21.391999999999999,"15min":21.391999999999999},"last":1.8320000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":2.129,"15min":2.129},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.955,"5min":14.955,"15min":14.955},"last":5.0389999999999997},{"interface":"front","average":{"1min":1.901,"5min":1.901,"15min":1.901},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":10.579000000000001,"5min":10.579000000000001,"15min":10.579000000000001},"last":1.3999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.639,"15min":1.639},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":2.0539999999999998},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":11.036,"5min":11.036,"15min":11.036},"last":2.0659999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.8440000000000001,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.8179999999999996,"5min":9.8179999999999996,"15min":9.8179999999999996},"last":0.56799999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.308},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.34200000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7910000000000004,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.40999999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.44700000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757967,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4867672,"kb_used_data":3378448,"kb_used_omap":0,"kb_used_meta":1489216,"kb_avail":88877480,"statfs":{"total":95995035648,"available":91010539520,"internally_reserved":0,"allocated":3459530752,"data_stored":3457879972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1524957184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:20 2023","interfaces":[{"interface":"back","average":{"1min":13.412000000000001,"5min":3.2010000000000001,"15min":2.121},"min":{"1min":0.252,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":103.745,"5min":103.745,"15min":106.465},"last":0.97699999999999998},{"interface":"front","average":{"1min":14.061999999999999,"5min":3.4969999999999999,"15min":2.2629999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.245},"max":{"1min":103.872,"5min":103.872,"15min":106.54600000000001},"last":0.34200000000000003}]},{"osd":1,"last update":"Mon Oct 9 05:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":4.8449999999999998,"15min":3.476},"min":{"1min":0.314,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":4.9240000000000004,"5min":262.26900000000001,"15min":262.26900000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.876,"5min":4.7000000000000002,"15min":3.6000000000000001},"min":{"1min":0.28899999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.569,"5min":262.291,"15min":262.291},"last":3.8210000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:04:38 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":3.0089999999999999,"15min":1.794},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.27000000000000002},"max":{"1min":10.382999999999999,"5min":106.238,"15min":106.238},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.8170000000000002,"15min":2.0219999999999998},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":8.4960000000000004,"5min":104.31699999999999,"15min":104.31699999999999},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:04:00 2023","interfaces":[{"interface":"back","average":{"1min":17.306999999999999,"5min":5.2629999999999999,"15min":2.9590000000000001},"min":{"1min":0.379,"5min":0.318,"15min":0.318},"max":{"1min":125.824,"5min":125.824,"15min":125.824},"last":0.54600000000000004},{"interface":"front","average":{"1min":16.542999999999999,"5min":5.391,"15min":2.9489999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.316},"max":{"1min":125.761,"5min":125.761,"15min":125.761},"last":0.79400000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.316},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.49299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.76400000000000001,"15min":0.82899999999999996},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23300000000000001},"max":{"1min":0.69999999999999996,"5min":11.093,"15min":11.093},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.63100000000000001,"15min":0.91900000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.73099999999999998,"5min":5.9329999999999998,"15min":13.083},"last":0.378}]},{"osd":7,"last update":"Mon Oct 9 05:04:00 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.74299999999999999,"15min":1.77},"min":{"1min":0.314,"5min":0.314,"15min":0.26700000000000002},"max":{"1min":1.982,"5min":8.3520000000000003,"15min":110.521},"last":0.438},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.95499999999999996,"15min":1.8260000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":0.90800000000000003,"5min":11.791,"15min":110.746},"last":0.39400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":797442048,"data_stored":797442048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1068105728,"data_stored":1068105728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2872639488,"data_stored":2872639488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1464532992,"data_stored":1464532992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2799702016,"data_stored":2799697944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3451588608,"data_stored":3451584536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1033834496,"data_stored":1033830424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1949892608,"data_stored":1949892608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:04:57.249 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:04:57.300 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:57.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:57.409 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:04:57.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:04:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:57 smithi177 ceph-mon[195797]: pgmap v2108: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 191 op/s; 60583/154485 objects degraded (39.216%); 2696/154485 objects misplaced (1.745%); 10 MiB/s, 189 objects/s recovering 2023-10-09T05:04:57.585 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:58.689 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:04:58.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:58.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:04:58.827 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:04:58.853 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:58.902 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:04:58.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:58.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:04:59.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:04:59.090 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:59.142 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:59.182 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:59.316 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:04:59.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:59.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:04:59.437 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:04:59.519 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:04:59.631 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:04:59.896 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:04:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:04:59.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:04:59.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:59 smithi029 ceph-mon[216903]: pgmap v2109: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 60209/153096 objects degraded (39.328%); 2696/153096 objects misplaced (1.761%); 10 MiB/s, 189 objects/s recovering 2023-10-09T05:04:59.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:59.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:04:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:04:59.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:59 smithi029 ceph-mon[223858]: pgmap v2109: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 60209/153096 objects degraded (39.328%); 2696/153096 objects misplaced (1.761%); 10 MiB/s, 189 objects/s recovering 2023-10-09T05:04:59.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:04:59.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:04:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:00.005 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:59 smithi177 ceph-mon[195797]: pgmap v2109: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 219 op/s; 60209/153096 objects degraded (39.328%); 2696/153096 objects misplaced (1.761%); 10 MiB/s, 189 objects/s recovering 2023-10-09T05:05:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:04:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:00.143 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:00.724 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:05:00.777 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:00 smithi029 ceph-mon[216903]: pgmap v2110: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 59668/151638 objects degraded (39.349%); 2043/151638 objects misplaced (1.347%); 9.3 MiB/s, 178 objects/s recovering 2023-10-09T05:05:00.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3603231912' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:05:00.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:00 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:05:00.778 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:00 smithi029 ceph-mon[223858]: pgmap v2110: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 59668/151638 objects degraded (39.349%); 2043/151638 objects misplaced (1.347%); 9.3 MiB/s, 178 objects/s recovering 2023-10-09T05:05:00.778 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3603231912' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:05:00.778 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:00 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:05:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:00 smithi177 ceph-mon[195797]: pgmap v2110: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 197 op/s; 59668/151638 objects degraded (39.349%); 2043/151638 objects misplaced (1.347%); 9.3 MiB/s, 178 objects/s recovering 2023-10-09T05:05:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3603231912' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:05:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:00 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:05:01.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[223858]: failed to encode map e660 with expected crc 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[223858]: osdmap e660: 8 total, 8 up, 7 in 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 59668/151638 objects degraded (39.349%), 28 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[216903]: failed to encode map e660 with expected crc 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:05:01.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[216903]: osdmap e660: 8 total, 8 up, 7 in 2023-10-09T05:05:01.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 59668/151638 objects degraded (39.349%), 28 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:01 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:01 smithi177 ceph-mon[195797]: failed to encode map e660 with expected crc 2023-10-09T05:05:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:01 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:05:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:01 smithi177 ceph-mon[195797]: osdmap e660: 8 total, 8 up, 7 in 2023-10-09T05:05:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 59668/151638 objects degraded (39.349%), 28 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:02.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:05:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:05:02] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[216903]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[216903]: osdmap e661: 8 total, 8 up, 7 in 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[216903]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[216903]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[216903]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[216903]: pgmap v2113: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 59668/151638 objects degraded (39.349%); 2043/151638 objects misplaced (1.347%); 8.1 MiB/s, 141 objects/s recovering 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[223858]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[223858]: osdmap e661: 8 total, 8 up, 7 in 2023-10-09T05:05:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[223858]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[223858]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[223858]: failed to encode map e661 with expected crc 2023-10-09T05:05:02.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:02 smithi029 ceph-mon[223858]: pgmap v2113: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 59668/151638 objects degraded (39.349%); 2043/151638 objects misplaced (1.347%); 8.1 MiB/s, 141 objects/s recovering 2023-10-09T05:05:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:02 smithi177 ceph-mon[195797]: failed to encode map e661 with expected crc 2023-10-09T05:05:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:02 smithi177 ceph-mon[195797]: osdmap e661: 8 total, 8 up, 7 in 2023-10-09T05:05:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:02 smithi177 ceph-mon[195797]: failed to encode map e661 with expected crc 2023-10-09T05:05:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:02 smithi177 ceph-mon[195797]: failed to encode map e661 with expected crc 2023-10-09T05:05:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:02 smithi177 ceph-mon[195797]: failed to encode map e661 with expected crc 2023-10-09T05:05:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:02 smithi177 ceph-mon[195797]: pgmap v2113: 119 pgs: 2 active+recovering+undersized+remapped, 89 active+clean, 18 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 59668/151638 objects degraded (39.349%); 2043/151638 objects misplaced (1.347%); 8.1 MiB/s, 141 objects/s recovering 2023-10-09T05:05:03.458 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:03.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:03.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:03.574 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:03.734 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:04.591 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:04.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:04.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:04.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:04.737 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:04.782 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:04.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:04.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:04.917 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:04.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:04 smithi029 ceph-mon[216903]: pgmap v2114: 119 pgs: 3 active+recovering+undersized+remapped, 91 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 52113/149262 objects degraded (34.914%); 7988/149262 objects misplaced (5.352%); 8.7 MiB/s, 163 objects/s recovering 2023-10-09T05:05:04.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:04 smithi029 ceph-mon[223858]: pgmap v2114: 119 pgs: 3 active+recovering+undersized+remapped, 91 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 52113/149262 objects degraded (34.914%); 7988/149262 objects misplaced (5.352%); 8.7 MiB/s, 163 objects/s recovering 2023-10-09T05:05:04.963 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:04.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:04.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:04 smithi177 ceph-mon[195797]: pgmap v2114: 119 pgs: 3 active+recovering+undersized+remapped, 91 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 217 op/s; 52113/149262 objects degraded (34.914%); 7988/149262 objects misplaced (5.352%); 8.7 MiB/s, 163 objects/s recovering 2023-10-09T05:05:05.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:05.253 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:05.579 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:05:05.643 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:05:05.691 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:05.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:05.721 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:05:05.782 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:05:05.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:05.837 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:05:05.839 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:05:05.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1664298970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:05.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1664298970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:05.896 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:05:05.905 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:05:05.952 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:05.954 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:05:05.998 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:05:06.004 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:05:06.004 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:05:06.004 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:05:06.004 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:05:06.004 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:05:06.028 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.028 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.029 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.029 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.029 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.029 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.029 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.029 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:05:06.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1664298970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:06.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:05:07.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:06 smithi029 ceph-mon[216903]: pgmap v2115: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 54785/148998 objects degraded (36.769%); 5945/148998 objects misplaced (3.990%); 8.0 MiB/s, 152 objects/s recovering 2023-10-09T05:05:07.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:06 smithi029 ceph-mon[216903]: from='client.47513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:07.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 54785/148998 objects degraded (36.769%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:07.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:06 smithi029 ceph-mon[223858]: pgmap v2115: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 54785/148998 objects degraded (36.769%); 5945/148998 objects misplaced (3.990%); 8.0 MiB/s, 152 objects/s recovering 2023-10-09T05:05:07.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:06 smithi029 ceph-mon[223858]: from='client.47513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:07.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 54785/148998 objects degraded (36.769%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:06 smithi177 ceph-mon[195797]: pgmap v2115: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 170 op/s; 54785/148998 objects degraded (36.769%); 5945/148998 objects misplaced (3.990%); 8.0 MiB/s, 152 objects/s recovering 2023-10-09T05:05:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:06 smithi177 ceph-mon[195797]: from='client.47513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:07.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 54785/148998 objects degraded (36.769%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:08.550 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:08.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:08.674 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:08.701 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:08.753 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:08.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:08.857 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:08 smithi029 ceph-mon[216903]: pgmap v2116: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 54704/148083 objects degraded (36.941%); 5816/148083 objects misplaced (3.928%); 4.4 MiB/s, 86 objects/s recovering 2023-10-09T05:05:08.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:08.976 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:08 smithi177 ceph-mon[195797]: pgmap v2116: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 54704/148083 objects degraded (36.941%); 5816/148083 objects misplaced (3.928%); 4.4 MiB/s, 86 objects/s recovering 2023-10-09T05:05:09.078 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2115,"stamp":"2023-10-09T05:05:05.759975+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3253868472,"num_objects":49666,"num_object_clones":0,"num_object_copies":148998,"num_objects_missing_on_primary":17449,"num_objects_missing":17449,"num_objects_degraded":54785,"num_objects_misplaced":5945,"num_objects_unfound":0,"num_objects_dirty":49666,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":105876,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69397,"num_bytes_recovered":4227041934,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":191181,"ondisk_log_size":191181,"up":357,"acting":339,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":486,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20031112,"kb_used_data":12125520,"kb_used_omap":34,"kb_used_meta":7905501,"kb_avail":636184952,"statfs":{"total":671965249536,"available":651453390848,"internally_reserved":0,"allocated":12416532480,"data_stored":12400584371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8095233418},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-89522176,"num_objects":-1366,"num_object_clones":0,"num_object_copies":-4098,"num_objects_missing_on_primary":-691,"num_objects_missing":-691,"num_objects_degraded":-5424,"num_objects_misplaced":3249,"num_objects_unfound":0,"num_objects_dirty":-1366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1366,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1225,"num_bytes_recovered":67371008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.011387"},"pg_stats":[{"pgid":"14.23","version":"661'5656","reported_seq":12275,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:02.768695+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:05:02.768695+0000","last_peered":"2023-10-09T05:05:02.768695+0000","last_clean":"2023-10-09T05:05:02.768695+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:05:02.768695+0000","last_undegraded":"2023-10-09T05:05:02.768695+0000","last_fullsized":"2023-10-09T05:05:02.768695+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1672,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"661'5564","reported_seq":11501,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.359728+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:05:02.359728+0000","last_peered":"2023-10-09T05:05:02.359728+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:05:02.359728+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5564,"log_dups_size":0,"ondisk_log_size":5564,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51019776,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":752,"num_objects_missing":752,"num_objects_degraded":2101,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1629,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":43892736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"3,4,5,7","objects":1},{"shards":"4,5,7","objects":594}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"661'5636","reported_seq":16327,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:01.773491+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:05:01.773491+0000","last_peered":"2023-10-09T05:05:01.773491+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:05:01.773491+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:05:01.773491+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50921472,"num_objects":777,"num_object_clones":0,"num_object_copies":2331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1653,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":777}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"661'5641","reported_seq":6680,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:02.110101+0000","last_change":"2023-10-09T05:03:26.464077+0000","last_active":"2023-10-09T05:05:02.110101+0000","last_peered":"2023-10-09T05:05:02.110101+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:05:02.110101+0000","last_undegraded":"2023-10-09T05:03:26.367100+0000","last_fullsized":"2023-10-09T05:05:02.110101+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5103,"log_dups_size":0,"ondisk_log_size":5103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51331072,"num_objects":783,"num_object_clones":0,"num_object_copies":2349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1986,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1648,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":206,"num_bytes_recovered":13516800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":734},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"661'5853","reported_seq":19408,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.649328+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:05:01.649328+0000","last_peered":"2023-10-09T05:05:01.649328+0000","last_clean":"2023-10-09T05:05:01.649328+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:05:01.649328+0000","last_undegraded":"2023-10-09T05:05:01.649328+0000","last_fullsized":"2023-10-09T05:05:01.649328+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103251968,"num_objects":1576,"num_object_clones":0,"num_object_copies":4728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3312,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":163,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638038+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:05:00.638038+0000","last_peered":"2023-10-09T05:05:00.638038+0000","last_clean":"2023-10-09T05:05:00.638038+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:05:00.638038+0000","last_undegraded":"2023-10-09T05:05:00.638038+0000","last_fullsized":"2023-10-09T05:05:00.638038+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":308,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.653651+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:05:00.653651+0000","last_peered":"2023-10-09T05:05:00.653651+0000","last_clean":"2023-10-09T05:05:00.653651+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:05:00.653651+0000","last_undegraded":"2023-10-09T05:05:00.653651+0000","last_fullsized":"2023-10-09T05:05:00.653651+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"661'5846","reported_seq":9067,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:03.433700+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:05:03.433700+0000","last_peered":"2023-10-09T05:05:03.433700+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:05:03.433700+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:05:03.433700+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101810176,"num_objects":1554,"num_object_clones":0,"num_object_copies":4662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3307,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":1554}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638714+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:05:00.638714+0000","last_peered":"2023-10-09T05:05:00.638714+0000","last_clean":"2023-10-09T05:05:00.638714+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:05:00.638714+0000","last_undegraded":"2023-10-09T05:05:00.638714+0000","last_fullsized":"2023-10-09T05:05:00.638714+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":173,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.652346+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:05:00.652346+0000","last_peered":"2023-10-09T05:05:00.652346+0000","last_clean":"2023-10-09T05:05:00.652346+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:05:00.652346+0000","last_undegraded":"2023-10-09T05:05:00.652346+0000","last_fullsized":"2023-10-09T05:05:00.652346+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"661'5826","reported_seq":17478,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:03.829808+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:05:03.829808+0000","last_peered":"2023-10-09T05:05:03.829808+0000","last_clean":"2023-10-09T05:05:03.829808+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:05:03.829808+0000","last_undegraded":"2023-10-09T05:05:03.829808+0000","last_fullsized":"2023-10-09T05:05:03.829808+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103219200,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3286,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":290,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.651247+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:05:00.651247+0000","last_peered":"2023-10-09T05:05:00.651247+0000","last_clean":"2023-10-09T05:05:00.651247+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:05:00.651247+0000","last_undegraded":"2023-10-09T05:05:00.651247+0000","last_fullsized":"2023-10-09T05:05:00.651247+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":187,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.642012+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:05:00.642012+0000","last_peered":"2023-10-09T05:05:00.642012+0000","last_clean":"2023-10-09T05:05:00.642012+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:05:00.642012+0000","last_undegraded":"2023-10-09T05:05:00.642012+0000","last_fullsized":"2023-10-09T05:05:00.642012+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"661'6055","reported_seq":23404,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.986479+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:05:01.986479+0000","last_peered":"2023-10-09T05:05:01.986479+0000","last_clean":"2023-10-09T05:05:01.986479+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:05:01.986479+0000","last_undegraded":"2023-10-09T05:05:01.986479+0000","last_fullsized":"2023-10-09T05:05:01.986479+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100728832,"num_objects":1537,"num_object_clones":0,"num_object_copies":4611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3234,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":35,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649547+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:05:00.649547+0000","last_peered":"2023-10-09T05:05:00.649547+0000","last_clean":"2023-10-09T05:05:00.649547+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:05:00.649547+0000","last_undegraded":"2023-10-09T05:05:00.649547+0000","last_fullsized":"2023-10-09T05:05:00.649547+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":281,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.651164+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:05:00.651164+0000","last_peered":"2023-10-09T05:05:00.651164+0000","last_clean":"2023-10-09T05:05:00.651164+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:05:00.651164+0000","last_undegraded":"2023-10-09T05:05:00.651164+0000","last_fullsized":"2023-10-09T05:05:00.651164+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"661'6350","reported_seq":11602,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.421503+0000","last_change":"2023-10-09T05:03:31.501660+0000","last_active":"2023-10-09T05:05:03.421503+0000","last_peered":"2023-10-09T05:05:03.421503+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:05:03.421503+0000","last_undegraded":"2023-10-09T05:03:27.167651+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4925,"log_dups_size":0,"ondisk_log_size":4925,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104202240,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":1973,"num_objects_missing":1973,"num_objects_degraded":5255,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1182,"num_bytes_recovered":77398016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":2},{"shards":"2,4,7","objects":1533},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":49,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649493+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:05:00.649493+0000","last_peered":"2023-10-09T05:05:00.649493+0000","last_clean":"2023-10-09T05:05:00.649493+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:05:00.649493+0000","last_undegraded":"2023-10-09T05:05:00.649493+0000","last_fullsized":"2023-10-09T05:05:00.649493+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":271,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649946+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:05:00.649946+0000","last_peered":"2023-10-09T05:05:00.649946+0000","last_clean":"2023-10-09T05:05:00.649946+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:05:00.649946+0000","last_undegraded":"2023-10-09T05:05:00.649946+0000","last_fullsized":"2023-10-09T05:05:00.649946+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"661'6000","reported_seq":9789,"reported_epoch":661,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:05:02.061885+0000","last_change":"2023-10-09T05:05:00.137617+0000","last_active":"2023-10-09T05:05:02.061885+0000","last_peered":"2023-10-09T05:05:02.061885+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:05:02.061885+0000","last_undegraded":"2023-10-09T05:05:02.061885+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4464,"log_dups_size":0,"ondisk_log_size":4464,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":1570,"num_objects_missing":1570,"num_objects_degraded":0,"num_objects_misplaced":4270,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3391,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":904,"num_bytes_recovered":59244544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,5,7","objects":1},{"shards":"1,3","objects":34},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1137}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":334,"reported_epoch":660,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:00.653511+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:05:00.653511+0000","last_peered":"2023-10-09T05:05:00.653511+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:05:00.653511+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:05:00.653511+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.639320+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:05:00.639320+0000","last_peered":"2023-10-09T05:05:00.639320+0000","last_clean":"2023-10-09T05:05:00.639320+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:05:00.639320+0000","last_undegraded":"2023-10-09T05:05:00.639320+0000","last_fullsized":"2023-10-09T05:05:00.639320+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"661'5850","reported_seq":17917,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:02.164917+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:05:02.164917+0000","last_peered":"2023-10-09T05:05:02.164917+0000","last_clean":"2023-10-09T05:05:02.164917+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:05:02.164917+0000","last_undegraded":"2023-10-09T05:05:02.164917+0000","last_fullsized":"2023-10-09T05:05:02.164917+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103383040,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3312,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":116,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.639489+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:05:00.639489+0000","last_peered":"2023-10-09T05:05:00.639489+0000","last_clean":"2023-10-09T05:05:00.639489+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:05:00.639489+0000","last_undegraded":"2023-10-09T05:05:00.639489+0000","last_fullsized":"2023-10-09T05:05:00.639489+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638461+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:05:00.638461+0000","last_peered":"2023-10-09T05:05:00.638461+0000","last_clean":"2023-10-09T05:05:00.638461+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:05:00.638461+0000","last_undegraded":"2023-10-09T05:05:00.638461+0000","last_fullsized":"2023-10-09T05:05:00.638461+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"661'6103","reported_seq":17626,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.568483+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:05:03.568483+0000","last_peered":"2023-10-09T05:05:03.568483+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:05:03.568483+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101482496,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2073,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3244,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2782,"num_bytes_recovered":177733632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":1045}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":137,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638410+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:05:00.638410+0000","last_peered":"2023-10-09T05:05:00.638410+0000","last_clean":"2023-10-09T05:05:00.638410+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:05:00.638410+0000","last_undegraded":"2023-10-09T05:05:00.638410+0000","last_fullsized":"2023-10-09T05:05:00.638410+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":244,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.653147+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:05:00.653147+0000","last_peered":"2023-10-09T05:05:00.653147+0000","last_clean":"2023-10-09T05:05:00.653147+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:05:00.653147+0000","last_undegraded":"2023-10-09T05:05:00.653147+0000","last_fullsized":"2023-10-09T05:05:00.653147+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"661'6119","reported_seq":10907,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.044956+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:05:02.044956+0000","last_peered":"2023-10-09T05:05:02.044956+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:05:02.044956+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3992,"log_dups_size":0,"ondisk_log_size":3992,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97583104,"num_objects":1489,"num_object_clones":0,"num_object_copies":4467,"num_objects_missing_on_primary":1594,"num_objects_missing":1594,"num_objects_degraded":3163,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3290,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1221,"num_bytes_recovered":79986688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,7","objects":-39},{"shards":"2,5,7","objects":1153},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.650780+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:05:00.650780+0000","last_peered":"2023-10-09T05:05:00.650780+0000","last_clean":"2023-10-09T05:05:00.650780+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:05:00.650780+0000","last_undegraded":"2023-10-09T05:05:00.650780+0000","last_fullsized":"2023-10-09T05:05:00.650780+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":39,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649307+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:05:00.649307+0000","last_peered":"2023-10-09T05:05:00.649307+0000","last_clean":"2023-10-09T05:05:00.649307+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:05:00.649307+0000","last_undegraded":"2023-10-09T05:05:00.649307+0000","last_fullsized":"2023-10-09T05:05:00.649307+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"661'6218","reported_seq":12653,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.029822+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:05:03.029822+0000","last_peered":"2023-10-09T05:05:03.029822+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:05:03.029822+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4455,"log_dups_size":0,"ondisk_log_size":4455,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101974016,"num_objects":1556,"num_object_clones":0,"num_object_copies":4668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3696,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3319,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1608,"num_bytes_recovered":92405760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5","objects":22},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":1452},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":100,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638109+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:05:00.638109+0000","last_peered":"2023-10-09T05:05:00.638109+0000","last_clean":"2023-10-09T05:05:00.638109+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:05:00.638109+0000","last_undegraded":"2023-10-09T05:05:00.638109+0000","last_fullsized":"2023-10-09T05:05:00.638109+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":298,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.652686+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:05:00.652686+0000","last_peered":"2023-10-09T05:05:00.652686+0000","last_clean":"2023-10-09T05:05:00.652686+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:05:00.652686+0000","last_undegraded":"2023-10-09T05:05:00.652686+0000","last_fullsized":"2023-10-09T05:05:00.652686+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"661'6121","reported_seq":12994,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:03.450576+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:05:03.450576+0000","last_peered":"2023-10-09T05:05:03.450576+0000","last_clean":"2023-10-09T05:05:03.450576+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:05:03.450576+0000","last_undegraded":"2023-10-09T05:05:03.450576+0000","last_fullsized":"2023-10-09T05:05:03.450576+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3828,"log_dups_size":0,"ondisk_log_size":3828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101220364,"num_objects":1545,"num_object_clones":0,"num_object_copies":4635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1545,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3259,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":295,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640160+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:05:00.640160+0000","last_peered":"2023-10-09T05:05:00.640160+0000","last_clean":"2023-10-09T05:05:00.640160+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:05:00.640160+0000","last_undegraded":"2023-10-09T05:05:00.640160+0000","last_fullsized":"2023-10-09T05:05:00.640160+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":156,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.667457+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:05:00.667457+0000","last_peered":"2023-10-09T05:05:00.667457+0000","last_clean":"2023-10-09T05:05:00.667457+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:05:00.667457+0000","last_undegraded":"2023-10-09T05:05:00.667457+0000","last_fullsized":"2023-10-09T05:05:00.667457+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"661'5600","reported_seq":11799,"reported_epoch":661,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:05:02.738520+0000","last_change":"2023-10-09T05:05:00.112229+0000","last_active":"2023-10-09T05:05:02.738520+0000","last_peered":"2023-10-09T05:05:02.738520+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.593716+0000","last_became_peered":"2023-10-09T05:03:28.593716+0000","last_unstale":"2023-10-09T05:05:02.738520+0000","last_undegraded":"2023-10-09T05:05:02.738520+0000","last_fullsized":"2023-10-09T05:03:26.339481+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5600,"log_dups_size":0,"ondisk_log_size":5600,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48660480,"num_objects":742,"num_object_clones":0,"num_object_copies":2226,"num_objects_missing_on_primary":676,"num_objects_missing":676,"num_objects_degraded":0,"num_objects_misplaced":1675,"num_objects_unfound":0,"num_objects_dirty":742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1665,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":48349184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":39},{"shards":"1,3","objects":97},{"shards":"3,4,5,6,7","objects":90},{"shards":"3,4,5,7","objects":2},{"shards":"4,5,7","objects":514}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":190,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638283+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:05:00.638283+0000","last_peered":"2023-10-09T05:05:00.638283+0000","last_clean":"2023-10-09T05:05:00.638283+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:05:00.638283+0000","last_undegraded":"2023-10-09T05:05:00.638283+0000","last_fullsized":"2023-10-09T05:05:00.638283+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.646706+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:05:00.646706+0000","last_peered":"2023-10-09T05:05:00.646706+0000","last_clean":"2023-10-09T05:05:00.646706+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:05:00.646706+0000","last_undegraded":"2023-10-09T05:05:00.646706+0000","last_fullsized":"2023-10-09T05:05:00.646706+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"661'5647","reported_seq":9073,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.881671+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:05:02.881671+0000","last_peered":"2023-10-09T05:05:02.881671+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:05:02.881671+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50397184,"num_objects":769,"num_object_clones":0,"num_object_copies":2307,"num_objects_missing_on_primary":947,"num_objects_missing":947,"num_objects_degraded":2259,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1663,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":453,"num_bytes_recovered":29655040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":549},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":18,"reported_epoch":660,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:00.638308+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:05:00.638308+0000","last_peered":"2023-10-09T05:05:00.638308+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:05:00.638308+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:05:00.638308+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":147,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638007+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:05:00.638007+0000","last_peered":"2023-10-09T05:05:00.638007+0000","last_clean":"2023-10-09T05:05:00.638007+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:05:00.638007+0000","last_undegraded":"2023-10-09T05:05:00.638007+0000","last_fullsized":"2023-10-09T05:05:00.638007+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"661'5670","reported_seq":9784,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:02.080407+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:05:02.080407+0000","last_peered":"2023-10-09T05:05:02.080407+0000","last_clean":"2023-10-09T05:05:02.080407+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:05:02.080407+0000","last_undegraded":"2023-10-09T05:05:02.080407+0000","last_fullsized":"2023-10-09T05:05:02.080407+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5132,"log_dups_size":0,"ondisk_log_size":5132,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49430528,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1677,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":43,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649176+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:05:00.649176+0000","last_peered":"2023-10-09T05:05:00.649176+0000","last_clean":"2023-10-09T05:05:00.649176+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:05:00.649176+0000","last_undegraded":"2023-10-09T05:05:00.649176+0000","last_fullsized":"2023-10-09T05:05:00.649176+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":207,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640292+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:05:00.640292+0000","last_peered":"2023-10-09T05:05:00.640292+0000","last_clean":"2023-10-09T05:05:00.640292+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:05:00.640292+0000","last_undegraded":"2023-10-09T05:05:00.640292+0000","last_fullsized":"2023-10-09T05:05:00.640292+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"661'5658","reported_seq":16349,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:02.723050+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:05:02.723050+0000","last_peered":"2023-10-09T05:05:02.723050+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:05:02.723050+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:05:02.723050+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5658,"log_dups_size":0,"ondisk_log_size":5658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1674,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":755}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":721,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649250+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:05:00.649250+0000","last_peered":"2023-10-09T05:05:00.649250+0000","last_clean":"2023-10-09T05:05:00.649250+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:05:00.649250+0000","last_undegraded":"2023-10-09T05:05:00.649250+0000","last_fullsized":"2023-10-09T05:05:00.649250+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":96,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.661905+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:05:00.661905+0000","last_peered":"2023-10-09T05:05:00.661905+0000","last_clean":"2023-10-09T05:05:00.661905+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:05:00.661905+0000","last_undegraded":"2023-10-09T05:05:00.661905+0000","last_fullsized":"2023-10-09T05:05:00.661905+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1249,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640192+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:05:00.640192+0000","last_peered":"2023-10-09T05:05:00.640192+0000","last_clean":"2023-10-09T05:05:00.640192+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:05:00.640192+0000","last_undegraded":"2023-10-09T05:05:00.640192+0000","last_fullsized":"2023-10-09T05:05:00.640192+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"661'5640","reported_seq":9395,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.431194+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:05:03.431194+0000","last_peered":"2023-10-09T05:05:03.431194+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:05:03.431194+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100433920,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":1845,"num_objects_missing":1845,"num_objects_degraded":4522,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3183,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":827,"num_bytes_recovered":54198272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":37},{"shards":"0,2,4,5","objects":1},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":1098}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":264,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.648467+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:05:00.648467+0000","last_peered":"2023-10-09T05:05:00.648467+0000","last_clean":"2023-10-09T05:05:00.648467+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:05:00.648467+0000","last_undegraded":"2023-10-09T05:05:00.648467+0000","last_fullsized":"2023-10-09T05:05:00.648467+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":99,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.639405+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:05:00.639405+0000","last_peered":"2023-10-09T05:05:00.639405+0000","last_clean":"2023-10-09T05:05:00.639405+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:05:00.639405+0000","last_undegraded":"2023-10-09T05:05:00.639405+0000","last_fullsized":"2023-10-09T05:05:00.639405+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1238,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640065+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:05:00.640065+0000","last_peered":"2023-10-09T05:05:00.640065+0000","last_clean":"2023-10-09T05:05:00.640065+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:05:00.640065+0000","last_undegraded":"2023-10-09T05:05:00.640065+0000","last_fullsized":"2023-10-09T05:05:00.640065+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"661'5867","reported_seq":14240,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.211879+0000","last_change":"2023-10-09T05:03:46.353847+0000","last_active":"2023-10-09T05:05:02.211879+0000","last_peered":"2023-10-09T05:05:02.211879+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:03:45.583447+0000","last_became_peered":"2023-10-09T05:03:45.583447+0000","last_unstale":"2023-10-09T05:05:02.211879+0000","last_undegraded":"2023-10-09T05:03:41.179518+0000","last_fullsized":"2023-10-09T05:03:38.964620+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102334464,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":328,"num_objects_missing":328,"num_objects_degraded":2781,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3325,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2158,"num_bytes_recovered":140443648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,2,4,5","objects":1197},{"shards":"2,4,5","objects":360},{"shards":"4,5","objects":-403}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":204,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640248+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:05:00.640248+0000","last_peered":"2023-10-09T05:05:00.640248+0000","last_clean":"2023-10-09T05:05:00.640248+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:05:00.640248+0000","last_undegraded":"2023-10-09T05:05:00.640248+0000","last_fullsized":"2023-10-09T05:05:00.640248+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":19,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638101+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:05:00.638101+0000","last_peered":"2023-10-09T05:05:00.638101+0000","last_clean":"2023-10-09T05:05:00.638101+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:05:00.638101+0000","last_undegraded":"2023-10-09T05:05:00.638101+0000","last_fullsized":"2023-10-09T05:05:00.638101+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"661'5863","reported_seq":9490,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.830260+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:05:02.830260+0000","last_peered":"2023-10-09T05:05:02.830260+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:05:02.830260+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5863,"log_dups_size":0,"ondisk_log_size":5863,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100696064,"num_objects":1536,"num_object_clones":0,"num_object_copies":4608,"num_objects_missing_on_primary":1143,"num_objects_missing":1143,"num_objects_degraded":2673,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3323,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":893,"num_bytes_recovered":58556416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,4,7","objects":1143},{"shards":"3,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":133,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638155+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:05:00.638155+0000","last_peered":"2023-10-09T05:05:00.638155+0000","last_clean":"2023-10-09T05:05:00.638155+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:05:00.638155+0000","last_undegraded":"2023-10-09T05:05:00.638155+0000","last_fullsized":"2023-10-09T05:05:00.638155+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638790+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:05:00.638790+0000","last_peered":"2023-10-09T05:05:00.638790+0000","last_clean":"2023-10-09T05:05:00.638790+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:05:00.638790+0000","last_undegraded":"2023-10-09T05:05:00.638790+0000","last_fullsized":"2023-10-09T05:05:00.638790+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1258,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640410+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:05:00.640410+0000","last_peered":"2023-10-09T05:05:00.640410+0000","last_clean":"2023-10-09T05:05:00.640410+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:05:00.640410+0000","last_undegraded":"2023-10-09T05:05:00.640410+0000","last_fullsized":"2023-10-09T05:05:00.640410+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"661'5888","reported_seq":19697,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:03.220670+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:05:03.220670+0000","last_peered":"2023-10-09T05:05:03.220670+0000","last_clean":"2023-10-09T05:05:03.220670+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:05:03.220670+0000","last_undegraded":"2023-10-09T05:05:03.220670+0000","last_fullsized":"2023-10-09T05:05:03.220670+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99155968,"num_objects":1513,"num_object_clones":0,"num_object_copies":4539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3348,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":179,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.650592+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:05:00.650592+0000","last_peered":"2023-10-09T05:05:00.650592+0000","last_clean":"2023-10-09T05:05:00.650592+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:05:00.650592+0000","last_undegraded":"2023-10-09T05:05:00.650592+0000","last_fullsized":"2023-10-09T05:05:00.650592+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":88,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.686799+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:05:00.686799+0000","last_peered":"2023-10-09T05:05:00.686799+0000","last_clean":"2023-10-09T05:05:00.686799+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:05:00.686799+0000","last_undegraded":"2023-10-09T05:05:00.686799+0000","last_fullsized":"2023-10-09T05:05:00.686799+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1264,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.641944+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:05:00.641944+0000","last_peered":"2023-10-09T05:05:00.641944+0000","last_clean":"2023-10-09T05:05:00.641944+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:05:00.641944+0000","last_undegraded":"2023-10-09T05:05:00.641944+0000","last_fullsized":"2023-10-09T05:05:00.641944+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"661'6010","reported_seq":10776,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.335991+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:05:03.335991+0000","last_peered":"2023-10-09T05:05:03.335991+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:05:03.335991+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6010,"log_dups_size":0,"ondisk_log_size":6010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104398848,"num_objects":1593,"num_object_clones":0,"num_object_copies":4779,"num_objects_missing_on_primary":1530,"num_objects_missing":1530,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3401,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1380,"num_bytes_recovered":90439680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,3,5,7","objects":475},{"shards":"0,3","objects":-458},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":738}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":76,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.659656+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:05:00.659656+0000","last_peered":"2023-10-09T05:05:00.659656+0000","last_clean":"2023-10-09T05:05:00.659656+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:05:00.659656+0000","last_undegraded":"2023-10-09T05:05:00.659656+0000","last_fullsized":"2023-10-09T05:05:00.659656+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649313+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:05:00.649313+0000","last_peered":"2023-10-09T05:05:00.649313+0000","last_clean":"2023-10-09T05:05:00.649313+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:05:00.649313+0000","last_undegraded":"2023-10-09T05:05:00.649313+0000","last_fullsized":"2023-10-09T05:05:00.649313+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":94,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.639252+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:05:00.639252+0000","last_peered":"2023-10-09T05:05:00.639252+0000","last_clean":"2023-10-09T05:05:00.639252+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:05:00.639252+0000","last_undegraded":"2023-10-09T05:05:00.639252+0000","last_fullsized":"2023-10-09T05:05:00.639252+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1247,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640198+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:05:00.640198+0000","last_peered":"2023-10-09T05:05:00.640198+0000","last_clean":"2023-10-09T05:05:00.640198+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:05:00.640198+0000","last_undegraded":"2023-10-09T05:05:00.640198+0000","last_fullsized":"2023-10-09T05:05:00.640198+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"661'6164","reported_seq":17812,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.716240+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:05:03.716240+0000","last_peered":"2023-10-09T05:05:03.716240+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:05:03.716240+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4483,"log_dups_size":0,"ondisk_log_size":4483,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97484800,"num_objects":1487,"num_object_clones":0,"num_object_copies":4461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2053,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3304,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2844,"num_bytes_recovered":181796864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":357},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":1004}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":745,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.647885+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:05:00.647885+0000","last_peered":"2023-10-09T05:05:00.647885+0000","last_clean":"2023-10-09T05:05:00.647885+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:05:00.647885+0000","last_undegraded":"2023-10-09T05:05:00.647885+0000","last_fullsized":"2023-10-09T05:05:00.647885+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638223+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:05:00.638223+0000","last_peered":"2023-10-09T05:05:00.638223+0000","last_clean":"2023-10-09T05:05:00.638223+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:05:00.638223+0000","last_undegraded":"2023-10-09T05:05:00.638223+0000","last_fullsized":"2023-10-09T05:05:00.638223+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":76,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.659579+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:05:00.659579+0000","last_peered":"2023-10-09T05:05:00.659579+0000","last_clean":"2023-10-09T05:05:00.659579+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:05:00.659579+0000","last_undegraded":"2023-10-09T05:05:00.659579+0000","last_fullsized":"2023-10-09T05:05:00.659579+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1258,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640331+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:05:00.640331+0000","last_peered":"2023-10-09T05:05:00.640331+0000","last_clean":"2023-10-09T05:05:00.640331+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:05:00.640331+0000","last_undegraded":"2023-10-09T05:05:00.640331+0000","last_fullsized":"2023-10-09T05:05:00.640331+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"661'6159","reported_seq":11933,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:03.435961+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:05:03.435961+0000","last_peered":"2023-10-09T05:05:03.435961+0000","last_clean":"2023-10-09T05:05:03.435961+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:05:03.435961+0000","last_undegraded":"2023-10-09T05:05:03.435961+0000","last_fullsized":"2023-10-09T05:05:03.435961+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3866,"log_dups_size":0,"ondisk_log_size":3866,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98729996,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3297,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":155,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649580+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:05:00.649580+0000","last_peered":"2023-10-09T05:05:00.649580+0000","last_clean":"2023-10-09T05:05:00.649580+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:05:00.649580+0000","last_undegraded":"2023-10-09T05:05:00.649580+0000","last_fullsized":"2023-10-09T05:05:00.649580+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":74,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.660510+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:05:00.660510+0000","last_peered":"2023-10-09T05:05:00.660510+0000","last_clean":"2023-10-09T05:05:00.660510+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:05:00.660510+0000","last_undegraded":"2023-10-09T05:05:00.660510+0000","last_fullsized":"2023-10-09T05:05:00.660510+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.686755+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:05:00.686755+0000","last_peered":"2023-10-09T05:05:00.686755+0000","last_clean":"2023-10-09T05:05:00.686755+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:05:00.686755+0000","last_undegraded":"2023-10-09T05:05:00.686755+0000","last_fullsized":"2023-10-09T05:05:00.686755+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"661'6030","reported_seq":10530,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.846794+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:05:02.846794+0000","last_peered":"2023-10-09T05:05:02.846794+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:05:02.846794+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3903,"log_dups_size":0,"ondisk_log_size":3903,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":1225,"num_objects_missing":1225,"num_objects_degraded":3106,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3201,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1121,"num_bytes_recovered":73433088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":1134},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":40}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":49,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.646655+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:05:00.646655+0000","last_peered":"2023-10-09T05:05:00.646655+0000","last_clean":"2023-10-09T05:05:00.646655+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:05:00.646655+0000","last_undegraded":"2023-10-09T05:05:00.646655+0000","last_fullsized":"2023-10-09T05:05:00.646655+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1264,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.639941+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:05:00.639941+0000","last_peered":"2023-10-09T05:05:00.639941+0000","last_clean":"2023-10-09T05:05:00.639941+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:05:00.639941+0000","last_undegraded":"2023-10-09T05:05:00.639941+0000","last_fullsized":"2023-10-09T05:05:00.639941+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":153,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.650862+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:05:00.650862+0000","last_peered":"2023-10-09T05:05:00.650862+0000","last_clean":"2023-10-09T05:05:00.650862+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:05:00.650862+0000","last_undegraded":"2023-10-09T05:05:00.650862+0000","last_fullsized":"2023-10-09T05:05:00.650862+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640854+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:05:00.640854+0000","last_peered":"2023-10-09T05:05:00.640854+0000","last_clean":"2023-10-09T05:05:00.640854+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:05:00.640854+0000","last_undegraded":"2023-10-09T05:05:00.640854+0000","last_fullsized":"2023-10-09T05:05:00.640854+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1237,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640727+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:05:00.640727+0000","last_peered":"2023-10-09T05:05:00.640727+0000","last_clean":"2023-10-09T05:05:00.640727+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:05:00.640727+0000","last_undegraded":"2023-10-09T05:05:00.640727+0000","last_fullsized":"2023-10-09T05:05:00.640727+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"661'6238","reported_seq":14494,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.274687+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:05:03.274687+0000","last_peered":"2023-10-09T05:05:03.274687+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:05:03.274687+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4475,"log_dups_size":0,"ondisk_log_size":4475,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100663296,"num_objects":1536,"num_object_clones":0,"num_object_copies":4608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3339,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2202,"num_bytes_recovered":131989504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":550},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":939}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":74,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.660589+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:05:00.660589+0000","last_peered":"2023-10-09T05:05:00.660589+0000","last_clean":"2023-10-09T05:05:00.660589+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:05:00.660589+0000","last_undegraded":"2023-10-09T05:05:00.660589+0000","last_fullsized":"2023-10-09T05:05:00.660589+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":19,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.637863+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:05:00.637863+0000","last_peered":"2023-10-09T05:05:00.637863+0000","last_clean":"2023-10-09T05:05:00.637863+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:05:00.637863+0000","last_undegraded":"2023-10-09T05:05:00.637863+0000","last_fullsized":"2023-10-09T05:05:00.637863+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":112,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.651515+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:05:00.651515+0000","last_peered":"2023-10-09T05:05:00.651515+0000","last_clean":"2023-10-09T05:05:00.651515+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:05:00.651515+0000","last_undegraded":"2023-10-09T05:05:00.651515+0000","last_fullsized":"2023-10-09T05:05:00.651515+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1258,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.642946+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:05:00.642946+0000","last_peered":"2023-10-09T05:05:00.642946+0000","last_clean":"2023-10-09T05:05:00.642946+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:05:00.642946+0000","last_undegraded":"2023-10-09T05:05:00.642946+0000","last_fullsized":"2023-10-09T05:05:00.642946+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"661'5853","reported_seq":17882,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:02.185273+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:05:02.185273+0000","last_peered":"2023-10-09T05:05:02.185273+0000","last_clean":"2023-10-09T05:05:02.185273+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:05:02.185273+0000","last_undegraded":"2023-10-09T05:05:02.185273+0000","last_fullsized":"2023-10-09T05:05:02.185273+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103186432,"num_objects":1574,"num_object_clones":0,"num_object_copies":4722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3314,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":126,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638577+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:05:00.638577+0000","last_peered":"2023-10-09T05:05:00.638577+0000","last_clean":"2023-10-09T05:05:00.638577+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:05:00.638577+0000","last_undegraded":"2023-10-09T05:05:00.638577+0000","last_fullsized":"2023-10-09T05:05:00.638577+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":85,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.659418+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:05:00.659418+0000","last_peered":"2023-10-09T05:05:00.659418+0000","last_clean":"2023-10-09T05:05:00.659418+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:05:00.659418+0000","last_undegraded":"2023-10-09T05:05:00.659418+0000","last_fullsized":"2023-10-09T05:05:00.659418+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":39,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.647996+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:05:00.647996+0000","last_peered":"2023-10-09T05:05:00.647996+0000","last_clean":"2023-10-09T05:05:00.647996+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:05:00.647996+0000","last_undegraded":"2023-10-09T05:05:00.647996+0000","last_fullsized":"2023-10-09T05:05:00.647996+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1261,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.640739+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:05:00.640739+0000","last_peered":"2023-10-09T05:05:00.640739+0000","last_clean":"2023-10-09T05:05:00.640739+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:05:00.640739+0000","last_undegraded":"2023-10-09T05:05:00.640739+0000","last_fullsized":"2023-10-09T05:05:00.640739+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"661'6008","reported_seq":23358,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:02.178699+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:05:02.178699+0000","last_peered":"2023-10-09T05:05:02.178699+0000","last_clean":"2023-10-09T05:05:02.178699+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:05:02.178699+0000","last_undegraded":"2023-10-09T05:05:02.178699+0000","last_fullsized":"2023-10-09T05:05:02.178699+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103809024,"num_objects":1584,"num_object_clones":0,"num_object_copies":4752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3187,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":284,"reported_epoch":660,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:00.681309+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:05:00.681309+0000","last_peered":"2023-10-09T05:05:00.681309+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:05:00.681309+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:05:00.681309+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.666840+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:05:00.666840+0000","last_peered":"2023-10-09T05:05:00.666840+0000","last_clean":"2023-10-09T05:05:00.666840+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:05:00.666840+0000","last_undegraded":"2023-10-09T05:05:00.666840+0000","last_fullsized":"2023-10-09T05:05:00.666840+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":164,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.650504+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:05:00.650504+0000","last_peered":"2023-10-09T05:05:00.650504+0000","last_clean":"2023-10-09T05:05:00.650504+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:05:00.650504+0000","last_undegraded":"2023-10-09T05:05:00.650504+0000","last_fullsized":"2023-10-09T05:05:00.650504+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1217,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.746766+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:05:01.746766+0000","last_peered":"2023-10-09T05:05:01.746766+0000","last_clean":"2023-10-09T05:05:01.746766+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:05:01.746766+0000","last_undegraded":"2023-10-09T05:05:01.746766+0000","last_fullsized":"2023-10-09T05:05:01.746766+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"661'6367","reported_seq":11688,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.448092+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:05:03.448092+0000","last_peered":"2023-10-09T05:05:03.448092+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:05:03.448092+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4942,"log_dups_size":0,"ondisk_log_size":4942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103088128,"num_objects":1573,"num_object_clones":0,"num_object_copies":4719,"num_objects_missing_on_primary":1965,"num_objects_missing":1965,"num_objects_degraded":5199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3407,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1198,"num_bytes_recovered":78512128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-2},{"shards":"0,2,4,7","objects":1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":1503},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":177,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.642092+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:05:00.642092+0000","last_peered":"2023-10-09T05:05:00.642092+0000","last_clean":"2023-10-09T05:05:00.642092+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:05:00.642092+0000","last_undegraded":"2023-10-09T05:05:00.642092+0000","last_fullsized":"2023-10-09T05:05:00.642092+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":95,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.666705+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:05:00.666705+0000","last_peered":"2023-10-09T05:05:00.666705+0000","last_clean":"2023-10-09T05:05:00.666705+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:05:00.666705+0000","last_undegraded":"2023-10-09T05:05:00.666705+0000","last_fullsized":"2023-10-09T05:05:00.666705+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.637982+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:05:00.637982+0000","last_peered":"2023-10-09T05:05:00.637982+0000","last_clean":"2023-10-09T05:05:00.637982+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:05:00.637982+0000","last_undegraded":"2023-10-09T05:05:00.637982+0000","last_fullsized":"2023-10-09T05:05:00.637982+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"661'5697","reported_seq":9685,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:03.398963+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:05:03.398963+0000","last_peered":"2023-10-09T05:05:03.398963+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:05:03.398963+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96698368,"num_objects":1476,"num_object_clones":0,"num_object_copies":4428,"num_objects_missing_on_primary":1900,"num_objects_missing":1900,"num_objects_degraded":4605,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3241,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":883,"num_bytes_recovered":57868288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-36},{"shards":"0,2,3","objects":2},{"shards":"0,2,4,5","objects":1},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":1124}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":126,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638379+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:05:00.638379+0000","last_peered":"2023-10-09T05:05:00.638379+0000","last_clean":"2023-10-09T05:05:00.638379+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:05:00.638379+0000","last_undegraded":"2023-10-09T05:05:00.638379+0000","last_fullsized":"2023-10-09T05:05:00.638379+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":76,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.659501+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:05:00.659501+0000","last_peered":"2023-10-09T05:05:00.659501+0000","last_clean":"2023-10-09T05:05:00.659501+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:05:00.659501+0000","last_undegraded":"2023-10-09T05:05:00.659501+0000","last_fullsized":"2023-10-09T05:05:00.659501+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"661'5782","reported_seq":10617,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:02.178966+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:05:02.178966+0000","last_peered":"2023-10-09T05:05:02.178966+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:05:02.178966+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:05:02.178966+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105644032,"num_objects":1612,"num_object_clones":0,"num_object_copies":4836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3247,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1612}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":796,"reported_epoch":660,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:00.649331+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:05:00.649331+0000","last_peered":"2023-10-09T05:05:00.649331+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:05:00.649331+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:05:00.649331+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":19,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.638179+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:05:00.638179+0000","last_peered":"2023-10-09T05:05:00.638179+0000","last_clean":"2023-10-09T05:05:00.638179+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:05:00.638179+0000","last_undegraded":"2023-10-09T05:05:00.638179+0000","last_fullsized":"2023-10-09T05:05:00.638179+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"661'5824","reported_seq":10243,"reported_epoch":661,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:02.109358+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:05:02.109358+0000","last_peered":"2023-10-09T05:05:02.109358+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:05:02.109358+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5286,"log_dups_size":0,"ondisk_log_size":5286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102924288,"num_objects":1571,"num_object_clones":0,"num_object_copies":4713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3293,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":924}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":742,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649537+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:05:00.649537+0000","last_peered":"2023-10-09T05:05:00.649537+0000","last_clean":"2023-10-09T05:05:00.649537+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:05:00.649537+0000","last_undegraded":"2023-10-09T05:05:00.649537+0000","last_fullsized":"2023-10-09T05:05:00.649537+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.637935+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:05:00.637935+0000","last_peered":"2023-10-09T05:05:00.637935+0000","last_clean":"2023-10-09T05:05:00.637935+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:05:00.637935+0000","last_undegraded":"2023-10-09T05:05:00.637935+0000","last_fullsized":"2023-10-09T05:05:00.637935+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"661'5817","reported_seq":16617,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:02.319750+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:05:02.319750+0000","last_peered":"2023-10-09T05:05:02.319750+0000","last_clean":"2023-10-09T05:05:02.319750+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:05:02.319750+0000","last_undegraded":"2023-10-09T05:05:02.319750+0000","last_fullsized":"2023-10-09T05:05:02.319750+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5817,"log_dups_size":0,"ondisk_log_size":5817,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103481344,"num_objects":1579,"num_object_clones":0,"num_object_copies":4737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3285,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":281,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.652598+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:05:00.652598+0000","last_peered":"2023-10-09T05:05:00.652598+0000","last_clean":"2023-10-09T05:05:00.652598+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:05:00.652598+0000","last_undegraded":"2023-10-09T05:05:00.652598+0000","last_fullsized":"2023-10-09T05:05:00.652598+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":289,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.650034+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:05:00.650034+0000","last_peered":"2023-10-09T05:05:00.650034+0000","last_clean":"2023-10-09T05:05:00.650034+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:05:00.650034+0000","last_undegraded":"2023-10-09T05:05:00.650034+0000","last_fullsized":"2023-10-09T05:05:00.650034+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"661'5755","reported_seq":17357,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:03.181365+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:05:03.181365+0000","last_peered":"2023-10-09T05:05:03.181365+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:05:03.181365+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:05:03.181365+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102170624,"num_objects":1559,"num_object_clones":0,"num_object_copies":4677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3256,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":1559}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":41,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.649642+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:05:00.649642+0000","last_peered":"2023-10-09T05:05:00.649642+0000","last_clean":"2023-10-09T05:05:00.649642+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:05:00.649642+0000","last_undegraded":"2023-10-09T05:05:00.649642+0000","last_fullsized":"2023-10-09T05:05:00.649642+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-10-09T05:05:00.637959+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:05:00.637959+0000","last_peered":"2023-10-09T05:05:00.637959+0000","last_clean":"2023-10-09T05:05:00.637959+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:05:00.637959+0000","last_undegraded":"2023-10-09T05:05:00.637959+0000","last_fullsized":"2023-10-09T05:05:00.637959+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":3253272600,"num_objects":49642,"num_object_clones":0,"num_object_copies":148926,"num_objects_missing_on_primary":17448,"num_objects_missing":17448,"num_objects_degraded":54780,"num_objects_misplaced":5945,"num_objects_unfound":0,"num_objects_dirty":49642,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":105576,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69228,"num_bytes_recovered":4219928576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15153115136,"data_stored":15153102920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":189355,"ondisk_log_size":189355,"up":108,"acting":90,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234692,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3391144,"kb_used_data":1869600,"kb_used_omap":0,"kb_used_meta":1521536,"kb_avail":90354008,"statfs":{"total":95995035648,"available":92522504192,"internally_reserved":0,"allocated":1914470400,"data_stored":1912807992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1558052864},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:14 2023","interfaces":[{"interface":"back","average":{"1min":11.875999999999999,"5min":3.4199999999999999,"15min":1.633},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":123.04000000000001,"5min":123.04000000000001,"15min":123.04000000000001},"last":10.1},{"interface":"front","average":{"1min":11.739000000000001,"5min":3.5819999999999999,"15min":1.9039999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":123.059,"5min":123.059,"15min":123.059},"last":2.0190000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:04:14 2023","interfaces":[{"interface":"back","average":{"1min":4.3799999999999999,"5min":2.3450000000000002,"15min":2.29},"min":{"1min":0.29999999999999999,"5min":0.28899999999999998,"15min":0.246},"max":{"1min":40.651000000000003,"5min":43.158000000000001,"15min":122.56},"last":2.1349999999999998},{"interface":"front","average":{"1min":3.5350000000000001,"5min":1.976,"15min":2.214},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":43.331000000000003,"5min":43.331000000000003,"15min":122.666},"last":10.419}]},{"osd":2,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.4569999999999999,"15min":3.8889999999999998},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.30299999999999999},"max":{"1min":10.545,"5min":154.76400000000001,"15min":207.96600000000001},"last":1.7569999999999999},{"interface":"front","average":{"1min":2.8029999999999999,"5min":4.4340000000000002,"15min":4.0110000000000001},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.29699999999999999},"max":{"1min":10.532,"5min":154.80000000000001,"15min":207.97900000000001},"last":1.72}]},{"osd":3,"last update":"Mon Oct 9 05:04:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":2.3730000000000002,"15min":1.8959999999999999},"min":{"1min":0.36299999999999999,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":8.5950000000000006,"5min":43.103000000000002,"15min":74.259},"last":1.915},{"interface":"front","average":{"1min":1.319,"5min":2.2810000000000001,"15min":1.9159999999999999},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.29299999999999998},"max":{"1min":5.593,"5min":32.804000000000002,"15min":74.100999999999999},"last":3.2440000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.32900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.6579999999999999,"15min":1.7929999999999999},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":5.5970000000000004,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.443},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.8169999999999999,"15min":2.149},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":0.57899999999999996,"5min":53.225000000000001,"15min":135.74199999999999},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":1.4730000000000001,"15min":1.23},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.57299999999999995,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45500000000000002,"5min":1.736,"15min":1.4339999999999999},"min":{"1min":0.34899999999999998,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":0.56100000000000005,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.41099999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496325,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2196160,"kb_used_data":1009592,"kb_used_omap":0,"kb_used_meta":1186560,"kb_avail":91548992,"statfs":{"total":95995035648,"available":93746167808,"internally_reserved":0,"allocated":1033822208,"data_stored":1032152359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1215037440},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":4.3780000000000001,"5min":5.883,"15min":2.4319999999999999},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":47.750999999999998,"5min":321.346,"15min":321.346},"last":0.255},{"interface":"front","average":{"1min":3.718,"5min":4.1790000000000003,"15min":2.0510000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":47.645000000000003,"5min":169.119,"15min":169.119},"last":0.84199999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":2.4590000000000001,"15min":4.657},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":6.5759999999999996,"5min":72.840999999999994,"15min":260.738},"last":0.96599999999999997},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.7370000000000001,"15min":4.6319999999999997},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.2770000000000001,"5min":72.832999999999998,"15min":261.26100000000002},"last":0.55600000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.4399999999999999,"15min":1.994},"min":{"1min":0.35299999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.032999999999999,"5min":23.943000000000001,"15min":133.535},"last":2.5430000000000001},{"interface":"front","average":{"1min":1.591,"5min":1.421,"15min":1.9390000000000001},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.30599999999999999},"max":{"1min":8.5920000000000005,"5min":25.978999999999999,"15min":133.51900000000001},"last":2.5539999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":5.9359999999999999,"5min":2.4500000000000002,"15min":2.6579999999999999},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":64.355999999999995,"5min":64.355999999999995,"15min":147.43600000000001},"last":2.512},{"interface":"front","average":{"1min":5.7930000000000001,"5min":2.2589999999999999,"15min":2.7370000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":56.133000000000003,"5min":56.133000000000003,"15min":147.41999999999999},"last":5.2939999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1710000000000003,"5min":7.1710000000000003,"15min":7.1710000000000003},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":11.102,"5min":11.102,"15min":11.102},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.085,"15min":1.3100000000000001},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.8300000000000001,"5min":25.934000000000001,"15min":66.995999999999995},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.85999999999999999,"15min":1.3120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.254},"max":{"1min":7.2220000000000004,"5min":16.789000000000001,"15min":67.189999999999998},"last":0.32900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.66900000000000004,"15min":1.1180000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":0.59299999999999997,"5min":5.835,"15min":36.204999999999998},"last":0.378},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.002,"15min":1.1259999999999999},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":2.8069999999999999,"5min":20.041,"15min":20.041},"last":0.434}]}]},{"osd":1,"up_from":118,"seq":506806141392,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2304200,"kb_used_data":1032704,"kb_used_omap":4,"kb_used_meta":1271483,"kb_avail":91440952,"statfs":{"total":95995035648,"available":93635534848,"internally_reserved":0,"allocated":1057488896,"data_stored":1053985726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1301999502},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8949999999999996,"5min":2.431,"15min":1.361},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":85.753,"5min":85.753,"15min":85.753},"last":2.7599999999999998},{"interface":"front","average":{"1min":5.9269999999999996,"5min":2.4049999999999998,"15min":1.355},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.214},"max":{"1min":85.227999999999994,"5min":85.227999999999994,"15min":85.227999999999994},"last":2.6890000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1639999999999999,"15min":2.4140000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":6.3129999999999997,"5min":12.460000000000001,"15min":263.089},"last":2.734},{"interface":"front","average":{"1min":2.2509999999999999,"5min":1.4870000000000001,"15min":2.4689999999999999},"min":{"1min":0.254,"5min":0.252,"15min":0.222},"max":{"1min":8.5250000000000004,"5min":13.933999999999999,"15min":254.50899999999999},"last":3.2269999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:04:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":1.6040000000000001,"15min":1.6379999999999999},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":28.797000000000001,"5min":28.797000000000001,"15min":77.819999999999993},"last":5.5250000000000004},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.0189999999999999,"15min":1.331},"min":{"1min":0.28499999999999998,"5min":0.249,"15min":0.249},"max":{"1min":5.8319999999999999,"5min":7.0860000000000003,"15min":77.796000000000006},"last":3.266}]},{"osd":4,"last update":"Mon Oct 9 05:04:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.7800000000000002,"5min":5.7800000000000002,"15min":5.7800000000000002},"last":0.436},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":1.3799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.03,"15min":1.1779999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":5.8109999999999999,"5min":7.79,"15min":7.79},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.4510000000000001,"15min":1.2350000000000001},"min":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":10.554,"5min":10.554,"15min":19.597999999999999},"last":1.345}]},{"osd":6,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.125,"15min":1.157},"min":{"1min":0.41099999999999998,"5min":0.39700000000000002,"15min":0.35699999999999998},"max":{"1min":5.7489999999999997,"5min":7.4370000000000003,"15min":7.4370000000000003},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.4430000000000001,"5min":1.3440000000000001,"15min":1.252},"min":{"1min":0.39500000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.472,"5min":10.472,"15min":19.548999999999999},"last":1.4059999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.484,"15min":1.722},"min":{"1min":0.42399999999999999,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":10.583,"5min":10.583,"15min":59.972000000000001},"last":1.3080000000000001},{"interface":"front","average":{"1min":0.745,"5min":1.2150000000000001,"15min":1.6279999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":3.5670000000000002,"5min":11.903,"15min":60.011000000000003},"last":2.1480000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024476,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1887812,"kb_used_data":787644,"kb_used_omap":5,"kb_used_meta":1100154,"kb_avail":91857340,"statfs":{"total":95995035648,"available":94061916160,"internally_reserved":0,"allocated":806547456,"data_stored":803701010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1126558330},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:04:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":2.2400000000000002,"15min":4.407},"min":{"1min":0.222,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":16.731000000000002,"5min":30.007999999999999,"15min":247.68899999999999},"last":2.081},{"interface":"front","average":{"1min":2.77,"5min":1.921,"15min":4.3559999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":16.823,"5min":29.940000000000001,"15min":246.38},"last":0.84099999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.9399999999999999,"15min":2.2930000000000001},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.9340000000000002,"5min":26.556999999999999,"15min":151.06399999999999},"last":1.5569999999999999},{"interface":"front","average":{"1min":4.4400000000000004,"5min":2.2400000000000002,"15min":2.4820000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":20.239000000000001,"5min":29.032,"15min":151.22399999999999},"last":0.81100000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:04:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":5.3319999999999999,"15min":3.5870000000000002},"min":{"1min":0.47599999999999998,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":11.872,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.33300000000000002},{"interface":"front","average":{"1min":3.4510000000000001,"5min":5.3380000000000001,"15min":3.6469999999999998},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":14.541,"5min":290.73899999999998,"15min":290.73899999999998},"last":0.44900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:04:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":1.8520000000000001,"15min":1.8520000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.9320000000000004,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":2.7149999999999999},{"interface":"front","average":{"1min":1.5349999999999999,"5min":1.5349999999999999,"15min":1.5349999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.9249999999999998,"5min":6.9249999999999998,"15min":6.9249999999999998},"last":0.36599999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:04:17 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.3939999999999999,"15min":1.1830000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33500000000000002},"max":{"1min":16.888000000000002,"5min":16.888000000000002,"15min":16.888000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.659,"15min":1.1619999999999999},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":7.9909999999999997,"5min":14.036,"15min":14.036},"last":2.8109999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:17 2023","interfaces":[{"interface":"back","average":{"1min":2.609,"5min":1.6160000000000001,"15min":1.3120000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.34799999999999998},"max":{"1min":16.917000000000002,"5min":20.129999999999999,"15min":39.594999999999999},"last":0.505},{"interface":"front","average":{"1min":2.1629999999999998,"5min":2.1989999999999998,"15min":1.4850000000000001},"min":{"1min":0.436,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":8.0619999999999994,"5min":47.241,"15min":47.241},"last":2.7570000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:04:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":1.909,"15min":1.381},"min":{"1min":0.47499999999999998,"5min":0.41899999999999998,"15min":0.38100000000000001},"max":{"1min":8.0269999999999992,"5min":21.658000000000001,"15min":21.658000000000001},"last":2.786},{"interface":"front","average":{"1min":1.702,"5min":1.736,"15min":1.383},"min":{"1min":0.52800000000000002,"5min":0.41999999999999998,"15min":0.38200000000000001},"max":{"1min":6.8730000000000002,"5min":21.68,"15min":21.68},"last":0.41199999999999998}]}]},{"osd":2,"up_from":0,"seq":949187772754,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356448,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2336284,"kb_used_data":1404884,"kb_used_omap":2,"kb_used_meta":931389,"kb_avail":91408868,"statfs":{"total":95995035648,"available":93602680832,"internally_reserved":0,"allocated":1438601216,"data_stored":1435671110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":953742389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":6.5350000000000001,"5min":2.077,"15min":1.0669999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":108.631,"5min":108.631,"15min":108.631},"last":0.33000000000000002},{"interface":"front","average":{"1min":6.5250000000000004,"5min":2.226,"15min":1.1859999999999999},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.23499999999999999},"max":{"1min":110.012,"5min":110.012,"15min":110.012},"last":0.39600000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.788,"5min":1.347,"15min":2.427},"min":{"1min":0.223,"5min":0.221,"15min":0.221},"max":{"1min":15.218,"5min":15.218,"15min":125.233},"last":4.2309999999999999},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.391,"15min":2.4689999999999999},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.21299999999999999},"max":{"1min":17.544,"5min":26.692,"15min":126.102},"last":1.994}]},{"osd":2,"last update":"Mon Oct 9 05:04:36 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.3009999999999999,"15min":2.1389999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":6.4160000000000004,"5min":15.606999999999999,"15min":212.065},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.0800000000000001,"5min":1.3899999999999999,"15min":2.1789999999999998},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":12.987,"5min":12.987,"15min":211.98400000000001},"last":3.9809999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.72,"15min":1.72},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.193,"5min":11.193,"15min":11.193},"last":0.441},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.6030000000000002,"5min":3.6030000000000002,"15min":3.6030000000000002},"last":1.6100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":0.90600000000000003,"15min":1.161},"min":{"1min":0.36499999999999999,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":7.6779999999999999,"5min":7.6779999999999999,"15min":28.302},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.2070000000000001,"15min":1.3440000000000001},"min":{"1min":0.40300000000000002,"5min":0.372,"15min":0.33600000000000002},"max":{"1min":3.7869999999999999,"5min":7.3040000000000003,"15min":28.280000000000001},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.2789999999999999,"15min":1.0229999999999999},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.8149999999999999,"5min":6.383,"15min":6.383},"last":0.495},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.2529999999999999,"15min":1.2470000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.731,"5min":14.731,"15min":24.916},"last":1.585}]},{"osd":7,"last update":"Mon Oct 9 05:04:08 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.353,"15min":1.349},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.375},"max":{"1min":14.784000000000001,"5min":14.784000000000001,"15min":25.004000000000001},"last":1.544},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.165,"15min":1.081},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.758,"5min":7.758,"15min":7.9009999999999998},"last":0.58499999999999996}]}]},{"osd":4,"up_from":626,"seq":2688649527317,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3078856,"kb_used_data":2677760,"kb_used_omap":22,"kb_used_meta":401065,"kb_avail":90666296,"statfs":{"total":95995035648,"available":92842287104,"internally_reserved":0,"allocated":2742026240,"data_stored":2740348564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":410691405},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":2.2909999999999999,"15min":2.2909999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":21.292000000000002,"5min":21.292000000000002,"15min":21.292000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":3.1709999999999998,"5min":3.1709999999999998,"15min":3.1709999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":21.391999999999999,"5min":21.391999999999999,"15min":21.391999999999999},"last":0.29899999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":2.129,"15min":2.129},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.955,"5min":14.955,"15min":14.955},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.901,"5min":1.901,"15min":1.901},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":10.579000000000001,"5min":10.579000000000001,"15min":10.579000000000001},"last":0.312}]},{"osd":2,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.639,"15min":1.639},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":0.38},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":11.036,"5min":11.036,"15min":11.036},"last":0.48299999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":270.767},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.8440000000000001,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.8179999999999996,"5min":9.8179999999999996,"15min":9.8179999999999996},"last":270.274}]},{"osd":5,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7910000000000004,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.44},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42599999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757969,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4836656,"kb_used_data":3343336,"kb_used_omap":0,"kb_used_meta":1493312,"kb_avail":88908496,"statfs":{"total":95995035648,"available":91042299904,"internally_reserved":0,"allocated":3423576064,"data_stored":3421917610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1529151488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:20 2023","interfaces":[{"interface":"back","average":{"1min":13.412000000000001,"5min":3.2010000000000001,"15min":2.121},"min":{"1min":0.252,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":103.745,"5min":103.745,"15min":106.465},"last":2.4649999999999999},{"interface":"front","average":{"1min":14.061999999999999,"5min":3.4969999999999999,"15min":2.2629999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.245},"max":{"1min":103.872,"5min":103.872,"15min":106.54600000000001},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":4.8449999999999998,"15min":3.476},"min":{"1min":0.314,"5min":0.26300000000000001,"15min":0.23200000000000001},"max":{"1min":4.9240000000000004,"5min":262.26900000000001,"15min":262.26900000000001},"last":6.0529999999999999},{"interface":"front","average":{"1min":0.876,"5min":4.7000000000000002,"15min":3.6000000000000001},"min":{"1min":0.28899999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.569,"5min":262.291,"15min":262.291},"last":2.8279999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:04:38 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":3.0089999999999999,"15min":1.794},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.27000000000000002},"max":{"1min":10.382999999999999,"5min":106.238,"15min":106.238},"last":1.998},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.8170000000000002,"15min":2.0219999999999998},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":8.4960000000000004,"5min":104.31699999999999,"15min":104.31699999999999},"last":0.42999999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:05:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":5.7110000000000003,"15min":3.0899999999999999},"min":{"1min":0.41599999999999998,"5min":0.318,"15min":0.318},"max":{"1min":10.385,"5min":125.824,"15min":125.824},"last":2.3620000000000001},{"interface":"front","average":{"1min":1.8640000000000001,"5min":5.6420000000000003,"15min":2.9870000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":9.8520000000000003,"5min":125.761,"15min":125.761},"last":2.9649999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.318},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.39000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.76400000000000001,"15min":0.82899999999999996},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23300000000000001},"max":{"1min":0.69999999999999996,"5min":11.093,"15min":11.093},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.63100000000000001,"15min":0.91900000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.73099999999999998,"5min":5.9329999999999998,"15min":13.083},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:05:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.71899999999999997,"15min":1.734},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.746,"5min":8.3520000000000003,"15min":110.521},"last":0.378},{"interface":"front","average":{"1min":0.442,"5min":0.90300000000000002,"15min":1.7729999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29499999999999998},"max":{"1min":0.72199999999999998,"5min":11.791,"15min":110.746},"last":0.45400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":796262400,"data_stored":796262400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1047461888,"data_stored":1047461888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2804219904,"data_stored":2804219904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1428750336,"data_stored":1428750336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2731151360,"data_stored":2731147288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3415347200,"data_stored":3415343128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1024593920,"data_stored":1024589848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1905328128,"data_stored":1905328128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:05:09.086 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:05:09.114 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:05:09.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:08 smithi029 ceph-mon[223858]: pgmap v2116: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 54704/148083 objects degraded (36.941%); 5816/148083 objects misplaced (3.928%); 4.4 MiB/s, 86 objects/s recovering 2023-10-09T05:05:09.262 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:09.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:09.375 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:09.414 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:09.593 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:09.732 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:09.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:09.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:09.971 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:10.173 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:10.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:10.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:10.292 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:10.438 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:10.638 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:10.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:10.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:10.757 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:10.784 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:10.832 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:10.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:10.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:11.037 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:11.145 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:05:11.145 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:11.147 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:10 smithi029 ceph-mon[216903]: pgmap v2117: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 54013/146031 objects degraded (36.987%); 5492/146031 objects misplaced (3.761%); 6.6 MiB/s, 120 objects/s recovering 2023-10-09T05:05:11.148 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:10 smithi029 ceph-mon[223858]: pgmap v2117: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 54013/146031 objects degraded (36.987%); 5492/146031 objects misplaced (3.761%); 6.6 MiB/s, 120 objects/s recovering 2023-10-09T05:05:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:11.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:11.249 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:11.272 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:10 smithi177 ceph-mon[195797]: pgmap v2117: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 204 op/s; 54013/146031 objects degraded (36.987%); 5492/146031 objects misplaced (3.761%); 6.6 MiB/s, 120 objects/s recovering 2023-10-09T05:05:11.275 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:11.318 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:11.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:11.424 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:11.509 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:11.851 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:11.854 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:11.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:11.886 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:11.974 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:12.118 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:12.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 54013/146031 objects degraded (36.987%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:12.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 54013/146031 objects degraded (36.987%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:12.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 54013/146031 objects degraded (36.987%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:12.421 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:12.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:12.551 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:12.577 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:12.624 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:12.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:12.730 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:12.758 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:12.759 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:12.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:12.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:12.909 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:13.049 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:13 smithi029 ceph-mon[223858]: pgmap v2118: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 54013/146031 objects degraded (36.987%); 5492/146031 objects misplaced (3.761%); 6.0 MiB/s, 108 objects/s recovering 2023-10-09T05:05:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:13.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:13.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:05:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:05:12] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-10-09T05:05:13.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:13 smithi029 ceph-mon[216903]: pgmap v2118: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 54013/146031 objects degraded (36.987%); 5492/146031 objects misplaced (3.761%); 6.0 MiB/s, 108 objects/s recovering 2023-10-09T05:05:13.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:13.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:13.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:13 smithi177 ceph-mon[195797]: pgmap v2118: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 3.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 54013/146031 objects degraded (36.987%); 5492/146031 objects misplaced (3.761%); 6.0 MiB/s, 108 objects/s recovering 2023-10-09T05:05:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:13.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:13.995 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:14.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:14.110 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:14.137 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:14.182 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:14.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:14.287 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:14.406 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:05:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:05:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3659781769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:14.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:14 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3659781769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:05:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:05:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:15.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:14 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3659781769' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:15.294 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:15.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:15.305 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:05:15.402 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:15.429 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:15.474 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:15.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:15.581 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:15.609 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:15.655 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:15.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:15.762 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:15 smithi177 ceph-mon[195797]: pgmap v2119: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 53686/144123 objects degraded (37.250%); 4984/144123 objects misplaced (3.458%); 6.9 MiB/s, 120 objects/s recovering 2023-10-09T05:05:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:15 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3659781769' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:05:15.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:15 smithi177 ceph-mon[195797]: osdmap e662: 8 total, 8 up, 7 in 2023-10-09T05:05:15.788 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:15.835 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:15.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[223858]: pgmap v2119: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 53686/144123 objects degraded (37.250%); 4984/144123 objects misplaced (3.458%); 6.9 MiB/s, 120 objects/s recovering 2023-10-09T05:05:15.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3659781769' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:05:15.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[223858]: osdmap e662: 8 total, 8 up, 7 in 2023-10-09T05:05:15.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[216903]: pgmap v2119: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 53686/144123 objects degraded (37.250%); 4984/144123 objects misplaced (3.458%); 6.9 MiB/s, 120 objects/s recovering 2023-10-09T05:05:15.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:15.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3659781769' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:05:15.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:15 smithi029 ceph-mon[216903]: osdmap e662: 8 total, 8 up, 7 in 2023-10-09T05:05:15.940 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:16.069 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:16.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:16 smithi029 ceph-mon[216903]: failed to encode map e662 with expected crc 2023-10-09T05:05:16.905 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 53686/144123 objects degraded (37.250%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:16.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:16 smithi029 ceph-mon[223858]: failed to encode map e662 with expected crc 2023-10-09T05:05:16.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 53686/144123 objects degraded (37.250%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:16 smithi177 ceph-mon[195797]: failed to encode map e662 with expected crc 2023-10-09T05:05:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 53686/144123 objects degraded (37.250%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:17.083 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:17.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:17.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:17.204 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:17.231 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:17.277 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:17.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:17.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:17.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:17.412 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:17.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:17 smithi029 ceph-mon[216903]: pgmap v2121: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 53186/143262 objects degraded (37.125%); 4984/143262 objects misplaced (3.479%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T05:05:17.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:17 smithi029 ceph-mon[223858]: pgmap v2121: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 53186/143262 objects degraded (37.125%); 4984/143262 objects misplaced (3.479%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T05:05:17.924 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:17.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:17 smithi177 ceph-mon[195797]: pgmap v2121: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 190 op/s; 53186/143262 objects degraded (37.125%); 4984/143262 objects misplaced (3.479%); 6.4 MiB/s, 101 objects/s recovering 2023-10-09T05:05:18.029 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:18.123 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:18.293 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:18.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:18.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:18.538 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:18.571 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92266 14.3 14:c00c27be:::benchmark_data_smithi029_550670_object14655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e614)", 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:58.952190+0000", 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 139.61278644199999, 2023-10-09T05:05:18.572 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9514774369999999, 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92266 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952190+0000", 2023-10-09T05:05:18.573 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952190+0000", 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952188+0000", 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.574 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952196+0000", 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9359999999999999e-06 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.958844+0000", 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066482440000000002 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.575 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.958866+0000", 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2206000000000001e-05 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.964761+0000", 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058949500000000004 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.964773+0000", 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2123000000000001e-05 2023-10-09T05:05:18.576 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.132948+0000", 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1681750040000001 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.134184+0000", 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012359319999999999 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.577 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.872115+0000", 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73793104899999995 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.873515+0000", 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013995679999999999 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.878743+0000", 2023-10-09T05:05:18.578 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052280269999999997 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.879886+0000", 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001143126 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.579 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.882208+0000", 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023221399999999999 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.884422+0000", 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022137929999999999 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.884806+0000", 2023-10-09T05:05:18.580 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000384292 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.899658+0000", 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014851544 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.900363+0000", 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000705027 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.581 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.903667+0000", 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033043550000000001 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93386 14.b 14:d69ddaa2:::benchmark_data_smithi029_550670_object15775:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.236556634, 2023-10-09T05:05:18.582 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1644230159999998, 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93386 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.583 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.584 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328422+0000", 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3869999999999999e-06 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328429+0000", 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2019999999999998e-06 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.585 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328435+0000", 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328494+0000", 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9197999999999998e-05 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328505+0000", 2023-10-09T05:05:18.586 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0804e-05 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340759+0000", 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012254243999999999 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340767+0000", 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1489999999999997e-06 2023-10-09T05:05:18.587 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.490908+0000", 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1501412469999996 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.492842+0000", 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001934479 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.588 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93396 14.1b 14:d8f43944:::benchmark_data_smithi029_550670_object15785:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.23652952899999, 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6073650639999997, 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93396 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.589 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.590 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328449+0000", 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5550000000000001e-06 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.591 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328455+0000", 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8199999999999999e-06 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328461+0000", 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4519999999999996e-06 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328571+0000", 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109665 2023-10-09T05:05:18.592 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328578+0000", 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4159999999999998e-06 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340728+0000", 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012150466 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.593 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340737+0000", 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2320000000000001e-06 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.555267+0000", 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2145304489999997 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.594 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.560223+0000", 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004956088 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.578749+0000", 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018526206999999999 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.583563+0000", 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048131340000000002 2023-10-09T05:05:18.595 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.890933+0000", 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30737034099999999 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.893611+0000", 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026780860000000001 2023-10-09T05:05:18.596 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.898539+0000", 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049277770000000004 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.927690+0000", 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029151626 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.597 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.927958+0000", 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000267335 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.928819+0000", 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00086092500000000001 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.930225+0000", 2023-10-09T05:05:18.598 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001406584 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.935811+0000", 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005586298 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.599 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93399 14.b 14:d3f904cb:::benchmark_data_smithi029_550670_object15788:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.236450508, 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4290976730000002, 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93399 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.600 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.601 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328528+0000", 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2449999999999998e-06 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328536+0000", 2023-10-09T05:05:18.602 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5859999999999999e-06 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328542+0000", 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.28e-06 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328609+0000", 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6457000000000004e-05 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.603 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328617+0000", 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0630000000000006e-06 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340788+0000", 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01217118 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340795+0000", 2023-10-09T05:05:18.604 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9949999999999999e-06 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.511929+0000", 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1711344940000004 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515822+0000", 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038923809999999999 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.605 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.537272+0000", 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021450818 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.540947+0000", 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036743560000000001 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.606 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.644480+0000", 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10353306700000001 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.647474+0000", 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002994343 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.723802+0000", 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076327964999999998 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.607 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.741136+0000", 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017333634000000001 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.751071+0000", 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.009935078 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.751762+0000", 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00069107999999999995 2023-10-09T05:05:18.608 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.752883+0000", 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011206230000000001 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.757623+0000", 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047405010000000003 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.609 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93409 14.b 14:d4aef484:::benchmark_data_smithi029_550670_object15798:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328604+0000", 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.23637216100001, 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6414218070000004, 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.610 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93409 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328604+0000", 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328604+0000", 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.611 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328603+0000", 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328606+0000", 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.136e-06 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.612 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328611+0000", 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8850000000000004e-06 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328617+0000", 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0880000000000002e-06 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328645+0000", 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8212e-05 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.613 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328652+0000", 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6320000000000008e-06 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340814+0000", 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012161929 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.614 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340821+0000", 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9299999999999997e-06 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.516308+0000", 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1754863630000001 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.519571+0000", 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032632640000000001 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.615 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.585755+0000", 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.066184280999999998 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.587974+0000", 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022187729999999998 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.616 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.937859+0000", 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34988536999999997 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.941380+0000", 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003520787 2023-10-09T05:05:18.617 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.948853+0000", 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074730439999999999 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.954870+0000", 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006017166 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.618 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.955270+0000", 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000399792 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.961354+0000", 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060840929999999996 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.619 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.963250+0000", 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001896107 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.970026+0000", 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067752860000000002 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.620 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93412 14.b 14:d2c07cf8:::benchmark_data_smithi029_550670_object15801:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.236353651, 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4372284039999998, 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.621 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93412 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.622 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.623 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328624+0000", 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1090000000000001e-06 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328630+0000", 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8420000000000001e-06 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.624 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328635+0000", 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.13e-06 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328677+0000", 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2333999999999998e-05 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:05:18.625 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328685+0000", 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6750000000000002e-06 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340840+0000", 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012154531999999999 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340846+0000", 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2430000000000002e-06 2023-10-09T05:05:18.626 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.519895+0000", 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.179048892 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.523331+0000", 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034361249999999999 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.627 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.552508+0000", 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029177214 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.554502+0000", 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019938569999999999 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.628 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.741382+0000", 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.186879822 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.743460+0000", 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020784990000000001 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.629 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.747621+0000", 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041610090000000002 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.758320+0000", 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010698322 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.630 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.758544+0000", 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000224291 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.759477+0000", 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00093306699999999999 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.760812+0000", 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001334855 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.631 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.765851+0000", 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050388869999999997 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93416 14.16 14:684cfe34:::benchmark_data_smithi029_550670_object15805:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:05:18.632 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.23633673800001, 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2858180489999995, 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93416 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:05:18.633 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.634 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328641+0000", 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.063e-06 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328647+0000", 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8769999999999998e-06 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.635 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328652+0000", 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2009999999999998e-06 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.329007+0000", 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035455600000000001 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.329031+0000", 2023-10-09T05:05:18.636 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4559999999999999e-05 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.337619+0000", 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085875819999999999 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.337630+0000", 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0822000000000001e-05 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.637 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.449356+0000", 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1117265169999992 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452444+0000", 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030876950000000001 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.638 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.472192+0000", 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019747752 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.473616+0000", 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014240419999999999 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.639 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.586181+0000", 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11256505999999999 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.589201+0000", 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030202639999999999 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.641 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.595371+0000", 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061696779999999996 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.603917+0000", 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085462880000000008 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.642 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.604966+0000", 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00104896 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.605313+0000", 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034676100000000001 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.607032+0000", 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017197169999999999 2023-10-09T05:05:18.643 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.614457+0000", 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074249579999999997 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 629/597 [PushOp(14:cc36aedf:::benchmark_data_smithi029_550670_object46560:head, version: 445'2960, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:cc36aedf:::benchmark_data_smithi029_550670_object46560:head@445'2960, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:26.410300+0000", 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.154676161, 2023-10-09T05:05:18.644 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1977970630000003, 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410300+0000", 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410300+0000", 2023-10-09T05:05:18.645 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410299+0000", 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410337+0000", 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8713e-05 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.646 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410353+0000", 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5715000000000001e-05 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410370+0000", 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7251000000000001e-05 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.606870+0000", 2023-10-09T05:05:18.647 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1965000789999998 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.606878+0000", 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9400000000000002e-06 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.608097+0000", 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001218528 2023-10-09T05:05:18.648 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 629/597 [PushOp(14:c9d12152:::benchmark_data_smithi029_550670_object46566:head, version: 445'2961, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:c9d12152:::benchmark_data_smithi029_550670_object46566:head@445'2961, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:26.423230+0000", 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 112.141745813, 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1900266359999998, 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.649 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423230+0000", 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423230+0000", 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.650 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423229+0000", 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423269+0000", 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0085e-05 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.651 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423290+0000", 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1591e-05 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423309+0000", 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8742000000000001e-05 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.611347+0000", 2023-10-09T05:05:18.652 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.188037445 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.611359+0000", 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2364000000000001e-05 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.613257+0000", 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001897805 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.653 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93422 14.13 e629/597 14:c932bd54:::benchmark_data_smithi029_550670_object15811:head v 629'5452, mlcod=445'2858)", 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:28.668401+0000", 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 109.896574847, 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8474744859999999, 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.654 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668401+0000", 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668401+0000", 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.655 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668400+0000", 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668411+0000", 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1674e-05 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668430+0000", 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9023999999999999e-05 2023-10-09T05:05:18.656 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668442+0000", 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1928e-05 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.506278+0000", 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8378357049999998 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.657 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.506305+0000", 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7076e-05 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515837+0000", 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095318950000000003 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515876+0000", 2023-10-09T05:05:18.658 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8557999999999999e-05 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93428 14.13 e629/597 14:c93fb668:::benchmark_data_smithi029_550670_object15817:head v 629'5453, mlcod=445'2858)", 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:28.714692+0000", 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 109.85028431000001, 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8012098910000001, 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.659 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714692+0000", 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714692+0000", 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714690+0000", 2023-10-09T05:05:18.660 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714701+0000", 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0254999999999999e-05 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714714+0000", 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3319000000000001e-05 2023-10-09T05:05:18.661 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714724+0000", 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7990000000000005e-06 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.508239+0000", 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7935153970000002 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.662 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.508266+0000", 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6548999999999999e-05 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515879+0000", 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007613519 2023-10-09T05:05:18.663 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515902+0000", 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2342999999999999e-05 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93442 14.1b 14:dd3cf304:::benchmark_data_smithi029_550670_object15831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:28.905854+0000", 2023-10-09T05:05:18.664 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 109.659122385, 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0445410229999998, 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93442 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905854+0000", 2023-10-09T05:05:18.665 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905854+0000", 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905851+0000", 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.666 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905861+0000", 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3379999999999994e-06 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905876+0000", 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5471000000000001e-05 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905887+0000", 2023-10-09T05:05:18.667 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0397000000000001e-05 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.560607+0000", 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.654720014 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.563852+0000", 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032456310000000001 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.668 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.599630+0000", 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.035777825999999999 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.601885+0000", 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022552140000000002 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.915030+0000", 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.313144804 2023-10-09T05:05:18.669 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.917683+0000", 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002652387 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.922705+0000", 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050228260000000002 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.670 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.936744+0000", 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014038185999999999 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.937094+0000", 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035057099999999999 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.940419+0000", 2023-10-09T05:05:18.671 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033246009999999999 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.942291+0000", 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018717810000000001 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.950395+0000", 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081040590000000003 2023-10-09T05:05:18.672 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93447 14.16 14:6acb3bd1:::benchmark_data_smithi029_550670_object15836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:30.831436+0000", 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 107.73354034099999, 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.975447398, 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.673 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93447 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831436+0000", 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.674 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831436+0000", 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831433+0000", 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831446+0000", 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2469999999999999e-05 2023-10-09T05:05:18.675 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831464+0000", 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7943000000000001e-05 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831478+0000", 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4232e-05 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.676 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452902+0000", 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6214245329999999 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.456340+0000", 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034377259999999999 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.677 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.498891+0000", 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042551154000000001 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.500720+0000", 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018290450000000001 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.678 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.747804+0000", 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.247083261 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.750643+0000", 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028398680000000002 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.757126+0000", 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064828990000000003 2023-10-09T05:05:18.679 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.764610+0000", 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074833529999999999 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.767128+0000", 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025186230000000002 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.680 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.797584+0000", 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030455612 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.799461+0000", 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018771689999999999 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.681 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.806883+0000", 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007421871 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93455 14.16 14:6990cb23:::benchmark_data_smithi029_550670_object15844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:31.524915+0000", 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 107.04006051099999, 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1630371140000002, 2023-10-09T05:05:18.682 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93455 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524915+0000", 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.683 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524915+0000", 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524914+0000", 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524922+0000", 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0220000000000004e-06 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.684 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524937+0000", 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.467e-05 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524947+0000", 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0023999999999999e-05 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.456674+0000", 2023-10-09T05:05:18.685 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.931727075 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.459565+0000", 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002890798 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.484873+0000", 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025308074 2023-10-09T05:05:18.686 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.486610+0000", 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017378389999999999 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.655312+0000", 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16870205799999999 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.687 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.658075+0000", 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027629289999999999 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.662487+0000", 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044119500000000004 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.688 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.672469+0000", 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099814220000000002 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.674550+0000", 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020812579999999999 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.679292+0000", 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047416660000000003 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.689 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.680828+0000", 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015360129999999999 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.687953+0000", 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007124816 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.690 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93459 14.6 14:6034b7a4:::benchmark_data_smithi029_550670_object15848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:31.539802+0000", 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 107.025174206, 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2219217410000001, 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.691 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93459 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539802+0000", 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.692 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539802+0000", 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539800+0000", 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539809+0000", 2023-10-09T05:05:18.693 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2410000000000001e-06 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539826+0000", 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6875e-05 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539839+0000", 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2816000000000001e-05 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.694 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.487287+0000", 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9474478159999999 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.490853+0000", 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035662910000000001 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.515785+0000", 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024931496000000001 2023-10-09T05:05:18.695 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.518582+0000", 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027977129999999998 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.720360+0000", 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20177731900000001 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.696 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.723968+0000", 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036078809999999998 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.730738+0000", 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006770463 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.746843+0000", 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016105209999999998 2023-10-09T05:05:18.697 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.749571+0000", 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002728252 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.751066+0000", 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014948209999999999 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.698 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.752990+0000", 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019237620000000001 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.761723+0000", 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087334360000000007 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.699 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93460 14.16 14:686632a8:::benchmark_data_smithi029_550670_object15849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:31.638258+0000", 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 106.926717571, 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1443809819999999, 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.700 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.701 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.701 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93460 2023-10-09T05:05:18.701 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.701 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.701 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.701 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638258+0000", 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638258+0000", 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.702 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638256+0000", 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638268+0000", 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1698e-05 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.703 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638286+0000", 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7572999999999999e-05 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638298+0000", 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.202e-05 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.501793+0000", 2023-10-09T05:05:18.704 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.863495178 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.504460+0000", 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026674910000000001 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.523685+0000", 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019225123 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.705 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.525268+0000", 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015823790000000001 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.731965+0000", 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20669738900000001 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.706 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.734608+0000", 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002642564 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.740445+0000", 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005837676 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.762584+0000", 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022138415000000002 2023-10-09T05:05:18.707 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.763646+0000", 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010624269999999999 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.774272+0000", 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010625874 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.708 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.775711+0000", 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014388789999999999 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.782639+0000", 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069283929999999997 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.709 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93503 14.6 14:6687bdb8:::benchmark_data_smithi029_550670_object15892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:32.188842+0000", 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 106.376134385, 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6964581760000002, 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93503 2023-10-09T05:05:18.710 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188842+0000", 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.711 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188842+0000", 2023-10-09T05:05:18.712 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.712 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.712 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.712 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.712 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188840+0000", 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188849+0000", 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2369999999999998e-06 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188867+0000", 2023-10-09T05:05:18.713 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8096000000000001e-05 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188877+0000", 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0236e-05 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.519062+0000", 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3301848039999999 2023-10-09T05:05:18.714 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.523363+0000", 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043013169999999998 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.538989+0000", 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015626225000000001 2023-10-09T05:05:18.715 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.540399+0000", 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014099169999999999 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.820654+0000", 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.28025441899999998 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.716 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.823802+0000", 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031477179999999999 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.830337+0000", 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065356770000000002 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.717 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.874290+0000", 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043952721 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.875405+0000", 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001115322 2023-10-09T05:05:18.718 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.875761+0000", 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035534100000000001 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.877530+0000", 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017697640000000001 2023-10-09T05:05:18.719 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.885300+0000", 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077693809999999997 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93510 14.6 14:6787e907:::benchmark_data_smithi029_550670_object15899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:05:18.720 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:32.323702+0000", 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 106.241273479, 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5738554680000001, 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93510 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.721 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323702+0000", 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323702+0000", 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.722 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323700+0000", 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323712+0000", 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1868000000000001e-05 2023-10-09T05:05:18.723 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323729+0000", 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6815000000000001e-05 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323740+0000", 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1317e-05 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.724 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.540685+0000", 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2169450450000001 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.544160+0000", 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034742140000000002 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.725 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.588340+0000", 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044180187000000003 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.589693+0000", 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001353314 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.726 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.859742+0000", 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27004854900000003 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.863107+0000", 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033654710000000001 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.869068+0000", 2023-10-09T05:05:18.727 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005960941 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.886409+0000", 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017341354 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.886728+0000", 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031839100000000001 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.728 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.887739+0000", 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010109030000000001 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.889526+0000", 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001787723 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.729 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.897558+0000", 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080314870000000003 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93476 14.1e e630/628 14:7b4313ab:::benchmark_data_smithi029_550670_object15865:head v 630'5763, mlcod=0'0)", 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:33.290870+0000", 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 105.274106296, 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1616698269999999, 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.730 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290870+0000", 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290870+0000", 2023-10-09T05:05:18.731 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290869+0000", 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290876+0000", 2023-10-09T05:05:18.732 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4189999999999996e-06 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290891+0000", 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4423000000000001e-05 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290901+0000", 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.91e-06 2023-10-09T05:05:18.733 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.447763+0000", 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.156862324 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.447787+0000", 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3946000000000001e-05 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.734 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452499+0000", 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047118480000000003 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452539+0000", 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0788999999999999e-05 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.735 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93508 14.1e e630/628 14:7b545f8b:::benchmark_data_smithi029_550670_object15897:head v 630'5764, mlcod=0'0)", 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:33.293751+0000", 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 105.27122462600001, 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1588168030000001, 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293751+0000", 2023-10-09T05:05:18.736 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293751+0000", 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293750+0000", 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:05:18.737 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293760+0000", 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8030000000000008e-06 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293772+0000", 2023-10-09T05:05:18.738 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2007e-05 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293781+0000", 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0059999999999998e-06 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.448529+0000", 2023-10-09T05:05:18.739 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.154747612 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.448543+0000", 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4457999999999999e-05 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452543+0000", 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039998309999999997 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:05:18.740 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452568+0000", 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5148e-05 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:05:18.741 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:05:18.805 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:18.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:18 smithi029 ceph-mon[216903]: pgmap v2122: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 53083/142263 objects degraded (37.313%); 4839/142263 objects misplaced (3.401%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:05:18.815 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:18 smithi029 ceph-mon[223858]: pgmap v2122: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 53083/142263 objects degraded (37.313%); 4839/142263 objects misplaced (3.401%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:05:18.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:18.863 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:05:18.864 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 53083/142263 objects degraded (37.313%), 26 pgs degraded, 16 pgs undersized","count":42},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2855,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":662,"num_osds":8,"num_up_osds":8,"osd_up_since":1696827803,"num_in_osds":7,"osd_in_since":1696827750,"num_remapped_pgs":18},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":91},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":16},{"state_name":"active+recovery_wait+degraded","count":10},{"state_name":"active+recovering+undersized+remapped","count":2}],"num_pgs":119,"num_pools":5,"num_objects":47421,"data_bytes":3106740152,"bytes_used":20228890624,"bytes_avail":651736358912,"bytes_total":671965249536,"degraded_objects":53083,"degraded_total":142263,"degraded_ratio":0.37313285956292219,"misplaced_objects":4839,"misplaced_total":142263,"misplaced_ratio":0.034014466164779315,"recovering_objects_per_sec":103,"recovering_bytes_per_sec":6787288,"recovering_keys_per_sec":0,"num_objects_recovered":1037,"num_bytes_recovered":67960832,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":193},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":36,"modified":"2023-10-09T05:04:23.723180+0000","services":{}},"progress_events":{"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8039e601-7bd2-4b88-bccf-3b879c49d7d2":{"message":"Global Recovery Event (11m)\n [=====================.......] (remaining: 3m)","progress":0.76470589637756348,"add_to_ceph_s":true}}} 2023-10-09T05:05:18.916 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:18.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:05:18.942 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:18.993 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:18.996 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:18 smithi177 ceph-mon[195797]: pgmap v2122: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 53083/142263 objects degraded (37.313%); 4839/142263 objects misplaced (3.401%); 6.5 MiB/s, 103 objects/s recovering 2023-10-09T05:05:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:18.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:19.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:19.270 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:20.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2961675446' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:05:20.011 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3616992840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:20.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2961675446' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:05:20.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3616992840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2961675446' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:05:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3616992840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:20.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:05:20.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:05:20.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:05:20.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:05:20.284 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:05:20.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:05:20.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:05:20.285 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:05:20.285 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.285 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.285 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.286 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.286 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.286 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.286 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.286 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:05:20.492 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:05:20.730 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:20.742 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:20 smithi029 ceph-mon[216903]: pgmap v2123: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 52386/140217 objects degraded (37.361%); 4493/140217 objects misplaced (3.204%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:05:20.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:20.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:20.845 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:20.990 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:20 smithi029 ceph-mon[223858]: pgmap v2123: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 52386/140217 objects degraded (37.361%); 4493/140217 objects misplaced (3.204%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:05:20.993 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:20 smithi177 ceph-mon[195797]: pgmap v2123: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 52386/140217 objects degraded (37.361%); 4493/140217 objects misplaced (3.204%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:05:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 52386/140217 objects degraded (37.361%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:22.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 52386/140217 objects degraded (37.361%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:22.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 52386/140217 objects degraded (37.361%), 26 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:22.188 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:22.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:22.297 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:22.325 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:22.367 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:05:22.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:05:22.370 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:22.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:22.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:22.503 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:22.594 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:22.629 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:22.770 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:22 smithi029 ceph-mon[216903]: pgmap v2124: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 52386/140217 objects degraded (37.361%); 4493/140217 objects misplaced (3.204%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:05:22.952 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:22.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:22 smithi177 ceph-mon[195797]: pgmap v2124: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 52386/140217 objects degraded (37.361%); 4493/140217 objects misplaced (3.204%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:05:23.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:23.084 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:22 smithi029 ceph-mon[223858]: pgmap v2124: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 52386/140217 objects degraded (37.361%); 4493/140217 objects misplaced (3.204%); 6.6 MiB/s, 104 objects/s recovering 2023-10-09T05:05:23.086 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:05:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:05:22] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2023-10-09T05:05:23.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:23.142 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:23.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:23.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:23.250 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:23.371 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:24.306 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:05:25.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:24 smithi029 ceph-mon[216903]: pgmap v2125: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 206 op/s; 52007/137922 objects degraded (37.708%); 3891/137922 objects misplaced (2.821%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:05:25.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:24 smithi029 ceph-mon[223858]: pgmap v2125: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 206 op/s; 52007/137922 objects degraded (37.708%); 3891/137922 objects misplaced (2.821%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:05:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:24 smithi177 ceph-mon[195797]: pgmap v2125: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 206 op/s; 52007/137922 objects degraded (37.708%); 3891/137922 objects misplaced (2.821%); 6.9 MiB/s, 110 objects/s recovering 2023-10-09T05:05:25.752 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:25.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:25.863 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:26.009 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:26.205 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:26.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:26.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:26.318 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:26.347 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:26.397 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:26.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:26.425 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:26.504 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:26.539 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:26.670 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:26.937 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:26.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:27.058 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:27.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:26 smithi029 ceph-mon[216903]: pgmap v2126: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 51492/137040 objects degraded (37.574%); 3891/137040 objects misplaced (2.839%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:05:27.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 51492/137040 objects degraded (37.574%), 26 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:27.144 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:26 smithi029 ceph-mon[223858]: pgmap v2126: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 51492/137040 objects degraded (37.574%); 3891/137040 objects misplaced (2.839%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:05:27.144 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 51492/137040 objects degraded (37.574%), 26 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:27.150 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:26 smithi177 ceph-mon[195797]: pgmap v2126: 119 pgs: 2 active+recovering+undersized+remapped, 91 active+clean, 16 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 51492/137040 objects degraded (37.574%); 3891/137040 objects misplaced (2.839%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:05:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 51492/137040 objects degraded (37.574%), 26 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:27.442 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:27.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:27.572 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:27.731 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:27.736 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:05:27.736 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:05:27.990 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:05:28.028 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3694514086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:05:28.029 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3694514086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:05:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3694514086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:05:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: pgmap v2127: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 49172/136074 objects degraded (36.136%); 5112/136074 objects misplaced (3.757%); 6.3 MiB/s, 102 objects/s recovering 2023-10-09T05:05:29.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: failed to encode map e663 with expected crc 2023-10-09T05:05:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: failed to encode map e663 with expected crc 2023-10-09T05:05:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3694514086' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:05:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: osdmap e663: 8 total, 8 up, 7 in 2023-10-09T05:05:29.279 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: pgmap v2127: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 49172/136074 objects degraded (36.136%); 5112/136074 objects misplaced (3.757%); 6.3 MiB/s, 102 objects/s recovering 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: failed to encode map e663 with expected crc 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: failed to encode map e663 with expected crc 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3694514086' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: osdmap e663: 8 total, 8 up, 7 in 2023-10-09T05:05:29.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: pgmap v2127: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 15 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 199 op/s; 49172/136074 objects degraded (36.136%); 5112/136074 objects misplaced (3.757%); 6.3 MiB/s, 102 objects/s recovering 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: failed to encode map e663 with expected crc 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: failed to encode map e663 with expected crc 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3694514086' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: osdmap e663: 8 total, 8 up, 7 in 2023-10-09T05:05:29.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:29.690 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:29.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:29.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:29.834 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:29 smithi029 ceph-mon[223858]: failed to encode map e664 with expected crc 2023-10-09T05:05:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:29 smithi029 ceph-mon[223858]: osdmap e664: 8 total, 8 up, 7 in 2023-10-09T05:05:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:29 smithi029 ceph-mon[223858]: failed to encode map e664 with expected crc 2023-10-09T05:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:29 smithi029 ceph-mon[216903]: failed to encode map e664 with expected crc 2023-10-09T05:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:29 smithi029 ceph-mon[216903]: osdmap e664: 8 total, 8 up, 7 in 2023-10-09T05:05:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:29 smithi029 ceph-mon[216903]: failed to encode map e664 with expected crc 2023-10-09T05:05:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:29 smithi177 ceph-mon[195797]: failed to encode map e664 with expected crc 2023-10-09T05:05:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:29 smithi177 ceph-mon[195797]: osdmap e664: 8 total, 8 up, 7 in 2023-10-09T05:05:30.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:29 smithi177 ceph-mon[195797]: failed to encode map e664 with expected crc 2023-10-09T05:05:30.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:05:30.561 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:30.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:30.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:30.666 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:30.695 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:30.741 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:30.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:30.849 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:31.005 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:31.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:30 smithi177 ceph-mon[195797]: pgmap v2130: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 43660/133896 objects degraded (32.607%); 9084/133896 objects misplaced (6.784%); 9.4 MiB/s, 171 objects/s recovering 2023-10-09T05:05:31.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:30 smithi029 ceph-mon[216903]: pgmap v2130: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 43660/133896 objects degraded (32.607%); 9084/133896 objects misplaced (6.784%); 9.4 MiB/s, 171 objects/s recovering 2023-10-09T05:05:31.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:30 smithi029 ceph-mon[223858]: pgmap v2130: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 43660/133896 objects degraded (32.607%); 9084/133896 objects misplaced (6.784%); 9.4 MiB/s, 171 objects/s recovering 2023-10-09T05:05:31.451 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:31.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:31.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:31.556 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:31.586 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:32.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 43660/133896 objects degraded (32.607%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:32.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 43660/133896 objects degraded (32.607%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:32.183 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:32.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:32.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 43660/133896 objects degraded (32.607%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:32.299 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:32.389 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:32.573 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:32.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:32.686 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:32.816 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:33.039 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:33 smithi029 ceph-mon[216903]: pgmap v2131: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 43660/133896 objects degraded (32.607%); 9084/133896 objects misplaced (6.784%); 6.2 MiB/s, 120 objects/s recovering 2023-10-09T05:05:33.040 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:05:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:05:32] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.43.0" 2023-10-09T05:05:33.051 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:33.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:33.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:33.238 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:33.342 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:33 smithi029 ceph-mon[223858]: pgmap v2131: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 43660/133896 objects degraded (32.607%); 9084/133896 objects misplaced (6.784%); 6.2 MiB/s, 120 objects/s recovering 2023-10-09T05:05:33.363 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:05:33.385 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:05:33.389 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:33.458 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:05:33.523 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:05:33.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:33 smithi177 ceph-mon[195797]: pgmap v2131: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 91 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 167 op/s; 43660/133896 objects degraded (32.607%); 9084/133896 objects misplaced (6.784%); 6.2 MiB/s, 120 objects/s recovering 2023-10-09T05:05:33.579 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:05:33.643 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:05:33.689 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:05:33.720 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:05:33.767 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:33.769 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:05:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:33.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:33.827 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:05:33.853 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.853 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.853 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.854 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:05:33.902 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:33.929 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:33.976 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:33.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:34.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:34 smithi029 ceph-mon[216903]: from='client.57148 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:34.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3745077421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:34.082 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:34.210 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:34.226 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:05:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:34 smithi177 ceph-mon[195797]: from='client.57148 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:34.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3745077421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:34.341 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:34 smithi029 ceph-mon[223858]: from='client.57148 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:34.341 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3745077421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:35.160 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:35.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:35.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:35.295 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:35.340 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:35.346 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:35 smithi177 ceph-mon[195797]: pgmap v2132: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 43324/131325 objects degraded (32.990%); 8868/131325 objects misplaced (6.753%); 6.9 MiB/s, 145 objects/s recovering 2023-10-09T05:05:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:35.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:35.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:35 smithi029 ceph-mon[216903]: pgmap v2132: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 43324/131325 objects degraded (32.990%); 8868/131325 objects misplaced (6.753%); 6.9 MiB/s, 145 objects/s recovering 2023-10-09T05:05:35.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:35 smithi029 ceph-mon[223858]: pgmap v2132: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 43324/131325 objects degraded (32.990%); 8868/131325 objects misplaced (6.753%); 6.9 MiB/s, 145 objects/s recovering 2023-10-09T05:05:35.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:35.475 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:35.521 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:35.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:35.626 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:35.721 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:36.099 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2131,"stamp":"2023-10-09T05:05:31.791972+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2923960248,"num_objects":44632,"num_object_clones":0,"num_object_copies":133896,"num_objects_missing_on_primary":15203,"num_objects_missing":15203,"num_objects_degraded":43660,"num_objects_misplaced":9084,"num_objects_unfound":0,"num_objects_dirty":44632,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":110910,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72310,"num_bytes_recovered":4406545038,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195874,"ondisk_log_size":195874,"up":357,"acting":340,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":485,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19474700,"kb_used_data":11519060,"kb_used_omap":34,"kb_used_meta":7955549,"kb_avail":636741364,"statfs":{"total":671965249536,"available":652023156736,"internally_reserved":0,"allocated":11795517440,"data_stored":11779496297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8146482570},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-87949312,"num_objects":-1342,"num_object_clones":0,"num_object_copies":-4026,"num_objects_missing_on_primary":-689,"num_objects_missing":-689,"num_objects_degraded":-8347,"num_objects_misplaced":5193,"num_objects_unfound":0,"num_objects_dirty":-1342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1342,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":963,"num_bytes_recovered":51707904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.015547"},"pg_stats":[{"pgid":"14.23","version":"662'5733","reported_seq":12354,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.959554+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:05:27.959554+0000","last_peered":"2023-10-09T05:05:27.959554+0000","last_clean":"2023-10-09T05:05:27.959554+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:05:27.959554+0000","last_undegraded":"2023-10-09T05:05:27.959554+0000","last_fullsized":"2023-10-09T05:05:27.959554+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44761088,"num_objects":683,"num_object_clones":0,"num_object_copies":2049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1749,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"663'5660","reported_seq":11982,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.051055+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:05:28.051055+0000","last_peered":"2023-10-09T05:05:28.051055+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:05:28.051055+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5660,"log_dups_size":0,"ondisk_log_size":5660,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44728320,"num_objects":683,"num_object_clones":0,"num_object_copies":2049,"num_objects_missing_on_primary":657,"num_objects_missing":657,"num_objects_degraded":1814,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1725,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":800,"num_bytes_recovered":50184192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"4,5,7","objects":499}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"662'5727","reported_seq":16421,"reported_epoch":662,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:27.960722+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:05:27.960722+0000","last_peered":"2023-10-09T05:05:27.960722+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:05:27.960722+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:05:27.960722+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44957696,"num_objects":686,"num_object_clones":0,"num_object_copies":2058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1744,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":686}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"662'5714","reported_seq":6976,"reported_epoch":662,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:26.963137+0000","last_change":"2023-10-09T05:05:26.963137+0000","last_active":"2023-10-09T05:05:26.963137+0000","last_peered":"2023-10-09T05:05:26.963137+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:05:26.963137+0000","last_undegraded":"2023-10-09T05:05:26.962565+0000","last_fullsized":"2023-10-09T05:05:26.963137+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5176,"log_dups_size":0,"ondisk_log_size":5176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46546944,"num_objects":710,"num_object_clones":0,"num_object_copies":2130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1721,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":279,"num_bytes_recovered":18300928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":661},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"662'5991","reported_seq":19547,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.203080+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:05:27.203080+0000","last_peered":"2023-10-09T05:05:27.203080+0000","last_clean":"2023-10-09T05:05:27.203080+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:05:27.203080+0000","last_undegraded":"2023-10-09T05:05:27.203080+0000","last_fullsized":"2023-10-09T05:05:27.203080+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5991,"log_dups_size":0,"ondisk_log_size":5991,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94208000,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3450,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":166,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943704+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:05:27.943704+0000","last_peered":"2023-10-09T05:05:27.943704+0000","last_clean":"2023-10-09T05:05:27.943704+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:05:27.943704+0000","last_undegraded":"2023-10-09T05:05:27.943704+0000","last_fullsized":"2023-10-09T05:05:27.943704+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":310,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.270725+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:05:15.270725+0000","last_peered":"2023-10-09T05:05:15.270725+0000","last_clean":"2023-10-09T05:05:15.270725+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:05:15.270725+0000","last_undegraded":"2023-10-09T05:05:15.270725+0000","last_fullsized":"2023-10-09T05:05:15.270725+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"663'5999","reported_seq":9222,"reported_epoch":663,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:28.702945+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:05:28.702945+0000","last_peered":"2023-10-09T05:05:28.702945+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:05:28.702945+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:05:28.702945+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5999,"log_dups_size":0,"ondisk_log_size":5999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91783168,"num_objects":1401,"num_object_clones":0,"num_object_copies":4203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3460,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":1401}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636335+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:05:01.636335+0000","last_peered":"2023-10-09T05:05:01.636335+0000","last_clean":"2023-10-09T05:05:01.636335+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:05:01.636335+0000","last_undegraded":"2023-10-09T05:05:01.636335+0000","last_fullsized":"2023-10-09T05:05:01.636335+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":175,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.270499+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:05:15.270499+0000","last_peered":"2023-10-09T05:05:15.270499+0000","last_clean":"2023-10-09T05:05:15.270499+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:05:15.270499+0000","last_undegraded":"2023-10-09T05:05:15.270499+0000","last_fullsized":"2023-10-09T05:05:15.270499+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"663'6001","reported_seq":17656,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.128040+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:05:28.128040+0000","last_peered":"2023-10-09T05:05:28.128040+0000","last_clean":"2023-10-09T05:05:28.128040+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:05:28.128040+0000","last_undegraded":"2023-10-09T05:05:28.128040+0000","last_fullsized":"2023-10-09T05:05:28.128040+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3461,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":292,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.271722+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:05:15.271722+0000","last_peered":"2023-10-09T05:05:15.271722+0000","last_clean":"2023-10-09T05:05:15.271722+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:05:15.271722+0000","last_undegraded":"2023-10-09T05:05:15.271722+0000","last_fullsized":"2023-10-09T05:05:15.271722+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":189,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.254280+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:05:15.254280+0000","last_peered":"2023-10-09T05:05:15.254280+0000","last_clean":"2023-10-09T05:05:15.254280+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:05:15.254280+0000","last_undegraded":"2023-10-09T05:05:15.254280+0000","last_fullsized":"2023-10-09T05:05:15.254280+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"662'6211","reported_seq":23561,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:26.976280+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:05:26.976280+0000","last_peered":"2023-10-09T05:05:26.976280+0000","last_clean":"2023-10-09T05:05:26.976280+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:05:26.976280+0000","last_undegraded":"2023-10-09T05:05:26.976280+0000","last_fullsized":"2023-10-09T05:05:26.976280+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90505216,"num_objects":1381,"num_object_clones":0,"num_object_copies":4143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.649198+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:05:01.649198+0000","last_peered":"2023-10-09T05:05:01.649198+0000","last_clean":"2023-10-09T05:05:01.649198+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:05:01.649198+0000","last_undegraded":"2023-10-09T05:05:01.649198+0000","last_fullsized":"2023-10-09T05:05:01.649198+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":283,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.271640+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:05:15.271640+0000","last_peered":"2023-10-09T05:05:15.271640+0000","last_clean":"2023-10-09T05:05:15.271640+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:05:15.271640+0000","last_undegraded":"2023-10-09T05:05:15.271640+0000","last_fullsized":"2023-10-09T05:05:15.271640+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"664'6517","reported_seq":12442,"reported_epoch":664,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:05:29.054682+0000","last_change":"2023-10-09T05:05:26.975807+0000","last_active":"2023-10-09T05:05:29.054682+0000","last_peered":"2023-10-09T05:05:29.054682+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:05:29.054682+0000","last_undegraded":"2023-10-09T05:05:29.054682+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93257728,"num_objects":1423,"num_object_clones":0,"num_object_copies":4269,"num_objects_missing_on_primary":1805,"num_objects_missing":1805,"num_objects_degraded":0,"num_objects_misplaced":4207,"num_objects_unfound":0,"num_objects_dirty":1423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3557,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1349,"num_bytes_recovered":88342528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":2},{"shards":"0,2,4,7","objects":1},{"shards":"2,4,7","objects":1365},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":50,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.648987+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:05:01.648987+0000","last_peered":"2023-10-09T05:05:01.648987+0000","last_clean":"2023-10-09T05:05:01.648987+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:05:01.648987+0000","last_undegraded":"2023-10-09T05:05:01.648987+0000","last_fullsized":"2023-10-09T05:05:01.648987+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":273,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.272402+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:05:15.272402+0000","last_peered":"2023-10-09T05:05:15.272402+0000","last_clean":"2023-10-09T05:05:15.272402+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:05:15.272402+0000","last_undegraded":"2023-10-09T05:05:15.272402+0000","last_fullsized":"2023-10-09T05:05:15.272402+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"662'6146","reported_seq":10993,"reported_epoch":662,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:05:27.276536+0000","last_change":"2023-10-09T05:05:00.137617+0000","last_active":"2023-10-09T05:05:27.276536+0000","last_peered":"2023-10-09T05:05:27.276536+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:05:27.276536+0000","last_undegraded":"2023-10-09T05:05:27.276536+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4610,"log_dups_size":0,"ondisk_log_size":4610,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":1306,"num_objects_missing":1306,"num_objects_degraded":0,"num_objects_misplaced":3478,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3537,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1168,"num_bytes_recovered":76546048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,5,7","objects":1},{"shards":"1,3","objects":152},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":873}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":336,"reported_epoch":662,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:15.270572+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:05:15.270572+0000","last_peered":"2023-10-09T05:05:15.270572+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:05:15.270572+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:05:15.270572+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":86,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.638743+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:05:01.638743+0000","last_peered":"2023-10-09T05:05:01.638743+0000","last_clean":"2023-10-09T05:05:01.638743+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:05:01.638743+0000","last_undegraded":"2023-10-09T05:05:01.638743+0000","last_fullsized":"2023-10-09T05:05:01.638743+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"662'6027","reported_seq":18095,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.087467+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:05:27.087467+0000","last_peered":"2023-10-09T05:05:27.087467+0000","last_clean":"2023-10-09T05:05:27.087467+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:05:27.087467+0000","last_undegraded":"2023-10-09T05:05:27.087467+0000","last_fullsized":"2023-10-09T05:05:27.087467+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91783168,"num_objects":1401,"num_object_clones":0,"num_object_copies":4203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3489,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":119,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.944221+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:05:27.944221+0000","last_peered":"2023-10-09T05:05:27.944221+0000","last_clean":"2023-10-09T05:05:27.944221+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:05:27.944221+0000","last_undegraded":"2023-10-09T05:05:27.944221+0000","last_fullsized":"2023-10-09T05:05:27.944221+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":134,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943358+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:05:27.943358+0000","last_peered":"2023-10-09T05:05:27.943358+0000","last_clean":"2023-10-09T05:05:27.943358+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:05:27.943358+0000","last_undegraded":"2023-10-09T05:05:27.943358+0000","last_fullsized":"2023-10-09T05:05:27.943358+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"663'6261","reported_seq":18102,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.414181+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:05:28.414181+0000","last_peered":"2023-10-09T05:05:28.414181+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:05:28.414181+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4580,"log_dups_size":0,"ondisk_log_size":4580,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91127808,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3402,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2940,"num_bytes_recovered":188088320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":887}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":140,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943322+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:05:27.943322+0000","last_peered":"2023-10-09T05:05:27.943322+0000","last_clean":"2023-10-09T05:05:27.943322+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:05:27.943322+0000","last_undegraded":"2023-10-09T05:05:27.943322+0000","last_fullsized":"2023-10-09T05:05:27.943322+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":246,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.281056+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:05:15.281056+0000","last_peered":"2023-10-09T05:05:15.281056+0000","last_clean":"2023-10-09T05:05:15.281056+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:05:15.281056+0000","last_undegraded":"2023-10-09T05:05:15.281056+0000","last_fullsized":"2023-10-09T05:05:15.281056+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"662'6290","reported_seq":11596,"reported_epoch":662,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:27.238627+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:05:27.238627+0000","last_peered":"2023-10-09T05:05:27.238627+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:05:27.238627+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4163,"log_dups_size":0,"ondisk_log_size":4163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86376448,"num_objects":1318,"num_object_clones":0,"num_object_copies":3954,"num_objects_missing_on_primary":1422,"num_objects_missing":1422,"num_objects_degraded":2819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3461,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1393,"num_bytes_recovered":91258880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,7","objects":-38},{"shards":"2,5,7","objects":981},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.272846+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:05:15.272846+0000","last_peered":"2023-10-09T05:05:15.272846+0000","last_clean":"2023-10-09T05:05:15.272846+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:05:15.272846+0000","last_undegraded":"2023-10-09T05:05:15.272846+0000","last_fullsized":"2023-10-09T05:05:15.272846+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":40,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.648382+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:05:01.648382+0000","last_peered":"2023-10-09T05:05:01.648382+0000","last_clean":"2023-10-09T05:05:01.648382+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:05:01.648382+0000","last_undegraded":"2023-10-09T05:05:01.648382+0000","last_fullsized":"2023-10-09T05:05:01.648382+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"663'6365","reported_seq":13244,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.451013+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:05:28.451013+0000","last_peered":"2023-10-09T05:05:28.451013+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:05:28.451013+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4602,"log_dups_size":0,"ondisk_log_size":4602,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92340224,"num_objects":1409,"num_object_clones":0,"num_object_copies":4227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3466,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1755,"num_bytes_recovered":102039552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":1327},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":103,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942873+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:05:27.942873+0000","last_peered":"2023-10-09T05:05:27.942873+0000","last_clean":"2023-10-09T05:05:27.942873+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:05:27.942873+0000","last_undegraded":"2023-10-09T05:05:27.942873+0000","last_fullsized":"2023-10-09T05:05:27.942873+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":300,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.270956+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:05:15.270956+0000","last_peered":"2023-10-09T05:05:15.270956+0000","last_clean":"2023-10-09T05:05:15.270956+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:05:15.270956+0000","last_undegraded":"2023-10-09T05:05:15.270956+0000","last_fullsized":"2023-10-09T05:05:15.270956+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"663'6278","reported_seq":13153,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.435392+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:05:28.435392+0000","last_peered":"2023-10-09T05:05:28.435392+0000","last_clean":"2023-10-09T05:05:28.435392+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:05:28.435392+0000","last_undegraded":"2023-10-09T05:05:28.435392+0000","last_fullsized":"2023-10-09T05:05:28.435392+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":3985,"log_dups_size":0,"ondisk_log_size":3985,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90931212,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3416,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":297,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253457+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:05:15.253457+0000","last_peered":"2023-10-09T05:05:15.253457+0000","last_clean":"2023-10-09T05:05:15.253457+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:05:15.253457+0000","last_undegraded":"2023-10-09T05:05:15.253457+0000","last_fullsized":"2023-10-09T05:05:15.253457+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":158,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.281793+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:05:15.281793+0000","last_peered":"2023-10-09T05:05:15.281793+0000","last_clean":"2023-10-09T05:05:15.281793+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:05:15.281793+0000","last_undegraded":"2023-10-09T05:05:15.281793+0000","last_fullsized":"2023-10-09T05:05:15.281793+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"662'5668","reported_seq":14531,"reported_epoch":663,"state":"peering","last_fresh":"2023-10-09T05:05:27.949043+0000","last_change":"2023-10-09T05:05:27.948824+0000","last_active":"2023-10-09T05:05:27.948563+0000","last_peered":"2023-10-09T05:05:27.183004+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.593716+0000","last_became_peered":"2023-10-09T05:03:28.593716+0000","last_unstale":"2023-10-09T05:05:27.949043+0000","last_undegraded":"2023-10-09T05:05:27.949043+0000","last_fullsized":"2023-10-09T05:05:27.949043+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5327,"log_dups_size":0,"ondisk_log_size":5327,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44204032,"num_objects":674,"num_object_clones":0,"num_object_copies":2022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1733,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":193,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943520+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:05:27.943520+0000","last_peered":"2023-10-09T05:05:27.943520+0000","last_clean":"2023-10-09T05:05:27.943520+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:05:27.943520+0000","last_undegraded":"2023-10-09T05:05:27.943520+0000","last_fullsized":"2023-10-09T05:05:27.943520+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.647180+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:05:01.647180+0000","last_peered":"2023-10-09T05:05:01.647180+0000","last_clean":"2023-10-09T05:05:01.647180+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:05:01.647180+0000","last_undegraded":"2023-10-09T05:05:01.647180+0000","last_fullsized":"2023-10-09T05:05:01.647180+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"663'5723","reported_seq":9463,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:29.017973+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:05:29.017973+0000","last_peered":"2023-10-09T05:05:29.017973+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:05:29.017973+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45416448,"num_objects":693,"num_object_clones":0,"num_object_copies":2079,"num_objects_missing_on_primary":868,"num_objects_missing":868,"num_objects_degraded":2026,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1739,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":530,"num_bytes_recovered":34701312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":14},{"shards":"0,2,5,7","objects":2},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":470},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":21,"reported_epoch":663,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:27.943443+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:05:27.943443+0000","last_peered":"2023-10-09T05:05:27.943443+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:05:27.943443+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:05:27.943443+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":150,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942790+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:05:27.942790+0000","last_peered":"2023-10-09T05:05:27.942790+0000","last_clean":"2023-10-09T05:05:27.942790+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:05:27.942790+0000","last_undegraded":"2023-10-09T05:05:27.942790+0000","last_fullsized":"2023-10-09T05:05:27.942790+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"662'5735","reported_seq":9850,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.227598+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:05:27.227598+0000","last_peered":"2023-10-09T05:05:27.227598+0000","last_clean":"2023-10-09T05:05:27.227598+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:05:27.227598+0000","last_undegraded":"2023-10-09T05:05:27.227598+0000","last_fullsized":"2023-10-09T05:05:27.227598+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5197,"log_dups_size":0,"ondisk_log_size":5197,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45170688,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1742,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":44,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.648307+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:05:01.648307+0000","last_peered":"2023-10-09T05:05:01.648307+0000","last_clean":"2023-10-09T05:05:01.648307+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:05:01.648307+0000","last_undegraded":"2023-10-09T05:05:01.648307+0000","last_fullsized":"2023-10-09T05:05:01.648307+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":209,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253581+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:05:15.253581+0000","last_peered":"2023-10-09T05:05:15.253581+0000","last_clean":"2023-10-09T05:05:15.253581+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:05:15.253581+0000","last_undegraded":"2023-10-09T05:05:15.253581+0000","last_fullsized":"2023-10-09T05:05:15.253581+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"663'5748","reported_seq":16442,"reported_epoch":663,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:27.966884+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:05:27.966884+0000","last_peered":"2023-10-09T05:05:27.966884+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:05:27.966884+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:05:27.966884+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43581440,"num_objects":665,"num_object_clones":0,"num_object_copies":1995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1764,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":665}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":722,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.648039+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:05:01.648039+0000","last_peered":"2023-10-09T05:05:01.648039+0000","last_clean":"2023-10-09T05:05:01.648039+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:05:01.648039+0000","last_undegraded":"2023-10-09T05:05:01.648039+0000","last_fullsized":"2023-10-09T05:05:01.648039+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.638813+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:05:01.638813+0000","last_peered":"2023-10-09T05:05:01.638813+0000","last_clean":"2023-10-09T05:05:01.638813+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:05:01.638813+0000","last_undegraded":"2023-10-09T05:05:01.638813+0000","last_fullsized":"2023-10-09T05:05:01.638813+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1251,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253482+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:05:15.253482+0000","last_peered":"2023-10-09T05:05:15.253482+0000","last_clean":"2023-10-09T05:05:15.253482+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:05:15.253482+0000","last_undegraded":"2023-10-09T05:05:15.253482+0000","last_fullsized":"2023-10-09T05:05:15.253482+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"663'5790","reported_seq":10144,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.170184+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:05:28.170184+0000","last_peered":"2023-10-09T05:05:28.170184+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:05:28.170184+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5790,"log_dups_size":0,"ondisk_log_size":5790,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90603520,"num_objects":1382,"num_object_clones":0,"num_object_copies":4146,"num_objects_missing_on_primary":1697,"num_objects_missing":1697,"num_objects_degraded":4076,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3333,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":976,"num_bytes_recovered":63963136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":950}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":266,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.269885+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:05:15.269885+0000","last_peered":"2023-10-09T05:05:15.269885+0000","last_clean":"2023-10-09T05:05:15.269885+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:05:15.269885+0000","last_undegraded":"2023-10-09T05:05:15.269885+0000","last_fullsized":"2023-10-09T05:05:15.269885+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":102,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.944140+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:05:27.944140+0000","last_peered":"2023-10-09T05:05:27.944140+0000","last_clean":"2023-10-09T05:05:27.944140+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:05:27.944140+0000","last_undegraded":"2023-10-09T05:05:27.944140+0000","last_fullsized":"2023-10-09T05:05:27.944140+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1240,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253655+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:05:15.253655+0000","last_peered":"2023-10-09T05:05:15.253655+0000","last_clean":"2023-10-09T05:05:15.253655+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:05:15.253655+0000","last_undegraded":"2023-10-09T05:05:15.253655+0000","last_fullsized":"2023-10-09T05:05:15.253655+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"662'6025","reported_seq":14422,"reported_epoch":662,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:27.302644+0000","last_change":"2023-10-09T05:05:26.966418+0000","last_active":"2023-10-09T05:05:27.302644+0000","last_peered":"2023-10-09T05:05:27.302644+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:03:45.583447+0000","last_became_peered":"2023-10-09T05:03:45.583447+0000","last_unstale":"2023-10-09T05:05:27.302644+0000","last_undegraded":"2023-10-09T05:03:41.179518+0000","last_fullsized":"2023-10-09T05:03:38.964620+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91979776,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":321,"num_objects_missing":321,"num_objects_degraded":565,"num_objects_misplaced":1399,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3483,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2165,"num_bytes_recovered":140443648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["2"],"object_location_counts":[{"shards":"1,2,4,5","objects":1039},{"shards":"2,4,5","objects":360},{"shards":"4,5","objects":-561}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":206,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253505+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:05:15.253505+0000","last_peered":"2023-10-09T05:05:15.253505+0000","last_clean":"2023-10-09T05:05:15.253505+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:05:15.253505+0000","last_undegraded":"2023-10-09T05:05:15.253505+0000","last_fullsized":"2023-10-09T05:05:15.253505+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942987+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:05:27.942987+0000","last_peered":"2023-10-09T05:05:27.942987+0000","last_clean":"2023-10-09T05:05:27.942987+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:05:27.942987+0000","last_undegraded":"2023-10-09T05:05:27.942987+0000","last_fullsized":"2023-10-09T05:05:27.942987+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"663'6006","reported_seq":10073,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.172970+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:05:28.172970+0000","last_peered":"2023-10-09T05:05:28.172970+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:05:28.172970+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91324416,"num_objects":1393,"num_object_clones":0,"num_object_copies":4179,"num_objects_missing_on_primary":996,"num_objects_missing":996,"num_objects_degraded":2381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3466,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1038,"num_bytes_recovered":68059136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":366},{"shards":"2,4,7","objects":996},{"shards":"3,7","objects":31}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":136,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942846+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:05:27.942846+0000","last_peered":"2023-10-09T05:05:27.942846+0000","last_clean":"2023-10-09T05:05:27.942846+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:05:27.942846+0000","last_undegraded":"2023-10-09T05:05:27.942846+0000","last_fullsized":"2023-10-09T05:05:27.942846+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":137,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943878+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:05:27.943878+0000","last_peered":"2023-10-09T05:05:27.943878+0000","last_clean":"2023-10-09T05:05:27.943878+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:05:27.943878+0000","last_undegraded":"2023-10-09T05:05:27.943878+0000","last_fullsized":"2023-10-09T05:05:27.943878+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1260,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253791+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:05:15.253791+0000","last_peered":"2023-10-09T05:05:15.253791+0000","last_clean":"2023-10-09T05:05:15.253791+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:05:15.253791+0000","last_undegraded":"2023-10-09T05:05:15.253791+0000","last_fullsized":"2023-10-09T05:05:15.253791+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"663'6026","reported_seq":19838,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.650571+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:05:28.650571+0000","last_peered":"2023-10-09T05:05:28.650571+0000","last_clean":"2023-10-09T05:05:28.650571+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:05:28.650571+0000","last_undegraded":"2023-10-09T05:05:28.650571+0000","last_fullsized":"2023-10-09T05:05:28.650571+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5780,"log_dups_size":0,"ondisk_log_size":5780,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112000,"num_objects":1375,"num_object_clones":0,"num_object_copies":4125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3486,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":181,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.271062+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:05:15.271062+0000","last_peered":"2023-10-09T05:05:15.271062+0000","last_clean":"2023-10-09T05:05:15.271062+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:05:15.271062+0000","last_undegraded":"2023-10-09T05:05:15.271062+0000","last_fullsized":"2023-10-09T05:05:15.271062+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":89,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.637900+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:05:01.637900+0000","last_peered":"2023-10-09T05:05:01.637900+0000","last_clean":"2023-10-09T05:05:01.637900+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:05:01.637900+0000","last_undegraded":"2023-10-09T05:05:01.637900+0000","last_fullsized":"2023-10-09T05:05:01.637900+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1266,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.254199+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:05:15.254199+0000","last_peered":"2023-10-09T05:05:15.254199+0000","last_clean":"2023-10-09T05:05:15.254199+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:05:15.254199+0000","last_undegraded":"2023-10-09T05:05:15.254199+0000","last_fullsized":"2023-10-09T05:05:15.254199+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"663'6185","reported_seq":10954,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.767697+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:05:28.767697+0000","last_peered":"2023-10-09T05:05:28.767697+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:05:28.767697+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92930048,"num_objects":1418,"num_object_clones":0,"num_object_copies":4254,"num_objects_missing_on_primary":1530,"num_objects_missing":1530,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3576,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1380,"num_bytes_recovered":90439680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,3,5,7","objects":300},{"shards":"0,3","objects":-633},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":738}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":77,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636700+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:05:01.636700+0000","last_peered":"2023-10-09T05:05:01.636700+0000","last_clean":"2023-10-09T05:05:01.636700+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:05:01.636700+0000","last_undegraded":"2023-10-09T05:05:01.636700+0000","last_fullsized":"2023-10-09T05:05:01.636700+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":40,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.648459+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:05:01.648459+0000","last_peered":"2023-10-09T05:05:01.648459+0000","last_clean":"2023-10-09T05:05:01.648459+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:05:01.648459+0000","last_undegraded":"2023-10-09T05:05:01.648459+0000","last_fullsized":"2023-10-09T05:05:01.648459+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.638735+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:05:01.638735+0000","last_peered":"2023-10-09T05:05:01.638735+0000","last_clean":"2023-10-09T05:05:01.638735+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:05:01.638735+0000","last_undegraded":"2023-10-09T05:05:01.638735+0000","last_fullsized":"2023-10-09T05:05:01.638735+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1249,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253432+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:05:15.253432+0000","last_peered":"2023-10-09T05:05:15.253432+0000","last_clean":"2023-10-09T05:05:15.253432+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:05:15.253432+0000","last_undegraded":"2023-10-09T05:05:15.253432+0000","last_fullsized":"2023-10-09T05:05:15.253432+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"663'6319","reported_seq":18277,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.660669+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:05:28.660669+0000","last_peered":"2023-10-09T05:05:28.660669+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:05:28.660669+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4638,"log_dups_size":0,"ondisk_log_size":4638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87326720,"num_objects":1332,"num_object_clones":0,"num_object_copies":3996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1899,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3459,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2998,"num_bytes_recovered":191889408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":850}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":746,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.647937+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:05:01.647937+0000","last_peered":"2023-10-09T05:05:01.647937+0000","last_clean":"2023-10-09T05:05:01.647937+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:05:01.647937+0000","last_undegraded":"2023-10-09T05:05:01.647937+0000","last_fullsized":"2023-10-09T05:05:01.647937+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":129,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943102+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:05:27.943102+0000","last_peered":"2023-10-09T05:05:27.943102+0000","last_clean":"2023-10-09T05:05:27.943102+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:05:27.943102+0000","last_undegraded":"2023-10-09T05:05:27.943102+0000","last_fullsized":"2023-10-09T05:05:27.943102+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":77,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636631+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:05:01.636631+0000","last_peered":"2023-10-09T05:05:01.636631+0000","last_clean":"2023-10-09T05:05:01.636631+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:05:01.636631+0000","last_undegraded":"2023-10-09T05:05:01.636631+0000","last_fullsized":"2023-10-09T05:05:01.636631+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1260,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253688+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:05:15.253688+0000","last_peered":"2023-10-09T05:05:15.253688+0000","last_clean":"2023-10-09T05:05:15.253688+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:05:15.253688+0000","last_undegraded":"2023-10-09T05:05:15.253688+0000","last_fullsized":"2023-10-09T05:05:15.253688+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"663'6296","reported_seq":12072,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.377911+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:05:28.377911+0000","last_peered":"2023-10-09T05:05:28.377911+0000","last_clean":"2023-10-09T05:05:28.377911+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:05:28.377911+0000","last_undegraded":"2023-10-09T05:05:28.377911+0000","last_fullsized":"2023-10-09T05:05:28.377911+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4003,"log_dups_size":0,"ondisk_log_size":4003,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89751564,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3434,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":157,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.272068+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:05:15.272068+0000","last_peered":"2023-10-09T05:05:15.272068+0000","last_clean":"2023-10-09T05:05:15.272068+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:05:15.272068+0000","last_undegraded":"2023-10-09T05:05:15.272068+0000","last_fullsized":"2023-10-09T05:05:15.272068+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636368+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:05:01.636368+0000","last_peered":"2023-10-09T05:05:01.636368+0000","last_clean":"2023-10-09T05:05:01.636368+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:05:01.636368+0000","last_undegraded":"2023-10-09T05:05:01.636368+0000","last_fullsized":"2023-10-09T05:05:01.636368+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.637829+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:05:01.637829+0000","last_peered":"2023-10-09T05:05:01.637829+0000","last_clean":"2023-10-09T05:05:01.637829+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:05:01.637829+0000","last_undegraded":"2023-10-09T05:05:01.637829+0000","last_fullsized":"2023-10-09T05:05:01.637829+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"662'6205","reported_seq":11228,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:27.989726+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:05:27.989726+0000","last_peered":"2023-10-09T05:05:27.989726+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:05:27.989726+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4078,"log_dups_size":0,"ondisk_log_size":4078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91947008,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":1052,"num_objects_missing":1052,"num_objects_degraded":2759,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3376,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1295,"num_bytes_recovered":84836352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,3,5,7","objects":1},{"shards":"2,5,7","objects":961},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":50,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.647514+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:05:01.647514+0000","last_peered":"2023-10-09T05:05:01.647514+0000","last_clean":"2023-10-09T05:05:01.647514+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:05:01.647514+0000","last_undegraded":"2023-10-09T05:05:01.647514+0000","last_fullsized":"2023-10-09T05:05:01.647514+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1266,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253267+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:05:15.253267+0000","last_peered":"2023-10-09T05:05:15.253267+0000","last_clean":"2023-10-09T05:05:15.253267+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:05:15.253267+0000","last_undegraded":"2023-10-09T05:05:15.253267+0000","last_fullsized":"2023-10-09T05:05:15.253267+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.272924+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:05:15.272924+0000","last_peered":"2023-10-09T05:05:15.272924+0000","last_clean":"2023-10-09T05:05:15.272924+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:05:15.272924+0000","last_undegraded":"2023-10-09T05:05:15.272924+0000","last_fullsized":"2023-10-09T05:05:15.272924+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":183,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.254016+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:05:15.254016+0000","last_peered":"2023-10-09T05:05:15.254016+0000","last_clean":"2023-10-09T05:05:15.254016+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:05:15.254016+0000","last_undegraded":"2023-10-09T05:05:15.254016+0000","last_fullsized":"2023-10-09T05:05:15.254016+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1239,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253755+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:05:15.253755+0000","last_peered":"2023-10-09T05:05:15.253755+0000","last_clean":"2023-10-09T05:05:15.253755+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:05:15.253755+0000","last_undegraded":"2023-10-09T05:05:15.253755+0000","last_fullsized":"2023-10-09T05:05:15.253755+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"663'6373","reported_seq":14632,"reported_epoch":664,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.941080+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:05:28.941080+0000","last_peered":"2023-10-09T05:05:28.941080+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:05:28.941080+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4610,"log_dups_size":0,"ondisk_log_size":4610,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91815936,"num_objects":1401,"num_object_clones":0,"num_object_copies":4203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3474,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2202,"num_bytes_recovered":131989504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":415},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":939}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":75,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636404+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:05:01.636404+0000","last_peered":"2023-10-09T05:05:01.636404+0000","last_clean":"2023-10-09T05:05:01.636404+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:05:01.636404+0000","last_undegraded":"2023-10-09T05:05:01.636404+0000","last_fullsized":"2023-10-09T05:05:01.636404+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":22,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942627+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:05:27.942627+0000","last_peered":"2023-10-09T05:05:27.942627+0000","last_clean":"2023-10-09T05:05:27.942627+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:05:27.942627+0000","last_undegraded":"2023-10-09T05:05:27.942627+0000","last_fullsized":"2023-10-09T05:05:27.942627+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":115,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.961035+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:05:27.961035+0000","last_peered":"2023-10-09T05:05:27.961035+0000","last_clean":"2023-10-09T05:05:27.961035+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:05:27.961035+0000","last_undegraded":"2023-10-09T05:05:27.961035+0000","last_fullsized":"2023-10-09T05:05:27.961035+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1260,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.254639+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:05:15.254639+0000","last_peered":"2023-10-09T05:05:15.254639+0000","last_clean":"2023-10-09T05:05:15.254639+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:05:15.254639+0000","last_undegraded":"2023-10-09T05:05:15.254639+0000","last_fullsized":"2023-10-09T05:05:15.254639+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"662'6021","reported_seq":18051,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.237420+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:05:27.237420+0000","last_peered":"2023-10-09T05:05:27.237420+0000","last_clean":"2023-10-09T05:05:27.237420+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:05:27.237420+0000","last_undegraded":"2023-10-09T05:05:27.237420+0000","last_fullsized":"2023-10-09T05:05:27.237420+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92176384,"num_objects":1406,"num_object_clones":0,"num_object_copies":4218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3482,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":129,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943497+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:05:27.943497+0000","last_peered":"2023-10-09T05:05:27.943497+0000","last_clean":"2023-10-09T05:05:27.943497+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:05:27.943497+0000","last_undegraded":"2023-10-09T05:05:27.943497+0000","last_fullsized":"2023-10-09T05:05:27.943497+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":86,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636484+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:05:01.636484+0000","last_peered":"2023-10-09T05:05:01.636484+0000","last_clean":"2023-10-09T05:05:01.636484+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:05:01.636484+0000","last_undegraded":"2023-10-09T05:05:01.636484+0000","last_fullsized":"2023-10-09T05:05:01.636484+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":40,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.647584+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:05:01.647584+0000","last_peered":"2023-10-09T05:05:01.647584+0000","last_clean":"2023-10-09T05:05:01.647584+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:05:01.647584+0000","last_undegraded":"2023-10-09T05:05:01.647584+0000","last_fullsized":"2023-10-09T05:05:01.647584+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1263,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253993+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:05:15.253993+0000","last_peered":"2023-10-09T05:05:15.253993+0000","last_clean":"2023-10-09T05:05:15.253993+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:05:15.253993+0000","last_undegraded":"2023-10-09T05:05:15.253993+0000","last_fullsized":"2023-10-09T05:05:15.253993+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"662'6164","reported_seq":23515,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:26.933519+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:05:26.933519+0000","last_peered":"2023-10-09T05:05:26.933519+0000","last_clean":"2023-10-09T05:05:26.933519+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:05:26.933519+0000","last_undegraded":"2023-10-09T05:05:26.933519+0000","last_fullsized":"2023-10-09T05:05:26.933519+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5764,"log_dups_size":0,"ondisk_log_size":5764,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93585408,"num_objects":1428,"num_object_clones":0,"num_object_copies":4284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3343,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":287,"reported_epoch":663,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:27.979187+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:05:27.979187+0000","last_peered":"2023-10-09T05:05:27.979187+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:05:27.979187+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:05:27.979187+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":92,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.637401+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:05:01.637401+0000","last_peered":"2023-10-09T05:05:01.637401+0000","last_clean":"2023-10-09T05:05:01.637401+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:05:01.637401+0000","last_undegraded":"2023-10-09T05:05:01.637401+0000","last_fullsized":"2023-10-09T05:05:01.637401+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":166,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.270970+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:05:15.270970+0000","last_peered":"2023-10-09T05:05:15.270970+0000","last_clean":"2023-10-09T05:05:15.270970+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:05:15.270970+0000","last_undegraded":"2023-10-09T05:05:15.270970+0000","last_fullsized":"2023-10-09T05:05:15.270970+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1219,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.253829+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:05:15.253829+0000","last_peered":"2023-10-09T05:05:15.253829+0000","last_clean":"2023-10-09T05:05:15.253829+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:05:15.253829+0000","last_undegraded":"2023-10-09T05:05:15.253829+0000","last_fullsized":"2023-10-09T05:05:15.253829+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"663'6540","reported_seq":12556,"reported_epoch":664,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:29.027938+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:05:29.027938+0000","last_peered":"2023-10-09T05:05:29.027938+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:05:29.027938+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5115,"log_dups_size":0,"ondisk_log_size":5115,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":1792,"num_objects_missing":1792,"num_objects_degraded":4680,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3580,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1371,"num_bytes_recovered":89849856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-2},{"shards":"0,2,4,7","objects":1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":1330},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":179,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.254367+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:05:15.254367+0000","last_peered":"2023-10-09T05:05:15.254367+0000","last_clean":"2023-10-09T05:05:15.254367+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:05:15.254367+0000","last_undegraded":"2023-10-09T05:05:15.254367+0000","last_fullsized":"2023-10-09T05:05:15.254367+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.637283+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:05:01.637283+0000","last_peered":"2023-10-09T05:05:01.637283+0000","last_clean":"2023-10-09T05:05:01.637283+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:05:01.637283+0000","last_undegraded":"2023-10-09T05:05:01.637283+0000","last_fullsized":"2023-10-09T05:05:01.637283+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":139,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942760+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:05:27.942760+0000","last_peered":"2023-10-09T05:05:27.942760+0000","last_clean":"2023-10-09T05:05:27.942760+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:05:27.942760+0000","last_undegraded":"2023-10-09T05:05:27.942760+0000","last_fullsized":"2023-10-09T05:05:27.942760+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"663'5842","reported_seq":10413,"reported_epoch":663,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:28.431926+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:05:28.431926+0000","last_peered":"2023-10-09T05:05:28.431926+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:05:28.431926+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87195648,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":1756,"num_objects_missing":1756,"num_objects_degraded":4171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3386,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1028,"num_bytes_recovered":67371008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-36},{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":980}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":129,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.943253+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:05:27.943253+0000","last_peered":"2023-10-09T05:05:27.943253+0000","last_clean":"2023-10-09T05:05:27.943253+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:05:27.943253+0000","last_undegraded":"2023-10-09T05:05:27.943253+0000","last_fullsized":"2023-10-09T05:05:27.943253+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":77,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.636559+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:05:01.636559+0000","last_peered":"2023-10-09T05:05:01.636559+0000","last_clean":"2023-10-09T05:05:01.636559+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:05:01.636559+0000","last_undegraded":"2023-10-09T05:05:01.636559+0000","last_fullsized":"2023-10-09T05:05:01.636559+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"662'5943","reported_seq":10780,"reported_epoch":662,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:27.935189+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:05:27.935189+0000","last_peered":"2023-10-09T05:05:27.935189+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:05:27.935189+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:05:27.935189+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95092736,"num_objects":1451,"num_object_clones":0,"num_object_copies":4353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3408,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1451}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":797,"reported_epoch":661,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:01.648126+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:05:01.648126+0000","last_peered":"2023-10-09T05:05:01.648126+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:05:01.648126+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:05:01.648126+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":22,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942927+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:05:27.942927+0000","last_peered":"2023-10-09T05:05:27.942927+0000","last_clean":"2023-10-09T05:05:27.942927+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:05:27.942927+0000","last_undegraded":"2023-10-09T05:05:27.942927+0000","last_fullsized":"2023-10-09T05:05:27.942927+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"662'5990","reported_seq":10410,"reported_epoch":662,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:27.227523+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:05:27.227523+0000","last_peered":"2023-10-09T05:05:27.227523+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:05:27.227523+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5452,"log_dups_size":0,"ondisk_log_size":5452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92045312,"num_objects":1405,"num_object_clones":0,"num_object_copies":4215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3459,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":758}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":743,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.648795+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:05:01.648795+0000","last_peered":"2023-10-09T05:05:01.648795+0000","last_clean":"2023-10-09T05:05:01.648795+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:05:01.648795+0000","last_undegraded":"2023-10-09T05:05:01.648795+0000","last_fullsized":"2023-10-09T05:05:01.648795+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942930+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:05:27.942930+0000","last_peered":"2023-10-09T05:05:27.942930+0000","last_clean":"2023-10-09T05:05:27.942930+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:05:27.942930+0000","last_undegraded":"2023-10-09T05:05:27.942930+0000","last_fullsized":"2023-10-09T05:05:27.942930+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"662'5974","reported_seq":16776,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.948886+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:05:27.948886+0000","last_peered":"2023-10-09T05:05:27.948886+0000","last_clean":"2023-10-09T05:05:27.948886+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:05:27.948886+0000","last_undegraded":"2023-10-09T05:05:27.948886+0000","last_fullsized":"2023-10-09T05:05:27.948886+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93192192,"num_objects":1422,"num_object_clones":0,"num_object_copies":4266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3442,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":283,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.270865+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:05:15.270865+0000","last_peered":"2023-10-09T05:05:15.270865+0000","last_clean":"2023-10-09T05:05:15.270865+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:05:15.270865+0000","last_undegraded":"2023-10-09T05:05:15.270865+0000","last_fullsized":"2023-10-09T05:05:15.270865+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":291,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-10-09T05:05:15.272479+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:05:15.272479+0000","last_peered":"2023-10-09T05:05:15.272479+0000","last_clean":"2023-10-09T05:05:15.272479+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:05:15.272479+0000","last_undegraded":"2023-10-09T05:05:15.272479+0000","last_fullsized":"2023-10-09T05:05:15.272479+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"663'5916","reported_seq":17520,"reported_epoch":663,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:28.295498+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:05:28.295498+0000","last_peered":"2023-10-09T05:05:28.295498+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:05:28.295498+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:05:28.295498+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91619328,"num_objects":1398,"num_object_clones":0,"num_object_copies":4194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3417,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":1398}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":42,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-10-09T05:05:01.649245+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:05:01.649245+0000","last_peered":"2023-10-09T05:05:01.649245+0000","last_clean":"2023-10-09T05:05:01.649245+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:05:01.649245+0000","last_undegraded":"2023-10-09T05:05:01.649245+0000","last_fullsized":"2023-10-09T05:05:01.649245+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":140,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-10-09T05:05:27.942709+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:05:27.942709+0000","last_peered":"2023-10-09T05:05:27.942709+0000","last_clean":"2023-10-09T05:05:27.942709+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:05:27.942709+0000","last_undegraded":"2023-10-09T05:05:27.942709+0000","last_fullsized":"2023-10-09T05:05:27.942709+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":2923364376,"num_objects":44608,"num_object_clones":0,"num_object_copies":133824,"num_objects_missing_on_primary":15202,"num_objects_missing":15202,"num_objects_degraded":43655,"num_objects_misplaced":9084,"num_objects_unfound":0,"num_objects_dirty":44608,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110610,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72141,"num_bytes_recovered":4399431680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14369239040,"data_stored":14369226824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":194048,"ondisk_log_size":194048,"up":108,"acting":91,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234698,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3233508,"kb_used_data":1724252,"kb_used_omap":0,"kb_used_meta":1509248,"kb_avail":90511644,"statfs":{"total":95995035648,"available":92683923456,"internally_reserved":0,"allocated":1765634048,"data_stored":1763952397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545469952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7190000000000003,"5min":4.2489999999999997,"15min":1.9099999999999999},"min":{"1min":0.23699999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":35.673000000000002,"5min":123.04000000000001,"15min":123.04000000000001},"last":2.0129999999999999},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.4390000000000001,"15min":2.1880000000000002},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":35.622999999999998,"5min":123.059,"15min":123.059},"last":0.41799999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:05:15 2023","interfaces":[{"interface":"back","average":{"1min":6.0919999999999996,"5min":3.4500000000000002,"15min":2.6579999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.25800000000000001},"max":{"1min":60.377000000000002,"5min":60.377000000000002,"15min":122.56},"last":2.2559999999999998},{"interface":"front","average":{"1min":6.5810000000000004,"5min":3.1829999999999998,"15min":2.6120000000000001},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":59.689999999999998,"5min":59.689999999999998,"15min":122.666},"last":0.30599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.4569999999999999,"15min":3.8889999999999998},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.30299999999999999},"max":{"1min":10.545,"5min":154.76400000000001,"15min":207.96600000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.8029999999999999,"5min":4.4340000000000002,"15min":4.0110000000000001},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.29699999999999999},"max":{"1min":10.532,"5min":154.80000000000001,"15min":207.97900000000001},"last":0.52600000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:05:08 2023","interfaces":[{"interface":"back","average":{"1min":16.815999999999999,"5min":5.5110000000000001,"15min":2.8839999999999999},"min":{"1min":0.35199999999999998,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":126.70399999999999,"5min":126.70399999999999,"15min":126.70399999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":15.994,"5min":5.3739999999999997,"15min":2.9060000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":125.991,"5min":125.991,"15min":125.991},"last":2.5299999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38400000000000001,"15min":0.38600000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.497,"15min":0.48099999999999998},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.923,"5min":2.923,"15min":2.923},"last":0.28000000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.6579999999999999,"15min":1.7929999999999999},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":5.5970000000000004,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.8169999999999999,"15min":2.149},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":0.57899999999999996,"5min":53.225000000000001,"15min":135.74199999999999},"last":0.32400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":1.4730000000000001,"15min":1.23},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.57299999999999995,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.45500000000000002,"5min":1.736,"15min":1.4339999999999999},"min":{"1min":0.34899999999999998,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":0.56100000000000005,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.38}]}]},{"osd":6,"up_from":46,"seq":197568496330,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2117068,"kb_used_data":934596,"kb_used_omap":0,"kb_used_meta":1182464,"kb_avail":91628084,"statfs":{"total":95995035648,"available":93827158016,"internally_reserved":0,"allocated":957026304,"data_stored":955352618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1210843136},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":4.3780000000000001,"5min":5.883,"15min":2.4319999999999999},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":47.750999999999998,"5min":321.346,"15min":321.346},"last":1.3580000000000001},{"interface":"front","average":{"1min":3.718,"5min":4.1790000000000003,"15min":2.0510000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":47.645000000000003,"5min":169.119,"15min":169.119},"last":5.9459999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":2.4590000000000001,"15min":4.657},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":6.5759999999999996,"5min":72.840999999999994,"15min":260.738},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.7370000000000001,"15min":4.6319999999999997},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.2770000000000001,"5min":72.832999999999998,"15min":261.26100000000002},"last":10.02}]},{"osd":2,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.4399999999999999,"15min":1.994},"min":{"1min":0.35299999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.032999999999999,"5min":23.943000000000001,"15min":133.535},"last":1.155},{"interface":"front","average":{"1min":1.591,"5min":1.421,"15min":1.9390000000000001},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.30599999999999999},"max":{"1min":8.5920000000000005,"5min":25.978999999999999,"15min":133.51900000000001},"last":1.1639999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":5.9359999999999999,"5min":2.4500000000000002,"15min":2.6579999999999999},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":64.355999999999995,"5min":64.355999999999995,"15min":147.43600000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":5.7930000000000001,"5min":2.2589999999999999,"15min":2.7370000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":56.133000000000003,"5min":56.133000000000003,"15min":147.41999999999999},"last":0.52500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":0.96699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":18.186,"5min":18.186,"15min":18.186},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.548,"5min":1.1120000000000001,"15min":1.04},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.502}]},{"osd":5,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.085,"15min":1.3100000000000001},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.8300000000000001,"5min":25.934000000000001,"15min":66.995999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.85999999999999999,"15min":1.3120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.254},"max":{"1min":7.2220000000000004,"5min":16.789000000000001,"15min":67.189999999999998},"last":0.372}]},{"osd":7,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.66900000000000004,"15min":1.1180000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":0.59299999999999997,"5min":5.835,"15min":36.204999999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.002,"15min":1.1259999999999999},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":2.8069999999999999,"5min":20.041,"15min":20.041},"last":0.46200000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141397,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2324008,"kb_used_data":1012320,"kb_used_omap":4,"kb_used_meta":1311675,"kb_avail":91421144,"statfs":{"total":95995035648,"available":93615251456,"internally_reserved":0,"allocated":1036615680,"data_stored":1033097148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1343156110},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":5.8949999999999996,"5min":2.431,"15min":1.361},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":85.753,"5min":85.753,"15min":85.753},"last":5.9269999999999996},{"interface":"front","average":{"1min":5.9269999999999996,"5min":2.4049999999999998,"15min":1.355},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.214},"max":{"1min":85.227999999999994,"5min":85.227999999999994,"15min":85.227999999999994},"last":1.919}]},{"osd":2,"last update":"Mon Oct 9 05:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1639999999999999,"15min":2.4140000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":6.3129999999999997,"5min":12.460000000000001,"15min":263.089},"last":5.5640000000000001},{"interface":"front","average":{"1min":2.2509999999999999,"5min":1.4870000000000001,"15min":2.4689999999999999},"min":{"1min":0.254,"5min":0.252,"15min":0.222},"max":{"1min":8.5250000000000004,"5min":13.933999999999999,"15min":254.50899999999999},"last":7.1200000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:05:16 2023","interfaces":[{"interface":"back","average":{"1min":24.670999999999999,"5min":6.3529999999999998,"15min":3.2160000000000002},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":235.43600000000001,"5min":235.43600000000001,"15min":235.43600000000001},"last":5.7469999999999999},{"interface":"front","average":{"1min":24.478000000000002,"5min":5.7560000000000002,"15min":2.9140000000000001},"min":{"1min":0.31900000000000001,"5min":0.249,"15min":0.249},"max":{"1min":234.32499999999999,"5min":234.32499999999999,"15min":234.32499999999999},"last":0.60499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:04:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.7800000000000002,"5min":5.7800000000000002,"15min":5.7800000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.03,"15min":1.1779999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":5.8109999999999999,"5min":7.79,"15min":7.79},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.4770000000000001,"5min":1.4510000000000001,"15min":1.2350000000000001},"min":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":10.554,"5min":10.554,"15min":19.597999999999999},"last":0.503}]},{"osd":6,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.125,"15min":1.157},"min":{"1min":0.41099999999999998,"5min":0.39700000000000002,"15min":0.35699999999999998},"max":{"1min":5.7489999999999997,"5min":7.4370000000000003,"15min":7.4370000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.4430000000000001,"5min":1.3440000000000001,"15min":1.252},"min":{"1min":0.39500000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.472,"5min":10.472,"15min":19.548999999999999},"last":0.78300000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":1.484,"15min":1.722},"min":{"1min":0.42399999999999999,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":10.583,"5min":10.583,"15min":59.972000000000001},"last":0.746},{"interface":"front","average":{"1min":0.745,"5min":1.2150000000000001,"15min":1.6279999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":3.5670000000000002,"5min":11.903,"15min":60.011000000000003},"last":1.389}]}]},{"osd":0,"up_from":91,"seq":390842024481,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1829536,"kb_used_data":715416,"kb_used_omap":5,"kb_used_meta":1114106,"kb_avail":91915616,"statfs":{"total":95995035648,"available":94121590784,"internally_reserved":0,"allocated":732585984,"data_stored":729728012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1140845178},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:05:12 2023","interfaces":[{"interface":"back","average":{"1min":6.1740000000000004,"5min":3.0640000000000001,"15min":4.7590000000000003},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":53.176000000000002,"5min":53.176000000000002,"15min":247.68899999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":7.3929999999999998,"5min":3.1240000000000001,"15min":4.7720000000000002},"min":{"1min":0.36499999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":53.372999999999998,"5min":53.372999999999998,"15min":246.38},"last":0.377}]},{"osd":2,"last update":"Mon Oct 9 05:04:31 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.9399999999999999,"15min":2.2930000000000001},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":6.9340000000000002,"5min":26.556999999999999,"15min":151.06399999999999},"last":0.318},{"interface":"front","average":{"1min":4.4400000000000004,"5min":2.2400000000000002,"15min":2.4820000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":20.239000000000001,"5min":29.032,"15min":151.22399999999999},"last":0.34799999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:04:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":5.3319999999999999,"15min":3.5870000000000002},"min":{"1min":0.47599999999999998,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":11.872,"5min":290.89100000000002,"15min":290.89100000000002},"last":57.337000000000003},{"interface":"front","average":{"1min":3.4510000000000001,"5min":5.3380000000000001,"15min":3.6469999999999998},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":14.541,"5min":290.73899999999998,"15min":290.73899999999998},"last":63.084000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":1.9570000000000001,"15min":1.887},"min":{"1min":0.443,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.5329999999999995,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.663,"15min":1.577},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.81,"5min":11.81,"15min":11.81},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0089999999999999,"5min":1.78,"15min":1.3100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.039999999999999,"5min":16.888000000000002,"15min":16.888000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.569,"5min":1.835,"15min":1.288},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":9.5779999999999994,"5min":14.036,"15min":14.036},"last":0.49399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0089999999999999,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.074999999999999,"5min":20.129999999999999,"15min":39.594999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.573,"5min":2.3759999999999999,"15min":1.6020000000000001},"min":{"1min":0.45400000000000001,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":9.6419999999999995,"5min":47.241,"15min":47.241},"last":0.61299999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.0870000000000002,"15min":1.496},"min":{"1min":0.51600000000000001,"5min":0.41899999999999998,"15min":0.39000000000000001},"max":{"1min":9.6120000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.986,"15min":1.4790000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.38200000000000001},"max":{"1min":11.842000000000001,"5min":21.68,"15min":21.68},"last":0.56100000000000005}]}]},{"osd":2,"up_from":0,"seq":949187772759,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356453,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2258808,"kb_used_data":1327408,"kb_used_omap":2,"kb_used_meta":931389,"kb_avail":91486344,"statfs":{"total":95995035648,"available":93682016256,"internally_reserved":0,"allocated":1359265792,"data_stored":1356324160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":953742389},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":2.6030000000000002,"15min":1.218},"min":{"1min":0.33000000000000002,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":15.021000000000001,"5min":108.631,"15min":108.631},"last":1.3700000000000001},{"interface":"front","average":{"1min":3.5089999999999999,"5min":2.7719999999999998,"15min":1.365},"min":{"1min":0.247,"5min":0.247,"15min":0.23499999999999999},"max":{"1min":11.728,"5min":110.012,"15min":110.012},"last":1.645}]},{"osd":1,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6139999999999999,"5min":1.8300000000000001,"15min":2.29},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":10.345000000000001,"5min":15.218,"15min":125.233},"last":1.6020000000000001},{"interface":"front","average":{"1min":2.972,"5min":1.839,"15min":2.2770000000000001},"min":{"1min":0.313,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":8.0039999999999996,"5min":26.692,"15min":126.102},"last":1.3979999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:36 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.3009999999999999,"15min":2.1389999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":6.4160000000000004,"5min":15.606999999999999,"15min":212.065},"last":1.3380000000000001},{"interface":"front","average":{"1min":2.0800000000000001,"5min":1.3899999999999999,"15min":2.1789999999999998},"min":{"1min":0.33100000000000002,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":12.987,"5min":12.987,"15min":211.98400000000001},"last":1.679}]},{"osd":4,"last update":"Mon Oct 9 05:04:27 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.72,"15min":1.72},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.193,"5min":11.193,"15min":11.193},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.6030000000000002,"5min":3.6030000000000002,"15min":3.6030000000000002},"last":0.59099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":5.0220000000000002,"5min":1.798,"15min":1.431},"min":{"1min":0.44900000000000001,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":57.804000000000002,"5min":57.804000000000002,"15min":57.804000000000002},"last":0.71199999999999997},{"interface":"front","average":{"1min":5.452,"5min":2.0990000000000002,"15min":1.6419999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":57.442,"5min":57.442,"15min":57.442},"last":0.68799999999999994}]},{"osd":6,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5779999999999998,"5min":1.6220000000000001,"15min":1.1299999999999999},"min":{"1min":0.40100000000000002,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":11.252000000000001,"5min":11.252000000000001,"15min":11.252000000000001},"last":0.627},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.526,"15min":1.3320000000000001},"min":{"1min":0.44800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.481,"5min":14.731,"15min":24.916},"last":0.55100000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.6240000000000001,"15min":1.425},"min":{"1min":0.46800000000000003,"5min":0.39400000000000002,"15min":0.375},"max":{"1min":10.42,"5min":14.784000000000001,"15min":25.004000000000001},"last":0.497},{"interface":"front","average":{"1min":1.9159999999999999,"5min":1.3819999999999999,"15min":1.1339999999999999},"min":{"1min":0.47999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.798,"5min":7.758,"15min":7.758},"last":0.751}]}]},{"osd":4,"up_from":626,"seq":2688649527322,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2968224,"kb_used_data":2558936,"kb_used_omap":22,"kb_used_meta":409257,"kb_avail":90776928,"statfs":{"total":95995035648,"available":92955574272,"internally_reserved":0,"allocated":2620350464,"data_stored":2618665085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":419080013},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":2.2909999999999999,"15min":2.2909999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":21.292000000000002,"5min":21.292000000000002,"15min":21.292000000000002},"last":6.4470000000000001},{"interface":"front","average":{"1min":3.1709999999999998,"5min":3.1709999999999998,"15min":3.1709999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":21.391999999999999,"5min":21.391999999999999,"15min":21.391999999999999},"last":1.9399999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":2.129,"5min":2.129,"15min":2.129},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.955,"5min":14.955,"15min":14.955},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.901,"5min":1.901,"15min":1.901},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":10.579000000000001,"5min":10.579000000000001,"15min":10.579000000000001},"last":2.1989999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.639,"5min":1.639,"15min":1.639},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.051,"5min":11.051,"15min":11.051},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":11.036,"5min":11.036,"15min":11.036},"last":0.80900000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.175,"15min":1.175},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":2.0640000000000001},{"interface":"front","average":{"1min":1.8440000000000001,"5min":1.8440000000000001,"15min":1.8440000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.8179999999999996,"5min":9.8179999999999996,"15min":9.8179999999999996},"last":0.48999999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.47699999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7910000000000004,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.63800000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:04:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.59399999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757974,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4743548,"kb_used_data":3246132,"kb_used_omap":0,"kb_used_meta":1497408,"kb_avail":89001604,"statfs":{"total":95995035648,"available":91137642496,"internally_reserved":0,"allocated":3324039168,"data_stored":3322376877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1533345792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.4119999999999999,"15min":2.137},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.0309999999999997,"5min":103.745,"15min":106.465},"last":9.1440000000000001},{"interface":"front","average":{"1min":1.5880000000000001,"5min":3.71,"15min":2.3090000000000002},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.245},"max":{"1min":10.882999999999999,"5min":103.872,"15min":106.54600000000001},"last":9.2520000000000007}]},{"osd":1,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":5.0410000000000004,"15min":3.556},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.23200000000000001},"max":{"1min":6.0529999999999999,"5min":262.26900000000001,"15min":262.26900000000001},"last":1.3859999999999999},{"interface":"front","average":{"1min":2.7709999999999999,"5min":5.0860000000000003,"15min":3.7040000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26900000000000002},"max":{"1min":10.462,"5min":262.291,"15min":262.291},"last":0.35499999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:04:38 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":3.0089999999999999,"15min":1.794},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.27000000000000002},"max":{"1min":10.382999999999999,"5min":106.238,"15min":106.238},"last":2.4359999999999999},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.8170000000000002,"15min":2.0219999999999998},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":8.4960000000000004,"5min":104.31699999999999,"15min":104.31699999999999},"last":4.7720000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:05:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":5.7110000000000003,"15min":3.0899999999999999},"min":{"1min":0.41599999999999998,"5min":0.318,"15min":0.318},"max":{"1min":10.385,"5min":125.824,"15min":125.824},"last":1.4019999999999999},{"interface":"front","average":{"1min":1.8640000000000001,"5min":5.6420000000000003,"15min":2.9870000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":9.8520000000000003,"5min":125.761,"15min":125.761},"last":1.51}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.623,"5min":0.873,"15min":0.873},"last":0.46000000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.76400000000000001,"15min":0.82899999999999996},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23300000000000001},"max":{"1min":0.69999999999999996,"5min":11.093,"15min":11.093},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.44,"5min":0.63100000000000001,"15min":0.91900000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.73099999999999998,"5min":5.9329999999999998,"15min":13.083},"last":0.48999999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:05:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.71899999999999997,"15min":1.734},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.746,"5min":8.3520000000000003,"15min":110.521},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.90300000000000002,"15min":1.7729999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29499999999999998},"max":{"1min":0.72199999999999998,"5min":11.791,"15min":110.746},"last":0.47499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":722665472,"data_stored":722665472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1025638400,"data_stored":1025638400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2649161728,"data_stored":2649161728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1351024640,"data_stored":1351024640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2610630656,"data_stored":2610626584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3313963008,"data_stored":3313958936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":944902144,"data_stored":944898072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1751252992,"data_stored":1751252992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:05:36.118 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:36.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:36.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:36.324 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 43324/131325 objects degraded (32.990%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:36.325 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 43324/131325 objects degraded (32.990%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:36.331 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:36.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 43324/131325 objects degraded (32.990%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:36.757 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:36.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:36.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:36.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:36.987 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:37.054 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:37.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:37.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:37.184 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:37.231 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:37.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:37.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:37.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:37 smithi029 ceph-mon[223858]: pgmap v2133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 42824/130173 objects degraded (32.898%); 8752/130173 objects misplaced (6.723%); 7.7 MiB/s, 157 objects/s recovering 2023-10-09T05:05:37.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:37 smithi029 ceph-mon[216903]: pgmap v2133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 42824/130173 objects degraded (32.898%); 8752/130173 objects misplaced (6.723%); 7.7 MiB/s, 157 objects/s recovering 2023-10-09T05:05:37.453 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:37.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:37 smithi177 ceph-mon[195797]: pgmap v2133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 42824/130173 objects degraded (32.898%); 8752/130173 objects misplaced (6.723%); 7.7 MiB/s, 157 objects/s recovering 2023-10-09T05:05:37.754 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:37.754 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:37.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:37.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:37.895 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:37.924 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:37.972 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:37.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:37.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:38.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:38.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:05:38.163 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:38.492 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:38.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:38.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:38.620 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:38.647 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:38.698 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:38.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:38.804 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:38.922 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:38.952 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:05:38.952 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:05:38.952 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:05:38.952 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:05:38.952 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:05:39.080 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:39.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:39.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:05:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:39 smithi029 ceph-mon[216903]: pgmap v2134: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 42732/128919 objects degraded (33.146%); 8528/128919 objects misplaced (6.615%); 7.1 MiB/s, 152 objects/s recovering 2023-10-09T05:05:39.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:39 smithi029 ceph-mon[223858]: pgmap v2134: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 42732/128919 objects degraded (33.146%); 8528/128919 objects misplaced (6.615%); 7.1 MiB/s, 152 objects/s recovering 2023-10-09T05:05:39.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:39.211 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:39.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:39.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:39.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:39.367 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:39.502 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:39.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:39 smithi177 ceph-mon[195797]: pgmap v2134: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 241 op/s; 42732/128919 objects degraded (33.146%); 8528/128919 objects misplaced (6.615%); 7.1 MiB/s, 152 objects/s recovering 2023-10-09T05:05:41.213 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:41.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:41.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:41 smithi029 ceph-mon[223858]: pgmap v2135: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 41975/126258 objects degraded (33.245%); 8404/126258 objects misplaced (6.656%); 6.8 MiB/s, 128 objects/s recovering 2023-10-09T05:05:41.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 41975/126258 objects degraded (33.245%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:41.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:41 smithi029 ceph-mon[216903]: pgmap v2135: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 41975/126258 objects degraded (33.245%); 8404/126258 objects misplaced (6.656%); 6.8 MiB/s, 128 objects/s recovering 2023-10-09T05:05:41.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 41975/126258 objects degraded (33.245%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:41.329 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:05:41.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:41.367 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:41.413 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:41.417 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:41 smithi177 ceph-mon[195797]: pgmap v2135: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 261 op/s; 41975/126258 objects degraded (33.245%); 8404/126258 objects misplaced (6.656%); 6.8 MiB/s, 128 objects/s recovering 2023-10-09T05:05:41.417 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 41975/126258 objects degraded (33.245%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:41.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:41.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:41.519 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:41.546 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:41.596 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:41.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:41.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:41.700 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:41.800 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:41.819 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:41.819 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:41.955 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:41.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:42.066 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:42.193 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:42.404 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:42.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:42.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:42.532 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:42.559 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:42.605 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:42.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:42.711 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:42.755 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:42.816 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:42.924 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:05:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:05:42] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.43.0" 2023-10-09T05:05:43.237 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:05:43.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:43 smithi029 ceph-mon[216903]: pgmap v2136: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 41975/126258 objects degraded (33.245%); 8404/126258 objects misplaced (6.656%); 5.2 MiB/s, 101 objects/s recovering 2023-10-09T05:05:43.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/763789821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:43.415 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:43 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:43.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:43 smithi029 ceph-mon[223858]: pgmap v2136: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 41975/126258 objects degraded (33.245%); 8404/126258 objects misplaced (6.656%); 5.2 MiB/s, 101 objects/s recovering 2023-10-09T05:05:43.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/763789821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:43.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:43 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:43 smithi177 ceph-mon[195797]: pgmap v2136: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 41975/126258 objects degraded (33.245%); 8404/126258 objects misplaced (6.656%); 5.2 MiB/s, 101 objects/s recovering 2023-10-09T05:05:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/763789821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:43 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:05:44.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:05:44.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: failed to encode map e665 with expected crc 2023-10-09T05:05:44.425 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: osdmap e665: 8 total, 8 up, 7 in 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: failed to encode map e665 with expected crc 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: failed to encode map e665 with expected crc 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:05:44.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: osdmap e665: 8 total, 8 up, 7 in 2023-10-09T05:05:44.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: failed to encode map e665 with expected crc 2023-10-09T05:05:44.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:44.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: failed to encode map e665 with expected crc 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: osdmap e665: 8 total, 8 up, 7 in 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: failed to encode map e665 with expected crc 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:05:44.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:44.606 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:44.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:44.758 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:44.783 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:44.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:44.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:44.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:44.932 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:44.960 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:45.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:45.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:45.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:45.120 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:45.255 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:45.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:45 smithi029 ceph-mon[216903]: pgmap v2138: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 41647/123606 objects degraded (33.693%); 8158/123606 objects misplaced (6.600%); 6.4 MiB/s, 131 objects/s recovering 2023-10-09T05:05:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:45 smithi029 ceph-mon[223858]: pgmap v2138: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 41647/123606 objects degraded (33.693%); 8158/123606 objects misplaced (6.600%); 6.4 MiB/s, 131 objects/s recovering 2023-10-09T05:05:45.997 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:46.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:46.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:45 smithi177 ceph-mon[195797]: pgmap v2138: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 41647/123606 objects degraded (33.693%); 8158/123606 objects misplaced (6.600%); 6.4 MiB/s, 131 objects/s recovering 2023-10-09T05:05:46.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:46.243 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:46.734 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:46.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:46.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:46.872 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:46.916 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 41647/123606 objects degraded (33.693%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 41647/123606 objects degraded (33.693%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:46.921 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 41647/123606 objects degraded (33.693%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:46.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:47.023 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:47.052 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:47.094 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:47.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:47.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:47.201 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:05:47.227 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:47.269 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:05:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:47.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:05:47.376 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:47.405 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:47.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:47.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:47.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:47.587 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:47.634 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:47.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:47.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:47.740 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:47.767 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:47.810 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:47.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:47.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:47 smithi029 ceph-mon[216903]: pgmap v2139: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 41226/122631 objects degraded (33.618%); 8049/122631 objects misplaced (6.564%); 6.1 MiB/s, 128 objects/s recovering 2023-10-09T05:05:47.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:47 smithi029 ceph-mon[223858]: pgmap v2139: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 41226/122631 objects degraded (33.618%); 8049/122631 objects misplaced (6.564%); 6.1 MiB/s, 128 objects/s recovering 2023-10-09T05:05:47.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:47 smithi177 ceph-mon[195797]: pgmap v2139: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 41226/122631 objects degraded (33.618%); 8049/122631 objects misplaced (6.564%); 6.1 MiB/s, 128 objects/s recovering 2023-10-09T05:05:48.061 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:48.260 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:48.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:48.303 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:05:48.378 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:48.381 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:05:48.474 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:05:48.501 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:05:48.517 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:48.560 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:05:48.571 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:05:48.575 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:05:48.638 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:05:48.723 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:05:48.772 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:05:48.773 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:05:48.773 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:05:48.830 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:05:48.841 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.841 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.841 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.841 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.842 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.842 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.842 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.842 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:05:48.879 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:48.892 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/338318356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:48.893 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/338318356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:48.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:48.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/338318356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:05:49.105 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:49 smithi177 ceph-mon[195797]: pgmap v2140: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 41102/121533 objects degraded (33.820%); 7495/121533 objects misplaced (6.167%); 8.3 MiB/s, 153 objects/s recovering 2023-10-09T05:05:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:49 smithi177 ceph-mon[195797]: from='client.37650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:50.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:49 smithi029 ceph-mon[223858]: pgmap v2140: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 41102/121533 objects degraded (33.820%); 7495/121533 objects misplaced (6.167%); 8.3 MiB/s, 153 objects/s recovering 2023-10-09T05:05:50.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:49 smithi029 ceph-mon[223858]: from='client.37650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:50.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:49 smithi029 ceph-mon[216903]: pgmap v2140: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 245 op/s; 41102/121533 objects degraded (33.820%); 7495/121533 objects misplaced (6.167%); 8.3 MiB/s, 153 objects/s recovering 2023-10-09T05:05:50.128 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:49 smithi029 ceph-mon[216903]: from='client.37650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:05:50.204 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:50.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:50.321 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:50.471 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:50.914 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:50 smithi029 ceph-mon[216903]: pgmap v2141: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 40347/119121 objects degraded (33.871%); 7368/119121 objects misplaced (6.185%); 8.3 MiB/s, 153 objects/s recovering 2023-10-09T05:05:50.915 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2140,"stamp":"2023-10-09T05:05:47.818328+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2653886392,"num_objects":40511,"num_object_clones":0,"num_object_copies":121533,"num_objects_missing_on_primary":13677,"num_objects_missing":13677,"num_objects_degraded":41102,"num_objects_misplaced":7495,"num_objects_unfound":0,"num_objects_dirty":40511,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":115031,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74683,"num_bytes_recovered":4534930062,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":199995,"ondisk_log_size":199995,"up":357,"acting":340,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":484,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18881232,"kb_used_data":10925592,"kb_used_omap":34,"kb_used_meta":7955549,"kb_avail":637334832,"statfs":{"total":671965249536,"available":652630867968,"internally_reserved":0,"allocated":11187806208,"data_stored":11171711712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8146482570},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,6,4,5,5,3,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-161349632,"num_objects":-2462,"num_object_clones":0,"num_object_copies":-7386,"num_objects_missing_on_primary":-839,"num_objects_missing":-839,"num_objects_degraded":-1630,"num_objects_misplaced":-1033,"num_objects_unfound":0,"num_objects_dirty":-2462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2462,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1534,"num_bytes_recovered":87621632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.020214"},"pg_stats":[{"pgid":"14.23","version":"665'5789","reported_seq":12412,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.189380+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:05:43.189380+0000","last_peered":"2023-10-09T05:05:43.189380+0000","last_clean":"2023-10-09T05:05:43.189380+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:05:43.189380+0000","last_undegraded":"2023-10-09T05:05:43.189380+0000","last_fullsized":"2023-10-09T05:05:43.189380+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5789,"log_dups_size":0,"ondisk_log_size":5789,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41091072,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1805,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"664'5713","reported_seq":12250,"reported_epoch":664,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.102869+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:05:43.102869+0000","last_peered":"2023-10-09T05:05:43.102869+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:05:43.102869+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5713,"log_dups_size":0,"ondisk_log_size":5713,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41254912,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":604,"num_objects_missing":604,"num_objects_degraded":1655,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1778,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":853,"num_bytes_recovered":53657600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"4,5,7","objects":446}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"664'5776","reported_seq":16472,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:42.801051+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:05:42.801051+0000","last_peered":"2023-10-09T05:05:42.801051+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:05:42.801051+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:05:42.801051+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5776,"log_dups_size":0,"ondisk_log_size":5776,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41746432,"num_objects":637,"num_object_clones":0,"num_object_copies":1911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1793,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":637}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"665'5782","reported_seq":7251,"reported_epoch":665,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:46.973496+0000","last_change":"2023-10-09T05:05:26.963137+0000","last_active":"2023-10-09T05:05:46.973496+0000","last_peered":"2023-10-09T05:05:46.973496+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:05:46.973496+0000","last_undegraded":"2023-10-09T05:05:26.962565+0000","last_fullsized":"2023-10-09T05:05:46.973496+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5244,"log_dups_size":0,"ondisk_log_size":5244,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42090496,"num_objects":642,"num_object_clones":0,"num_object_copies":1926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1789,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":347,"num_bytes_recovered":22757376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":593},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"665'6156","reported_seq":19715,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:47.291573+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:05:47.291573+0000","last_peered":"2023-10-09T05:05:47.291573+0000","last_clean":"2023-10-09T05:05:47.291573+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:05:47.291573+0000","last_undegraded":"2023-10-09T05:05:47.291573+0000","last_fullsized":"2023-10-09T05:05:47.291573+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6156,"log_dups_size":0,"ondisk_log_size":6156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83394560,"num_objects":1273,"num_object_clones":0,"num_object_copies":3819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3615,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":167,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940028+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:05:28.940028+0000","last_peered":"2023-10-09T05:05:28.940028+0000","last_clean":"2023-10-09T05:05:28.940028+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:05:28.940028+0000","last_undegraded":"2023-10-09T05:05:28.940028+0000","last_fullsized":"2023-10-09T05:05:28.940028+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":312,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.014600+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:05:29.014600+0000","last_peered":"2023-10-09T05:05:29.014600+0000","last_clean":"2023-10-09T05:05:29.014600+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:05:29.014600+0000","last_undegraded":"2023-10-09T05:05:29.014600+0000","last_fullsized":"2023-10-09T05:05:29.014600+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"665'6106","reported_seq":9331,"reported_epoch":665,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:43.409540+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:05:43.409540+0000","last_peered":"2023-10-09T05:05:43.409540+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:05:43.409540+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:05:43.409540+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84770816,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3567,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":1294}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":89,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.011372+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:05:29.011372+0000","last_peered":"2023-10-09T05:05:29.011372+0000","last_clean":"2023-10-09T05:05:29.011372+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:05:29.011372+0000","last_undegraded":"2023-10-09T05:05:29.011372+0000","last_fullsized":"2023-10-09T05:05:29.011372+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.189624+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:05:43.189624+0000","last_peered":"2023-10-09T05:05:43.189624+0000","last_clean":"2023-10-09T05:05:43.189624+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:05:43.189624+0000","last_undegraded":"2023-10-09T05:05:43.189624+0000","last_fullsized":"2023-10-09T05:05:43.189624+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"665'6118","reported_seq":17774,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.691035+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:05:43.691035+0000","last_peered":"2023-10-09T05:05:43.691035+0000","last_clean":"2023-10-09T05:05:43.691035+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:05:43.691035+0000","last_undegraded":"2023-10-09T05:05:43.691035+0000","last_fullsized":"2023-10-09T05:05:43.691035+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5872,"log_dups_size":0,"ondisk_log_size":5872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3578,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":294,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.117564+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:05:29.117564+0000","last_peered":"2023-10-09T05:05:29.117564+0000","last_clean":"2023-10-09T05:05:29.117564+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:05:29.117564+0000","last_undegraded":"2023-10-09T05:05:29.117564+0000","last_fullsized":"2023-10-09T05:05:29.117564+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":191,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942503+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:05:28.942503+0000","last_peered":"2023-10-09T05:05:28.942503+0000","last_clean":"2023-10-09T05:05:28.942503+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:05:28.942503+0000","last_undegraded":"2023-10-09T05:05:28.942503+0000","last_fullsized":"2023-10-09T05:05:28.942503+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"665'6376","reported_seq":23728,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:47.317580+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:05:47.317580+0000","last_peered":"2023-10-09T05:05:47.317580+0000","last_clean":"2023-10-09T05:05:47.317580+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:05:47.317580+0000","last_undegraded":"2023-10-09T05:05:47.317580+0000","last_fullsized":"2023-10-09T05:05:47.317580+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5976,"log_dups_size":0,"ondisk_log_size":5976,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79691776,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3555,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":39,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945393+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:05:28.945393+0000","last_peered":"2023-10-09T05:05:28.945393+0000","last_clean":"2023-10-09T05:05:28.945393+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:05:28.945393+0000","last_undegraded":"2023-10-09T05:05:28.945393+0000","last_fullsized":"2023-10-09T05:05:28.945393+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":285,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.117493+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:05:29.117493+0000","last_peered":"2023-10-09T05:05:29.117493+0000","last_clean":"2023-10-09T05:05:29.117493+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:05:29.117493+0000","last_undegraded":"2023-10-09T05:05:29.117493+0000","last_fullsized":"2023-10-09T05:05:29.117493+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"665'6633","reported_seq":13024,"reported_epoch":665,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:05:44.281927+0000","last_change":"2023-10-09T05:05:26.975807+0000","last_active":"2023-10-09T05:05:44.281927+0000","last_peered":"2023-10-09T05:05:44.281927+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:05:44.281927+0000","last_undegraded":"2023-10-09T05:05:44.281927+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5208,"log_dups_size":0,"ondisk_log_size":5208,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85655552,"num_objects":1307,"num_object_clones":0,"num_object_copies":3921,"num_objects_missing_on_primary":1688,"num_objects_missing":1688,"num_objects_degraded":0,"num_objects_misplaced":3858,"num_objects_unfound":0,"num_objects_dirty":1307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3673,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1465,"num_bytes_recovered":95944704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":2},{"shards":"0,2,4,7","objects":2},{"shards":"2,4,7","objects":1248},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":53,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.944716+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:05:28.944716+0000","last_peered":"2023-10-09T05:05:28.944716+0000","last_clean":"2023-10-09T05:05:28.944716+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:05:28.944716+0000","last_undegraded":"2023-10-09T05:05:28.944716+0000","last_fullsized":"2023-10-09T05:05:28.944716+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":275,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.118567+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:05:29.118567+0000","last_peered":"2023-10-09T05:05:29.118567+0000","last_clean":"2023-10-09T05:05:29.118567+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:05:29.118567+0000","last_undegraded":"2023-10-09T05:05:29.118567+0000","last_fullsized":"2023-10-09T05:05:29.118567+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"665'6309","reported_seq":12162,"reported_epoch":665,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:05:47.538417+0000","last_change":"2023-10-09T05:05:00.137617+0000","last_active":"2023-10-09T05:05:47.538417+0000","last_peered":"2023-10-09T05:05:47.538417+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:05:47.538417+0000","last_undegraded":"2023-10-09T05:05:47.538417+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4773,"log_dups_size":0,"ondisk_log_size":4773,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84803584,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":1055,"num_objects_missing":1055,"num_objects_degraded":0,"num_objects_misplaced":2725,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3700,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1419,"num_bytes_recovered":92995584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,5,7","objects":1},{"shards":"1,3","objects":240},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":622}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":338,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:29.117170+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:05:29.117170+0000","last_peered":"2023-10-09T05:05:29.117170+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:05:29.117170+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:05:29.117170+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":89,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.012066+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:05:29.012066+0000","last_peered":"2023-10-09T05:05:29.012066+0000","last_clean":"2023-10-09T05:05:29.012066+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:05:29.012066+0000","last_undegraded":"2023-10-09T05:05:29.012066+0000","last_fullsized":"2023-10-09T05:05:29.012066+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"665'6185","reported_seq":18256,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:46.703162+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:05:46.703162+0000","last_peered":"2023-10-09T05:05:46.703162+0000","last_clean":"2023-10-09T05:05:46.703162+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:05:46.703162+0000","last_undegraded":"2023-10-09T05:05:46.703162+0000","last_fullsized":"2023-10-09T05:05:46.703162+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81428480,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3647,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":120,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.941866+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:05:28.941866+0000","last_peered":"2023-10-09T05:05:28.941866+0000","last_clean":"2023-10-09T05:05:28.941866+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:05:28.941866+0000","last_undegraded":"2023-10-09T05:05:28.941866+0000","last_fullsized":"2023-10-09T05:05:28.941866+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":135,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940400+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:05:28.940400+0000","last_peered":"2023-10-09T05:05:28.940400+0000","last_clean":"2023-10-09T05:05:28.940400+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:05:28.940400+0000","last_undegraded":"2023-10-09T05:05:28.940400+0000","last_fullsized":"2023-10-09T05:05:28.940400+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"664'6369","reported_seq":18428,"reported_epoch":664,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.170813+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:05:43.170813+0000","last_peered":"2023-10-09T05:05:43.170813+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:05:43.170813+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84049920,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1807,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3510,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3048,"num_bytes_recovered":195166208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":779}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":141,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940372+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:05:28.940372+0000","last_peered":"2023-10-09T05:05:28.940372+0000","last_clean":"2023-10-09T05:05:28.940372+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:05:28.940372+0000","last_undegraded":"2023-10-09T05:05:28.940372+0000","last_fullsized":"2023-10-09T05:05:28.940372+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":248,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.129580+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:05:29.129580+0000","last_peered":"2023-10-09T05:05:29.129580+0000","last_clean":"2023-10-09T05:05:29.129580+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:05:29.129580+0000","last_undegraded":"2023-10-09T05:05:29.129580+0000","last_fullsized":"2023-10-09T05:05:29.129580+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"665'6461","reported_seq":12280,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:47.485127+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:05:47.485127+0000","last_peered":"2023-10-09T05:05:47.485127+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:05:47.485127+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4334,"log_dups_size":0,"ondisk_log_size":4334,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75169792,"num_objects":1147,"num_object_clones":0,"num_object_copies":3441,"num_objects_missing_on_primary":1252,"num_objects_missing":1252,"num_objects_degraded":2479,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3632,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1563,"num_bytes_recovered":102400000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,7","objects":-39},{"shards":"2,5,7","objects":811},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.118297+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:05:29.118297+0000","last_peered":"2023-10-09T05:05:29.118297+0000","last_clean":"2023-10-09T05:05:29.118297+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:05:29.118297+0000","last_undegraded":"2023-10-09T05:05:29.118297+0000","last_fullsized":"2023-10-09T05:05:29.118297+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":43,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945437+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:05:28.945437+0000","last_peered":"2023-10-09T05:05:28.945437+0000","last_clean":"2023-10-09T05:05:28.945437+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:05:28.945437+0000","last_undegraded":"2023-10-09T05:05:28.945437+0000","last_fullsized":"2023-10-09T05:05:28.945437+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"665'6472","reported_seq":13673,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.740897+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:05:43.740897+0000","last_peered":"2023-10-09T05:05:43.740897+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:05:43.740897+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3188,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3573,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1862,"num_bytes_recovered":109051904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":1220},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":104,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939936+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:05:28.939936+0000","last_peered":"2023-10-09T05:05:28.939936+0000","last_clean":"2023-10-09T05:05:28.939936+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:05:28.939936+0000","last_undegraded":"2023-10-09T05:05:28.939936+0000","last_fullsized":"2023-10-09T05:05:28.939936+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":302,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.117632+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:05:29.117632+0000","last_peered":"2023-10-09T05:05:29.117632+0000","last_clean":"2023-10-09T05:05:29.117632+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:05:29.117632+0000","last_undegraded":"2023-10-09T05:05:29.117632+0000","last_fullsized":"2023-10-09T05:05:29.117632+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"665'6412","reported_seq":13289,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.725079+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:05:43.725079+0000","last_peered":"2023-10-09T05:05:43.725079+0000","last_clean":"2023-10-09T05:05:43.725079+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:05:43.725079+0000","last_undegraded":"2023-10-09T05:05:43.725079+0000","last_fullsized":"2023-10-09T05:05:43.725079+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4119,"log_dups_size":0,"ondisk_log_size":4119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82149388,"num_objects":1254,"num_object_clones":0,"num_object_copies":3762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1254,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3550,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":299,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942070+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:05:28.942070+0000","last_peered":"2023-10-09T05:05:28.942070+0000","last_clean":"2023-10-09T05:05:28.942070+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:05:28.942070+0000","last_undegraded":"2023-10-09T05:05:28.942070+0000","last_fullsized":"2023-10-09T05:05:28.942070+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":160,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.127061+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:05:29.127061+0000","last_peered":"2023-10-09T05:05:29.127061+0000","last_clean":"2023-10-09T05:05:29.127061+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:05:29.127061+0000","last_undegraded":"2023-10-09T05:05:29.127061+0000","last_fullsized":"2023-10-09T05:05:29.127061+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"665'5715","reported_seq":14588,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.181063+0000","last_change":"2023-10-09T05:05:29.132074+0000","last_active":"2023-10-09T05:05:43.181063+0000","last_peered":"2023-10-09T05:05:43.181063+0000","last_clean":"2023-10-09T05:05:43.181063+0000","last_became_active":"2023-10-09T05:05:29.125934+0000","last_became_peered":"2023-10-09T05:05:29.125934+0000","last_unstale":"2023-10-09T05:05:43.181063+0000","last_undegraded":"2023-10-09T05:05:43.181063+0000","last_fullsized":"2023-10-09T05:05:43.181063+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41123840,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1780,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":194,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940961+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:05:28.940961+0000","last_peered":"2023-10-09T05:05:28.940961+0000","last_clean":"2023-10-09T05:05:28.940961+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:05:28.940961+0000","last_undegraded":"2023-10-09T05:05:28.940961+0000","last_fullsized":"2023-10-09T05:05:28.940961+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":53,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.943812+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:05:28.943812+0000","last_peered":"2023-10-09T05:05:28.943812+0000","last_clean":"2023-10-09T05:05:28.943812+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:05:28.943812+0000","last_undegraded":"2023-10-09T05:05:28.943812+0000","last_fullsized":"2023-10-09T05:05:28.943812+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"665'5778","reported_seq":9734,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:44.200846+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:05:44.200846+0000","last_peered":"2023-10-09T05:05:44.200846+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:05:44.200846+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5778,"log_dups_size":0,"ondisk_log_size":5778,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41811968,"num_objects":638,"num_object_clones":0,"num_object_copies":1914,"num_objects_missing_on_primary":815,"num_objects_missing":815,"num_objects_degraded":1865,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1794,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":584,"num_bytes_recovered":38240256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"0,2,5,7","objects":1},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":417},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":22,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:28.940547+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:05:28.940547+0000","last_peered":"2023-10-09T05:05:28.940547+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:05:28.940547+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:05:28.940547+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":151,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939852+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:05:28.939852+0000","last_peered":"2023-10-09T05:05:28.939852+0000","last_clean":"2023-10-09T05:05:28.939852+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:05:28.939852+0000","last_undegraded":"2023-10-09T05:05:28.939852+0000","last_fullsized":"2023-10-09T05:05:28.939852+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"665'5825","reported_seq":9943,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:46.397431+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:05:46.397431+0000","last_peered":"2023-10-09T05:05:46.397431+0000","last_clean":"2023-10-09T05:05:46.397431+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:05:46.397431+0000","last_undegraded":"2023-10-09T05:05:46.397431+0000","last_fullsized":"2023-10-09T05:05:46.397431+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5287,"log_dups_size":0,"ondisk_log_size":5287,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39272448,"num_objects":599,"num_object_clones":0,"num_object_copies":1797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1832,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":47,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945390+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:05:28.945390+0000","last_peered":"2023-10-09T05:05:28.945390+0000","last_clean":"2023-10-09T05:05:28.945390+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:05:28.945390+0000","last_undegraded":"2023-10-09T05:05:28.945390+0000","last_fullsized":"2023-10-09T05:05:28.945390+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":211,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942214+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:05:28.942214+0000","last_peered":"2023-10-09T05:05:28.942214+0000","last_clean":"2023-10-09T05:05:28.942214+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:05:28.942214+0000","last_undegraded":"2023-10-09T05:05:28.942214+0000","last_fullsized":"2023-10-09T05:05:28.942214+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"664'5795","reported_seq":16491,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:43.019854+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:05:43.019854+0000","last_peered":"2023-10-09T05:05:43.019854+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:05:43.019854+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:05:43.019854+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5795,"log_dups_size":0,"ondisk_log_size":5795,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40501248,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1811,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":618}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":725,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.944429+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:05:28.944429+0000","last_peered":"2023-10-09T05:05:28.944429+0000","last_clean":"2023-10-09T05:05:28.944429+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:05:28.944429+0000","last_undegraded":"2023-10-09T05:05:28.944429+0000","last_fullsized":"2023-10-09T05:05:28.944429+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":100,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.011205+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:05:29.011205+0000","last_peered":"2023-10-09T05:05:29.011205+0000","last_clean":"2023-10-09T05:05:29.011205+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:05:29.011205+0000","last_undegraded":"2023-10-09T05:05:29.011205+0000","last_fullsized":"2023-10-09T05:05:29.011205+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1253,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942554+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:05:28.942554+0000","last_peered":"2023-10-09T05:05:28.942554+0000","last_clean":"2023-10-09T05:05:28.942554+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:05:28.942554+0000","last_undegraded":"2023-10-09T05:05:28.942554+0000","last_fullsized":"2023-10-09T05:05:28.942554+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"665'5908","reported_seq":10735,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:44.315550+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:05:44.315550+0000","last_peered":"2023-10-09T05:05:44.315550+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:05:44.315550+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82870272,"num_objects":1264,"num_object_clones":0,"num_object_copies":3792,"num_objects_missing_on_primary":1579,"num_objects_missing":1579,"num_objects_degraded":3722,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3451,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1094,"num_bytes_recovered":71696384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":832}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":269,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:38.284971+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:05:38.284971+0000","last_peered":"2023-10-09T05:05:38.284971+0000","last_clean":"2023-10-09T05:05:38.284971+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:05:38.284971+0000","last_undegraded":"2023-10-09T05:05:38.284971+0000","last_fullsized":"2023-10-09T05:05:38.284971+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.941782+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:05:28.941782+0000","last_peered":"2023-10-09T05:05:28.941782+0000","last_clean":"2023-10-09T05:05:28.941782+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:05:28.941782+0000","last_undegraded":"2023-10-09T05:05:28.941782+0000","last_fullsized":"2023-10-09T05:05:28.941782+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1242,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.941902+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:05:28.941902+0000","last_peered":"2023-10-09T05:05:28.941902+0000","last_clean":"2023-10-09T05:05:28.941902+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:05:28.941902+0000","last_undegraded":"2023-10-09T05:05:28.941902+0000","last_fullsized":"2023-10-09T05:05:28.941902+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"665'6184","reported_seq":16453,"reported_epoch":665,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:47.339008+0000","last_change":"2023-10-09T05:05:26.966418+0000","last_active":"2023-10-09T05:05:47.339008+0000","last_peered":"2023-10-09T05:05:47.339008+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:03:45.583447+0000","last_became_peered":"2023-10-09T05:03:45.583447+0000","last_unstale":"2023-10-09T05:05:47.339008+0000","last_undegraded":"2023-10-09T05:03:41.179518+0000","last_fullsized":"2023-10-09T05:03:38.964620+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6184,"log_dups_size":0,"ondisk_log_size":6184,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81559552,"num_objects":1244,"num_object_clones":0,"num_object_copies":3732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":702,"num_objects_misplaced":912,"num_objects_unfound":0,"num_objects_dirty":1244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3642,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2929,"num_bytes_recovered":163381248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4],"avail_no_missing":["5","2","4"],"object_location_counts":[{"shards":"1,2,4,5","objects":552},{"shards":"2,4,5","objects":360},{"shards":"4,5","objects":-370}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":208,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.941933+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:05:28.941933+0000","last_peered":"2023-10-09T05:05:28.941933+0000","last_clean":"2023-10-09T05:05:28.941933+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:05:28.941933+0000","last_undegraded":"2023-10-09T05:05:28.941933+0000","last_fullsized":"2023-10-09T05:05:28.941933+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939935+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:05:28.939935+0000","last_peered":"2023-10-09T05:05:28.939935+0000","last_clean":"2023-10-09T05:05:28.939935+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:05:28.939935+0000","last_undegraded":"2023-10-09T05:05:28.939935+0000","last_fullsized":"2023-10-09T05:05:28.939935+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"665'6130","reported_seq":10563,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.256344+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:05:43.256344+0000","last_peered":"2023-10-09T05:05:43.256344+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:05:43.256344+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6130,"log_dups_size":0,"ondisk_log_size":6130,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83197952,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":876,"num_objects_missing":876,"num_objects_degraded":2139,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3590,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1160,"num_bytes_recovered":76054528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,4,7","objects":876},{"shards":"3,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":137,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939935+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:05:28.939935+0000","last_peered":"2023-10-09T05:05:28.939935+0000","last_clean":"2023-10-09T05:05:28.939935+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:05:28.939935+0000","last_undegraded":"2023-10-09T05:05:28.939935+0000","last_fullsized":"2023-10-09T05:05:28.939935+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":138,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940837+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:05:28.940837+0000","last_peered":"2023-10-09T05:05:28.940837+0000","last_clean":"2023-10-09T05:05:28.940837+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:05:28.940837+0000","last_undegraded":"2023-10-09T05:05:28.940837+0000","last_fullsized":"2023-10-09T05:05:28.940837+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1262,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942344+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:05:28.942344+0000","last_peered":"2023-10-09T05:05:28.942344+0000","last_clean":"2023-10-09T05:05:28.942344+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:05:28.942344+0000","last_undegraded":"2023-10-09T05:05:28.942344+0000","last_fullsized":"2023-10-09T05:05:28.942344+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"665'6152","reported_seq":19965,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.733430+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:05:43.733430+0000","last_peered":"2023-10-09T05:05:43.733430+0000","last_clean":"2023-10-09T05:05:43.733430+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:05:43.733430+0000","last_undegraded":"2023-10-09T05:05:43.733430+0000","last_fullsized":"2023-10-09T05:05:43.733430+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5906,"log_dups_size":0,"ondisk_log_size":5906,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81854464,"num_objects":1249,"num_object_clones":0,"num_object_copies":3747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3612,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":183,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.014614+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:05:29.014614+0000","last_peered":"2023-10-09T05:05:29.014614+0000","last_clean":"2023-10-09T05:05:29.014614+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:05:29.014614+0000","last_undegraded":"2023-10-09T05:05:29.014614+0000","last_fullsized":"2023-10-09T05:05:29.014614+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.125166+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:05:29.125166+0000","last_peered":"2023-10-09T05:05:29.125166+0000","last_clean":"2023-10-09T05:05:29.125166+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:05:29.125166+0000","last_undegraded":"2023-10-09T05:05:29.125166+0000","last_fullsized":"2023-10-09T05:05:29.125166+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1268,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942424+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:05:28.942424+0000","last_peered":"2023-10-09T05:05:28.942424+0000","last_clean":"2023-10-09T05:05:28.942424+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:05:28.942424+0000","last_undegraded":"2023-10-09T05:05:28.942424+0000","last_fullsized":"2023-10-09T05:05:28.942424+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"665'6301","reported_seq":11071,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:44.178590+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:05:44.178590+0000","last_peered":"2023-10-09T05:05:44.178590+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:05:44.178590+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":1530,"num_objects_missing":1530,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3692,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1380,"num_bytes_recovered":90439680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,3,5,7","objects":184},{"shards":"0,3","objects":-749},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":738}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":80,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.124531+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:05:29.124531+0000","last_peered":"2023-10-09T05:05:29.124531+0000","last_clean":"2023-10-09T05:05:29.124531+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:05:29.124531+0000","last_undegraded":"2023-10-09T05:05:29.124531+0000","last_fullsized":"2023-10-09T05:05:29.124531+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":43,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945481+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:05:28.945481+0000","last_peered":"2023-10-09T05:05:28.945481+0000","last_clean":"2023-10-09T05:05:28.945481+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:05:28.945481+0000","last_undegraded":"2023-10-09T05:05:28.945481+0000","last_fullsized":"2023-10-09T05:05:28.945481+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.011988+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:05:29.011988+0000","last_peered":"2023-10-09T05:05:29.011988+0000","last_clean":"2023-10-09T05:05:29.011988+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:05:29.011988+0000","last_undegraded":"2023-10-09T05:05:29.011988+0000","last_fullsized":"2023-10-09T05:05:29.011988+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1251,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942166+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:05:28.942166+0000","last_peered":"2023-10-09T05:05:28.942166+0000","last_clean":"2023-10-09T05:05:28.942166+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:05:28.942166+0000","last_undegraded":"2023-10-09T05:05:28.942166+0000","last_fullsized":"2023-10-09T05:05:28.942166+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"665'6440","reported_seq":18642,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.708802+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:05:43.708802+0000","last_peered":"2023-10-09T05:05:43.708802+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:05:43.708802+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4759,"log_dups_size":0,"ondisk_log_size":4759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79396864,"num_objects":1211,"num_object_clones":0,"num_object_copies":3633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1778,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3580,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3119,"num_bytes_recovered":199819264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":729}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":749,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945281+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:05:28.945281+0000","last_peered":"2023-10-09T05:05:28.945281+0000","last_clean":"2023-10-09T05:05:28.945281+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:05:28.945281+0000","last_undegraded":"2023-10-09T05:05:28.945281+0000","last_fullsized":"2023-10-09T05:05:28.945281+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940149+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:05:28.940149+0000","last_peered":"2023-10-09T05:05:28.940149+0000","last_clean":"2023-10-09T05:05:28.940149+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:05:28.940149+0000","last_undegraded":"2023-10-09T05:05:28.940149+0000","last_fullsized":"2023-10-09T05:05:28.940149+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.124492+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:05:29.124492+0000","last_peered":"2023-10-09T05:05:29.124492+0000","last_clean":"2023-10-09T05:05:29.124492+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:05:29.124492+0000","last_undegraded":"2023-10-09T05:05:29.124492+0000","last_fullsized":"2023-10-09T05:05:29.124492+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1262,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942320+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:05:28.942320+0000","last_peered":"2023-10-09T05:05:28.942320+0000","last_clean":"2023-10-09T05:05:28.942320+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:05:28.942320+0000","last_undegraded":"2023-10-09T05:05:28.942320+0000","last_fullsized":"2023-10-09T05:05:28.942320+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"665'6415","reported_seq":12193,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.181258+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:05:43.181258+0000","last_peered":"2023-10-09T05:05:43.181258+0000","last_clean":"2023-10-09T05:05:43.181258+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:05:43.181258+0000","last_undegraded":"2023-10-09T05:05:43.181258+0000","last_fullsized":"2023-10-09T05:05:43.181258+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4122,"log_dups_size":0,"ondisk_log_size":4122,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81952780,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3553,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":160,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.197144+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:05:43.197144+0000","last_peered":"2023-10-09T05:05:43.197144+0000","last_clean":"2023-10-09T05:05:43.197144+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:05:43.197144+0000","last_undegraded":"2023-10-09T05:05:43.197144+0000","last_fullsized":"2023-10-09T05:05:43.197144+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.011725+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:05:29.011725+0000","last_peered":"2023-10-09T05:05:29.011725+0000","last_clean":"2023-10-09T05:05:29.011725+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:05:29.011725+0000","last_undegraded":"2023-10-09T05:05:29.011725+0000","last_fullsized":"2023-10-09T05:05:29.011725+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.125102+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:05:29.125102+0000","last_peered":"2023-10-09T05:05:29.125102+0000","last_clean":"2023-10-09T05:05:29.125102+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:05:29.125102+0000","last_undegraded":"2023-10-09T05:05:29.125102+0000","last_fullsized":"2023-10-09T05:05:29.125102+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"665'6311","reported_seq":11655,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.276891+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:05:43.276891+0000","last_peered":"2023-10-09T05:05:43.276891+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:05:43.276891+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4184,"log_dups_size":0,"ondisk_log_size":4184,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":944,"num_objects_missing":944,"num_objects_degraded":2545,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3482,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1401,"num_bytes_recovered":91783168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,3,5,7","objects":3},{"shards":"2,5,7","objects":853},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":53,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.943975+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:05:28.943975+0000","last_peered":"2023-10-09T05:05:28.943975+0000","last_clean":"2023-10-09T05:05:28.943975+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:05:28.943975+0000","last_undegraded":"2023-10-09T05:05:28.943975+0000","last_fullsized":"2023-10-09T05:05:28.943975+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1268,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.941808+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:05:28.941808+0000","last_peered":"2023-10-09T05:05:28.941808+0000","last_clean":"2023-10-09T05:05:28.941808+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:05:28.941808+0000","last_undegraded":"2023-10-09T05:05:28.941808+0000","last_fullsized":"2023-10-09T05:05:28.941808+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.118387+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:05:29.118387+0000","last_peered":"2023-10-09T05:05:29.118387+0000","last_clean":"2023-10-09T05:05:29.118387+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:05:29.118387+0000","last_undegraded":"2023-10-09T05:05:29.118387+0000","last_fullsized":"2023-10-09T05:05:29.118387+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":185,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.943078+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:05:28.943078+0000","last_peered":"2023-10-09T05:05:28.943078+0000","last_clean":"2023-10-09T05:05:28.943078+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:05:28.943078+0000","last_undegraded":"2023-10-09T05:05:28.943078+0000","last_fullsized":"2023-10-09T05:05:28.943078+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1241,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942313+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:05:28.942313+0000","last_peered":"2023-10-09T05:05:28.942313+0000","last_clean":"2023-10-09T05:05:28.942313+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:05:28.942313+0000","last_undegraded":"2023-10-09T05:05:28.942313+0000","last_fullsized":"2023-10-09T05:05:28.942313+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"665'6485","reported_seq":14745,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:43.493708+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:05:43.493708+0000","last_peered":"2023-10-09T05:05:43.493708+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:05:43.493708+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4722,"log_dups_size":0,"ondisk_log_size":4722,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84475904,"num_objects":1289,"num_object_clones":0,"num_object_copies":3867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3586,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2202,"num_bytes_recovered":131989504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":303},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":939}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":78,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.011794+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:05:29.011794+0000","last_peered":"2023-10-09T05:05:29.011794+0000","last_clean":"2023-10-09T05:05:29.011794+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:05:29.011794+0000","last_undegraded":"2023-10-09T05:05:29.011794+0000","last_fullsized":"2023-10-09T05:05:29.011794+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":23,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939711+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:05:28.939711+0000","last_peered":"2023-10-09T05:05:28.939711+0000","last_clean":"2023-10-09T05:05:28.939711+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:05:28.939711+0000","last_undegraded":"2023-10-09T05:05:28.939711+0000","last_fullsized":"2023-10-09T05:05:28.939711+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":116,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.016769+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:05:29.016769+0000","last_peered":"2023-10-09T05:05:29.016769+0000","last_clean":"2023-10-09T05:05:29.016769+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:05:29.016769+0000","last_undegraded":"2023-10-09T05:05:29.016769+0000","last_fullsized":"2023-10-09T05:05:29.016769+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1262,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942802+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:05:28.942802+0000","last_peered":"2023-10-09T05:05:28.942802+0000","last_clean":"2023-10-09T05:05:28.942802+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:05:28.942802+0000","last_undegraded":"2023-10-09T05:05:28.942802+0000","last_fullsized":"2023-10-09T05:05:28.942802+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"665'6176","reported_seq":18209,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:46.961511+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:05:46.961511+0000","last_peered":"2023-10-09T05:05:46.961511+0000","last_clean":"2023-10-09T05:05:46.961511+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:05:46.961511+0000","last_undegraded":"2023-10-09T05:05:46.961511+0000","last_fullsized":"2023-10-09T05:05:46.961511+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6176,"log_dups_size":0,"ondisk_log_size":6176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82018304,"num_objects":1251,"num_object_clones":0,"num_object_copies":3753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3637,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":130,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940695+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:05:28.940695+0000","last_peered":"2023-10-09T05:05:28.940695+0000","last_clean":"2023-10-09T05:05:28.940695+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:05:28.940695+0000","last_undegraded":"2023-10-09T05:05:28.940695+0000","last_fullsized":"2023-10-09T05:05:28.940695+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":89,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.124398+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:05:29.124398+0000","last_peered":"2023-10-09T05:05:29.124398+0000","last_clean":"2023-10-09T05:05:29.124398+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:05:29.124398+0000","last_undegraded":"2023-10-09T05:05:29.124398+0000","last_fullsized":"2023-10-09T05:05:29.124398+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":43,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.944522+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:05:28.944522+0000","last_peered":"2023-10-09T05:05:28.944522+0000","last_clean":"2023-10-09T05:05:28.944522+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:05:28.944522+0000","last_undegraded":"2023-10-09T05:05:28.944522+0000","last_fullsized":"2023-10-09T05:05:28.944522+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1265,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.943085+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:05:28.943085+0000","last_peered":"2023-10-09T05:05:28.943085+0000","last_clean":"2023-10-09T05:05:28.943085+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:05:28.943085+0000","last_undegraded":"2023-10-09T05:05:28.943085+0000","last_fullsized":"2023-10-09T05:05:28.943085+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"665'6326","reported_seq":23680,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:46.791824+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:05:46.791824+0000","last_peered":"2023-10-09T05:05:46.791824+0000","last_clean":"2023-10-09T05:05:46.791824+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:05:46.791824+0000","last_undegraded":"2023-10-09T05:05:46.791824+0000","last_fullsized":"2023-10-09T05:05:46.791824+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5926,"log_dups_size":0,"ondisk_log_size":5926,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82968576,"num_objects":1266,"num_object_clones":0,"num_object_copies":3798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3505,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":289,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:29.135683+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:05:29.135683+0000","last_peered":"2023-10-09T05:05:29.135683+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:05:29.135683+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:05:29.135683+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":95,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.124685+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:05:29.124685+0000","last_peered":"2023-10-09T05:05:29.124685+0000","last_clean":"2023-10-09T05:05:29.124685+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:05:29.124685+0000","last_undegraded":"2023-10-09T05:05:29.124685+0000","last_fullsized":"2023-10-09T05:05:29.124685+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":665,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.190339+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:05:43.190339+0000","last_peered":"2023-10-09T05:05:43.190339+0000","last_clean":"2023-10-09T05:05:43.190339+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:05:43.190339+0000","last_undegraded":"2023-10-09T05:05:43.190339+0000","last_fullsized":"2023-10-09T05:05:43.190339+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1221,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.943176+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:05:28.943176+0000","last_peered":"2023-10-09T05:05:28.943176+0000","last_clean":"2023-10-09T05:05:28.943176+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:05:28.943176+0000","last_undegraded":"2023-10-09T05:05:28.943176+0000","last_fullsized":"2023-10-09T05:05:28.943176+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"665'6662","reported_seq":13166,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:44.359902+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:05:44.359902+0000","last_peered":"2023-10-09T05:05:44.359902+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:05:44.359902+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5237,"log_dups_size":0,"ondisk_log_size":5237,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":1670,"num_objects_missing":1670,"num_objects_degraded":4313,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3702,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1493,"num_bytes_recovered":97845248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-2},{"shards":"0,1,2,4,7","objects":1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":1208},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":181,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.942598+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:05:28.942598+0000","last_peered":"2023-10-09T05:05:28.942598+0000","last_clean":"2023-10-09T05:05:28.942598+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:05:28.942598+0000","last_undegraded":"2023-10-09T05:05:28.942598+0000","last_fullsized":"2023-10-09T05:05:28.942598+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":99,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.124627+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:05:29.124627+0000","last_peered":"2023-10-09T05:05:29.124627+0000","last_clean":"2023-10-09T05:05:29.124627+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:05:29.124627+0000","last_undegraded":"2023-10-09T05:05:29.124627+0000","last_fullsized":"2023-10-09T05:05:29.124627+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":140,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939836+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:05:28.939836+0000","last_peered":"2023-10-09T05:05:28.939836+0000","last_clean":"2023-10-09T05:05:28.939836+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:05:28.939836+0000","last_undegraded":"2023-10-09T05:05:28.939836+0000","last_fullsized":"2023-10-09T05:05:28.939836+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"665'5935","reported_seq":10880,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:44.215499+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:05:44.215499+0000","last_peered":"2023-10-09T05:05:44.215499+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:05:44.215499+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81100800,"num_objects":1238,"num_object_clones":0,"num_object_copies":3714,"num_objects_missing_on_primary":1663,"num_objects_missing":1663,"num_objects_degraded":3892,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3479,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1121,"num_bytes_recovered":73465856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-36},{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":887}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":130,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.940294+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:05:28.940294+0000","last_peered":"2023-10-09T05:05:28.940294+0000","last_clean":"2023-10-09T05:05:28.940294+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:05:28.940294+0000","last_undegraded":"2023-10-09T05:05:28.940294+0000","last_fullsized":"2023-10-09T05:05:28.940294+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.124441+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:05:29.124441+0000","last_peered":"2023-10-09T05:05:29.124441+0000","last_clean":"2023-10-09T05:05:29.124441+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:05:29.124441+0000","last_undegraded":"2023-10-09T05:05:29.124441+0000","last_fullsized":"2023-10-09T05:05:29.124441+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"664'6084","reported_seq":10923,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:43.029423+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:05:43.029423+0000","last_peered":"2023-10-09T05:05:43.029423+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:05:43.029423+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:05:43.029423+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6084,"log_dups_size":0,"ondisk_log_size":6084,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85852160,"num_objects":1310,"num_object_clones":0,"num_object_copies":3930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3549,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1310}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":800,"reported_epoch":664,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:28.944993+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:05:28.944993+0000","last_peered":"2023-10-09T05:05:28.944993+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:05:28.944993+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:05:28.944993+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939936+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:05:28.939936+0000","last_peered":"2023-10-09T05:05:28.939936+0000","last_clean":"2023-10-09T05:05:28.939936+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:05:28.939936+0000","last_undegraded":"2023-10-09T05:05:28.939936+0000","last_fullsized":"2023-10-09T05:05:28.939936+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"665'6123","reported_seq":10546,"reported_epoch":665,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:05:46.996129+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:05:46.996129+0000","last_peered":"2023-10-09T05:05:46.996129+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:05:46.996129+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5585,"log_dups_size":0,"ondisk_log_size":5585,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83329024,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3592,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":625}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":746,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945695+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:05:28.945695+0000","last_peered":"2023-10-09T05:05:28.945695+0000","last_clean":"2023-10-09T05:05:28.945695+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:05:28.945695+0000","last_undegraded":"2023-10-09T05:05:28.945695+0000","last_fullsized":"2023-10-09T05:05:28.945695+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939782+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:05:28.939782+0000","last_peered":"2023-10-09T05:05:28.939782+0000","last_clean":"2023-10-09T05:05:28.939782+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:05:28.939782+0000","last_undegraded":"2023-10-09T05:05:28.939782+0000","last_fullsized":"2023-10-09T05:05:28.939782+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"664'6102","reported_seq":16906,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:43.143620+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:05:43.143620+0000","last_peered":"2023-10-09T05:05:43.143620+0000","last_clean":"2023-10-09T05:05:43.143620+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:05:43.143620+0000","last_undegraded":"2023-10-09T05:05:43.143620+0000","last_fullsized":"2023-10-09T05:05:43.143620+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6102,"log_dups_size":0,"ondisk_log_size":6102,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84803584,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3570,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":285,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.117545+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:05:29.117545+0000","last_peered":"2023-10-09T05:05:29.117545+0000","last_clean":"2023-10-09T05:05:29.117545+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:05:29.117545+0000","last_undegraded":"2023-10-09T05:05:29.117545+0000","last_fullsized":"2023-10-09T05:05:29.117545+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":293,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:29.118640+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:05:29.118640+0000","last_peered":"2023-10-09T05:05:29.118640+0000","last_clean":"2023-10-09T05:05:29.118640+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:05:29.118640+0000","last_undegraded":"2023-10-09T05:05:29.118640+0000","last_fullsized":"2023-10-09T05:05:29.118640+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"665'6026","reported_seq":17632,"reported_epoch":665,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:43.543538+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:05:43.543538+0000","last_peered":"2023-10-09T05:05:43.543538+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:05:43.543538+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:05:43.543538+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84410368,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3527,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":1288}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":45,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.945484+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:05:28.945484+0000","last_peered":"2023-10-09T05:05:28.945484+0000","last_clean":"2023-10-09T05:05:28.945484+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:05:28.945484+0000","last_undegraded":"2023-10-09T05:05:28.945484+0000","last_fullsized":"2023-10-09T05:05:28.945484+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":141,"reported_epoch":664,"state":"active+clean","last_fresh":"2023-10-09T05:05:28.939799+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:05:28.939799+0000","last_peered":"2023-10-09T05:05:28.939799+0000","last_clean":"2023-10-09T05:05:28.939799+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:05:28.939799+0000","last_undegraded":"2023-10-09T05:05:28.939799+0000","last_fullsized":"2023-10-09T05:05:28.939799+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":2653290520,"num_objects":40487,"num_object_clones":0,"num_object_copies":121461,"num_objects_missing_on_primary":13676,"num_objects_missing":13676,"num_objects_degraded":41097,"num_objects_misplaced":7495,"num_objects_unfound":0,"num_objects_dirty":40487,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114731,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74514,"num_bytes_recovered":4527816704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13710995456,"data_stored":13710983240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":198169,"ondisk_log_size":198169,"up":108,"acting":91,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234701,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3143536,"kb_used_data":1630184,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":90601616,"statfs":{"total":95995035648,"available":92776054784,"internally_reserved":0,"allocated":1669308416,"data_stored":1667622899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7190000000000003,"5min":4.2489999999999997,"15min":1.9099999999999999},"min":{"1min":0.23699999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":35.673000000000002,"5min":123.04000000000001,"15min":123.04000000000001},"last":2.157},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.4390000000000001,"15min":2.1880000000000002},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":35.622999999999998,"5min":123.059,"15min":123.059},"last":3.8090000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:05:15 2023","interfaces":[{"interface":"back","average":{"1min":6.0919999999999996,"5min":3.4500000000000002,"15min":2.6579999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.25800000000000001},"max":{"1min":60.377000000000002,"5min":60.377000000000002,"15min":122.56},"last":0.372},{"interface":"front","average":{"1min":6.5810000000000004,"5min":3.1829999999999998,"15min":2.6120000000000001},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":59.689999999999998,"5min":59.689999999999998,"15min":122.666},"last":0.42399999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":4.4569999999999999,"15min":3.8889999999999998},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.30299999999999999},"max":{"1min":10.545,"5min":154.76400000000001,"15min":207.96600000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.8029999999999999,"5min":4.4340000000000002,"15min":4.0110000000000001},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.29699999999999999},"max":{"1min":10.532,"5min":154.80000000000001,"15min":207.97900000000001},"last":0.58599999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:05:08 2023","interfaces":[{"interface":"back","average":{"1min":16.815999999999999,"5min":5.5110000000000001,"15min":2.8839999999999999},"min":{"1min":0.35199999999999998,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":126.70399999999999,"5min":126.70399999999999,"15min":126.70399999999999},"last":1.9890000000000001},{"interface":"front","average":{"1min":15.994,"5min":5.3739999999999997,"15min":2.9060000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":125.991,"5min":125.991,"15min":125.991},"last":0.28399999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38400000000000001,"15min":0.38600000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.497,"15min":0.48099999999999998},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.923,"5min":2.923,"15min":2.923},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.6579999999999999,"15min":1.7929999999999999},"min":{"1min":0.33700000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":5.5970000000000004,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46000000000000002,"5min":1.8169999999999999,"15min":2.149},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":0.57899999999999996,"5min":53.225000000000001,"15min":135.74199999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:04:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":1.4730000000000001,"15min":1.23},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.57299999999999995,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":1.736,"15min":1.4339999999999999},"min":{"1min":0.34899999999999998,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":0.56100000000000005,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.38800000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496333,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2044272,"kb_used_data":849512,"kb_used_omap":0,"kb_used_meta":1194752,"kb_avail":91700880,"statfs":{"total":95995035648,"available":93901701120,"internally_reserved":0,"allocated":869900288,"data_stored":868215004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1223426048},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":4.3780000000000001,"5min":5.883,"15min":2.4319999999999999},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":47.750999999999998,"5min":321.346,"15min":321.346},"last":22.187000000000001},{"interface":"front","average":{"1min":3.718,"5min":4.1790000000000003,"15min":2.0510000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":47.645000000000003,"5min":169.119,"15min":169.119},"last":7.2530000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":2.4590000000000001,"15min":4.657},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":6.5759999999999996,"5min":72.840999999999994,"15min":260.738},"last":2.952},{"interface":"front","average":{"1min":2.3969999999999998,"5min":2.7370000000000001,"15min":4.6319999999999997},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":7.2770000000000001,"5min":72.832999999999998,"15min":261.26100000000002},"last":0.42499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.4399999999999999,"15min":1.994},"min":{"1min":0.35299999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.032999999999999,"5min":23.943000000000001,"15min":133.535},"last":5.0090000000000003},{"interface":"front","average":{"1min":1.591,"5min":1.421,"15min":1.9390000000000001},"min":{"1min":0.39400000000000002,"5min":0.35699999999999998,"15min":0.30599999999999999},"max":{"1min":8.5920000000000005,"5min":25.978999999999999,"15min":133.51900000000001},"last":4.9359999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":5.9359999999999999,"5min":2.4500000000000002,"15min":2.6579999999999999},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":64.355999999999995,"5min":64.355999999999995,"15min":147.43600000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":5.7930000000000001,"5min":2.2589999999999999,"15min":2.7370000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":56.133000000000003,"5min":56.133000000000003,"15min":147.41999999999999},"last":7.1040000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":0.96699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":18.186,"5min":18.186,"15min":18.186},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.548,"5min":1.1120000000000001,"15min":1.04},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:05:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":1.085,"15min":1.3100000000000001},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.8300000000000001,"5min":25.934000000000001,"15min":66.995999999999995},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.85999999999999999,"15min":1.3120000000000001},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.254},"max":{"1min":7.2220000000000004,"5min":16.789000000000001,"15min":67.189999999999998},"last":0.38900000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:04:41 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.66900000000000004,"15min":1.1180000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":0.59299999999999997,"5min":5.835,"15min":36.204999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.002,"15min":1.1259999999999999},"min":{"1min":0.42199999999999999,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":2.8069999999999999,"5min":20.041,"15min":20.041},"last":0.38}]}]},{"osd":1,"up_from":118,"seq":506806141401,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2256344,"kb_used_data":948752,"kb_used_omap":4,"kb_used_meta":1307579,"kb_avail":91488808,"statfs":{"total":95995035648,"available":93684539392,"internally_reserved":0,"allocated":971522048,"data_stored":967980399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1338961806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":10.287000000000001,"5min":4.125,"15min":2.0139999999999998},"min":{"1min":0.29999999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":100.247,"5min":100.247,"15min":100.247},"last":1.2729999999999999},{"interface":"front","average":{"1min":12.194000000000001,"5min":4.5540000000000003,"15min":2.129},"min":{"1min":0.52600000000000002,"5min":0.247,"15min":0.214},"max":{"1min":99.935000000000002,"5min":99.935000000000002,"15min":99.935000000000002},"last":2.282}]},{"osd":2,"last update":"Mon Oct 9 05:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.1639999999999999,"15min":2.4140000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":6.3129999999999997,"5min":12.460000000000001,"15min":263.089},"last":1.044},{"interface":"front","average":{"1min":2.2509999999999999,"5min":1.4870000000000001,"15min":2.4689999999999999},"min":{"1min":0.254,"5min":0.252,"15min":0.222},"max":{"1min":8.5250000000000004,"5min":13.933999999999999,"15min":254.50899999999999},"last":2.238}]},{"osd":3,"last update":"Mon Oct 9 05:05:16 2023","interfaces":[{"interface":"back","average":{"1min":24.670999999999999,"5min":6.3529999999999998,"15min":3.2160000000000002},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":235.43600000000001,"5min":235.43600000000001,"15min":235.43600000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":24.478000000000002,"5min":5.7560000000000002,"15min":2.9140000000000001},"min":{"1min":0.31900000000000001,"5min":0.249,"15min":0.249},"max":{"1min":234.32499999999999,"5min":234.32499999999999,"15min":234.32499999999999},"last":0.76500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.4930000000000001,"15min":1.347},"min":{"1min":0.42899999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.9060000000000001,"5min":1.339,"15min":1.0780000000000001},"min":{"1min":0.46500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.521000000000001,"5min":10.521000000000001,"15min":10.521000000000001},"last":15.382}]},{"osd":5,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.2929999999999999,"15min":1.278},"min":{"1min":0.36899999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":10.353,"5min":10.353,"15min":10.353},"last":0.71699999999999997},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.542,"15min":1.345},"min":{"1min":0.503,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":6.9459999999999997,"5min":10.554,"15min":19.597999999999999},"last":15.468999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.383,"15min":1.2689999999999999},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.35699999999999998},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.4390000000000001,"15min":1.3580000000000001},"min":{"1min":0.54400000000000004,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.915,"5min":10.472,"15min":19.548999999999999},"last":15.435}]},{"osd":7,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.5649999999999999,"15min":1.8180000000000001},"min":{"1min":0.54300000000000004,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.8330000000000002,"5min":10.583,"15min":59.972000000000001},"last":15.502000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.351,"15min":1.714},"min":{"1min":0.51500000000000001,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":9.7170000000000005,"5min":11.903,"15min":60.011000000000003},"last":0.54100000000000004}]}]},{"osd":0,"up_from":91,"seq":390842024484,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1789880,"kb_used_data":663472,"kb_used_omap":5,"kb_used_meta":1126394,"kb_avail":91955272,"statfs":{"total":95995035648,"available":94162198528,"internally_reserved":0,"allocated":679395328,"data_stored":676529624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1153428090},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2,3,3,3,2],"upper_bound":1024},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:05:12 2023","interfaces":[{"interface":"back","average":{"1min":6.1740000000000004,"5min":3.0640000000000001,"15min":4.7590000000000003},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":53.176000000000002,"5min":53.176000000000002,"15min":247.68899999999999},"last":3.5489999999999999},{"interface":"front","average":{"1min":7.3929999999999998,"5min":3.1240000000000001,"15min":4.7720000000000002},"min":{"1min":0.36499999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":53.372999999999998,"5min":53.372999999999998,"15min":246.38},"last":4.944}]},{"osd":2,"last update":"Mon Oct 9 05:05:32 2023","interfaces":[{"interface":"back","average":{"1min":3.7530000000000001,"5min":2.3969999999999998,"15min":2.516},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.762,"5min":26.556999999999999,"15min":151.06399999999999},"last":2.5169999999999999},{"interface":"front","average":{"1min":3.367,"5min":2.645,"15min":2.6789999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.7720000000000002,"5min":29.032,"15min":151.22399999999999},"last":4.8970000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:04:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":5.3319999999999999,"15min":3.5870000000000002},"min":{"1min":0.47599999999999998,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":11.872,"5min":290.89100000000002,"15min":290.89100000000002},"last":2.609},{"interface":"front","average":{"1min":3.4510000000000001,"5min":5.3380000000000001,"15min":3.6469999999999998},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":14.541,"5min":290.73899999999998,"15min":290.73899999999998},"last":16.620999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":1.9570000000000001,"15min":1.887},"min":{"1min":0.443,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.5329999999999995,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":1.1639999999999999},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.663,"15min":1.577},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.81,"5min":11.81,"15min":11.81},"last":2.758}]},{"osd":5,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0089999999999999,"5min":1.78,"15min":1.3100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.039999999999999,"5min":16.888000000000002,"15min":16.888000000000002},"last":5.0049999999999999},{"interface":"front","average":{"1min":2.569,"5min":1.835,"15min":1.288},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":9.5779999999999994,"5min":14.036,"15min":14.036},"last":1.204}]},{"osd":6,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0089999999999999,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.074999999999999,"5min":20.129999999999999,"15min":39.594999999999999},"last":5.0439999999999996},{"interface":"front","average":{"1min":2.573,"5min":2.3759999999999999,"15min":1.6020000000000001},"min":{"1min":0.45400000000000001,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":9.6419999999999995,"5min":47.241,"15min":47.241},"last":1.232}]},{"osd":7,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.0870000000000002,"15min":1.496},"min":{"1min":0.51600000000000001,"5min":0.41899999999999998,"15min":0.39000000000000001},"max":{"1min":9.6120000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":1.2549999999999999},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.986,"15min":1.4790000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.38200000000000001},"max":{"1min":11.842000000000001,"5min":21.68,"15min":21.68},"last":2.8079999999999998}]}]},{"osd":2,"up_from":0,"seq":949187772762,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356456,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2176008,"kb_used_data":1252800,"kb_used_omap":2,"kb_used_meta":923197,"kb_avail":91569144,"statfs":{"total":95995035648,"available":93766803456,"internally_reserved":0,"allocated":1282867200,"data_stored":1279917858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":945353781},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2,1,1,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":2.6030000000000002,"15min":1.218},"min":{"1min":0.33000000000000002,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":15.021000000000001,"5min":108.631,"15min":108.631},"last":2.2200000000000002},{"interface":"front","average":{"1min":3.5089999999999999,"5min":2.7719999999999998,"15min":1.365},"min":{"1min":0.247,"5min":0.247,"15min":0.23499999999999999},"max":{"1min":11.728,"5min":110.012,"15min":110.012},"last":2.2080000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6139999999999999,"5min":1.8300000000000001,"15min":2.29},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":10.345000000000001,"5min":15.218,"15min":125.233},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.972,"5min":1.839,"15min":2.2770000000000001},"min":{"1min":0.313,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":8.0039999999999996,"5min":26.692,"15min":126.102},"last":2.4660000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:05:41 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.3620000000000001,"15min":2.2309999999999999},"min":{"1min":0.35499999999999998,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":6.843,"5min":11.121,"15min":212.065},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.4649999999999999,"5min":1.6910000000000001,"15min":2.3159999999999998},"min":{"1min":0.32500000000000001,"5min":0.253,"15min":0.23499999999999999},"max":{"1min":6.3120000000000003,"5min":12.987,"15min":211.98400000000001},"last":0.70899999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:05:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":1.835,"15min":1.758},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.0099999999999998,"5min":11.193,"15min":11.193},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.3319999999999999,"5min":1.1839999999999999,"15min":0.99199999999999999},"min":{"1min":0.39700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":10.452,"5min":10.452,"15min":10.452},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":5.0220000000000002,"5min":1.798,"15min":1.431},"min":{"1min":0.44900000000000001,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":57.804000000000002,"5min":57.804000000000002,"15min":57.804000000000002},"last":0.69099999999999995},{"interface":"front","average":{"1min":5.452,"5min":2.0990000000000002,"15min":1.6419999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":57.442,"5min":57.442,"15min":57.442},"last":0.64200000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5779999999999998,"5min":1.6220000000000001,"15min":1.1299999999999999},"min":{"1min":0.40100000000000002,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":11.252000000000001,"5min":11.252000000000001,"15min":11.252000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.526,"15min":1.3320000000000001},"min":{"1min":0.44800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.481,"5min":14.731,"15min":24.916},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.6240000000000001,"15min":1.425},"min":{"1min":0.46800000000000003,"5min":0.39400000000000002,"15min":0.375},"max":{"1min":10.42,"5min":14.784000000000001,"15min":25.004000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.9159999999999999,"5min":1.3819999999999999,"15min":1.1339999999999999},"min":{"1min":0.47999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.798,"5min":7.758,"15min":7.758},"last":0.77600000000000002}]}]},{"osd":4,"up_from":626,"seq":2688649527325,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2855480,"kb_used_data":2458480,"kb_used_omap":22,"kb_used_meta":396969,"kb_avail":90889672,"statfs":{"total":95995035648,"available":93071024128,"internally_reserved":0,"allocated":2517483520,"data_stored":2515790409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":406497101},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":2.3330000000000002,"15min":2.3050000000000002},"min":{"1min":0.27700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":17.135000000000002,"5min":21.292000000000002,"15min":21.292000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.544,"5min":3.0449999999999999,"15min":3.129},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.030999999999999,"5min":21.391999999999999,"15min":21.391999999999999},"last":0.433}]},{"osd":1,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":2.1909999999999998,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.9879999999999995,"5min":14.955,"15min":14.955},"last":0.746},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.7829999999999999,"15min":1.861},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.585,"5min":10.579000000000001,"15min":10.579000000000001},"last":0.38300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.667,"15min":1.6479999999999999},"min":{"1min":0.377,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.3819999999999997,"5min":11.051,"15min":11.051},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.736,"15min":1.7030000000000001},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":5.3959999999999999,"5min":11.036,"15min":11.036},"last":0.53700000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":20.216000000000001,"5min":4.9829999999999997,"15min":2.444},"min":{"1min":0.45100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":270.767,"5min":270.767,"15min":270.767},"last":0.59899999999999998},{"interface":"front","average":{"1min":19.672000000000001,"5min":5.4089999999999998,"15min":3.032},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":270.274,"5min":270.274,"15min":270.274},"last":3.5089999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.39100000000000001,"15min":0.38700000000000001},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.313},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":0.371}]},{"osd":6,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.624,"15min":0.65000000000000002},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.5409999999999999,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.46800000000000003,"15min":0.45800000000000002},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.35499999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.36199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.48799999999999999,"15min":0.47599999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.45300000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757978,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4615712,"kb_used_data":3122392,"kb_used_omap":0,"kb_used_meta":1493312,"kb_avail":89129440,"statfs":{"total":95995035648,"available":91268546560,"internally_reserved":0,"allocated":3197329408,"data_stored":3195655519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1529151488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.4119999999999999,"15min":2.137},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.0309999999999997,"5min":103.745,"15min":106.465},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.5880000000000001,"5min":3.71,"15min":2.3090000000000002},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.245},"max":{"1min":10.882999999999999,"5min":103.872,"15min":106.54600000000001},"last":2.1629999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":5.0410000000000004,"15min":3.556},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.23200000000000001},"max":{"1min":6.0529999999999999,"5min":262.26900000000001,"15min":262.26900000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.7709999999999999,"5min":5.0860000000000003,"15min":3.7040000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26900000000000002},"max":{"1min":10.462,"5min":262.291,"15min":262.291},"last":2.0720000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":2.6499999999999999,"15min":1.875},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":8.1340000000000003,"5min":106.238,"15min":106.238},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.9550000000000001,"5min":2.7109999999999999,"15min":2.1269999999999998},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":8.4390000000000001,"5min":104.31699999999999,"15min":104.31699999999999},"last":0.443}]},{"osd":3,"last update":"Mon Oct 9 05:05:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":5.7110000000000003,"15min":3.0899999999999999},"min":{"1min":0.41599999999999998,"5min":0.318,"15min":0.318},"max":{"1min":10.385,"5min":125.824,"15min":125.824},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.8640000000000001,"5min":5.6420000000000003,"15min":2.9870000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":9.8520000000000003,"5min":125.761,"15min":125.761},"last":1.9450000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.623,"5min":0.873,"15min":0.873},"last":0.40899999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.67500000000000004,"15min":0.79700000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":11.093,"15min":11.093},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.61899999999999999,"15min":0.88800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.69699999999999995,"5min":5.9329999999999998,"15min":13.083},"last":0.48799999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:05:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.71899999999999997,"15min":1.734},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.746,"5min":8.3520000000000003,"15min":110.521},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.90300000000000002,"15min":1.7729999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.29499999999999998},"max":{"1min":0.72199999999999998,"5min":11.791,"15min":110.746},"last":0.433}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":670629888,"data_stored":670629888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":960561152,"data_stored":960561152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2592735232,"data_stored":2592735232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1271136256,"data_stored":1271136256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2507542528,"data_stored":2507538456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3189051392,"data_stored":3189047320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":862392320,"data_stored":862388248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1656946688,"data_stored":1656946688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:05:50.923 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:05:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:50 smithi029 ceph-mon[223858]: pgmap v2141: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 40347/119121 objects degraded (33.871%); 7368/119121 objects misplaced (6.185%); 8.3 MiB/s, 153 objects/s recovering 2023-10-09T05:05:51.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:50 smithi177 ceph-mon[195797]: pgmap v2141: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 40347/119121 objects degraded (33.871%); 7368/119121 objects misplaced (6.185%); 8.3 MiB/s, 153 objects/s recovering 2023-10-09T05:05:51.069 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:51.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:51.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:51.196 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:51.280 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:51.713 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:51.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:51.813 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:05:51.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:05:51.951 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 40347/119121 objects degraded (33.871%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:51.951 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 40347/119121 objects degraded (33.871%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:51.989 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:52.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 40347/119121 objects degraded (33.871%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:52.246 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:05:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:52.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:05:52.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:52.468 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:52.474 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:52.512 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:52.642 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:52.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:52.766 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:52.791 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:52.837 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:52.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:52.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:52.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:52 smithi029 ceph-mon[216903]: pgmap v2142: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 40347/119121 objects degraded (33.871%); 7368/119121 objects misplaced (6.185%); 8.4 MiB/s, 153 objects/s recovering 2023-10-09T05:05:52.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:52 smithi029 ceph-mon[223858]: pgmap v2142: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 40347/119121 objects degraded (33.871%); 7368/119121 objects misplaced (6.185%); 8.4 MiB/s, 153 objects/s recovering 2023-10-09T05:05:52.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:05:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:05:52] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.43.0" 2023-10-09T05:05:52.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:52.970 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:53.022 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:53.025 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:52 smithi177 ceph-mon[195797]: pgmap v2142: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 237 op/s; 40347/119121 objects degraded (33.871%); 7368/119121 objects misplaced (6.185%); 8.4 MiB/s, 153 objects/s recovering 2023-10-09T05:05:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:53.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:53.127 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:53.153 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:53.199 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:53.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:53.306 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:05:53.331 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:53.350 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:53.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:05:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:53.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:05:53.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:05:53.634 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:54.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:05:54.396 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:05:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:54.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:05:54.506 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:54.550 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:54.646 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:54.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:54.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:54.750 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:05:54.778 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:54.826 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:05:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:54.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:05:54.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:54 smithi029 ceph-mon[216903]: pgmap v2143: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 39964/116541 objects degraded (34.292%); 6819/116541 objects misplaced (5.851%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:05:54.932 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:55.002 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:54 smithi177 ceph-mon[195797]: pgmap v2143: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 39964/116541 objects degraded (34.292%); 6819/116541 objects misplaced (5.851%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:05:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:54 smithi029 ceph-mon[223858]: pgmap v2143: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 39964/116541 objects degraded (34.292%); 6819/116541 objects misplaced (5.851%); 11 MiB/s, 175 objects/s recovering 2023-10-09T05:05:55.870 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:55.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:55.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:55.996 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:56.136 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:56.380 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:56.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:56.509 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:56.634 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:56 smithi177 ceph-mon[195797]: pgmap v2144: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 227 op/s; 39468/115404 objects degraded (34.200%); 6707/115404 objects misplaced (5.812%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T05:05:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 39468/115404 objects degraded (34.200%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:56 smithi029 ceph-mon[216903]: pgmap v2144: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 227 op/s; 39468/115404 objects degraded (34.200%); 6707/115404 objects misplaced (5.812%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T05:05:57.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 39468/115404 objects degraded (34.200%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:56 smithi029 ceph-mon[223858]: pgmap v2144: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 227 op/s; 39468/115404 objects degraded (34.200%); 6707/115404 objects misplaced (5.812%); 9.9 MiB/s, 159 objects/s recovering 2023-10-09T05:05:57.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 39468/115404 objects degraded (34.200%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:05:58.390 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:05:58.470 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:05:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:58.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:05:58.587 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:05:58.615 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:58.662 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:05:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:58.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:05:58.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:05:58.972 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:05:59.048 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:58 smithi029 ceph-mon[216903]: pgmap v2145: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 39349/114246 objects degraded (34.442%); 6016/114246 objects misplaced (5.266%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:58 smithi029 ceph-mon[223858]: pgmap v2145: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 39349/114246 objects degraded (34.442%); 6016/114246 objects misplaced (5.266%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:05:59.049 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:59.268 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:05:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:58 smithi177 ceph-mon[195797]: pgmap v2145: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 232 op/s; 39349/114246 objects degraded (34.442%); 6016/114246 objects misplaced (5.266%); 12 MiB/s, 193 objects/s recovering 2023-10-09T05:05:59.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:05:59.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:05:59.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:05:59.308 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:05:59.308 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 39349/114246 objects degraded (34.442%), 25 pgs degraded, 17 pgs undersized","count":42},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2895,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":665,"num_osds":8,"num_up_osds":8,"osd_up_since":1696827803,"num_in_osds":7,"osd_in_since":1696827750,"num_remapped_pgs":17},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":14},{"state_name":"active+recovery_wait+degraded","count":10},{"state_name":"active+recovering+undersized+remapped","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":38082,"data_bytes":2494699448,"bytes_used":19051606016,"bytes_avail":652913643520,"bytes_total":671965249536,"degraded_objects":39349,"degraded_total":114246,"degraded_ratio":0.34442343714440765,"misplaced_objects":6016,"misplaced_total":114246,"misplaced_ratio":0.052658298758818688,"recovering_objects_per_sec":193,"recovering_bytes_per_sec":12653466,"recovering_keys_per_sec":0,"num_objects_recovered":2325,"num_bytes_recovered":151977984,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":232},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":36,"modified":"2023-10-09T05:04:23.723180+0000","services":{}},"progress_events":{"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8039e601-7bd2-4b88-bccf-3b879c49d7d2":{"message":"Global Recovery Event (12m)\n [=====================.......] (remaining: 3m)","progress":0.77310925722122192,"add_to_ceph_s":true}}} 2023-10-09T05:05:59.380 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:05:59.421 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:00.031 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:06:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3491455732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:59 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:00.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:05:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1189786117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:06:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3491455732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:59 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:05:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1189786117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:06:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3491455732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:59 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:05:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1189786117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:06:00.285 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:06:00.286 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:06:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: pgmap v2146: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 38544/111570 objects degraded (34.547%); 5906/111570 objects misplaced (5.294%); 12 MiB/s, 185 objects/s recovering 2023-10-09T05:06:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:06:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: osdmap e666: 8 total, 8 up, 7 in 2023-10-09T05:06:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:00 smithi177 ceph-mon[195797]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: pgmap v2146: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 38544/111570 objects degraded (34.547%); 5906/111570 objects misplaced (5.294%); 12 MiB/s, 185 objects/s recovering 2023-10-09T05:06:01.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:01.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: osdmap e666: 8 total, 8 up, 7 in 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[216903]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: pgmap v2146: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 38544/111570 objects degraded (34.547%); 5906/111570 objects misplaced (5.294%); 12 MiB/s, 185 objects/s recovering 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:06:01.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: osdmap e666: 8 total, 8 up, 7 in 2023-10-09T05:06:01.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: failed to encode map e666 with expected crc 2023-10-09T05:06:01.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:00 smithi029 ceph-mon[223858]: failed to encode map e666 with expected crc 2023-10-09T05:06:02.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 38544/111570 objects degraded (34.547%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 38544/111570 objects degraded (34.547%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 38544/111570 objects degraded (34.547%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:02.632 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:02.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:02.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:02.769 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:02.819 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:02.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:02.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:06:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:06:02] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-10-09T05:06:02.927 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:02.957 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:03.006 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:03.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:03.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:03.113 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:03.142 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:03.300 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:03 smithi029 ceph-mon[223858]: pgmap v2148: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 38544/111570 objects degraded (34.547%); 5906/111570 objects misplaced (5.294%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:06:03.311 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:03 smithi029 ceph-mon[216903]: pgmap v2148: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 38544/111570 objects degraded (34.547%); 5906/111570 objects misplaced (5.294%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:06:03.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:06:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:03 smithi177 ceph-mon[195797]: pgmap v2148: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 38544/111570 objects degraded (34.547%); 5906/111570 objects misplaced (5.294%); 11 MiB/s, 183 objects/s recovering 2023-10-09T05:06:03.601 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:03.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:03.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:03.719 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:03.859 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:03.867 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:06:03.868 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:06:04.007 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:04.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:04.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:04.120 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:04.148 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:04.198 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:04.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:04.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:04.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:04.333 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:04.381 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:04.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:04.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:04.514 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:04.558 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:04.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:04.665 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:04.681 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:04.779 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:06:04.813 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:04.831 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:06:04.904 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:06:04.931 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:06:04.964 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:06:05.004 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:06:05.041 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:06:05.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:05 smithi029 ceph-mon[216903]: pgmap v2149: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 37907/108891 objects degraded (34.812%); 5593/108891 objects misplaced (5.136%); 9.9 MiB/s, 188 objects/s recovering 2023-10-09T05:06:05.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/910871221' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:05.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:05 smithi029 ceph-mon[223858]: pgmap v2149: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 37907/108891 objects degraded (34.812%); 5593/108891 objects misplaced (5.136%); 9.9 MiB/s, 188 objects/s recovering 2023-10-09T05:06:05.066 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/910871221' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:05.084 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:06:05.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.090 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.091 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.091 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.091 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:06:05.229 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:05.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:05.334 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:05.453 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:05 smithi177 ceph-mon[195797]: pgmap v2149: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 37907/108891 objects degraded (34.812%); 5593/108891 objects misplaced (5.136%); 9.9 MiB/s, 188 objects/s recovering 2023-10-09T05:06:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/910871221' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:05.861 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:06:06.062 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:06.114 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 37907/108891 objects degraded (34.812%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:06.171 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:06.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:06.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:06.305 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:06.331 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:06.373 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:06.376 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 37907/108891 objects degraded (34.812%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:06.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:06.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:06.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 37907/108891 objects degraded (34.812%), 25 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:06.478 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:06.508 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:07.300 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:07.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:07.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:07.424 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:07 smithi029 ceph-mon[223858]: pgmap v2150: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 37420/107736 objects degraded (34.733%); 5464/107736 objects misplaced (5.072%); 10 MiB/s, 188 objects/s recovering 2023-10-09T05:06:07.424 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:07 smithi029 ceph-mon[216903]: pgmap v2150: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 37420/107736 objects degraded (34.733%); 5464/107736 objects misplaced (5.072%); 10 MiB/s, 188 objects/s recovering 2023-10-09T05:06:07.472 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:07 smithi177 ceph-mon[195797]: pgmap v2150: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 37420/107736 objects degraded (34.733%); 5464/107736 objects misplaced (5.072%); 10 MiB/s, 188 objects/s recovering 2023-10-09T05:06:08.325 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:08.337 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[216903]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[216903]: osdmap e667: 8 total, 8 up, 7 in 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[216903]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[216903]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[216903]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[223858]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[223858]: osdmap e667: 8 total, 8 up, 7 in 2023-10-09T05:06:08.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[223858]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.339 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[223858]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.339 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:08 smithi029 ceph-mon[223858]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:08.440 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:08 smithi177 ceph-mon[195797]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:08 smithi177 ceph-mon[195797]: osdmap e667: 8 total, 8 up, 7 in 2023-10-09T05:06:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:08 smithi177 ceph-mon[195797]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:08 smithi177 ceph-mon[195797]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:08 smithi177 ceph-mon[195797]: failed to encode map e667 with expected crc 2023-10-09T05:06:08.637 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:08.840 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:08.869 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:08.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:08.979 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:09.007 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:09.054 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:09.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:09.160 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:09.190 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:09.238 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:09.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:09 smithi177 ceph-mon[195797]: pgmap v2152: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 207 op/s; 36848/106584 objects degraded (34.572%); 5229/106584 objects misplaced (4.906%); 5.8 MiB/s, 186 objects/s recovering 2023-10-09T05:06:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:09 smithi177 ceph-mon[195797]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:09 smithi177 ceph-mon[195797]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:09 smithi177 ceph-mon[195797]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:09 smithi177 ceph-mon[195797]: osdmap e668: 8 total, 8 up, 7 in 2023-10-09T05:06:09.344 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:09.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[216903]: pgmap v2152: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 207 op/s; 36848/106584 objects degraded (34.572%); 5229/106584 objects misplaced (4.906%); 5.8 MiB/s, 186 objects/s recovering 2023-10-09T05:06:09.404 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[216903]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[216903]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[216903]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.405 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[216903]: osdmap e668: 8 total, 8 up, 7 in 2023-10-09T05:06:09.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[223858]: pgmap v2152: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 207 op/s; 36848/106584 objects degraded (34.572%); 5229/106584 objects misplaced (4.906%); 5.8 MiB/s, 186 objects/s recovering 2023-10-09T05:06:09.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[223858]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[223858]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[223858]: failed to encode map e668 with expected crc 2023-10-09T05:06:09.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:09 smithi029 ceph-mon[223858]: osdmap e668: 8 total, 8 up, 7 in 2023-10-09T05:06:09.412 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:10.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:06:10.246 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:06:10.246 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:06:10.246 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:06:10.246 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:06:10.247 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:06:10.250 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:10.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:10.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:10.503 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:10.925 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:10.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:10.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:11.046 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:11.074 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:11.125 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:11.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:11.232 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:11.260 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:11.303 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:11.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:11.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:11.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:11.468 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:11 smithi029 ceph-mon[223858]: pgmap v2154: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 35959/103791 objects degraded (34.646%); 5079/103791 objects misplaced (4.893%); 9.3 MiB/s, 241 objects/s recovering 2023-10-09T05:06:11.468 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35959/103791 objects degraded (34.646%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:11.468 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:11 smithi029 ceph-mon[216903]: pgmap v2154: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 35959/103791 objects degraded (34.646%); 5079/103791 objects misplaced (4.893%); 9.3 MiB/s, 241 objects/s recovering 2023-10-09T05:06:11.469 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35959/103791 objects degraded (34.646%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:11.474 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:11 smithi177 ceph-mon[195797]: pgmap v2154: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 323 op/s; 35959/103791 objects degraded (34.646%); 5079/103791 objects misplaced (4.893%); 9.3 MiB/s, 241 objects/s recovering 2023-10-09T05:06:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35959/103791 objects degraded (34.646%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:11.998 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:12.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:12.112 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:12.132 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:06:12.189 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:12.460 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:12.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:12.562 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:12.593 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:12.640 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:12.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:12.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:12.777 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:12.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:12.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:12.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:12.934 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:13.053 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:13.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:06:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:06:12] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-10-09T05:06:13.197 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:06:13.233 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:06:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:13 smithi177 ceph-mon[195797]: pgmap v2155: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 35959/103791 objects degraded (34.646%); 5079/103791 objects misplaced (4.893%); 6.2 MiB/s, 156 objects/s recovering 2023-10-09T05:06:13.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:13 smithi029 ceph-mon[223858]: pgmap v2155: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 35959/103791 objects degraded (34.646%); 5079/103791 objects misplaced (4.893%); 6.2 MiB/s, 156 objects/s recovering 2023-10-09T05:06:13.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:13 smithi029 ceph-mon[216903]: pgmap v2155: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 92 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 212 op/s; 35959/103791 objects degraded (34.646%); 5079/103791 objects misplaced (4.893%); 6.2 MiB/s, 156 objects/s recovering 2023-10-09T05:06:14.037 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:14.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:14.147 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:14.174 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:14.222 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:14.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:14.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:14.361 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:14.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:14 smithi029 ceph-mon[216903]: from='client.47576 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:14.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:14.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:14.869 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:14.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:14 smithi029 ceph-mon[223858]: from='client.47576 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:14.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:14.881 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:14.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:14.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:14 smithi177 ceph-mon[195797]: from='client.47576 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:15.028 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:15.074 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2155,"stamp":"2023-10-09T05:06:11.843225+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2266306488,"num_objects":34597,"num_object_clones":0,"num_object_copies":103791,"num_objects_missing_on_primary":11818,"num_objects_missing":11818,"num_objects_degraded":35959,"num_objects_misplaced":5079,"num_objects_unfound":0,"num_objects_dirty":34597,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":120945,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78848,"num_bytes_recovered":4758735502,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":205175,"ondisk_log_size":205175,"up":357,"acting":341,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18545628,"kb_used_data":10221348,"kb_used_omap":34,"kb_used_meta":8323869,"kb_avail":637670436,"statfs":{"total":671965249536,"available":652974526464,"internally_reserved":0,"allocated":10466660352,"data_stored":10450492241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8523642250},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,3,0,1,7,5,4,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-111411200,"num_objects":-1700,"num_object_clones":0,"num_object_copies":-5100,"num_objects_missing_on_primary":-612,"num_objects_missing":-612,"num_objects_degraded":-1948,"num_objects_misplaced":-514,"num_objects_unfound":0,"num_objects_dirty":-1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1700,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1256,"num_bytes_recovered":52166656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.009105"},"pg_stats":[{"pgid":"14.23","version":"668'5888","reported_seq":12514,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151062+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:06:08.151062+0000","last_peered":"2023-10-09T05:06:08.151062+0000","last_clean":"2023-10-09T05:06:08.151062+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:06:08.151062+0000","last_undegraded":"2023-10-09T05:06:08.151062+0000","last_fullsized":"2023-10-09T05:06:08.151062+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34603008,"num_objects":528,"num_object_clones":0,"num_object_copies":1584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1904,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"668'5798","reported_seq":12679,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:08.517990+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:06:08.517990+0000","last_peered":"2023-10-09T05:06:08.517990+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:06:08.517990+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35684352,"num_objects":545,"num_object_clones":0,"num_object_copies":1635,"num_objects_missing_on_primary":518,"num_objects_missing":518,"num_objects_degraded":1399,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1863,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":938,"num_bytes_recovered":59228160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"3,4,5,7","objects":1},{"shards":"4,5,7","objects":360}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"668'5878","reported_seq":16577,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.154930+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:06:08.154930+0000","last_peered":"2023-10-09T05:06:08.154930+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:06:08.154930+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:06:08.154930+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5878,"log_dups_size":0,"ondisk_log_size":5878,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35061760,"num_objects":535,"num_object_clones":0,"num_object_copies":1605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1895,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":535}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"667'5860","reported_seq":7565,"reported_epoch":667,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:07.226980+0000","last_change":"2023-10-09T05:05:26.963137+0000","last_active":"2023-10-09T05:06:07.226980+0000","last_peered":"2023-10-09T05:06:07.226980+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:06:07.226980+0000","last_undegraded":"2023-10-09T05:05:26.962565+0000","last_fullsized":"2023-10-09T05:06:07.226980+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5322,"log_dups_size":0,"ondisk_log_size":5322,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36978688,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1548,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1867,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":425,"num_bytes_recovered":27869184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":515},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"667'6285","reported_seq":19846,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.194444+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:06:07.194444+0000","last_peered":"2023-10-09T05:06:07.194444+0000","last_clean":"2023-10-09T05:06:07.194444+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:06:07.194444+0000","last_undegraded":"2023-10-09T05:06:07.194444+0000","last_fullsized":"2023-10-09T05:06:07.194444+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6285,"log_dups_size":0,"ondisk_log_size":6285,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74940416,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3744,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":170,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147125+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:06:07.147125+0000","last_peered":"2023-10-09T05:06:07.147125+0000","last_clean":"2023-10-09T05:06:07.147125+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:06:07.147125+0000","last_undegraded":"2023-10-09T05:06:07.147125+0000","last_fullsized":"2023-10-09T05:06:07.147125+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":315,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.153717+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:06:07.153717+0000","last_peered":"2023-10-09T05:06:07.153717+0000","last_clean":"2023-10-09T05:06:07.153717+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:06:07.153717+0000","last_undegraded":"2023-10-09T05:06:07.153717+0000","last_fullsized":"2023-10-09T05:06:07.153717+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"668'6303","reported_seq":9531,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.644362+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:06:08.644362+0000","last_peered":"2023-10-09T05:06:08.644362+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:06:08.644362+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:06:08.644362+0000","mapping_epoch":638,"log_start":"444'156","ondisk_log_start":"444'156","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6147,"log_dups_size":0,"ondisk_log_size":6147,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71860224,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3764,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":1097}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":91,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.035551+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:06:00.035551+0000","last_peered":"2023-10-09T05:06:00.035551+0000","last_clean":"2023-10-09T05:06:00.035551+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:06:00.035551+0000","last_undegraded":"2023-10-09T05:06:00.035551+0000","last_fullsized":"2023-10-09T05:06:00.035551+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.156610+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:06:07.156610+0000","last_peered":"2023-10-09T05:06:07.156610+0000","last_clean":"2023-10-09T05:06:07.156610+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:06:07.156610+0000","last_undegraded":"2023-10-09T05:06:07.156610+0000","last_fullsized":"2023-10-09T05:06:07.156610+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"668'6316","reported_seq":17975,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.761731+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:06:08.761731+0000","last_peered":"2023-10-09T05:06:08.761731+0000","last_clean":"2023-10-09T05:06:08.761731+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:06:08.761731+0000","last_undegraded":"2023-10-09T05:06:08.761731+0000","last_fullsized":"2023-10-09T05:06:08.761731+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71106560,"num_objects":1085,"num_object_clones":0,"num_object_copies":3255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3776,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":297,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.158438+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:06:07.158438+0000","last_peered":"2023-10-09T05:06:07.158438+0000","last_clean":"2023-10-09T05:06:07.158438+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:06:07.158438+0000","last_undegraded":"2023-10-09T05:06:07.158438+0000","last_fullsized":"2023-10-09T05:06:07.158438+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":194,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.150801+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:06:07.150801+0000","last_peered":"2023-10-09T05:06:07.150801+0000","last_clean":"2023-10-09T05:06:07.150801+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:06:07.150801+0000","last_undegraded":"2023-10-09T05:06:07.150801+0000","last_fullsized":"2023-10-09T05:06:07.150801+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"666'6518","reported_seq":23873,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.112426+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:06:07.112426+0000","last_peered":"2023-10-09T05:06:07.112426+0000","last_clean":"2023-10-09T05:06:07.112426+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:06:07.112426+0000","last_undegraded":"2023-10-09T05:06:07.112426+0000","last_fullsized":"2023-10-09T05:06:07.112426+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6118,"log_dups_size":0,"ondisk_log_size":6118,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70385664,"num_objects":1074,"num_object_clones":0,"num_object_copies":3222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3697,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":41,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.985905+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:05:59.985905+0000","last_peered":"2023-10-09T05:05:59.985905+0000","last_clean":"2023-10-09T05:05:59.985905+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:05:59.985905+0000","last_undegraded":"2023-10-09T05:05:59.985905+0000","last_fullsized":"2023-10-09T05:05:59.985905+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":289,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.158344+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:06:07.158344+0000","last_peered":"2023-10-09T05:06:07.158344+0000","last_clean":"2023-10-09T05:06:07.158344+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:06:07.158344+0000","last_undegraded":"2023-10-09T05:06:07.158344+0000","last_fullsized":"2023-10-09T05:06:07.158344+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"668'6842","reported_seq":14075,"reported_epoch":668,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:06:09.715487+0000","last_change":"2023-10-09T05:05:26.975807+0000","last_active":"2023-10-09T05:06:09.715487+0000","last_peered":"2023-10-09T05:06:09.715487+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:06:09.715487+0000","last_undegraded":"2023-10-09T05:06:09.715487+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5417,"log_dups_size":0,"ondisk_log_size":5417,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":1479,"num_objects_missing":1479,"num_objects_degraded":0,"num_objects_misplaced":3230,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3882,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1674,"num_bytes_recovered":109641728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":2},{"shards":"0,2,4,6,7","objects":1},{"shards":"0,2,4,7","objects":1},{"shards":"2,4,7","objects":1039},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":55,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.985394+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:05:59.985394+0000","last_peered":"2023-10-09T05:05:59.985394+0000","last_clean":"2023-10-09T05:05:59.985394+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:05:59.985394+0000","last_undegraded":"2023-10-09T05:05:59.985394+0000","last_fullsized":"2023-10-09T05:05:59.985394+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":279,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.158621+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:06:07.158621+0000","last_peered":"2023-10-09T05:06:07.158621+0000","last_clean":"2023-10-09T05:06:07.158621+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:06:07.158621+0000","last_undegraded":"2023-10-09T05:06:07.158621+0000","last_fullsized":"2023-10-09T05:06:07.158621+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"667'6476","reported_seq":13499,"reported_epoch":667,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:06:07.642758+0000","last_change":"2023-10-09T05:05:00.137617+0000","last_active":"2023-10-09T05:06:07.642758+0000","last_peered":"2023-10-09T05:06:07.642758+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:06:07.642758+0000","last_undegraded":"2023-10-09T05:06:07.642758+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4940,"log_dups_size":0,"ondisk_log_size":4940,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73859072,"num_objects":1127,"num_object_clones":0,"num_object_copies":3381,"num_objects_missing_on_primary":763,"num_objects_missing":763,"num_objects_degraded":0,"num_objects_misplaced":1849,"num_objects_unfound":0,"num_objects_dirty":1127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3867,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1711,"num_bytes_recovered":112132096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,5,7","objects":1},{"shards":"1,3","objects":365},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":330}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":341,"reported_epoch":667,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:07.153604+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:06:07.153604+0000","last_peered":"2023-10-09T05:06:07.153604+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:06:07.153604+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:06:07.153604+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.033533+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:06:00.033533+0000","last_peered":"2023-10-09T05:06:00.033533+0000","last_clean":"2023-10-09T05:06:00.033533+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:06:00.033533+0000","last_undegraded":"2023-10-09T05:06:00.033533+0000","last_fullsized":"2023-10-09T05:06:00.033533+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"666'6346","reported_seq":18419,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.017369+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:06:07.017369+0000","last_peered":"2023-10-09T05:06:07.017369+0000","last_clean":"2023-10-09T05:06:07.017369+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:06:07.017369+0000","last_undegraded":"2023-10-09T05:06:07.017369+0000","last_fullsized":"2023-10-09T05:06:07.017369+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6346,"log_dups_size":0,"ondisk_log_size":6346,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70877184,"num_objects":1082,"num_object_clones":0,"num_object_copies":3246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3808,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":123,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.148558+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:06:07.148558+0000","last_peered":"2023-10-09T05:06:07.148558+0000","last_clean":"2023-10-09T05:06:07.148558+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:06:07.148558+0000","last_undegraded":"2023-10-09T05:06:07.148558+0000","last_fullsized":"2023-10-09T05:06:07.148558+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147618+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:06:07.147618+0000","last_peered":"2023-10-09T05:06:07.147618+0000","last_clean":"2023-10-09T05:06:07.147618+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:06:07.147618+0000","last_undegraded":"2023-10-09T05:06:07.147618+0000","last_fullsized":"2023-10-09T05:06:07.147618+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"668'6567","reported_seq":19025,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:08.730195+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:06:08.730195+0000","last_peered":"2023-10-09T05:06:08.730195+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:06:08.730195+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4886,"log_dups_size":0,"ondisk_log_size":4886,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71073792,"num_objects":1085,"num_object_clones":0,"num_object_copies":3255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1609,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3708,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3246,"num_bytes_recovered":208142336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":581}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":144,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147562+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:06:07.147562+0000","last_peered":"2023-10-09T05:06:07.147562+0000","last_clean":"2023-10-09T05:06:07.147562+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:06:07.147562+0000","last_undegraded":"2023-10-09T05:06:07.147562+0000","last_fullsized":"2023-10-09T05:06:07.147562+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.193536+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:06:07.193536+0000","last_peered":"2023-10-09T05:06:07.193536+0000","last_clean":"2023-10-09T05:06:07.193536+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:06:07.193536+0000","last_undegraded":"2023-10-09T05:06:07.193536+0000","last_fullsized":"2023-10-09T05:06:07.193536+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"667'6603","reported_seq":12852,"reported_epoch":667,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:07.308995+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:06:07.308995+0000","last_peered":"2023-10-09T05:06:07.308995+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:06:07.308995+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4476,"log_dups_size":0,"ondisk_log_size":4476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":1109,"num_objects_missing":1109,"num_objects_degraded":2194,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3774,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1705,"num_bytes_recovered":111706112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,5,7","objects":1},{"shards":"1,7","objects":-39},{"shards":"2,5,7","objects":668},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":171,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.161075+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:06:07.161075+0000","last_peered":"2023-10-09T05:06:07.161075+0000","last_clean":"2023-10-09T05:06:07.161075+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:06:07.161075+0000","last_undegraded":"2023-10-09T05:06:07.161075+0000","last_fullsized":"2023-10-09T05:06:07.161075+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":45,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.986495+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:05:59.986495+0000","last_peered":"2023-10-09T05:05:59.986495+0000","last_clean":"2023-10-09T05:05:59.986495+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:05:59.986495+0000","last_undegraded":"2023-10-09T05:05:59.986495+0000","last_fullsized":"2023-10-09T05:05:59.986495+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"668'6671","reported_seq":14472,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.071154+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:06:09.071154+0000","last_peered":"2023-10-09T05:06:09.071154+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:06:09.071154+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72286208,"num_objects":1103,"num_object_clones":0,"num_object_copies":3309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2790,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3772,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2061,"num_bytes_recovered":122093568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":1021},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":107,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147941+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:06:07.147941+0000","last_peered":"2023-10-09T05:06:07.147941+0000","last_clean":"2023-10-09T05:06:07.147941+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:06:07.147941+0000","last_undegraded":"2023-10-09T05:06:07.147941+0000","last_fullsized":"2023-10-09T05:06:07.147941+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":305,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.157054+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:06:07.157054+0000","last_peered":"2023-10-09T05:06:07.157054+0000","last_clean":"2023-10-09T05:06:07.157054+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:06:07.157054+0000","last_undegraded":"2023-10-09T05:06:07.157054+0000","last_fullsized":"2023-10-09T05:06:07.157054+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"668'6600","reported_seq":13480,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.490304+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:06:08.490304+0000","last_peered":"2023-10-09T05:06:08.490304+0000","last_clean":"2023-10-09T05:06:08.490304+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:06:08.490304+0000","last_undegraded":"2023-10-09T05:06:08.490304+0000","last_fullsized":"2023-10-09T05:06:08.490304+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4307,"log_dups_size":0,"ondisk_log_size":4307,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69828620,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3738,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":302,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.152276+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:06:07.152276+0000","last_peered":"2023-10-09T05:06:07.152276+0000","last_clean":"2023-10-09T05:06:07.152276+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:06:07.152276+0000","last_undegraded":"2023-10-09T05:06:07.152276+0000","last_fullsized":"2023-10-09T05:06:07.152276+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":163,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.157509+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:06:07.157509+0000","last_peered":"2023-10-09T05:06:07.157509+0000","last_clean":"2023-10-09T05:06:07.157509+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:06:07.157509+0000","last_undegraded":"2023-10-09T05:06:07.157509+0000","last_fullsized":"2023-10-09T05:06:07.157509+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"668'5818","reported_seq":14694,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.589949+0000","last_change":"2023-10-09T05:05:29.132074+0000","last_active":"2023-10-09T05:06:08.589949+0000","last_peered":"2023-10-09T05:06:08.589949+0000","last_clean":"2023-10-09T05:06:08.589949+0000","last_became_active":"2023-10-09T05:05:29.125934+0000","last_became_peered":"2023-10-09T05:05:29.125934+0000","last_unstale":"2023-10-09T05:06:08.589949+0000","last_undegraded":"2023-10-09T05:06:08.589949+0000","last_fullsized":"2023-10-09T05:06:08.589949+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5477,"log_dups_size":0,"ondisk_log_size":5477,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34373632,"num_objects":524,"num_object_clones":0,"num_object_copies":1572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1883,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":197,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147932+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:06:07.147932+0000","last_peered":"2023-10-09T05:06:07.147932+0000","last_clean":"2023-10-09T05:06:07.147932+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:06:07.147932+0000","last_undegraded":"2023-10-09T05:06:07.147932+0000","last_fullsized":"2023-10-09T05:06:07.147932+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.983097+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:05:59.983097+0000","last_peered":"2023-10-09T05:05:59.983097+0000","last_clean":"2023-10-09T05:05:59.983097+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:05:59.983097+0000","last_undegraded":"2023-10-09T05:05:59.983097+0000","last_fullsized":"2023-10-09T05:05:59.983097+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"668'5877","reported_seq":10231,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.584623+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:06:09.584623+0000","last_peered":"2023-10-09T05:06:09.584623+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:06:09.584623+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5877,"log_dups_size":0,"ondisk_log_size":5877,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35323904,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":717,"num_objects_missing":717,"num_objects_degraded":1569,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1893,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":683,"num_bytes_recovered":44728320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":319},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":25,"reported_epoch":667,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:07.147348+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:06:07.147348+0000","last_peered":"2023-10-09T05:06:07.147348+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:06:07.147348+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:06:07.147348+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147865+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:06:07.147865+0000","last_peered":"2023-10-09T05:06:07.147865+0000","last_clean":"2023-10-09T05:06:07.147865+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:06:07.147865+0000","last_undegraded":"2023-10-09T05:06:07.147865+0000","last_fullsized":"2023-10-09T05:06:07.147865+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"667'5907","reported_seq":10027,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.221539+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:06:07.221539+0000","last_peered":"2023-10-09T05:06:07.221539+0000","last_clean":"2023-10-09T05:06:07.221539+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:06:07.221539+0000","last_undegraded":"2023-10-09T05:06:07.221539+0000","last_fullsized":"2023-10-09T05:06:07.221539+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5369,"log_dups_size":0,"ondisk_log_size":5369,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33898496,"num_objects":517,"num_object_clones":0,"num_object_copies":1551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1914,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":49,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.986529+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:05:59.986529+0000","last_peered":"2023-10-09T05:05:59.986529+0000","last_clean":"2023-10-09T05:05:59.986529+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:05:59.986529+0000","last_undegraded":"2023-10-09T05:05:59.986529+0000","last_fullsized":"2023-10-09T05:05:59.986529+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":214,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151754+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:06:07.151754+0000","last_peered":"2023-10-09T05:06:07.151754+0000","last_clean":"2023-10-09T05:06:07.151754+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:06:07.151754+0000","last_undegraded":"2023-10-09T05:06:07.151754+0000","last_fullsized":"2023-10-09T05:06:07.151754+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"668'5894","reported_seq":16593,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.336413+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:06:08.336413+0000","last_peered":"2023-10-09T05:06:08.336413+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:06:08.336413+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:06:08.336413+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34013184,"num_objects":519,"num_object_clones":0,"num_object_copies":1557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1910,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":519}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":727,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.984773+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:05:59.984773+0000","last_peered":"2023-10-09T05:05:59.984773+0000","last_clean":"2023-10-09T05:05:59.984773+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:05:59.984773+0000","last_undegraded":"2023-10-09T05:05:59.984773+0000","last_fullsized":"2023-10-09T05:05:59.984773+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":102,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.034997+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:06:00.034997+0000","last_peered":"2023-10-09T05:06:00.034997+0000","last_clean":"2023-10-09T05:06:00.034997+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:06:00.034997+0000","last_undegraded":"2023-10-09T05:06:00.034997+0000","last_fullsized":"2023-10-09T05:06:00.034997+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1256,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151175+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:06:07.151175+0000","last_peered":"2023-10-09T05:06:07.151175+0000","last_clean":"2023-10-09T05:06:07.151175+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:06:07.151175+0000","last_undegraded":"2023-10-09T05:06:07.151175+0000","last_fullsized":"2023-10-09T05:06:07.151175+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"668'6086","reported_seq":11631,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.711415+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:06:09.711415+0000","last_peered":"2023-10-09T05:06:09.711415+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:06:09.711415+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6086,"log_dups_size":0,"ondisk_log_size":6086,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71204864,"num_objects":1086,"num_object_clones":0,"num_object_copies":3258,"num_objects_missing_on_primary":1400,"num_objects_missing":1400,"num_objects_degraded":3185,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3629,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1273,"num_bytes_recovered":83427328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":37},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":653}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":272,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.157378+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:06:07.157378+0000","last_peered":"2023-10-09T05:06:07.157378+0000","last_clean":"2023-10-09T05:06:07.157378+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:06:07.157378+0000","last_undegraded":"2023-10-09T05:06:07.157378+0000","last_fullsized":"2023-10-09T05:06:07.157378+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":106,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.148461+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:06:07.148461+0000","last_peered":"2023-10-09T05:06:07.148461+0000","last_clean":"2023-10-09T05:06:07.148461+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:06:07.148461+0000","last_undegraded":"2023-10-09T05:06:07.148461+0000","last_fullsized":"2023-10-09T05:06:07.148461+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1245,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.152144+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:06:07.152144+0000","last_peered":"2023-10-09T05:06:07.152144+0000","last_clean":"2023-10-09T05:06:07.152144+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:06:07.152144+0000","last_undegraded":"2023-10-09T05:06:07.152144+0000","last_fullsized":"2023-10-09T05:06:07.152144+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"666'6342","reported_seq":19940,"reported_epoch":667,"state":"peering","last_fresh":"2023-10-09T05:06:07.151551+0000","last_change":"2023-10-09T05:06:07.152482+0000","last_active":"2023-10-09T05:06:07.151529+0000","last_peered":"2023-10-09T05:06:07.127867+0000","last_clean":"2023-10-09T04:55:08.779070+0000","last_became_active":"2023-10-09T05:03:45.583447+0000","last_became_peered":"2023-10-09T05:03:45.583447+0000","last_unstale":"2023-10-09T05:06:07.151551+0000","last_undegraded":"2023-10-09T05:06:07.151551+0000","last_fullsized":"2023-10-09T05:06:07.151551+0000","mapping_epoch":667,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6342,"log_dups_size":0,"ondisk_log_size":6342,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71204864,"num_objects":1086,"num_object_clones":0,"num_object_copies":3258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3800,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4591,"num_bytes_recovered":223150080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,2,3,4],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":211,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151682+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:06:07.151682+0000","last_peered":"2023-10-09T05:06:07.151682+0000","last_clean":"2023-10-09T05:06:07.151682+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:06:07.151682+0000","last_undegraded":"2023-10-09T05:06:07.151682+0000","last_fullsized":"2023-10-09T05:06:07.151682+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":26,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147030+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:06:07.147030+0000","last_peered":"2023-10-09T05:06:07.147030+0000","last_clean":"2023-10-09T05:06:07.147030+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:06:07.147030+0000","last_undegraded":"2023-10-09T05:06:07.147030+0000","last_fullsized":"2023-10-09T05:06:07.147030+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"668'6322","reported_seq":11335,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:08.539778+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:06:08.539778+0000","last_peered":"2023-10-09T05:06:08.539778+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:06:08.539778+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"444'206","ondisk_log_start":"444'206","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6116,"log_dups_size":0,"ondisk_log_size":6116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70615040,"num_objects":1077,"num_object_clones":0,"num_object_copies":3231,"num_objects_missing_on_primary":684,"num_objects_missing":684,"num_objects_degraded":1755,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3782,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1352,"num_bytes_recovered":88637440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,4,7","objects":684},{"shards":"3,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":140,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147922+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:06:07.147922+0000","last_peered":"2023-10-09T05:06:07.147922+0000","last_clean":"2023-10-09T05:06:07.147922+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:06:07.147922+0000","last_undegraded":"2023-10-09T05:06:07.147922+0000","last_fullsized":"2023-10-09T05:06:07.147922+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":141,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147697+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:06:07.147697+0000","last_peered":"2023-10-09T05:06:07.147697+0000","last_clean":"2023-10-09T05:06:07.147697+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:06:07.147697+0000","last_undegraded":"2023-10-09T05:06:07.147697+0000","last_fullsized":"2023-10-09T05:06:07.147697+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1265,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151932+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:06:07.151932+0000","last_peered":"2023-10-09T05:06:07.151932+0000","last_clean":"2023-10-09T05:06:07.151932+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:06:07.151932+0000","last_undegraded":"2023-10-09T05:06:07.151932+0000","last_fullsized":"2023-10-09T05:06:07.151932+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"668'6337","reported_seq":20153,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.947570+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:06:08.947570+0000","last_peered":"2023-10-09T05:06:08.947570+0000","last_clean":"2023-10-09T05:06:08.947570+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:06:08.947570+0000","last_undegraded":"2023-10-09T05:06:08.947570+0000","last_fullsized":"2023-10-09T05:06:08.947570+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69730304,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3797,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.153037+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:06:07.153037+0000","last_peered":"2023-10-09T05:06:07.153037+0000","last_clean":"2023-10-09T05:06:07.153037+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:06:07.153037+0000","last_undegraded":"2023-10-09T05:06:07.153037+0000","last_fullsized":"2023-10-09T05:06:07.153037+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":94,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.031856+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:06:00.031856+0000","last_peered":"2023-10-09T05:06:00.031856+0000","last_clean":"2023-10-09T05:06:00.031856+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:06:00.031856+0000","last_undegraded":"2023-10-09T05:06:00.031856+0000","last_fullsized":"2023-10-09T05:06:00.031856+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1271,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.150723+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:06:07.150723+0000","last_peered":"2023-10-09T05:06:07.150723+0000","last_clean":"2023-10-09T05:06:07.150723+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:06:07.150723+0000","last_undegraded":"2023-10-09T05:06:07.150723+0000","last_fullsized":"2023-10-09T05:06:07.150723+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"668'6504","reported_seq":11355,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.712771+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:06:09.712771+0000","last_peered":"2023-10-09T05:06:09.712771+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:06:09.712771+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6504,"log_dups_size":0,"ondisk_log_size":6504,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":1510,"num_objects_missing":1510,"num_objects_degraded":4338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3895,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1399,"num_bytes_recovered":91684864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,5,7","objects":1},{"shards":"0,3","objects":-933},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":718}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":82,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.972245+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:05:59.972245+0000","last_peered":"2023-10-09T05:05:59.972245+0000","last_clean":"2023-10-09T05:05:59.972245+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:05:59.972245+0000","last_undegraded":"2023-10-09T05:05:59.972245+0000","last_fullsized":"2023-10-09T05:05:59.972245+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":45,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.986576+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:05:59.986576+0000","last_peered":"2023-10-09T05:05:59.986576+0000","last_clean":"2023-10-09T05:05:59.986576+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:05:59.986576+0000","last_undegraded":"2023-10-09T05:05:59.986576+0000","last_fullsized":"2023-10-09T05:05:59.986576+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":100,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.033458+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:06:00.033458+0000","last_peered":"2023-10-09T05:06:00.033458+0000","last_clean":"2023-10-09T05:06:00.033458+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:06:00.033458+0000","last_undegraded":"2023-10-09T05:06:00.033458+0000","last_fullsized":"2023-10-09T05:06:00.033458+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1254,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151092+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:06:07.151092+0000","last_peered":"2023-10-09T05:06:07.151092+0000","last_clean":"2023-10-09T05:06:07.151092+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:06:07.151092+0000","last_undegraded":"2023-10-09T05:06:07.151092+0000","last_fullsized":"2023-10-09T05:06:07.151092+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"668'6607","reported_seq":19146,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:08.695046+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:06:08.695046+0000","last_peered":"2023-10-09T05:06:08.695046+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:06:08.695046+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4926,"log_dups_size":0,"ondisk_log_size":4926,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68452352,"num_objects":1044,"num_object_clones":0,"num_object_copies":3132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1611,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3747,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3286,"num_bytes_recovered":210763776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":562}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":751,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.985433+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:05:59.985433+0000","last_peered":"2023-10-09T05:05:59.985433+0000","last_clean":"2023-10-09T05:05:59.985433+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:05:59.985433+0000","last_undegraded":"2023-10-09T05:05:59.985433+0000","last_fullsized":"2023-10-09T05:05:59.985433+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147416+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:06:07.147416+0000","last_peered":"2023-10-09T05:06:07.147416+0000","last_clean":"2023-10-09T05:06:07.147416+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:06:07.147416+0000","last_undegraded":"2023-10-09T05:06:07.147416+0000","last_fullsized":"2023-10-09T05:06:07.147416+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.972174+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:05:59.972174+0000","last_peered":"2023-10-09T05:05:59.972174+0000","last_clean":"2023-10-09T05:05:59.972174+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:05:59.972174+0000","last_undegraded":"2023-10-09T05:05:59.972174+0000","last_fullsized":"2023-10-09T05:05:59.972174+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1265,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151835+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:06:07.151835+0000","last_peered":"2023-10-09T05:06:07.151835+0000","last_clean":"2023-10-09T05:06:07.151835+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:06:07.151835+0000","last_undegraded":"2023-10-09T05:06:07.151835+0000","last_fullsized":"2023-10-09T05:06:07.151835+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"668'6610","reported_seq":12391,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.630767+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:06:08.630767+0000","last_peered":"2023-10-09T05:06:08.630767+0000","last_clean":"2023-10-09T05:06:08.630767+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:06:08.630767+0000","last_undegraded":"2023-10-09T05:06:08.630767+0000","last_fullsized":"2023-10-09T05:06:08.630767+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4317,"log_dups_size":0,"ondisk_log_size":4317,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69173260,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3748,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":163,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.157968+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:06:07.157968+0000","last_peered":"2023-10-09T05:06:07.157968+0000","last_clean":"2023-10-09T05:06:07.157968+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:06:07.157968+0000","last_undegraded":"2023-10-09T05:06:07.157968+0000","last_fullsized":"2023-10-09T05:06:07.157968+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":80,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.971746+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:05:59.971746+0000","last_peered":"2023-10-09T05:05:59.971746+0000","last_clean":"2023-10-09T05:05:59.971746+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:05:59.971746+0000","last_undegraded":"2023-10-09T05:05:59.971746+0000","last_fullsized":"2023-10-09T05:05:59.971746+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.031782+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:06:00.031782+0000","last_peered":"2023-10-09T05:06:00.031782+0000","last_clean":"2023-10-09T05:06:00.031782+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:06:00.031782+0000","last_undegraded":"2023-10-09T05:06:00.031782+0000","last_fullsized":"2023-10-09T05:06:00.031782+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"668'6531","reported_seq":12539,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:08.573667+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:06:08.573667+0000","last_peered":"2023-10-09T05:06:08.573667+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:06:08.573667+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4404,"log_dups_size":0,"ondisk_log_size":4404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70582272,"num_objects":1077,"num_object_clones":0,"num_object_copies":3231,"num_objects_missing_on_primary":726,"num_objects_missing":726,"num_objects_degraded":2106,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3702,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1622,"num_bytes_recovered":106266624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,7","objects":635},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":40}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":55,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.984508+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:05:59.984508+0000","last_peered":"2023-10-09T05:05:59.984508+0000","last_clean":"2023-10-09T05:05:59.984508+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:05:59.984508+0000","last_undegraded":"2023-10-09T05:05:59.984508+0000","last_fullsized":"2023-10-09T05:05:59.984508+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1271,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.152022+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:06:07.152022+0000","last_peered":"2023-10-09T05:06:07.152022+0000","last_clean":"2023-10-09T05:06:07.152022+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:06:07.152022+0000","last_undegraded":"2023-10-09T05:06:07.152022+0000","last_fullsized":"2023-10-09T05:06:07.152022+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":161,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.161156+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:06:07.161156+0000","last_peered":"2023-10-09T05:06:07.161156+0000","last_clean":"2023-10-09T05:06:07.161156+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:06:07.161156+0000","last_undegraded":"2023-10-09T05:06:07.161156+0000","last_fullsized":"2023-10-09T05:06:07.161156+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.150626+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:06:07.150626+0000","last_peered":"2023-10-09T05:06:07.150626+0000","last_clean":"2023-10-09T05:06:07.150626+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:06:07.150626+0000","last_undegraded":"2023-10-09T05:06:07.150626+0000","last_fullsized":"2023-10-09T05:06:07.150626+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1244,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.152618+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:06:07.152618+0000","last_peered":"2023-10-09T05:06:07.152618+0000","last_clean":"2023-10-09T05:06:07.152618+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:06:07.152618+0000","last_undegraded":"2023-10-09T05:06:07.152618+0000","last_fullsized":"2023-10-09T05:06:07.152618+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"668'6700","reported_seq":14963,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.043971+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:06:09.043971+0000","last_peered":"2023-10-09T05:06:09.043971+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:06:09.043971+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70385664,"num_objects":1074,"num_object_clones":0,"num_object_copies":3222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3801,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2202,"num_bytes_recovered":131989504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":88},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":939}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":80,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.971828+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:05:59.971828+0000","last_peered":"2023-10-09T05:05:59.971828+0000","last_clean":"2023-10-09T05:05:59.971828+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:05:59.971828+0000","last_undegraded":"2023-10-09T05:05:59.971828+0000","last_fullsized":"2023-10-09T05:05:59.971828+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":26,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147723+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:06:07.147723+0000","last_peered":"2023-10-09T05:06:07.147723+0000","last_clean":"2023-10-09T05:06:07.147723+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:06:07.147723+0000","last_undegraded":"2023-10-09T05:06:07.147723+0000","last_fullsized":"2023-10-09T05:06:07.147723+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.191466+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:06:07.191466+0000","last_peered":"2023-10-09T05:06:07.191466+0000","last_clean":"2023-10-09T05:06:07.191466+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:06:07.191466+0000","last_undegraded":"2023-10-09T05:06:07.191466+0000","last_fullsized":"2023-10-09T05:06:07.191466+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1265,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.153274+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:06:07.153274+0000","last_peered":"2023-10-09T05:06:07.153274+0000","last_clean":"2023-10-09T05:06:07.153274+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:06:07.153274+0000","last_undegraded":"2023-10-09T05:06:07.153274+0000","last_fullsized":"2023-10-09T05:06:07.153274+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"667'6346","reported_seq":18381,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.281414+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:06:07.281414+0000","last_peered":"2023-10-09T05:06:07.281414+0000","last_clean":"2023-10-09T05:06:07.281414+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:06:07.281414+0000","last_undegraded":"2023-10-09T05:06:07.281414+0000","last_fullsized":"2023-10-09T05:06:07.281414+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6346,"log_dups_size":0,"ondisk_log_size":6346,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70877184,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3807,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":133,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147645+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:06:07.147645+0000","last_peered":"2023-10-09T05:06:07.147645+0000","last_clean":"2023-10-09T05:06:07.147645+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:06:07.147645+0000","last_undegraded":"2023-10-09T05:06:07.147645+0000","last_fullsized":"2023-10-09T05:06:07.147645+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":91,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.971992+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:05:59.971992+0000","last_peered":"2023-10-09T05:05:59.971992+0000","last_clean":"2023-10-09T05:05:59.971992+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:05:59.971992+0000","last_undegraded":"2023-10-09T05:05:59.971992+0000","last_fullsized":"2023-10-09T05:05:59.971992+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":45,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.984849+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:05:59.984849+0000","last_peered":"2023-10-09T05:05:59.984849+0000","last_clean":"2023-10-09T05:05:59.984849+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:05:59.984849+0000","last_undegraded":"2023-10-09T05:05:59.984849+0000","last_fullsized":"2023-10-09T05:05:59.984849+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1268,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.150555+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:06:07.150555+0000","last_peered":"2023-10-09T05:06:07.150555+0000","last_clean":"2023-10-09T05:06:07.150555+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:06:07.150555+0000","last_undegraded":"2023-10-09T05:06:07.150555+0000","last_fullsized":"2023-10-09T05:06:07.150555+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"667'6471","reported_seq":23827,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.200408+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:06:07.200408+0000","last_peered":"2023-10-09T05:06:07.200408+0000","last_clean":"2023-10-09T05:06:07.200408+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:06:07.200408+0000","last_undegraded":"2023-10-09T05:06:07.200408+0000","last_fullsized":"2023-10-09T05:06:07.200408+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73465856,"num_objects":1121,"num_object_clones":0,"num_object_copies":3363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3650,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":292,"reported_epoch":667,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:07.160375+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:06:07.160375+0000","last_peered":"2023-10-09T05:06:07.160375+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:06:07.160375+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:06:07.160375+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":97,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.034725+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:06:00.034725+0000","last_peered":"2023-10-09T05:06:00.034725+0000","last_clean":"2023-10-09T05:06:00.034725+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:06:00.034725+0000","last_undegraded":"2023-10-09T05:06:00.034725+0000","last_fullsized":"2023-10-09T05:06:00.034725+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":172,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.152928+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:06:07.152928+0000","last_peered":"2023-10-09T05:06:07.152928+0000","last_clean":"2023-10-09T05:06:07.152928+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:06:07.152928+0000","last_undegraded":"2023-10-09T05:06:07.152928+0000","last_fullsized":"2023-10-09T05:06:07.152928+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1224,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.151853+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:06:07.151853+0000","last_peered":"2023-10-09T05:06:07.151853+0000","last_clean":"2023-10-09T05:06:07.151853+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:06:07.151853+0000","last_undegraded":"2023-10-09T05:06:07.151853+0000","last_fullsized":"2023-10-09T05:06:07.151853+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"668'6884","reported_seq":14283,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.641558+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:06:09.641558+0000","last_peered":"2023-10-09T05:06:09.641558+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:06:09.641558+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":1448,"num_objects_missing":1448,"num_objects_degraded":3646,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3924,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1716,"num_bytes_recovered":112459776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":986},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":184,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.150986+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:06:07.150986+0000","last_peered":"2023-10-09T05:06:07.150986+0000","last_clean":"2023-10-09T05:06:07.150986+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:06:07.150986+0000","last_undegraded":"2023-10-09T05:06:07.150986+0000","last_fullsized":"2023-10-09T05:06:07.150986+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":101,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:06:00.034584+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:06:00.034584+0000","last_peered":"2023-10-09T05:06:00.034584+0000","last_clean":"2023-10-09T05:06:00.034584+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:06:00.034584+0000","last_undegraded":"2023-10-09T05:06:00.034584+0000","last_fullsized":"2023-10-09T05:06:00.034584+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":143,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147835+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:06:07.147835+0000","last_peered":"2023-10-09T05:06:07.147835+0000","last_clean":"2023-10-09T05:06:07.147835+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:06:07.147835+0000","last_undegraded":"2023-10-09T05:06:07.147835+0000","last_fullsized":"2023-10-09T05:06:07.147835+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"668'6134","reported_seq":11882,"reported_epoch":668,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:09.683519+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:06:09.683519+0000","last_peered":"2023-10-09T05:06:09.683519+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:06:09.683519+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68059136,"num_objects":1039,"num_object_clones":0,"num_object_copies":3117,"num_objects_missing_on_primary":1463,"num_objects_missing":1463,"num_objects_degraded":3292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3678,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1321,"num_bytes_recovered":86573056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-35},{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":687}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":133,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147531+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:06:07.147531+0000","last_peered":"2023-10-09T05:06:07.147531+0000","last_clean":"2023-10-09T05:06:07.147531+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:06:07.147531+0000","last_undegraded":"2023-10-09T05:06:07.147531+0000","last_fullsized":"2023-10-09T05:06:07.147531+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":82,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.972102+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:05:59.972102+0000","last_peered":"2023-10-09T05:05:59.972102+0000","last_clean":"2023-10-09T05:05:59.972102+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:05:59.972102+0000","last_undegraded":"2023-10-09T05:05:59.972102+0000","last_fullsized":"2023-10-09T05:05:59.972102+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"668'6284","reported_seq":11126,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.276405+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:06:08.276405+0000","last_peered":"2023-10-09T05:06:08.276405+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:06:08.276405+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:06:08.276405+0000","mapping_epoch":597,"log_start":"444'49","ondisk_log_start":"444'49","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6235,"log_dups_size":0,"ondisk_log_size":6235,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72744960,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3749,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":1110}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":802,"reported_epoch":666,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:05:59.985426+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:05:59.985426+0000","last_peered":"2023-10-09T05:05:59.985426+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:05:59.985426+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:05:59.985426+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":26,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147185+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:06:07.147185+0000","last_peered":"2023-10-09T05:06:07.147185+0000","last_clean":"2023-10-09T05:06:07.147185+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:06:07.147185+0000","last_undegraded":"2023-10-09T05:06:07.147185+0000","last_fullsized":"2023-10-09T05:06:07.147185+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"667'6284","reported_seq":10709,"reported_epoch":667,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:07.259908+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:06:07.259908+0000","last_peered":"2023-10-09T05:06:07.259908+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:06:07.259908+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72777728,"num_objects":1111,"num_object_clones":0,"num_object_copies":3333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3753,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":464}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":748,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.986823+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:05:59.986823+0000","last_peered":"2023-10-09T05:05:59.986823+0000","last_clean":"2023-10-09T05:05:59.986823+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:05:59.986823+0000","last_undegraded":"2023-10-09T05:05:59.986823+0000","last_fullsized":"2023-10-09T05:05:59.986823+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":26,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.146964+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:06:07.146964+0000","last_peered":"2023-10-09T05:06:07.146964+0000","last_clean":"2023-10-09T05:06:07.146964+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:06:07.146964+0000","last_undegraded":"2023-10-09T05:06:07.146964+0000","last_fullsized":"2023-10-09T05:06:07.146964+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"668'6339","reported_seq":17147,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.360937+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:06:08.360937+0000","last_peered":"2023-10-09T05:06:08.360937+0000","last_clean":"2023-10-09T05:06:08.360937+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:06:08.360937+0000","last_undegraded":"2023-10-09T05:06:08.360937+0000","last_fullsized":"2023-10-09T05:06:08.360937+0000","mapping_epoch":638,"log_start":"444'251","ondisk_log_start":"444'251","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6088,"log_dups_size":0,"ondisk_log_size":6088,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69271552,"num_objects":1057,"num_object_clones":0,"num_object_copies":3171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3807,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":288,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.156967+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:06:07.156967+0000","last_peered":"2023-10-09T05:06:07.156967+0000","last_clean":"2023-10-09T05:06:07.156967+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:06:07.156967+0000","last_undegraded":"2023-10-09T05:06:07.156967+0000","last_fullsized":"2023-10-09T05:06:07.156967+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":296,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.158694+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:06:07.158694+0000","last_peered":"2023-10-09T05:06:07.158694+0000","last_clean":"2023-10-09T05:06:07.158694+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:06:07.158694+0000","last_undegraded":"2023-10-09T05:06:07.158694+0000","last_fullsized":"2023-10-09T05:06:07.158694+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"668'6216","reported_seq":17825,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.419948+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:06:08.419948+0000","last_peered":"2023-10-09T05:06:08.419948+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:06:08.419948+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:06:08.419948+0000","mapping_epoch":597,"log_start":"444'72","ondisk_log_start":"444'72","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3717,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":1098}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":47,"reported_epoch":666,"state":"active+clean","last_fresh":"2023-10-09T05:05:59.985974+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:05:59.985974+0000","last_peered":"2023-10-09T05:05:59.985974+0000","last_clean":"2023-10-09T05:05:59.985974+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:05:59.985974+0000","last_undegraded":"2023-10-09T05:05:59.985974+0000","last_fullsized":"2023-10-09T05:05:59.985974+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":144,"reported_epoch":667,"state":"active+clean","last_fresh":"2023-10-09T05:06:07.147026+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:06:07.147026+0000","last_peered":"2023-10-09T05:06:07.147026+0000","last_clean":"2023-10-09T05:06:07.147026+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:06:07.147026+0000","last_undegraded":"2023-10-09T05:06:07.147026+0000","last_fullsized":"2023-10-09T05:06:07.147026+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":2265710616,"num_objects":34573,"num_object_clones":0,"num_object_copies":103719,"num_objects_missing_on_primary":11817,"num_objects_missing":11817,"num_objects_degraded":35954,"num_objects_misplaced":5079,"num_objects_unfound":0,"num_objects_dirty":34573,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120645,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78679,"num_bytes_recovered":4751622144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12948680704,"data_stored":12948668488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203349,"ondisk_log_size":203349,"up":108,"acting":92,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234706,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3042132,"kb_used_data":1524684,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":90703020,"statfs":{"total":95995035648,"available":92879892480,"internally_reserved":0,"allocated":1561276416,"data_stored":1559579243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7190000000000003,"5min":4.2489999999999997,"15min":1.9099999999999999},"min":{"1min":0.23699999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":35.673000000000002,"5min":123.04000000000001,"15min":123.04000000000001},"last":4.5709999999999997},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.4390000000000001,"15min":2.1880000000000002},"min":{"1min":0.33200000000000002,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":35.622999999999998,"5min":123.059,"15min":123.059},"last":0.34999999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:05:15 2023","interfaces":[{"interface":"back","average":{"1min":6.0919999999999996,"5min":3.4500000000000002,"15min":2.6579999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.25800000000000001},"max":{"1min":60.377000000000002,"5min":60.377000000000002,"15min":122.56},"last":1.5389999999999999},{"interface":"front","average":{"1min":6.5810000000000004,"5min":3.1829999999999998,"15min":2.6120000000000001},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":59.689999999999998,"5min":59.689999999999998,"15min":122.666},"last":2.629}]},{"osd":2,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.7069999999999999,"15min":3.8879999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9269999999999996,"5min":64.480999999999995,"15min":207.96600000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.524,"5min":2.6469999999999998,"15min":4.0019999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.8989999999999991,"5min":55.134,"15min":207.97900000000001},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:05:08 2023","interfaces":[{"interface":"back","average":{"1min":16.815999999999999,"5min":5.5110000000000001,"15min":2.8839999999999999},"min":{"1min":0.35199999999999998,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":126.70399999999999,"5min":126.70399999999999,"15min":126.70399999999999},"last":3.1000000000000001},{"interface":"front","average":{"1min":15.994,"5min":5.3739999999999997,"15min":2.9060000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":125.991,"5min":125.991,"15min":125.991},"last":0.441}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38400000000000001,"15min":0.38600000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.497,"15min":0.48099999999999998},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.923,"5min":2.923,"15min":2.923},"last":0.54100000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":1.6399999999999999,"15min":1.7230000000000001},"min":{"1min":0.308,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":0.57399999999999995,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.734,"15min":2.0139999999999998},"min":{"1min":0.29099999999999998,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":2.2210000000000001,"5min":53.225000000000001,"15min":135.74199999999999},"last":0.51000000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.458,"15min":1.1699999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.88200000000000001,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":1.651,"15min":1.3129999999999999},"min":{"1min":0.374,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":2.2770000000000001,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.57999999999999996}]}]},{"osd":6,"up_from":46,"seq":197568496338,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2237448,"kb_used_data":718656,"kb_used_omap":0,"kb_used_meta":1518784,"kb_avail":91507704,"statfs":{"total":95995035648,"available":93703888896,"internally_reserved":0,"allocated":735903744,"data_stored":734210699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1555234816},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":6.9930000000000003,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":49.659999999999997,"5min":321.346,"15min":321.346},"last":4.9450000000000003},{"interface":"front","average":{"1min":6.0620000000000003,"5min":5.2439999999999998,"15min":2.355},"min":{"1min":0.35799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":46.987000000000002,"5min":169.119,"15min":169.119},"last":0.27400000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":2.415,"15min":3.2170000000000001},"min":{"1min":0.39000000000000001,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":10.973000000000001,"5min":72.840999999999994,"15min":220.48400000000001},"last":2.9750000000000001},{"interface":"front","average":{"1min":2.2810000000000001,"5min":2.665,"15min":3.2000000000000002},"min":{"1min":0.34899999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":10.02,"5min":72.832999999999998,"15min":220.56200000000001},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:05:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.6799999999999999,"15min":2.0630000000000002},"min":{"1min":0.376,"5min":0.32900000000000001,"15min":0.25600000000000001},"max":{"1min":5.0090000000000003,"5min":23.943000000000001,"15min":133.535},"last":2.3319999999999999},{"interface":"front","average":{"1min":1.877,"5min":1.651,"15min":1.9850000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":4.9359999999999999,"5min":25.978999999999999,"15min":133.51900000000001},"last":2.3679999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":5.6349999999999998,"5min":3.3769999999999998,"15min":2.956},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":45.915999999999997,"5min":64.355999999999995,"15min":147.43600000000001},"last":4.5170000000000003},{"interface":"front","average":{"1min":6.0229999999999997,"5min":3.2040000000000002,"15min":3.0270000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":45.868000000000002,"5min":56.133000000000003,"15min":147.41999999999999},"last":4.4400000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":0.96699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":18.186,"5min":18.186,"15min":18.186},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.548,"5min":1.1120000000000001,"15min":1.04},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.378}]},{"osd":5,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":1.006,"15min":0.99099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.499,"5min":25.934000000000001,"15min":25.934000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.83599999999999997,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.254},"max":{"1min":0.57699999999999996,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.73899999999999999,"15min":1.052},"min":{"1min":0.314,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":7.2549999999999999,"5min":7.2549999999999999,"15min":36.204999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.99199999999999999,"15min":1.0269999999999999},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":0.58999999999999997,"5min":20.041,"15min":20.041},"last":0.41799999999999998}]}]},{"osd":1,"up_from":118,"seq":506806141405,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2193256,"kb_used_data":885536,"kb_used_omap":4,"kb_used_meta":1307579,"kb_avail":91551896,"statfs":{"total":95995035648,"available":93749141504,"internally_reserved":0,"allocated":906788864,"data_stored":903239497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1338961806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":10.287000000000001,"5min":4.125,"15min":2.0139999999999998},"min":{"1min":0.29999999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":100.247,"5min":100.247,"15min":100.247},"last":27.922000000000001},{"interface":"front","average":{"1min":12.194000000000001,"5min":4.5540000000000003,"15min":2.129},"min":{"1min":0.52600000000000002,"5min":0.247,"15min":0.214},"max":{"1min":99.935000000000002,"5min":99.935000000000002,"15min":99.935000000000002},"last":9.641}]},{"osd":2,"last update":"Mon Oct 9 05:05:48 2023","interfaces":[{"interface":"back","average":{"1min":3.202,"5min":1.5429999999999999,"15min":2.5939999999999999},"min":{"1min":0.309,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.478,"5min":12.460000000000001,"15min":263.089},"last":0.83099999999999996},{"interface":"front","average":{"1min":3.335,"5min":1.9319999999999999,"15min":2.653},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.222},"max":{"1min":10.385,"5min":13.933999999999999,"15min":254.50899999999999},"last":9.5579999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:05:16 2023","interfaces":[{"interface":"back","average":{"1min":24.670999999999999,"5min":6.3529999999999998,"15min":3.2160000000000002},"min":{"1min":0.29299999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":235.43600000000001,"5min":235.43600000000001,"15min":235.43600000000001},"last":10.640000000000001},{"interface":"front","average":{"1min":24.478000000000002,"5min":5.7560000000000002,"15min":2.9140000000000001},"min":{"1min":0.31900000000000001,"5min":0.249,"15min":0.249},"max":{"1min":234.32499999999999,"5min":234.32499999999999,"15min":234.32499999999999},"last":9.6709999999999994}]},{"osd":4,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.4930000000000001,"15min":1.347},"min":{"1min":0.42899999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":9.4570000000000007},{"interface":"front","average":{"1min":2.9060000000000001,"5min":1.339,"15min":1.0780000000000001},"min":{"1min":0.46500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.521000000000001,"5min":10.521000000000001,"15min":10.521000000000001},"last":4.484}]},{"osd":5,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.2929999999999999,"15min":1.278},"min":{"1min":0.36899999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":10.353,"5min":10.353,"15min":10.353},"last":9.5139999999999993},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.542,"15min":1.345},"min":{"1min":0.503,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":6.9459999999999997,"5min":10.554,"15min":19.597999999999999},"last":4.4109999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.383,"15min":1.2689999999999999},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.35699999999999998},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":9.6059999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.4390000000000001,"15min":1.3580000000000001},"min":{"1min":0.54400000000000004,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.915,"5min":10.472,"15min":19.548999999999999},"last":4.4530000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.5649999999999999,"15min":1.8180000000000001},"min":{"1min":0.54300000000000004,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.8330000000000002,"5min":10.583,"15min":59.972000000000001},"last":4.5140000000000002},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.351,"15min":1.714},"min":{"1min":0.51500000000000001,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":9.7170000000000005,"5min":11.903,"15min":60.011000000000003},"last":0.79000000000000004}]}]},{"osd":0,"up_from":91,"seq":390842024489,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1749276,"kb_used_data":630868,"kb_used_omap":5,"kb_used_meta":1118202,"kb_avail":91995876,"statfs":{"total":95995035648,"available":94203777024,"internally_reserved":0,"allocated":646008832,"data_stored":643131472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1145039482},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,0,6,5,4,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:05:12 2023","interfaces":[{"interface":"back","average":{"1min":6.1740000000000004,"5min":3.0640000000000001,"15min":4.7590000000000003},"min":{"1min":0.38500000000000001,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":53.176000000000002,"5min":53.176000000000002,"15min":247.68899999999999},"last":1.966},{"interface":"front","average":{"1min":7.3929999999999998,"5min":3.1240000000000001,"15min":4.7720000000000002},"min":{"1min":0.36499999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":53.372999999999998,"5min":53.372999999999998,"15min":246.38},"last":0.29999999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:05:32 2023","interfaces":[{"interface":"back","average":{"1min":3.7530000000000001,"5min":2.3969999999999998,"15min":2.516},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.762,"5min":26.556999999999999,"15min":151.06399999999999},"last":2.0830000000000002},{"interface":"front","average":{"1min":3.367,"5min":2.645,"15min":2.6789999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.7720000000000002,"5min":29.032,"15min":151.22399999999999},"last":4.141}]},{"osd":3,"last update":"Mon Oct 9 05:05:59 2023","interfaces":[{"interface":"back","average":{"1min":13.766999999999999,"5min":7.6890000000000001,"15min":4.4539999999999997},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":114.871,"5min":290.89100000000002,"15min":290.89100000000002},"last":2.1080000000000001},{"interface":"front","average":{"1min":15.404999999999999,"5min":7.9009999999999998,"15min":4.6449999999999996},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":115.375,"5min":290.73899999999998,"15min":290.73899999999998},"last":3.984}]},{"osd":4,"last update":"Mon Oct 9 05:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":1.9570000000000001,"15min":1.887},"min":{"1min":0.443,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.5329999999999995,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.663,"15min":1.577},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.81,"5min":11.81,"15min":11.81},"last":2.0139999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0089999999999999,"5min":1.78,"15min":1.3100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.039999999999999,"5min":16.888000000000002,"15min":16.888000000000002},"last":3.9340000000000002},{"interface":"front","average":{"1min":2.569,"5min":1.835,"15min":1.288},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":9.5779999999999994,"5min":14.036,"15min":14.036},"last":0.63500000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0070000000000001,"5min":2.0089999999999999,"15min":1.4359999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.074999999999999,"5min":20.129999999999999,"15min":39.594999999999999},"last":3.9649999999999999},{"interface":"front","average":{"1min":2.573,"5min":2.3759999999999999,"15min":1.6020000000000001},"min":{"1min":0.45400000000000001,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":9.6419999999999995,"5min":47.241,"15min":47.241},"last":0.66400000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:05:19 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":2.0870000000000002,"15min":1.496},"min":{"1min":0.51600000000000001,"5min":0.41899999999999998,"15min":0.39000000000000001},"max":{"1min":9.6120000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":0.69599999999999995},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.986,"15min":1.4790000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.38200000000000001},"max":{"1min":11.842000000000001,"5min":21.68,"15min":21.68},"last":2.052}]}]},{"osd":2,"up_from":0,"seq":949187772767,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356461,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2081460,"kb_used_data":1122156,"kb_used_omap":2,"kb_used_meta":959293,"kb_avail":91663692,"statfs":{"total":95995035648,"available":93863620608,"internally_reserved":0,"allocated":1149087744,"data_stored":1146119100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":982316085},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":2.6030000000000002,"15min":1.218},"min":{"1min":0.33000000000000002,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":15.021000000000001,"5min":108.631,"15min":108.631},"last":9.2189999999999994},{"interface":"front","average":{"1min":3.5089999999999999,"5min":2.7719999999999998,"15min":1.365},"min":{"1min":0.247,"5min":0.247,"15min":0.23499999999999999},"max":{"1min":11.728,"5min":110.012,"15min":110.012},"last":6.2610000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":3.6139999999999999,"5min":1.8300000000000001,"15min":2.29},"min":{"1min":0.32700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":10.345000000000001,"5min":15.218,"15min":125.233},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.972,"5min":1.839,"15min":2.2770000000000001},"min":{"1min":0.313,"5min":0.247,"15min":0.24299999999999999},"max":{"1min":8.0039999999999996,"5min":26.692,"15min":126.102},"last":1.766}]},{"osd":2,"last update":"Mon Oct 9 05:05:41 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.3620000000000001,"15min":2.2309999999999999},"min":{"1min":0.35499999999999998,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":6.843,"5min":11.121,"15min":212.065},"last":2.3050000000000002},{"interface":"front","average":{"1min":2.4649999999999999,"5min":1.6910000000000001,"15min":2.3159999999999998},"min":{"1min":0.32500000000000001,"5min":0.253,"15min":0.23499999999999999},"max":{"1min":6.3120000000000003,"5min":12.987,"15min":211.98400000000001},"last":1.778}]},{"osd":4,"last update":"Mon Oct 9 05:05:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":1.835,"15min":1.758},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.0099999999999998,"5min":11.193,"15min":11.193},"last":1.698},{"interface":"front","average":{"1min":2.3319999999999999,"5min":1.1839999999999999,"15min":0.99199999999999999},"min":{"1min":0.39700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":10.452,"5min":10.452,"15min":10.452},"last":0.47399999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":5.0220000000000002,"5min":1.798,"15min":1.431},"min":{"1min":0.44900000000000001,"5min":0.315,"15min":0.30599999999999999},"max":{"1min":57.804000000000002,"5min":57.804000000000002,"15min":57.804000000000002},"last":1.7989999999999999},{"interface":"front","average":{"1min":5.452,"5min":2.0990000000000002,"15min":1.6419999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":57.442,"5min":57.442,"15min":57.442},"last":1.75}]},{"osd":6,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5779999999999998,"5min":1.6220000000000001,"15min":1.1299999999999999},"min":{"1min":0.40100000000000002,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":11.252000000000001,"5min":11.252000000000001,"15min":11.252000000000001},"last":1.8440000000000001},{"interface":"front","average":{"1min":2.3559999999999999,"5min":1.526,"15min":1.3320000000000001},"min":{"1min":0.44800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.481,"5min":14.731,"15min":24.916},"last":0.49199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:05:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.6240000000000001,"15min":1.425},"min":{"1min":0.46800000000000003,"5min":0.39400000000000002,"15min":0.375},"max":{"1min":10.42,"5min":14.784000000000001,"15min":25.004000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.9159999999999999,"5min":1.3819999999999999,"15min":1.1339999999999999},"min":{"1min":0.47999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.798,"5min":7.758,"15min":7.758},"last":1.833}]}]},{"osd":4,"up_from":626,"seq":2688649527330,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2719516,"kb_used_data":2314324,"kb_used_omap":22,"kb_used_meta":405161,"kb_avail":91025636,"statfs":{"total":95995035648,"available":93210251264,"internally_reserved":0,"allocated":2369867776,"data_stored":2368163009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":414885709},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":2.3330000000000002,"15min":2.3050000000000002},"min":{"1min":0.27700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":17.135000000000002,"5min":21.292000000000002,"15min":21.292000000000002},"last":0.373},{"interface":"front","average":{"1min":2.544,"5min":3.0449999999999999,"15min":3.129},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.030999999999999,"5min":21.391999999999999,"15min":21.391999999999999},"last":1.1599999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":2.1909999999999998,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.9879999999999995,"5min":14.955,"15min":14.955},"last":2.0720000000000001},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.7829999999999999,"15min":1.861},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.585,"5min":10.579000000000001,"15min":10.579000000000001},"last":1.258}]},{"osd":2,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.667,"15min":1.6479999999999999},"min":{"1min":0.377,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.3819999999999997,"5min":11.051,"15min":11.051},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.736,"15min":1.7030000000000001},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":5.3959999999999999,"5min":11.036,"15min":11.036},"last":0.35199999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":20.216000000000001,"5min":4.9829999999999997,"15min":2.444},"min":{"1min":0.45100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":270.767,"5min":270.767,"15min":270.767},"last":0.68700000000000006},{"interface":"front","average":{"1min":19.672000000000001,"5min":5.4089999999999998,"15min":3.032},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":270.274,"5min":270.274,"15min":270.274},"last":0.47099999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.39100000000000001,"15min":0.38700000000000001},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":0.31900000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.624,"15min":0.65000000000000002},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.5409999999999999,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.46800000000000003,"15min":0.45800000000000002},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.30199999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.36199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.48799999999999999,"15min":0.47599999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.34300000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757982,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4522540,"kb_used_data":3025124,"kb_used_omap":0,"kb_used_meta":1497408,"kb_avail":89222612,"statfs":{"total":95995035648,"available":91363954688,"internally_reserved":0,"allocated":3097726976,"data_stored":3096049221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1533345792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.4119999999999999,"15min":2.137},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.0309999999999997,"5min":103.745,"15min":106.465},"last":124.504},{"interface":"front","average":{"1min":1.5880000000000001,"5min":3.71,"15min":2.3090000000000002},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.245},"max":{"1min":10.882999999999999,"5min":103.872,"15min":106.54600000000001},"last":124.678}]},{"osd":1,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":5.0410000000000004,"15min":3.556},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.23200000000000001},"max":{"1min":6.0529999999999999,"5min":262.26900000000001,"15min":262.26900000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.7709999999999999,"5min":5.0860000000000003,"15min":3.7040000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26900000000000002},"max":{"1min":10.462,"5min":262.291,"15min":262.291},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":2.6499999999999999,"15min":1.875},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":8.1340000000000003,"5min":106.238,"15min":106.238},"last":0.372},{"interface":"front","average":{"1min":2.9550000000000001,"5min":2.7109999999999999,"15min":2.1269999999999998},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":8.4390000000000001,"5min":104.31699999999999,"15min":104.31699999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":7.8579999999999997,"5min":7.0819999999999999,"15min":3.524},"min":{"1min":0.375,"5min":0.318,"15min":0.318},"max":{"1min":121.40600000000001,"5min":125.824,"15min":125.824},"last":121.40600000000001},{"interface":"front","average":{"1min":7.9930000000000003,"5min":6.9649999999999999,"15min":3.4710000000000001},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":121.53700000000001,"5min":125.761,"15min":125.761},"last":121.53700000000001}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.623,"5min":0.873,"15min":0.873},"last":0.44800000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.67500000000000004,"15min":0.79700000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":11.093,"15min":11.093},"last":0.313},{"interface":"front","average":{"1min":0.442,"5min":0.61899999999999999,"15min":0.88800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.69699999999999995,"5min":5.9329999999999998,"15min":13.083},"last":0.35399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.68999999999999995,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.948,"5min":8.3520000000000003,"15min":9.6809999999999992},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.89200000000000002,"15min":1.0589999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":0.90900000000000003,"5min":11.791,"15min":15.461},"last":0.41199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":633470976,"data_stored":633470976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":893648896,"data_stored":893648896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2566520832,"data_stored":2566520832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1136590848,"data_stored":1136590848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2355105792,"data_stored":2355101720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3088519168,"data_stored":3088515096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":721358848,"data_stored":721354776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1553465344,"data_stored":1553465344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:06:15.099 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:15.226 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:15.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:15.286 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:15.356 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:15.384 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:15.435 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:15.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:15.544 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:15.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:15 smithi029 ceph-mon[216903]: pgmap v2156: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 35488/100869 objects degraded (35.182%); 4866/100869 objects misplaced (4.824%); 7.1 MiB/s, 171 objects/s recovering 2023-10-09T05:06:15.640 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:06:15.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:15 smithi029 ceph-mon[223858]: pgmap v2156: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 35488/100869 objects degraded (35.182%); 4866/100869 objects misplaced (4.824%); 7.1 MiB/s, 171 objects/s recovering 2023-10-09T05:06:15.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:06:15.651 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:15.955 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:15.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:15 smithi177 ceph-mon[195797]: pgmap v2156: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 35488/100869 objects degraded (35.182%); 4866/100869 objects misplaced (4.824%); 7.1 MiB/s, 171 objects/s recovering 2023-10-09T05:06:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:06:16.071 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:16.103 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:16.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:16.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:16.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:16.369 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:16.539 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 35488/100869 objects degraded (35.182%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/796834472' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:16.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:16 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 35488/100869 objects degraded (35.182%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/796834472' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:16 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 35488/100869 objects degraded (35.182%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/796834472' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:16 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:17.123 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:06:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:17 smithi177 ceph-mon[195797]: pgmap v2157: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 34745/99501 objects degraded (34.919%); 4712/99501 objects misplaced (4.736%); 8.8 MiB/s, 194 objects/s recovering 2023-10-09T05:06:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:17 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:17 smithi177 ceph-mon[195797]: failed to encode map e669 with expected crc 2023-10-09T05:06:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:17 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:06:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:17 smithi177 ceph-mon[195797]: osdmap e669: 8 total, 8 up, 7 in 2023-10-09T05:06:18.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:06:18.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[216903]: pgmap v2157: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 34745/99501 objects degraded (34.919%); 4712/99501 objects misplaced (4.736%); 8.8 MiB/s, 194 objects/s recovering 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[216903]: failed to encode map e669 with expected crc 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[216903]: osdmap e669: 8 total, 8 up, 7 in 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[223858]: pgmap v2157: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 34745/99501 objects degraded (34.919%); 4712/99501 objects misplaced (4.736%); 8.8 MiB/s, 194 objects/s recovering 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:18.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[223858]: failed to encode map e669 with expected crc 2023-10-09T05:06:18.127 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:06:18.127 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:17 smithi029 ceph-mon[223858]: osdmap e669: 8 total, 8 up, 7 in 2023-10-09T05:06:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:18 smithi029 ceph-mon[216903]: failed to encode map e669 with expected crc 2023-10-09T05:06:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:18.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:18 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:18 smithi029 ceph-mon[223858]: failed to encode map e669 with expected crc 2023-10-09T05:06:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:18.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:18 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:18 smithi177 ceph-mon[195797]: failed to encode map e669 with expected crc 2023-10-09T05:06:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:18 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:19 smithi177 ceph-mon[195797]: pgmap v2159: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 34580/98088 objects degraded (35.254%); 4500/98088 objects misplaced (4.588%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T05:06:20.028 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:20.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:20.042 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:19 smithi029 ceph-mon[216903]: pgmap v2159: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 34580/98088 objects degraded (35.254%); 4500/98088 objects misplaced (4.588%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T05:06:20.042 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:19 smithi029 ceph-mon[223858]: pgmap v2159: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 291 op/s; 34580/98088 objects degraded (35.254%); 4500/98088 objects misplaced (4.588%); 8.1 MiB/s, 128 objects/s recovering 2023-10-09T05:06:20.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:20.144 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:20.174 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:20.224 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:20.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:20.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:20.426 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:20.763 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:20.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:20.841 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:20.886 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80535 14.3 14:cc98a142:::benchmark_data_smithi029_550670_object2924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 494.62797995400001, 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 40.148030116999998, 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80535 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249930+0000", 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249927+0000", 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249939+0000", 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1640999999999999e-05 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249963+0000", 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3714999999999999e-05 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.249976+0000", 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3699e-05 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.632019+0000", 2023-10-09T05:06:20.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.38204312600000001 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.653132+0000", 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021113060999999999 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.779084+0000", 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12595120000000001 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.891 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.797955+0000", 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018871354 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103793+0000", 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30583778099999998 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103806+0000", 2023-10-09T05:06:20.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3494e-05 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844372+0000", 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.74056582000000004 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844384+0000", 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2394e-05 2023-10-09T05:06:20.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.144509+0000", 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 38.300124437999997 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.170256+0000", 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025747190999999999 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.299425+0000", 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12916846400000001 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.304976+0000", 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055519480000000001 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.314789+0000", 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0098120340000000007 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.351252+0000", 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036463151999999999 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.353456+0000", 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022039910000000002 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.356260+0000", 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028044680000000001 2023-10-09T05:06:20.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.358811+0000", 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025511090000000002 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.365432+0000", 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066203989999999999 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.397960+0000", 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032528230999999998 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:80543 14.3 14:cf2f7cf9:::benchmark_data_smithi029_550670_object2932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 494.34863999200002, 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 39.820873380999998, 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.898 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 80543 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.899 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529270+0000", 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529268+0000", 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529279+0000", 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1258e-05 2023-10-09T05:06:20.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529301+0000", 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1912e-05 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.529314+0000", 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3030000000000001e-05 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.655681+0000", 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.12636664 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.679149+0000", 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.023468648000000002 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.887190+0000", 2023-10-09T05:06:20.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20804069 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:06.911521+0000", 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0243314 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103710+0000", 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19218885399999999 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.103727+0000", 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6439999999999998e-05 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844324+0000", 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.740597536 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:07.844341+0000", 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7054999999999998e-05 2023-10-09T05:06:20.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.011168+0000", 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 38.166826931000003 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for readable", 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.028691+0000", 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017522685 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.108149+0000", 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.079458184000000001 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.127919+0000", 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019770056000000001 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.216507+0000", 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.088587677000000004 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.223312+0000", 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068052829999999996 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,5,6", 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.233101+0000", 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097893839999999999 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.265633+0000", 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032531428000000001 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.323546+0000", 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.057912784000000002 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.325561+0000", 2023-10-09T05:06:20.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020157949999999999 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.327379+0000", 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001817639 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.330515+0000", 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031356869999999998 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T04:58:46.350143+0000", 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019628362 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93420 14.2 14:43ae63e0:::benchmark_data_smithi029_550670_object15809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:24.754998+0000", 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 176.12291208100001, 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9690727130000001, 2023-10-09T05:06:20.910 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93420 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.754998+0000", 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.754998+0000", 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.754996+0000", 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755004+0000", 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5299999999999999e-06 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755023+0000", 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9181000000000002e-05 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755031+0000", 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0649999999999999e-06 2023-10-09T05:06:20.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755088+0000", 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7707999999999997e-05 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755100+0000", 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1593000000000001e-05 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.335001+0000", 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.579900517 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.335020+0000", 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9627e-05 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.645432+0000", 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3104114999999998 2023-10-09T05:06:20.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.650356+0000", 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049244140000000002 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.662698+0000", 2023-10-09T05:06:20.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012341431 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.664260+0000", 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001562934 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.667511+0000", 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032506119999999999 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.687962+0000", 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020450585 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.688145+0000", 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018303899999999999 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.719106+0000", 2023-10-09T05:06:20.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030961059999999999 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.719995+0000", 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00088945599999999995 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.724070+0000", 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040750789999999997 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.920 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93755 14.14 e641/638 14:2dfdf15f:::benchmark_data_smithi029_550670_object16144:head v 641'5490, mlcod=594'5440)", 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.249417+0000", 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.62849315, 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.634247083, 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249417+0000", 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249417+0000", 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249415+0000", 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249427+0000", 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1328000000000001e-05 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249442+0000", 2023-10-09T05:06:20.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5837000000000001e-05 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249452+0000", 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2529999999999993e-06 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.215500+0000", 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9660483769999999 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.215533+0000", 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2830000000000002e-05 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.883635+0000", 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.66810195699999997 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.883664+0000", 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8776e-05 2023-10-09T05:06:20.925 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93770 14.4 14:211bc49a:::benchmark_data_smithi029_550670_object16159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.428219+0000", 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.44969065000001, 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7114670619999997, 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.926 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93770 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428219+0000", 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428219+0000", 2023-10-09T05:06:20.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428217+0000", 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428228+0000", 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0953e-05 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.928 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428256+0000", 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8501e-05 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428270+0000", 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4003e-05 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.900063+0000", 2023-10-09T05:06:20.929 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4717922569999997 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.903238+0000", 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031751729999999999 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.946608+0000", 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043369759000000001 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.930 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.949983+0000", 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033750970000000001 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.099731+0000", 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.149748467 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.102098+0000", 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023662840000000002 2023-10-09T05:06:20.931 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.107041+0000", 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049436259999999996 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.121280+0000", 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014238971 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.932 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.130592+0000", 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0093116850000000001 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.131401+0000", 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000808722 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.132729+0000", 2023-10-09T05:06:20.933 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013287170000000001 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.139686+0000", 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069568520000000003 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.934 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93774 14.4 14:279fca4c:::benchmark_data_smithi029_550670_object16163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.461282+0000", 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.41662720900001, 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7526535790000004, 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93774 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.935 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461282+0000", 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461282+0000", 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.936 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461280+0000", 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461290+0000", 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8290000000000003e-06 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.937 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461305+0000", 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4718e-05 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461320+0000", 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4907e-05 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.950440+0000", 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4891200979999999 2023-10-09T05:06:20.938 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.954595+0000", 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041548949999999996 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.016951+0000", 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062356379000000003 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.939 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.021414+0000", 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044629769999999999 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.158317+0000", 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13690313800000001 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.940 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.160690+0000", 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023728540000000002 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.165538+0000", 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004848102 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.195961+0000", 2023-10-09T05:06:20.941 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030422825000000001 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.196603+0000", 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064218499999999998 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.198402+0000", 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017984419999999999 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.942 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.201683+0000", 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032815539999999999 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.213936+0000", 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012252798000000001 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.943 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93775 14.f 14:f35a5edc:::benchmark_data_smithi029_550670_object16164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.416578453, 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1113186439999998, 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93775 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.944 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.945 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461334+0000", 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0879999999999999e-06 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461340+0000", 2023-10-09T05:06:20.946 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5409999999999997e-06 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461346+0000", 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2780000000000001e-06 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461427+0000", 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1403000000000006e-05 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.947 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461439+0000", 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2146e-05 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.855663+0000", 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.394224124 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.859257+0000", 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003593564 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.948 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.863603+0000", 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043460169999999998 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.868058+0000", 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044545899999999996 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.949 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.868427+0000", 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036936199999999998 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.564654+0000", 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6962266349999999 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.950 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.566002+0000", 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013480359999999999 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.572650+0000", 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066483480000000001 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93781 14.4 14:2296b9e3:::benchmark_data_smithi029_550670_object16170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:06:20.951 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.589249+0000", 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 149.28866074300001, 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6379777039999999, 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93781 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.952 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589249+0000", 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589249+0000", 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589247+0000", 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:20.953 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589256+0000", 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7790000000000001e-06 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589276+0000", 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9729999999999999e-05 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.954 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589287+0000", 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0974e-05 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.955385+0000", 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3660981110000003 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.955 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.959367+0000", 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039820539999999996 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.012478+0000", 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053111053999999998 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.956 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.016463+0000", 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039855580000000002 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.180920+0000", 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16445663399999999 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.183328+0000", 2023-10-09T05:06:20.957 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024079969999999998 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.188382+0000", 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050540079999999996 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.215773+0000", 2023-10-09T05:06:20.958 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027390687 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.216155+0000", 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000382643 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.217042+0000", 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00088696100000000002 2023-10-09T05:06:20.959 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.218823+0000", 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00178072 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.227227+0000", 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084035540000000006 2023-10-09T05:06:20.960 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:107745 14.f 14:f5cecc17:::benchmark_data_smithi029_550670_object30134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e661)", 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:14.312003+0000", 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 66.565906376000001, 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.52072821199999997, 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.961 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 107745 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.312003+0000", 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.962 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.312003+0000", 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.312002+0000", 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:20.963 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.312021+0000", 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9704e-05 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.312036+0000", 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.506e-05 2023-10-09T05:06:20.964 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.312046+0000", 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.876e-06 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.340854+0000", 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028808062999999998 2023-10-09T05:06:20.965 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.344534+0000", 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036794649999999998 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.374835+0000", 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030301623999999999 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.966 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.377782+0000", 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029467550000000001 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.536978+0000", 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.159196323 2023-10-09T05:06:20.967 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.539324+0000", 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023458440000000001 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.543671+0000", 2023-10-09T05:06:20.968 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043465999999999999 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.569322+0000", 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025651274000000002 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.969 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.571390+0000", 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002067401 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.824246+0000", 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.252856421 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.970 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.825867+0000", 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016204850000000001 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:14.832732+0000", 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068650120000000002 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.971 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108083 14.f 14:f619db8a:::benchmark_data_smithi029_550670_object30472:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:15.976930+0000", 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.900979573000001, 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.55834777700000005, 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.972 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108083 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.976930+0000", 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.976930+0000", 2023-10-09T05:06:20.973 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.976929+0000", 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.976935+0000", 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4640000000000003e-06 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.974 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.976946+0000", 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0407e-05 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.976954+0000", 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6289999999999999e-06 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.975 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.977716+0000", 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00076111500000000003 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:15.982516+0000", 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048009530000000002 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.015177+0000", 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032660389999999997 2023-10-09T05:06:20.976 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.026977+0000", 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011800312 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.255947+0000", 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.228969691 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.977 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.258694+0000", 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027466970000000002 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.264591+0000", 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058971759999999996 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.285962+0000", 2023-10-09T05:06:20.978 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021371049 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.513635+0000", 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.227673345 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.528870+0000", 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015234783999999999 2023-10-09T05:06:20.979 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.530228+0000", 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013583759999999999 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.535278+0000", 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050496170000000002 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.980 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108091 14.f 14:f6dcd6db:::benchmark_data_smithi029_550670_object30480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.000063+0000", 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.877846672999993, 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.54592259499999995, 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108091 2023-10-09T05:06:20.981 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.000063+0000", 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.000063+0000", 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.982 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.000061+0000", 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.000071+0000", 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1339999999999993e-06 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.000085+0000", 2023-10-09T05:06:20.983 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4453e-05 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.000094+0000", 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9269999999999994e-06 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.028940+0000", 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.028845826000000001 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.984 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.118283+0000", 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.089343682999999993 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.155117+0000", 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.036833636000000003 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.985 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.158728+0000", 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036109240000000002 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.286386+0000", 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.127658305 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.289107+0000", 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027204690000000001 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.986 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.296708+0000", 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076008209999999998 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.340133+0000", 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043425315999999999 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.987 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.536485+0000", 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.196352586 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.538205+0000", 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001719171 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.988 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.539603+0000", 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013979509999999999 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.545986+0000", 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0063830309999999999 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.989 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108097 14.f 14:f47740ba:::benchmark_data_smithi029_550670_object30486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.008530+0000", 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.869379960000003, 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.54681480100000002, 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108097 2023-10-09T05:06:20.990 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.008530+0000", 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.008530+0000", 2023-10-09T05:06:20.991 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.008529+0000", 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.008533+0000", 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4300000000000002e-06 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.992 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.008539+0000", 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6699999999999997e-06 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.008545+0000", 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2989999999999996e-06 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.120243+0000", 2023-10-09T05:06:20.993 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.111698084 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.123384+0000", 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031416679999999998 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.176738+0000", 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053353340999999999 2023-10-09T05:06:20.994 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.179723+0000", 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029855799999999998 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.314685+0000", 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13496208100000001 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.995 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.317302+0000", 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026161140000000001 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.322549+0000", 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052470329999999999 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:20.996 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.340467+0000", 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017918923 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.546738+0000", 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.206270541 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.547460+0000", 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00072222599999999997 2023-10-09T05:06:20.997 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.548837+0000", 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013765100000000001 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.555345+0000", 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065077729999999997 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.998 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108133 14.f 14:f3f59ff2:::benchmark_data_smithi029_550670_object30522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.218787+0000", 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.659122941999996, 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.60949035699999998, 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:20.999 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108133 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218787+0000", 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218787+0000", 2023-10-09T05:06:21.000 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218785+0000", 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218794+0000", 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6090000000000001e-06 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.001 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218807+0000", 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3541e-05 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218817+0000", 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4590000000000004e-06 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.219556+0000", 2023-10-09T05:06:21.002 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00073897100000000005 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.223282+0000", 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037261769999999998 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.237906+0000", 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014624513 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.241433+0000", 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035265940000000001 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.708381+0000", 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.46694833699999999 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.711125+0000", 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002743639 2023-10-09T05:06:21.004 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.715990+0000", 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048649770000000004 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.760270+0000", 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044279639000000003 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.005 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.819221+0000", 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.058951642999999998 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.820560+0000", 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013389369999999999 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.822350+0000", 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017902459999999999 2023-10-09T05:06:21.006 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.007 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.007 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.007 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.828277+0000", 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059267640000000002 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108140 14.f 14:f15fa152:::benchmark_data_smithi029_550670_object30529:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.238100+0000", 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.639809729000007, 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.60911499400000002, 2023-10-09T05:06:21.008 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108140 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.238100+0000", 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.009 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.238100+0000", 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.238098+0000", 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.010 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.238108+0000", 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7699999999999996e-06 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.238122+0000", 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4182000000000001e-05 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.011 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.238132+0000", 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0081000000000001e-05 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.270161+0000", 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.032028634 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.272784+0000", 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026228800000000002 2023-10-09T05:06:21.012 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.524377+0000", 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.251593864 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.527513+0000", 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003135844 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.013 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.736405+0000", 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20889195599999999 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.738896+0000", 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024906580000000002 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.743527+0000", 2023-10-09T05:06:21.014 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046310930000000002 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.772547+0000", 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029020469 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.829454+0000", 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.056906301999999999 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.015 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.830344+0000", 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00088986399999999998 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.831757+0000", 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014131230000000001 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.016 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.847215+0000", 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015458279 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108151 14.f 14:f3630f30:::benchmark_data_smithi029_550670_object30540:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.300585+0000", 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.577324873999999, 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.56320519899999999, 2023-10-09T05:06:21.017 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108151 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.300585+0000", 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.018 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.300585+0000", 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.300583+0000", 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.019 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.300593+0000", 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0264e-05 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.300620+0000", 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6376999999999998e-05 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.300630+0000", 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0353000000000001e-05 2023-10-09T05:06:21.020 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.567402+0000", 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26677246500000001 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.570894+0000", 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00349154 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.021 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.588181+0000", 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017287281000000002 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.591849+0000", 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036674189999999999 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.760583+0000", 2023-10-09T05:06:21.022 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16873463399999999 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.763229+0000", 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026460619999999998 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.768671+0000", 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054418260000000003 2023-10-09T05:06:21.023 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.784494+0000", 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015823095999999998 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.848278+0000", 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.063784186000000007 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.024 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.849234+0000", 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00095510199999999999 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.850762+0000", 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001528937 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.863790+0000", 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013027555999999999 2023-10-09T05:06:21.025 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108156 14.f 14:f4fd88a0:::benchmark_data_smithi029_550670_object30545:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.307420+0000", 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 64.570490061000001, 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.626578886, 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.026 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108156 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307420+0000", 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.027 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307420+0000", 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307418+0000", 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307425+0000", 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1570000000000003e-06 2023-10-09T05:06:21.028 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307438+0000", 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3005e-05 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307447+0000", 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9069999999999996e-06 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.029 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.571211+0000", 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26376351199999998 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.573982+0000", 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027715790000000001 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.614003+0000", 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040020695000000002 2023-10-09T05:06:21.030 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.617403+0000", 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033996040000000001 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.873892+0000", 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25648922499999999 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.031 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.877668+0000", 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037761040000000002 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.883641+0000", 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059727790000000001 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.895657+0000", 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012015836 2023-10-09T05:06:21.032 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.895925+0000", 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026844899999999998 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.921679+0000", 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025754174000000001 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.033 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.926718+0000", 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050388509999999996 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.933999+0000", 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072804990000000002 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.034 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108301 14.4 14:226cd067:::benchmark_data_smithi029_550670_object30690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:17.211157+0000", 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 63.666752482, 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.50996538899999999, 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108301 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.035 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.211157+0000", 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.211157+0000", 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.036 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.211156+0000", 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.211161+0000", 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.181e-06 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.211171+0000", 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0117e-05 2023-10-09T05:06:21.037 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.211177+0000", 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0680000000000002e-06 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.467489+0000", 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.256312076 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.038 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.470707+0000", 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00321777 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.509638+0000", 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038931118000000001 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.512959+0000", 2023-10-09T05:06:21.039 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033213069999999999 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.683814+0000", 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.17085429799999999 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.686460+0000", 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002645863 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.040 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.692590+0000", 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061302070000000004 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.711705+0000", 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019114776 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.712636+0000", 2023-10-09T05:06:21.041 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00093169499999999996 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.712924+0000", 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028786699999999997 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.714417+0000", 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001493208 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.042 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.721123+0000", 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067052090000000002 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108322 14.4 14:27a4d917:::benchmark_data_smithi029_550670_object30711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:17.231579+0000", 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 63.646330544000001, 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47913506500000003, 2023-10-09T05:06:21.043 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108322 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.231579+0000", 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.044 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.231579+0000", 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.231578+0000", 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.045 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.231583+0000", 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9699999999999998e-06 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.231591+0000", 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4589999999999998e-06 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.046 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.231597+0000", 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9839999999999998e-06 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.471119+0000", 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.23952214999999999 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.474399+0000", 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00327933 2023-10-09T05:06:21.047 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.492763+0000", 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018364347 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.495847+0000", 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030841789999999998 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.048 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.643334+0000", 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.14748718699999999 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.646191+0000", 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028564329999999998 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:21.049 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.651966+0000", 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057753500000000003 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.697774+0000", 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.045807358999999999 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.698664+0000", 2023-10-09T05:06:21.050 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00089022700000000005 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.699696+0000", 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00103236 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.701200+0000", 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015033640000000001 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.051 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:17.710714+0000", 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095146799999999993 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:112801 14.4 14:240393e5:::benchmark_data_smithi029_550670_object35190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e664)", 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:36.826878+0000", 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 44.051031383000002, 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.471652238, 2023-10-09T05:06:21.052 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112801 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826878+0000", 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.053 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826878+0000", 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826877+0000", 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.054 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826883+0000", 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1099999999999999e-06 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826897+0000", 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3771000000000001e-05 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826906+0000", 2023-10-09T05:06:21.055 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8549999999999997e-06 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.833174+0000", 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0062675040000000001 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.835417+0000", 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022432659999999998 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.056 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.886121+0000", 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.050703541999999997 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.889098+0000", 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029771770000000001 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.235839+0000", 2023-10-09T05:06:21.057 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34674081600000001 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.238288+0000", 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024496600000000002 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.243336+0000", 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050473619999999997 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.058 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.287456+0000", 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044120158 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.288287+0000", 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00083085500000000003 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.059 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.290778+0000", 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002491376 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.292002+0000", 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012234730000000001 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.298531+0000", 2023-10-09T05:06:21.060 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065289420000000003 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:112802 14.4 14:2376a096:::benchmark_data_smithi029_550670_object35191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e664)", 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:36.826915+0000", 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 44.050994435, 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47892136400000002, 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:21.061 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 112802 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826915+0000", 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.062 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826915+0000", 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826915+0000", 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826917+0000", 2023-10-09T05:06:21.063 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5840000000000002e-06 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826923+0000", 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5249999999999996e-06 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.826928+0000", 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8130000000000003e-06 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.064 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.900348+0000", 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.073419939000000004 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.903633+0000", 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003284974 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.065 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.921861+0000", 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018228316000000001 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:36.924862+0000", 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030013840000000002 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.267501+0000", 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34263896100000002 2023-10-09T05:06:21.066 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.269368+0000", 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001866403 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.273559+0000", 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041910059999999997 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.067 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.289044+0000", 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015485314 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.289716+0000", 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00067207400000000004 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.298841+0000", 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091243309999999994 2023-10-09T05:06:21.068 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.300074+0000", 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001233434 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.305837+0000", 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0057626769999999999 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.069 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.070 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:06:21.070 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:06:21.070 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:06:21.070 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:21.071 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:20 smithi177 ceph-mon[195797]: pgmap v2160: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 34142/95910 objects degraded (35.598%); 4500/95910 objects misplaced (4.692%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:06:21.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:20 smithi029 ceph-mon[216903]: pgmap v2160: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 34142/95910 objects degraded (35.598%); 4500/95910 objects misplaced (4.692%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:06:21.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:20 smithi029 ceph-mon[223858]: pgmap v2160: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 2.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 34142/95910 objects degraded (35.598%); 4500/95910 objects misplaced (4.692%); 6.6 MiB/s, 105 objects/s recovering 2023-10-09T05:06:21.215 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:21.417 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:06:21.459 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:06:21.527 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:06:21.580 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:06:21.626 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:06:21.647 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:21.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:21.690 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:06:21.733 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:06:21.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 34142/95910 objects degraded (35.598%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:21.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:21 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/752752663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:21.736 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 34142/95910 objects degraded (35.598%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:21.736 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:21 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/752752663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:21.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:21.778 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:06:21.795 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:21.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.811 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.812 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.812 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:06:21.843 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:21.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:21.948 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 34142/95910 objects degraded (35.598%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:21 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/752752663' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:22.063 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:22.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:06:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:22 smithi177 ceph-mon[195797]: pgmap v2161: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 33533/95124 objects degraded (35.252%); 4320/95124 objects misplaced (4.541%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:23.074 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:22 smithi029 ceph-mon[216903]: pgmap v2161: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 33533/95124 objects degraded (35.252%); 4320/95124 objects misplaced (4.541%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:23.074 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:22 smithi029 ceph-mon[223858]: pgmap v2161: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 33533/95124 objects degraded (35.252%); 4320/95124 objects misplaced (4.541%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:23.077 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:06:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:06:22] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.43.0" 2023-10-09T05:06:23.102 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:23.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:23.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:23.210 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:23.237 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:23.284 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:23.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:23.391 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:23.418 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:23.466 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:23.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:23.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:23.658 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:23.998 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:06:24.053 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:24.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:24.171 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:24.201 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:24.257 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:24.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:24.365 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:24.396 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:24.406 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:25.192 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:25.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:25.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:25.206 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:24 smithi029 ceph-mon[223858]: pgmap v2162: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 33059/92136 objects degraded (35.881%); 4130/92136 objects misplaced (4.483%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:25.207 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:24 smithi029 ceph-mon[216903]: pgmap v2162: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 33059/92136 objects degraded (35.881%); 4130/92136 objects misplaced (4.483%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:25.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:24 smithi177 ceph-mon[195797]: pgmap v2162: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 33059/92136 objects degraded (35.881%); 4130/92136 objects misplaced (4.483%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:25.307 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:25.340 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:25.387 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:25.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:25.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:25.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:25.587 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:25.633 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:25.708 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:25.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:25.820 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:25.851 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:25.903 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:25.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:25.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:26.006 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:26.034 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:26.068 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:06:26.084 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:26.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:26.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:26.215 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:26.260 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:26.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:26.363 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:26.392 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:26.438 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:26.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:26.541 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:26.661 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:26.800 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:26.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:26.868 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:06:26.912 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:26.938 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:26.979 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:26.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:26.998 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:06:27.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:27.168 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:27.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:26 smithi029 ceph-mon[216903]: pgmap v2163: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 287 op/s; 32335/90858 objects degraded (35.589%); 3953/90858 objects misplaced (4.351%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:06:27.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 32335/90858 objects degraded (35.589%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:27.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:26 smithi029 ceph-mon[223858]: pgmap v2163: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 287 op/s; 32335/90858 objects degraded (35.589%); 3953/90858 objects misplaced (4.351%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:06:27.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 32335/90858 objects degraded (35.589%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:26 smithi177 ceph-mon[195797]: pgmap v2163: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 287 op/s; 32335/90858 objects degraded (35.589%); 3953/90858 objects misplaced (4.351%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:06:27.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 32335/90858 objects degraded (35.589%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:28.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:27 smithi177 ceph-mon[195797]: from='client.47588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:28.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:27 smithi029 ceph-mon[216903]: from='client.47588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:28.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:27 smithi029 ceph-mon[223858]: from='client.47588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:28.469 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:28.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:28.583 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:28.614 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:28.663 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:28.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:28.767 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:28.798 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:28.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:28.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:28.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:28.926 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:06:28.926 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:06:28.960 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:28.989 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:29.193 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:29.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:28 smithi177 ceph-mon[195797]: pgmap v2164: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 32207/89577 objects degraded (35.955%); 3731/89577 objects misplaced (4.165%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:06:29.195 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:29.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:28 smithi029 ceph-mon[216903]: pgmap v2164: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 32207/89577 objects degraded (35.955%); 3731/89577 objects misplaced (4.165%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:06:29.268 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:29.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:28 smithi029 ceph-mon[223858]: pgmap v2164: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 306 op/s; 32207/89577 objects degraded (35.955%); 3731/89577 objects misplaced (4.165%); 8.5 MiB/s, 135 objects/s recovering 2023-10-09T05:06:29.269 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:29.296 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:29.321 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:29.364 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2163,"stamp":"2023-10-09T05:06:25.861701+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1983780792,"num_objects":30286,"num_object_clones":0,"num_object_copies":90858,"num_objects_missing_on_primary":10354,"num_objects_missing":10354,"num_objects_degraded":32335,"num_objects_misplaced":3953,"num_objects_unfound":0,"num_objects_dirty":30286,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":125256,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80825,"num_bytes_recovered":4888300174,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207895,"ondisk_log_size":207895,"up":357,"acting":341,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18097548,"kb_used_data":9709396,"kb_used_omap":34,"kb_used_meta":8387869,"kb_avail":638118516,"statfs":{"total":671965249536,"available":653433360384,"internally_reserved":0,"allocated":9942421504,"data_stored":9926183543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8589178250},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,0,0,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-188809216,"num_objects":-2881,"num_object_clones":0,"num_object_copies":-8643,"num_objects_missing_on_primary":-983,"num_objects_missing":-983,"num_objects_degraded":-2410,"num_objects_misplaced":-759,"num_objects_unfound":0,"num_objects_dirty":-2881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2881,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1322,"num_bytes_recovered":86638592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.014532"},"pg_stats":[{"pgid":"14.23","version":"669'5954","reported_seq":12581,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:23.169174+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:06:23.169174+0000","last_peered":"2023-10-09T05:06:23.169174+0000","last_clean":"2023-10-09T05:06:23.169174+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:06:23.169174+0000","last_undegraded":"2023-10-09T05:06:23.169174+0000","last_fullsized":"2023-10-09T05:06:23.169174+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5954,"log_dups_size":0,"ondisk_log_size":5954,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30277632,"num_objects":462,"num_object_clones":0,"num_object_copies":1386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1970,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"669'5878","reported_seq":13081,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:23.604843+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:06:23.604843+0000","last_peered":"2023-10-09T05:06:23.604843+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:06:23.604843+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5878,"log_dups_size":0,"ondisk_log_size":5878,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30441472,"num_objects":465,"num_object_clones":0,"num_object_copies":1395,"num_objects_missing_on_primary":438,"num_objects_missing":438,"num_objects_degraded":1159,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1943,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1018,"num_bytes_recovered":64471040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"3,4,5,7","objects":1},{"shards":"4,5,7","objects":280}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"669'5948","reported_seq":16648,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:23.458027+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:06:23.458027+0000","last_peered":"2023-10-09T05:06:23.458027+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:06:23.458027+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:06:23.458027+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30474240,"num_objects":465,"num_object_clones":0,"num_object_copies":1395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1965,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":465}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"669'5931","reported_seq":7851,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:21.630019+0000","last_change":"2023-10-09T05:05:26.963137+0000","last_active":"2023-10-09T05:06:21.630019+0000","last_peered":"2023-10-09T05:06:21.630019+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:06:21.630019+0000","last_undegraded":"2023-10-09T05:05:26.962565+0000","last_fullsized":"2023-10-09T05:06:21.630019+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32325632,"num_objects":493,"num_object_clones":0,"num_object_copies":1479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1406,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1938,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":496,"num_bytes_recovered":32522240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":444},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"669'6426","reported_seq":19989,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:22.087583+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:06:22.087583+0000","last_peered":"2023-10-09T05:06:22.087583+0000","last_clean":"2023-10-09T05:06:22.087583+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:06:22.087583+0000","last_undegraded":"2023-10-09T05:06:22.087583+0000","last_fullsized":"2023-10-09T05:06:22.087583+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6426,"log_dups_size":0,"ondisk_log_size":6426,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65699840,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3885,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":171,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150652+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:06:08.150652+0000","last_peered":"2023-10-09T05:06:08.150652+0000","last_clean":"2023-10-09T05:06:08.150652+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:06:08.150652+0000","last_undegraded":"2023-10-09T05:06:08.150652+0000","last_fullsized":"2023-10-09T05:06:08.150652+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":316,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151123+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:06:08.151123+0000","last_peered":"2023-10-09T05:06:08.151123+0000","last_clean":"2023-10-09T05:06:08.151123+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:06:08.151123+0000","last_undegraded":"2023-10-09T05:06:08.151123+0000","last_fullsized":"2023-10-09T05:06:08.151123+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"669'6426","reported_seq":9655,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:23.531492+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:06:23.531492+0000","last_peered":"2023-10-09T05:06:23.531492+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:06:23.531492+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:06:23.531492+0000","mapping_epoch":638,"log_start":"444'406","ondisk_log_start":"444'406","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63799296,"num_objects":974,"num_object_clones":0,"num_object_copies":2922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3887,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":974}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":93,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148570+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:06:08.148570+0000","last_peered":"2023-10-09T05:06:08.148570+0000","last_clean":"2023-10-09T05:06:08.148570+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:06:08.148570+0000","last_undegraded":"2023-10-09T05:06:08.148570+0000","last_fullsized":"2023-10-09T05:06:08.148570+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151571+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:06:08.151571+0000","last_peered":"2023-10-09T05:06:08.151571+0000","last_clean":"2023-10-09T05:06:08.151571+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:06:08.151571+0000","last_undegraded":"2023-10-09T05:06:08.151571+0000","last_fullsized":"2023-10-09T05:06:08.151571+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"669'6447","reported_seq":18107,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:24.003044+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:06:24.003044+0000","last_peered":"2023-10-09T05:06:24.003044+0000","last_clean":"2023-10-09T05:06:24.003044+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:06:24.003044+0000","last_undegraded":"2023-10-09T05:06:24.003044+0000","last_fullsized":"2023-10-09T05:06:24.003044+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6201,"log_dups_size":0,"ondisk_log_size":6201,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62521344,"num_objects":954,"num_object_clones":0,"num_object_copies":2862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3907,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":298,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152708+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:06:08.152708+0000","last_peered":"2023-10-09T05:06:08.152708+0000","last_clean":"2023-10-09T05:06:08.152708+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:06:08.152708+0000","last_undegraded":"2023-10-09T05:06:08.152708+0000","last_fullsized":"2023-10-09T05:06:08.152708+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155435+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:06:08.155435+0000","last_peered":"2023-10-09T05:06:08.155435+0000","last_clean":"2023-10-09T05:06:08.155435+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:06:08.155435+0000","last_undegraded":"2023-10-09T05:06:08.155435+0000","last_fullsized":"2023-10-09T05:06:08.155435+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"669'6640","reported_seq":23997,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:22.026390+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:06:22.026390+0000","last_peered":"2023-10-09T05:06:22.026390+0000","last_clean":"2023-10-09T05:06:22.026390+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:06:22.026390+0000","last_undegraded":"2023-10-09T05:06:22.026390+0000","last_fullsized":"2023-10-09T05:06:22.026390+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6240,"log_dups_size":0,"ondisk_log_size":6240,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62390272,"num_objects":952,"num_object_clones":0,"num_object_copies":2856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3819,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":43,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155933+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:06:08.155933+0000","last_peered":"2023-10-09T05:06:08.155933+0000","last_clean":"2023-10-09T05:06:08.155933+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:06:08.155933+0000","last_undegraded":"2023-10-09T05:06:08.155933+0000","last_fullsized":"2023-10-09T05:06:08.155933+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":290,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152640+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:06:08.152640+0000","last_peered":"2023-10-09T05:06:08.152640+0000","last_clean":"2023-10-09T05:06:08.152640+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:06:08.152640+0000","last_undegraded":"2023-10-09T05:06:08.152640+0000","last_fullsized":"2023-10-09T05:06:08.152640+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"669'6978","reported_seq":14893,"reported_epoch":669,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:06:24.885653+0000","last_change":"2023-10-09T05:05:26.975807+0000","last_active":"2023-10-09T05:06:24.885653+0000","last_peered":"2023-10-09T05:06:24.885653+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:06:24.885653+0000","last_undegraded":"2023-10-09T05:06:24.885653+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63045632,"num_objects":962,"num_object_clones":0,"num_object_copies":2886,"num_objects_missing_on_primary":1309,"num_objects_missing":1309,"num_objects_degraded":0,"num_objects_misplaced":2719,"num_objects_unfound":0,"num_objects_dirty":962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4018,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1845,"num_bytes_recovered":120848384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":37},{"shards":"0,2,4,7","objects":1},{"shards":"2,4,7","objects":869},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":57,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154103+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:06:08.154103+0000","last_peered":"2023-10-09T05:06:08.154103+0000","last_clean":"2023-10-09T05:06:08.154103+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:06:08.154103+0000","last_undegraded":"2023-10-09T05:06:08.154103+0000","last_fullsized":"2023-10-09T05:06:08.154103+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":280,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155470+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:06:08.155470+0000","last_peered":"2023-10-09T05:06:08.155470+0000","last_clean":"2023-10-09T05:06:08.155470+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:06:08.155470+0000","last_undegraded":"2023-10-09T05:06:08.155470+0000","last_fullsized":"2023-10-09T05:06:08.155470+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"669'6613","reported_seq":14459,"reported_epoch":669,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:06:22.794122+0000","last_change":"2023-10-09T05:05:00.137617+0000","last_active":"2023-10-09T05:06:22.794122+0000","last_peered":"2023-10-09T05:06:22.794122+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:06:22.794122+0000","last_undegraded":"2023-10-09T05:06:22.794122+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880640,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":558,"num_objects_missing":558,"num_objects_degraded":0,"num_objects_misplaced":1234,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4004,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1916,"num_bytes_recovered":125566976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,5,7","objects":1},{"shards":"1,3","objects":433},{"shards":"2,3,5","objects":424},{"shards":"2,3,5,7","objects":2},{"shards":"2,5,7","objects":125}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":342,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.151141+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:06:08.151141+0000","last_peered":"2023-10-09T05:06:08.151141+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:06:08.151141+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:06:08.151141+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150738+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:06:08.150738+0000","last_peered":"2023-10-09T05:06:08.150738+0000","last_clean":"2023-10-09T05:06:08.150738+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:06:08.150738+0000","last_undegraded":"2023-10-09T05:06:08.150738+0000","last_fullsized":"2023-10-09T05:06:08.150738+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"669'6465","reported_seq":18540,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:22.041324+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:06:22.041324+0000","last_peered":"2023-10-09T05:06:22.041324+0000","last_clean":"2023-10-09T05:06:22.041324+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:06:22.041324+0000","last_undegraded":"2023-10-09T05:06:22.041324+0000","last_fullsized":"2023-10-09T05:06:22.041324+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63078400,"num_objects":963,"num_object_clones":0,"num_object_copies":2889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3927,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":124,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152121+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:06:08.152121+0000","last_peered":"2023-10-09T05:06:08.152121+0000","last_clean":"2023-10-09T05:06:08.152121+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:06:08.152121+0000","last_undegraded":"2023-10-09T05:06:08.152121+0000","last_fullsized":"2023-10-09T05:06:08.152121+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":139,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151148+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:06:08.151148+0000","last_peered":"2023-10-09T05:06:08.151148+0000","last_clean":"2023-10-09T05:06:08.151148+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:06:08.151148+0000","last_undegraded":"2023-10-09T05:06:08.151148+0000","last_fullsized":"2023-10-09T05:06:08.151148+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"669'6701","reported_seq":19428,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:23.730307+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:06:23.730307+0000","last_peered":"2023-10-09T05:06:23.730307+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:06:23.730307+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5020,"log_dups_size":0,"ondisk_log_size":5020,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62291968,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1475,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3842,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3380,"num_bytes_recovered":216924160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":447}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":145,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151229+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:06:08.151229+0000","last_peered":"2023-10-09T05:06:08.151229+0000","last_clean":"2023-10-09T05:06:08.151229+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:06:08.151229+0000","last_undegraded":"2023-10-09T05:06:08.151229+0000","last_fullsized":"2023-10-09T05:06:08.151229+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":252,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.164657+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:06:08.164657+0000","last_peered":"2023-10-09T05:06:08.164657+0000","last_clean":"2023-10-09T05:06:08.164657+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:06:08.164657+0000","last_undegraded":"2023-10-09T05:06:08.164657+0000","last_fullsized":"2023-10-09T05:06:08.164657+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"669'6732","reported_seq":13369,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:22.632466+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:06:22.632466+0000","last_peered":"2023-10-09T05:06:22.632466+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:06:22.632466+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4605,"log_dups_size":0,"ondisk_log_size":4605,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57409536,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":981,"num_objects_missing":981,"num_objects_degraded":1937,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3903,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1834,"num_bytes_recovered":120160256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,7","objects":-39},{"shards":"2,5,7","objects":540},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.158571+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:06:08.158571+0000","last_peered":"2023-10-09T05:06:08.158571+0000","last_clean":"2023-10-09T05:06:08.158571+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:06:08.158571+0000","last_undegraded":"2023-10-09T05:06:08.158571+0000","last_fullsized":"2023-10-09T05:06:08.158571+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":47,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.153825+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:06:08.153825+0000","last_peered":"2023-10-09T05:06:08.153825+0000","last_clean":"2023-10-09T05:06:08.153825+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:06:08.153825+0000","last_undegraded":"2023-10-09T05:06:08.153825+0000","last_fullsized":"2023-10-09T05:06:08.153825+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"669'6815","reported_seq":15049,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:23.823563+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:06:23.823563+0000","last_peered":"2023-10-09T05:06:23.823563+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:06:23.823563+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5052,"log_dups_size":0,"ondisk_log_size":5052,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62849024,"num_objects":959,"num_object_clones":0,"num_object_copies":2877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2502,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3916,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2205,"num_bytes_recovered":131530752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":877},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":108,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150793+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:06:08.150793+0000","last_peered":"2023-10-09T05:06:08.150793+0000","last_clean":"2023-10-09T05:06:08.150793+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:06:08.150793+0000","last_undegraded":"2023-10-09T05:06:08.150793+0000","last_fullsized":"2023-10-09T05:06:08.150793+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":306,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152170+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:06:08.152170+0000","last_peered":"2023-10-09T05:06:08.152170+0000","last_clean":"2023-10-09T05:06:08.152170+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:06:08.152170+0000","last_undegraded":"2023-10-09T05:06:08.152170+0000","last_fullsized":"2023-10-09T05:06:08.152170+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"669'6745","reported_seq":13626,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:23.654464+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:06:23.654464+0000","last_peered":"2023-10-09T05:06:23.654464+0000","last_clean":"2023-10-09T05:06:23.654464+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:06:23.654464+0000","last_undegraded":"2023-10-09T05:06:23.654464+0000","last_fullsized":"2023-10-09T05:06:23.654464+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4452,"log_dups_size":0,"ondisk_log_size":4452,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60325900,"num_objects":921,"num_object_clones":0,"num_object_copies":2763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":921,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3883,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":303,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154642+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:06:08.154642+0000","last_peered":"2023-10-09T05:06:08.154642+0000","last_clean":"2023-10-09T05:06:08.154642+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:06:08.154642+0000","last_undegraded":"2023-10-09T05:06:08.154642+0000","last_fullsized":"2023-10-09T05:06:08.154642+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152061+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:06:08.152061+0000","last_peered":"2023-10-09T05:06:08.152061+0000","last_clean":"2023-10-09T05:06:08.152061+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:06:08.152061+0000","last_undegraded":"2023-10-09T05:06:08.152061+0000","last_fullsized":"2023-10-09T05:06:08.152061+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"669'5891","reported_seq":14768,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:23.097985+0000","last_change":"2023-10-09T05:05:29.132074+0000","last_active":"2023-10-09T05:06:23.097985+0000","last_peered":"2023-10-09T05:06:23.097985+0000","last_clean":"2023-10-09T05:06:23.097985+0000","last_became_active":"2023-10-09T05:05:29.125934+0000","last_became_peered":"2023-10-09T05:05:29.125934+0000","last_unstale":"2023-10-09T05:06:23.097985+0000","last_undegraded":"2023-10-09T05:06:23.097985+0000","last_fullsized":"2023-10-09T05:06:23.097985+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29589504,"num_objects":451,"num_object_clones":0,"num_object_copies":1353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1956,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":198,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151152+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:06:08.151152+0000","last_peered":"2023-10-09T05:06:08.151152+0000","last_clean":"2023-10-09T05:06:08.151152+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:06:08.151152+0000","last_undegraded":"2023-10-09T05:06:08.151152+0000","last_fullsized":"2023-10-09T05:06:08.151152+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152522+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:06:08.152522+0000","last_peered":"2023-10-09T05:06:08.152522+0000","last_clean":"2023-10-09T05:06:08.152522+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:06:08.152522+0000","last_undegraded":"2023-10-09T05:06:08.152522+0000","last_fullsized":"2023-10-09T05:06:08.152522+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"669'5944","reported_seq":10567,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:24.873484+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:06:24.873484+0000","last_peered":"2023-10-09T05:06:24.873484+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:06:24.873484+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30932992,"num_objects":472,"num_object_clones":0,"num_object_copies":1416,"num_objects_missing_on_primary":650,"num_objects_missing":650,"num_objects_degraded":1368,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1960,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":750,"num_bytes_recovered":49119232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":252},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":26,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.151126+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:06:08.151126+0000","last_peered":"2023-10-09T05:06:08.151126+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:06:08.151126+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:06:08.151126+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":155,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150652+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:06:08.150652+0000","last_peered":"2023-10-09T05:06:08.150652+0000","last_clean":"2023-10-09T05:06:08.150652+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:06:08.150652+0000","last_undegraded":"2023-10-09T05:06:08.150652+0000","last_fullsized":"2023-10-09T05:06:08.150652+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"669'5983","reported_seq":10105,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:22.050896+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:06:22.050896+0000","last_peered":"2023-10-09T05:06:22.050896+0000","last_clean":"2023-10-09T05:06:22.050896+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:06:22.050896+0000","last_undegraded":"2023-10-09T05:06:22.050896+0000","last_fullsized":"2023-10-09T05:06:22.050896+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5445,"log_dups_size":0,"ondisk_log_size":5445,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28917760,"num_objects":441,"num_object_clones":0,"num_object_copies":1323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1990,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":51,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.153082+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:06:08.153082+0000","last_peered":"2023-10-09T05:06:08.153082+0000","last_clean":"2023-10-09T05:06:08.153082+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:06:08.153082+0000","last_undegraded":"2023-10-09T05:06:08.153082+0000","last_fullsized":"2023-10-09T05:06:08.153082+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":215,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154908+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:06:08.154908+0000","last_peered":"2023-10-09T05:06:08.154908+0000","last_clean":"2023-10-09T05:06:08.154908+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:06:08.154908+0000","last_undegraded":"2023-10-09T05:06:08.154908+0000","last_fullsized":"2023-10-09T05:06:08.154908+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"669'5952","reported_seq":16652,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:23.358439+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:06:23.358439+0000","last_peered":"2023-10-09T05:06:23.358439+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:06:23.358439+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:06:23.358439+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30212096,"num_objects":461,"num_object_clones":0,"num_object_copies":1383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1968,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":461}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":729,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.153507+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:06:08.153507+0000","last_peered":"2023-10-09T05:06:08.153507+0000","last_clean":"2023-10-09T05:06:08.153507+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:06:08.153507+0000","last_undegraded":"2023-10-09T05:06:08.153507+0000","last_fullsized":"2023-10-09T05:06:08.153507+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":104,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.149099+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:06:08.149099+0000","last_peered":"2023-10-09T05:06:08.149099+0000","last_clean":"2023-10-09T05:06:08.149099+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:06:08.149099+0000","last_undegraded":"2023-10-09T05:06:08.149099+0000","last_fullsized":"2023-10-09T05:06:08.149099+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1257,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154833+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:06:08.154833+0000","last_peered":"2023-10-09T05:06:08.154833+0000","last_clean":"2023-10-09T05:06:08.154833+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:06:08.154833+0000","last_undegraded":"2023-10-09T05:06:08.154833+0000","last_fullsized":"2023-10-09T05:06:08.154833+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"669'6223","reported_seq":12315,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:24.879082+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:06:24.879082+0000","last_peered":"2023-10-09T05:06:24.879082+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:06:24.879082+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6223,"log_dups_size":0,"ondisk_log_size":6223,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62226432,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":1263,"num_objects_missing":1263,"num_objects_degraded":2776,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3766,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1409,"num_bytes_recovered":92340224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36},{"shards":"0,2,4,5","objects":1},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":516}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":273,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151301+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:06:08.151301+0000","last_peered":"2023-10-09T05:06:08.151301+0000","last_clean":"2023-10-09T05:06:08.151301+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:06:08.151301+0000","last_undegraded":"2023-10-09T05:06:08.151301+0000","last_fullsized":"2023-10-09T05:06:08.151301+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":107,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152092+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:06:08.152092+0000","last_peered":"2023-10-09T05:06:08.152092+0000","last_clean":"2023-10-09T05:06:08.152092+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:06:08.152092+0000","last_undegraded":"2023-10-09T05:06:08.152092+0000","last_fullsized":"2023-10-09T05:06:08.152092+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1246,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154488+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:06:08.154488+0000","last_peered":"2023-10-09T05:06:08.154488+0000","last_clean":"2023-10-09T05:06:08.154488+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:06:08.154488+0000","last_undegraded":"2023-10-09T05:06:08.154488+0000","last_fullsized":"2023-10-09T05:06:08.154488+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"669'6483","reported_seq":20091,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:21.667143+0000","last_change":"2023-10-09T05:06:08.182013+0000","last_active":"2023-10-09T05:06:21.667143+0000","last_peered":"2023-10-09T05:06:21.667143+0000","last_clean":"2023-10-09T05:06:21.667143+0000","last_became_active":"2023-10-09T05:06:08.171542+0000","last_became_peered":"2023-10-09T05:06:08.171542+0000","last_unstale":"2023-10-09T05:06:21.667143+0000","last_undegraded":"2023-10-09T05:06:21.667143+0000","last_fullsized":"2023-10-09T05:06:21.667143+0000","mapping_epoch":667,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6483,"log_dups_size":0,"ondisk_log_size":6483,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61964288,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3941,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4591,"num_bytes_recovered":223150080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":212,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154798+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:06:08.154798+0000","last_peered":"2023-10-09T05:06:08.154798+0000","last_clean":"2023-10-09T05:06:08.154798+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:06:08.154798+0000","last_undegraded":"2023-10-09T05:06:08.154798+0000","last_fullsized":"2023-10-09T05:06:08.154798+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":27,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150892+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:06:08.150892+0000","last_peered":"2023-10-09T05:06:08.150892+0000","last_clean":"2023-10-09T05:06:08.150892+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:06:08.150892+0000","last_undegraded":"2023-10-09T05:06:08.150892+0000","last_fullsized":"2023-10-09T05:06:08.150892+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"669'6454","reported_seq":11864,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:23.292634+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:06:23.292634+0000","last_peered":"2023-10-09T05:06:23.292634+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:06:23.292634+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"444'460","ondisk_log_start":"444'460","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5994,"log_dups_size":0,"ondisk_log_size":5994,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61964288,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":552,"num_objects_missing":552,"num_objects_degraded":1491,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3914,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":97288192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":364},{"shards":"2,4,7","objects":552},{"shards":"3,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":141,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150731+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:06:08.150731+0000","last_peered":"2023-10-09T05:06:08.150731+0000","last_clean":"2023-10-09T05:06:08.150731+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:06:08.150731+0000","last_undegraded":"2023-10-09T05:06:08.150731+0000","last_fullsized":"2023-10-09T05:06:08.150731+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":142,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151424+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:06:08.151424+0000","last_peered":"2023-10-09T05:06:08.151424+0000","last_clean":"2023-10-09T05:06:08.151424+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:06:08.151424+0000","last_undegraded":"2023-10-09T05:06:08.151424+0000","last_fullsized":"2023-10-09T05:06:08.151424+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1266,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155214+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:06:08.155214+0000","last_peered":"2023-10-09T05:06:08.155214+0000","last_clean":"2023-10-09T05:06:08.155214+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:06:08.155214+0000","last_undegraded":"2023-10-09T05:06:08.155214+0000","last_fullsized":"2023-10-09T05:06:08.155214+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"669'6478","reported_seq":20295,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:24.088620+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:06:24.088620+0000","last_peered":"2023-10-09T05:06:24.088620+0000","last_clean":"2023-10-09T05:06:24.088620+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:06:24.088620+0000","last_undegraded":"2023-10-09T05:06:24.088620+0000","last_fullsized":"2023-10-09T05:06:24.088620+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6232,"log_dups_size":0,"ondisk_log_size":6232,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60489728,"num_objects":923,"num_object_clones":0,"num_object_copies":2769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3938,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":188,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:13.621590+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:06:13.621590+0000","last_peered":"2023-10-09T05:06:13.621590+0000","last_clean":"2023-10-09T05:06:13.621590+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:06:13.621590+0000","last_undegraded":"2023-10-09T05:06:13.621590+0000","last_fullsized":"2023-10-09T05:06:13.621590+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":96,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.153521+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:06:08.153521+0000","last_peered":"2023-10-09T05:06:08.153521+0000","last_clean":"2023-10-09T05:06:08.153521+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:06:08.153521+0000","last_undegraded":"2023-10-09T05:06:08.153521+0000","last_fullsized":"2023-10-09T05:06:08.153521+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1272,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155339+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:06:08.155339+0000","last_peered":"2023-10-09T05:06:08.155339+0000","last_clean":"2023-10-09T05:06:08.155339+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:06:08.155339+0000","last_undegraded":"2023-10-09T05:06:08.155339+0000","last_fullsized":"2023-10-09T05:06:08.155339+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"669'6649","reported_seq":12079,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:24.624464+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:06:24.624464+0000","last_peered":"2023-10-09T05:06:24.624464+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:06:24.624464+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"444'218","ondisk_log_start":"444'218","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62521344,"num_objects":954,"num_object_clones":0,"num_object_copies":2862,"num_objects_missing_on_primary":1366,"num_objects_missing":1366,"num_objects_degraded":3904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4040,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1544,"num_bytes_recovered":101187584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,3","objects":-933},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":574}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":84,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148892+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:06:08.148892+0000","last_peered":"2023-10-09T05:06:08.148892+0000","last_clean":"2023-10-09T05:06:08.148892+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:06:08.148892+0000","last_undegraded":"2023-10-09T05:06:08.148892+0000","last_fullsized":"2023-10-09T05:06:08.148892+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":47,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.153903+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:06:08.153903+0000","last_peered":"2023-10-09T05:06:08.153903+0000","last_clean":"2023-10-09T05:06:08.153903+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:06:08.153903+0000","last_undegraded":"2023-10-09T05:06:08.153903+0000","last_fullsized":"2023-10-09T05:06:08.153903+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150659+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:06:08.150659+0000","last_peered":"2023-10-09T05:06:08.150659+0000","last_clean":"2023-10-09T05:06:08.150659+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:06:08.150659+0000","last_undegraded":"2023-10-09T05:06:08.150659+0000","last_fullsized":"2023-10-09T05:06:08.150659+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1255,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154753+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:06:08.154753+0000","last_peered":"2023-10-09T05:06:08.154753+0000","last_clean":"2023-10-09T05:06:08.154753+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:06:08.154753+0000","last_undegraded":"2023-10-09T05:06:08.154753+0000","last_fullsized":"2023-10-09T05:06:08.154753+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"669'6744","reported_seq":19558,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:23.517646+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:06:23.517646+0000","last_peered":"2023-10-09T05:06:23.517646+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:06:23.517646+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59473920,"num_objects":907,"num_object_clones":0,"num_object_copies":2721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1474,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3884,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3423,"num_bytes_recovered":219742208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":425}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":753,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154569+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:06:08.154569+0000","last_peered":"2023-10-09T05:06:08.154569+0000","last_clean":"2023-10-09T05:06:08.154569+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:06:08.154569+0000","last_undegraded":"2023-10-09T05:06:08.154569+0000","last_fullsized":"2023-10-09T05:06:08.154569+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150855+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:06:08.150855+0000","last_peered":"2023-10-09T05:06:08.150855+0000","last_clean":"2023-10-09T05:06:08.150855+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:06:08.150855+0000","last_undegraded":"2023-10-09T05:06:08.150855+0000","last_fullsized":"2023-10-09T05:06:08.150855+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":84,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148820+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:06:08.148820+0000","last_peered":"2023-10-09T05:06:08.148820+0000","last_clean":"2023-10-09T05:06:08.148820+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:06:08.148820+0000","last_undegraded":"2023-10-09T05:06:08.148820+0000","last_fullsized":"2023-10-09T05:06:08.148820+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1266,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154988+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:06:08.154988+0000","last_peered":"2023-10-09T05:06:08.154988+0000","last_clean":"2023-10-09T05:06:08.154988+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:06:08.154988+0000","last_undegraded":"2023-10-09T05:06:08.154988+0000","last_fullsized":"2023-10-09T05:06:08.154988+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"669'6737","reported_seq":12519,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:23.484254+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:06:23.484254+0000","last_peered":"2023-10-09T05:06:23.484254+0000","last_clean":"2023-10-09T05:06:23.484254+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:06:23.484254+0000","last_undegraded":"2023-10-09T05:06:23.484254+0000","last_fullsized":"2023-10-09T05:06:23.484254+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4444,"log_dups_size":0,"ondisk_log_size":4444,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60850188,"num_objects":929,"num_object_clones":0,"num_object_copies":2787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3875,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":164,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155171+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:06:08.155171+0000","last_peered":"2023-10-09T05:06:08.155171+0000","last_clean":"2023-10-09T05:06:08.155171+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:06:08.155171+0000","last_undegraded":"2023-10-09T05:06:08.155171+0000","last_fullsized":"2023-10-09T05:06:08.155171+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":82,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148431+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:06:08.148431+0000","last_peered":"2023-10-09T05:06:08.148431+0000","last_clean":"2023-10-09T05:06:08.148431+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:06:08.148431+0000","last_undegraded":"2023-10-09T05:06:08.148431+0000","last_fullsized":"2023-10-09T05:06:08.148431+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.153447+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:06:08.153447+0000","last_peered":"2023-10-09T05:06:08.153447+0000","last_clean":"2023-10-09T05:06:08.153447+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:06:08.153447+0000","last_undegraded":"2023-10-09T05:06:08.153447+0000","last_fullsized":"2023-10-09T05:06:08.153447+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"669'6667","reported_seq":13082,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:23.537941+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:06:23.537941+0000","last_peered":"2023-10-09T05:06:23.537941+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:06:23.537941+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4540,"log_dups_size":0,"ondisk_log_size":4540,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61669376,"num_objects":941,"num_object_clones":0,"num_object_copies":2823,"num_objects_missing_on_primary":591,"num_objects_missing":591,"num_objects_degraded":1836,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1757,"num_bytes_recovered":115113984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,7","objects":500},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":57,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152915+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:06:08.152915+0000","last_peered":"2023-10-09T05:06:08.152915+0000","last_clean":"2023-10-09T05:06:08.152915+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:06:08.152915+0000","last_undegraded":"2023-10-09T05:06:08.152915+0000","last_fullsized":"2023-10-09T05:06:08.152915+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1272,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154312+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:06:08.154312+0000","last_peered":"2023-10-09T05:06:08.154312+0000","last_clean":"2023-10-09T05:06:08.154312+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:06:08.154312+0000","last_undegraded":"2023-10-09T05:06:08.154312+0000","last_fullsized":"2023-10-09T05:06:08.154312+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":162,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.158651+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:06:08.158651+0000","last_peered":"2023-10-09T05:06:08.158651+0000","last_clean":"2023-10-09T05:06:08.158651+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:06:08.158651+0000","last_undegraded":"2023-10-09T05:06:08.158651+0000","last_fullsized":"2023-10-09T05:06:08.158651+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":189,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155607+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:06:08.155607+0000","last_peered":"2023-10-09T05:06:08.155607+0000","last_clean":"2023-10-09T05:06:08.155607+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:06:08.155607+0000","last_undegraded":"2023-10-09T05:06:08.155607+0000","last_fullsized":"2023-10-09T05:06:08.155607+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1245,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155289+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:06:08.155289+0000","last_peered":"2023-10-09T05:06:08.155289+0000","last_clean":"2023-10-09T05:06:08.155289+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:06:08.155289+0000","last_undegraded":"2023-10-09T05:06:08.155289+0000","last_fullsized":"2023-10-09T05:06:08.155289+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"669'6848","reported_seq":15190,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:24.097831+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:06:24.097831+0000","last_peered":"2023-10-09T05:06:24.097831+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:06:24.097831+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5085,"log_dups_size":0,"ondisk_log_size":5085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60686336,"num_objects":926,"num_object_clones":0,"num_object_copies":2778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3949,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2228,"num_bytes_recovered":133693440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":-34},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":913}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":82,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148506+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:06:08.148506+0000","last_peered":"2023-10-09T05:06:08.148506+0000","last_clean":"2023-10-09T05:06:08.148506+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:06:08.148506+0000","last_undegraded":"2023-10-09T05:06:08.148506+0000","last_fullsized":"2023-10-09T05:06:08.148506+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":27,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150488+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:06:08.150488+0000","last_peered":"2023-10-09T05:06:08.150488+0000","last_clean":"2023-10-09T05:06:08.150488+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:06:08.150488+0000","last_undegraded":"2023-10-09T05:06:08.150488+0000","last_fullsized":"2023-10-09T05:06:08.150488+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":121,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:14.794388+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:06:14.794388+0000","last_peered":"2023-10-09T05:06:14.794388+0000","last_clean":"2023-10-09T05:06:14.794388+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:06:14.794388+0000","last_undegraded":"2023-10-09T05:06:14.794388+0000","last_fullsized":"2023-10-09T05:06:14.794388+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1266,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.156169+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:06:08.156169+0000","last_peered":"2023-10-09T05:06:08.156169+0000","last_clean":"2023-10-09T05:06:08.156169+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:06:08.156169+0000","last_undegraded":"2023-10-09T05:06:08.156169+0000","last_fullsized":"2023-10-09T05:06:08.156169+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"669'6467","reported_seq":18504,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:22.054605+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:06:22.054605+0000","last_peered":"2023-10-09T05:06:22.054605+0000","last_clean":"2023-10-09T05:06:22.054605+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:06:22.054605+0000","last_undegraded":"2023-10-09T05:06:22.054605+0000","last_fullsized":"2023-10-09T05:06:22.054605+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62947328,"num_objects":960,"num_object_clones":0,"num_object_copies":2880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3928,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151385+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:06:08.151385+0000","last_peered":"2023-10-09T05:06:08.151385+0000","last_clean":"2023-10-09T05:06:08.151385+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:06:08.151385+0000","last_undegraded":"2023-10-09T05:06:08.151385+0000","last_fullsized":"2023-10-09T05:06:08.151385+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":93,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148663+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:06:08.148663+0000","last_peered":"2023-10-09T05:06:08.148663+0000","last_clean":"2023-10-09T05:06:08.148663+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:06:08.148663+0000","last_undegraded":"2023-10-09T05:06:08.148663+0000","last_fullsized":"2023-10-09T05:06:08.148663+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152828+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:06:08.152828+0000","last_peered":"2023-10-09T05:06:08.152828+0000","last_clean":"2023-10-09T05:06:08.152828+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:06:08.152828+0000","last_undegraded":"2023-10-09T05:06:08.152828+0000","last_fullsized":"2023-10-09T05:06:08.152828+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1269,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155551+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:06:08.155551+0000","last_peered":"2023-10-09T05:06:08.155551+0000","last_clean":"2023-10-09T05:06:08.155551+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:06:08.155551+0000","last_undegraded":"2023-10-09T05:06:08.155551+0000","last_fullsized":"2023-10-09T05:06:08.155551+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"669'6585","reported_seq":23943,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:21.971329+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:06:21.971329+0000","last_peered":"2023-10-09T05:06:21.971329+0000","last_clean":"2023-10-09T05:06:21.971329+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:06:21.971329+0000","last_undegraded":"2023-10-09T05:06:21.971329+0000","last_fullsized":"2023-10-09T05:06:21.971329+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65994752,"num_objects":1007,"num_object_clones":0,"num_object_copies":3021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3764,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":293,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.163966+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:06:08.163966+0000","last_peered":"2023-10-09T05:06:08.163966+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:06:08.163966+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:06:08.163966+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":99,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150024+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:06:08.150024+0000","last_peered":"2023-10-09T05:06:08.150024+0000","last_clean":"2023-10-09T05:06:08.150024+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:06:08.150024+0000","last_undegraded":"2023-10-09T05:06:08.150024+0000","last_fullsized":"2023-10-09T05:06:08.150024+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":173,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152046+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:06:08.152046+0000","last_peered":"2023-10-09T05:06:08.152046+0000","last_clean":"2023-10-09T05:06:08.152046+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:06:08.152046+0000","last_undegraded":"2023-10-09T05:06:08.152046+0000","last_fullsized":"2023-10-09T05:06:08.152046+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1225,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155309+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:06:08.155309+0000","last_peered":"2023-10-09T05:06:08.155309+0000","last_clean":"2023-10-09T05:06:08.155309+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:06:08.155309+0000","last_undegraded":"2023-10-09T05:06:08.155309+0000","last_fullsized":"2023-10-09T05:06:08.155309+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"669'7009","reported_seq":14905,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:24.415855+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:06:24.415855+0000","last_peered":"2023-10-09T05:06:24.415855+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:06:24.415855+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61014016,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":1324,"num_objects_missing":1324,"num_objects_degraded":3274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4049,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1840,"num_bytes_recovered":120586240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-2},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":862},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":185,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155521+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:06:08.155521+0000","last_peered":"2023-10-09T05:06:08.155521+0000","last_clean":"2023-10-09T05:06:08.155521+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:06:08.155521+0000","last_undegraded":"2023-10-09T05:06:08.155521+0000","last_fullsized":"2023-10-09T05:06:08.155521+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":103,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.149838+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:06:08.149838+0000","last_peered":"2023-10-09T05:06:08.149838+0000","last_clean":"2023-10-09T05:06:08.149838+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:06:08.149838+0000","last_undegraded":"2023-10-09T05:06:08.149838+0000","last_fullsized":"2023-10-09T05:06:08.149838+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":144,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150624+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:06:08.150624+0000","last_peered":"2023-10-09T05:06:08.150624+0000","last_clean":"2023-10-09T05:06:08.150624+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:06:08.150624+0000","last_undegraded":"2023-10-09T05:06:08.150624+0000","last_fullsized":"2023-10-09T05:06:08.150624+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"669'6276","reported_seq":12590,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:24.891223+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:06:24.891223+0000","last_peered":"2023-10-09T05:06:24.891223+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:06:24.891223+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6276,"log_dups_size":0,"ondisk_log_size":6276,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58753024,"num_objects":897,"num_object_clones":0,"num_object_copies":2691,"num_objects_missing_on_primary":1321,"num_objects_missing":1321,"num_objects_degraded":2868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3820,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1462,"num_bytes_recovered":95813632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-36},{"shards":"0,2,3","objects":2},{"shards":"0,2,4,5","objects":1},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":545}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":134,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.151029+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:06:08.151029+0000","last_peered":"2023-10-09T05:06:08.151029+0000","last_clean":"2023-10-09T05:06:08.151029+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:06:08.151029+0000","last_undegraded":"2023-10-09T05:06:08.151029+0000","last_fullsized":"2023-10-09T05:06:08.151029+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":84,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.148742+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:06:08.148742+0000","last_peered":"2023-10-09T05:06:08.148742+0000","last_clean":"2023-10-09T05:06:08.148742+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:06:08.148742+0000","last_undegraded":"2023-10-09T05:06:08.148742+0000","last_fullsized":"2023-10-09T05:06:08.148742+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"669'6420","reported_seq":11263,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:23.384344+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:06:23.384344+0000","last_peered":"2023-10-09T05:06:23.384344+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:06:23.384344+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:06:23.384344+0000","mapping_epoch":597,"log_start":"444'319","ondisk_log_start":"444'319","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6101,"log_dups_size":0,"ondisk_log_size":6101,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63832064,"num_objects":974,"num_object_clones":0,"num_object_copies":2922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3885,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":974}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":804,"reported_epoch":668,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:08.153615+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:06:08.153615+0000","last_peered":"2023-10-09T05:06:08.153615+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:06:08.153615+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:06:08.153615+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":27,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150640+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:06:08.150640+0000","last_peered":"2023-10-09T05:06:08.150640+0000","last_clean":"2023-10-09T05:06:08.150640+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:06:08.150640+0000","last_undegraded":"2023-10-09T05:06:08.150640+0000","last_fullsized":"2023-10-09T05:06:08.150640+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"669'6415","reported_seq":10842,"reported_epoch":669,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:22.068423+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:06:22.068423+0000","last_peered":"2023-10-09T05:06:22.068423+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:06:22.068423+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5877,"log_dups_size":0,"ondisk_log_size":5877,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64192512,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3884,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":333}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":750,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.154250+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:06:08.154250+0000","last_peered":"2023-10-09T05:06:08.154250+0000","last_clean":"2023-10-09T05:06:08.154250+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:06:08.154250+0000","last_undegraded":"2023-10-09T05:06:08.154250+0000","last_fullsized":"2023-10-09T05:06:08.154250+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":27,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150540+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:06:08.150540+0000","last_peered":"2023-10-09T05:06:08.150540+0000","last_clean":"2023-10-09T05:06:08.150540+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:06:08.150540+0000","last_undegraded":"2023-10-09T05:06:08.150540+0000","last_fullsized":"2023-10-09T05:06:08.150540+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"669'6482","reported_seq":17291,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:23.286515+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:06:23.286515+0000","last_peered":"2023-10-09T05:06:23.286515+0000","last_clean":"2023-10-09T05:06:23.286515+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:06:23.286515+0000","last_undegraded":"2023-10-09T05:06:23.286515+0000","last_fullsized":"2023-10-09T05:06:23.286515+0000","mapping_epoch":638,"log_start":"444'559","ondisk_log_start":"444'559","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59899904,"num_objects":914,"num_object_clones":0,"num_object_copies":2742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3950,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":289,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.152097+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:06:08.152097+0000","last_peered":"2023-10-09T05:06:08.152097+0000","last_clean":"2023-10-09T05:06:08.152097+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:06:08.152097+0000","last_undegraded":"2023-10-09T05:06:08.152097+0000","last_fullsized":"2023-10-09T05:06:08.152097+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":297,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155490+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:06:08.155490+0000","last_peered":"2023-10-09T05:06:08.155490+0000","last_clean":"2023-10-09T05:06:08.155490+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:06:08.155490+0000","last_undegraded":"2023-10-09T05:06:08.155490+0000","last_fullsized":"2023-10-09T05:06:08.155490+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"669'6349","reported_seq":17959,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:23.471983+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:06:23.471983+0000","last_peered":"2023-10-09T05:06:23.471983+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:06:23.471983+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:06:23.471983+0000","mapping_epoch":597,"log_start":"444'363","ondisk_log_start":"444'363","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5986,"log_dups_size":0,"ondisk_log_size":5986,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63242240,"num_objects":965,"num_object_clones":0,"num_object_copies":2895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3850,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":965}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":49,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.155420+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:06:08.155420+0000","last_peered":"2023-10-09T05:06:08.155420+0000","last_clean":"2023-10-09T05:06:08.155420+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:06:08.155420+0000","last_undegraded":"2023-10-09T05:06:08.155420+0000","last_fullsized":"2023-10-09T05:06:08.155420+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":145,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-10-09T05:06:08.150586+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:06:08.150586+0000","last_peered":"2023-10-09T05:06:08.150586+0000","last_clean":"2023-10-09T05:06:08.150586+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:06:08.150586+0000","last_undegraded":"2023-10-09T05:06:08.150586+0000","last_fullsized":"2023-10-09T05:06:08.150586+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":1983184920,"num_objects":30262,"num_object_clones":0,"num_object_copies":90786,"num_objects_missing_on_primary":10353,"num_objects_missing":10353,"num_objects_degraded":32330,"num_objects_misplaced":3953,"num_objects_unfound":0,"num_objects_dirty":30262,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124956,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80656,"num_bytes_recovered":4881186816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12401192960,"data_stored":12401180744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":206069,"ondisk_log_size":206069,"up":108,"acting":92,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234708,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3008160,"kb_used_data":1490712,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":90736992,"statfs":{"total":95995035648,"available":92914679808,"internally_reserved":0,"allocated":1526489088,"data_stored":1524788020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:16 2023","interfaces":[{"interface":"back","average":{"1min":11.641999999999999,"5min":6.4889999999999999,"15min":2.593},"min":{"1min":0.25900000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":221.91800000000001,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":11.976000000000001,"5min":6.7329999999999997,"15min":2.839},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":221.87799999999999,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.70999999999999996}]},{"osd":1,"last update":"Mon Oct 9 05:06:16 2023","interfaces":[{"interface":"back","average":{"1min":4.8289999999999997,"5min":4.2850000000000001,"15min":2.1459999999999999},"min":{"1min":0.32800000000000001,"5min":0.28899999999999998,"15min":0.25800000000000001},"max":{"1min":63.927999999999997,"5min":63.927999999999997,"15min":63.927999999999997},"last":0.44900000000000001},{"interface":"front","average":{"1min":4.7889999999999997,"5min":4.0110000000000001,"15min":2.0990000000000002},"min":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":64.334000000000003,"5min":64.334000000000003,"15min":64.334000000000003},"last":0.375}]},{"osd":2,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.7069999999999999,"15min":3.8879999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9269999999999996,"5min":64.480999999999995,"15min":207.96600000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.524,"5min":2.6469999999999998,"15min":4.0019999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.8989999999999991,"5min":55.134,"15min":207.97900000000001},"last":0.58799999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:06:12 2023","interfaces":[{"interface":"back","average":{"1min":15.56,"5min":8.3770000000000007,"15min":3.7829999999999999},"min":{"1min":0.36699999999999999,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":241.666,"5min":241.666,"15min":241.666},"last":1.7310000000000001},{"interface":"front","average":{"1min":14.212,"5min":7.8179999999999996,"15min":3.7149999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":230.751,"5min":230.751,"15min":230.751},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38400000000000001,"15min":0.38600000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.497,"15min":0.48099999999999998},"min":{"1min":0.26900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.923,"5min":2.923,"15min":2.923},"last":0.47499999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":1.6399999999999999,"15min":1.7230000000000001},"min":{"1min":0.308,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":0.57399999999999995,"5min":57.064999999999998,"15min":131.84899999999999},"last":0.499},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.734,"15min":2.0139999999999998},"min":{"1min":0.29099999999999998,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":2.2210000000000001,"5min":53.225000000000001,"15min":135.74199999999999},"last":0.53200000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.458,"15min":1.1699999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.88200000000000001,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":1.651,"15min":1.3129999999999999},"min":{"1min":0.374,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":2.2770000000000001,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.42299999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496341,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136160,"kb_used_data":625560,"kb_used_omap":0,"kb_used_meta":1510592,"kb_avail":91608992,"statfs":{"total":95995035648,"available":93807607808,"internally_reserved":0,"allocated":640573440,"data_stored":638872605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1546846208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":6.9930000000000003,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":49.659999999999997,"5min":321.346,"15min":321.346},"last":32.697000000000003},{"interface":"front","average":{"1min":6.0620000000000003,"5min":5.2439999999999998,"15min":2.355},"min":{"1min":0.35799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":46.987000000000002,"5min":169.119,"15min":169.119},"last":32.713000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":2.415,"15min":3.2170000000000001},"min":{"1min":0.39000000000000001,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":10.973000000000001,"5min":72.840999999999994,"15min":220.48400000000001},"last":0.376},{"interface":"front","average":{"1min":2.2810000000000001,"5min":2.665,"15min":3.2000000000000002},"min":{"1min":0.34899999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":10.02,"5min":72.832999999999998,"15min":220.56200000000001},"last":0.44800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:05:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.6799999999999999,"15min":2.0630000000000002},"min":{"1min":0.376,"5min":0.32900000000000001,"15min":0.25600000000000001},"max":{"1min":5.0090000000000003,"5min":23.943000000000001,"15min":133.535},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.877,"5min":1.651,"15min":1.9850000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":4.9359999999999999,"5min":25.978999999999999,"15min":133.51900000000001},"last":0.47499999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":5.6349999999999998,"5min":3.3769999999999998,"15min":2.956},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":45.915999999999997,"5min":64.355999999999995,"15min":147.43600000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":6.0229999999999997,"5min":3.2040000000000002,"15min":3.0270000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":45.868000000000002,"5min":56.133000000000003,"15min":147.41999999999999},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":0.96699999999999997,"15min":0.81200000000000006},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":18.186,"5min":18.186,"15min":18.186},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.548,"5min":1.1120000000000001,"15min":1.04},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":17.597000000000001,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.497}]},{"osd":5,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":1.006,"15min":0.99099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.499,"5min":25.934000000000001,"15min":25.934000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.83599999999999997,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.254},"max":{"1min":0.57699999999999996,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.51600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.73899999999999999,"15min":1.052},"min":{"1min":0.314,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":7.2549999999999999,"5min":7.2549999999999999,"15min":36.204999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.99199999999999999,"15min":1.0269999999999999},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":0.58999999999999997,"5min":20.041,"15min":20.041},"last":0.433}]}]},{"osd":1,"up_from":118,"seq":506806141408,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2172376,"kb_used_data":832656,"kb_used_omap":4,"kb_used_meta":1339579,"kb_avail":91572776,"statfs":{"total":95995035648,"available":93770522624,"internally_reserved":0,"allocated":852639744,"data_stored":849067173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1371729806},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":10.287000000000001,"5min":4.125,"15min":2.0139999999999998},"min":{"1min":0.29999999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":100.247,"5min":100.247,"15min":100.247},"last":1.97},{"interface":"front","average":{"1min":12.194000000000001,"5min":4.5540000000000003,"15min":2.129},"min":{"1min":0.52600000000000002,"5min":0.247,"15min":0.214},"max":{"1min":99.935000000000002,"5min":99.935000000000002,"15min":99.935000000000002},"last":0.32300000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:05:48 2023","interfaces":[{"interface":"back","average":{"1min":3.202,"5min":1.5429999999999999,"15min":2.5939999999999999},"min":{"1min":0.309,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.478,"5min":12.460000000000001,"15min":263.089},"last":1.732},{"interface":"front","average":{"1min":3.335,"5min":1.9319999999999999,"15min":2.653},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.222},"max":{"1min":10.385,"5min":13.933999999999999,"15min":254.50899999999999},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:06:19 2023","interfaces":[{"interface":"back","average":{"1min":10.869999999999999,"5min":8.1379999999999999,"15min":3.9049999999999998},"min":{"1min":0.41699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":102.729,"5min":235.43600000000001,"15min":235.43600000000001},"last":1.8089999999999999},{"interface":"front","average":{"1min":8.6829999999999998,"5min":7.2679999999999998,"15min":3.456},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":104.13500000000001,"5min":234.32499999999999,"15min":234.32499999999999},"last":0.53400000000000003}]},{"osd":4,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.4930000000000001,"15min":1.347},"min":{"1min":0.42899999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":10.25,"5min":10.25,"15min":10.25},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.9060000000000001,"5min":1.339,"15min":1.0780000000000001},"min":{"1min":0.46500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":10.521000000000001,"5min":10.521000000000001,"15min":10.521000000000001},"last":1.879}]},{"osd":5,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.2929999999999999,"15min":1.278},"min":{"1min":0.36899999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":10.353,"5min":10.353,"15min":10.353},"last":0.89600000000000002},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.542,"15min":1.345},"min":{"1min":0.503,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":6.9459999999999997,"5min":10.554,"15min":19.597999999999999},"last":1.8380000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":1.383,"15min":1.2689999999999999},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.35699999999999998},"max":{"1min":10.311999999999999,"5min":10.311999999999999,"15min":10.311999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.4390000000000001,"15min":1.3580000000000001},"min":{"1min":0.54400000000000004,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":6.915,"5min":10.472,"15min":19.548999999999999},"last":1.9099999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:05:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3879999999999999,"5min":1.5649999999999999,"15min":1.8180000000000001},"min":{"1min":0.54300000000000004,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.8330000000000002,"5min":10.583,"15min":59.972000000000001},"last":1.9370000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.351,"15min":1.714},"min":{"1min":0.51500000000000001,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":9.7170000000000005,"5min":11.903,"15min":60.011000000000003},"last":1.6100000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024492,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1734696,"kb_used_data":568032,"kb_used_omap":5,"kb_used_meta":1166586,"kb_avail":92010456,"statfs":{"total":95995035648,"available":94218706944,"internally_reserved":0,"allocated":581664768,"data_stored":578783513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1194584698},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:06:12 2023","interfaces":[{"interface":"back","average":{"1min":5.1909999999999998,"5min":3.673,"15min":2.5150000000000001},"min":{"1min":0.28299999999999997,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":19.042000000000002,"5min":53.176000000000002,"15min":155.411},"last":2.6160000000000001},{"interface":"front","average":{"1min":5.5419999999999998,"5min":3.859,"15min":2.5430000000000001},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.657,"5min":53.372999999999998,"15min":154.75},"last":3.6099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:05:32 2023","interfaces":[{"interface":"back","average":{"1min":3.7530000000000001,"5min":2.3969999999999998,"15min":2.516},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.762,"5min":26.556999999999999,"15min":151.06399999999999},"last":7.6539999999999999},{"interface":"front","average":{"1min":3.367,"5min":2.645,"15min":2.6789999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.7720000000000002,"5min":29.032,"15min":151.22399999999999},"last":3.7309999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:05:59 2023","interfaces":[{"interface":"back","average":{"1min":13.766999999999999,"5min":7.6890000000000001,"15min":4.4539999999999997},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":114.871,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.495},{"interface":"front","average":{"1min":15.404999999999999,"5min":7.9009999999999998,"15min":4.6449999999999996},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":115.375,"5min":290.73899999999998,"15min":290.73899999999998},"last":3.8730000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:05:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":1.9570000000000001,"15min":1.887},"min":{"1min":0.443,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.5329999999999995,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":7.4000000000000004},{"interface":"front","average":{"1min":2.1789999999999998,"5min":1.663,"15min":1.577},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.81,"5min":11.81,"15min":11.81},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.4300000000000002,"15min":1.494},"min":{"1min":0.42199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.51,"5min":16.888000000000002,"15min":16.888000000000002},"last":3.3940000000000001},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.0979999999999999,"15min":1.3460000000000001},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.5670000000000002,"5min":14.036,"15min":14.036},"last":7.4489999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":2.6629999999999998,"15min":1.6459999999999999},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.598000000000001,"5min":20.129999999999999,"15min":39.594999999999999},"last":3.4319999999999999},{"interface":"front","average":{"1min":2.1150000000000002,"5min":2.6299999999999999,"15min":1.6599999999999999},"min":{"1min":0.36299999999999999,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":7.6020000000000003,"5min":47.241,"15min":47.241},"last":7.5140000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.3460000000000001,"15min":1.524},"min":{"1min":0.441,"5min":0.41899999999999998,"15min":0.39400000000000002},"max":{"1min":7.6360000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":7.4820000000000002},{"interface":"front","average":{"1min":1.929,"5min":2.2120000000000002,"15min":1.508},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":9.9770000000000003,"5min":21.68,"15min":21.68},"last":0.56000000000000005}]}]},{"osd":2,"up_from":0,"seq":949187772770,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356464,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2014292,"kb_used_data":1042700,"kb_used_omap":2,"kb_used_meta":971581,"kb_avail":91730860,"statfs":{"total":95995035648,"available":93932400640,"internally_reserved":0,"allocated":1067724800,"data_stored":1064740670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":994898997},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":5.5090000000000003,"5min":3.3820000000000001,"15min":1.5600000000000001},"min":{"1min":0.25700000000000001,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":51.145000000000003,"5min":108.631,"15min":108.631},"last":0.255},{"interface":"front","average":{"1min":6.1710000000000003,"5min":3.6160000000000001,"15min":1.752},"min":{"1min":0.28599999999999998,"5min":0.247,"15min":0.247},"max":{"1min":51.191000000000003,"5min":110.012,"15min":110.012},"last":0.35199999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0190000000000001,"15min":1.391},"min":{"1min":0.26100000000000001,"5min":0.221,"15min":0.221},"max":{"1min":13.374000000000001,"5min":15.218,"15min":15.983000000000001},"last":13.103999999999999},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.754,"15min":1.294},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.245},"max":{"1min":4.54,"5min":17.544,"15min":26.692},"last":14.164}]},{"osd":2,"last update":"Mon Oct 9 05:05:41 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":1.3620000000000001,"15min":2.2309999999999999},"min":{"1min":0.35499999999999998,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":6.843,"5min":11.121,"15min":212.065},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.4649999999999999,"5min":1.6910000000000001,"15min":2.3159999999999998},"min":{"1min":0.32500000000000001,"5min":0.253,"15min":0.23499999999999999},"max":{"1min":6.3120000000000003,"5min":12.987,"15min":211.98400000000001},"last":0.32100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:05:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":1.835,"15min":1.758},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.0099999999999998,"5min":11.193,"15min":11.193},"last":0.434},{"interface":"front","average":{"1min":2.3319999999999999,"5min":1.1839999999999999,"15min":0.99199999999999999},"min":{"1min":0.39700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":10.452,"5min":10.452,"15min":10.452},"last":0.41799999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.855,"15min":1.395},"min":{"1min":0.46000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.6040000000000001,"5min":57.804000000000002,"15min":57.804000000000002},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.95,"5min":2.1269999999999998,"15min":1.659},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":6.383,"5min":57.442,"15min":57.442},"last":0.45500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.649,"15min":1.222},"min":{"1min":0.376,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":6.3659999999999997,"5min":11.252000000000001,"15min":11.252000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.4319999999999999,"15min":1.327},"min":{"1min":0.441,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.988,"5min":14.731,"15min":24.916},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.5369999999999999,"15min":1.3580000000000001},"min":{"1min":0.47399999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.9470000000000001,"5min":14.784000000000001,"15min":25.004000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.4319999999999999,"15min":1.1120000000000001},"min":{"1min":0.40300000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.758,"5min":7.758,"15min":7.758},"last":0.53800000000000003}]}]},{"osd":4,"up_from":626,"seq":2688649527333,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2605544,"kb_used_data":2208544,"kb_used_omap":22,"kb_used_meta":396969,"kb_avail":91139608,"statfs":{"total":95995035648,"available":93326958592,"internally_reserved":0,"allocated":2261549056,"data_stored":2259840394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":406497101},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":2.3330000000000002,"15min":2.3050000000000002},"min":{"1min":0.27700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":17.135000000000002,"5min":21.292000000000002,"15min":21.292000000000002},"last":3.306},{"interface":"front","average":{"1min":2.544,"5min":3.0449999999999999,"15min":3.129},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.030999999999999,"5min":21.391999999999999,"15min":21.391999999999999},"last":1.665}]},{"osd":1,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4390000000000001,"5min":2.1909999999999998,"15min":2.149},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.9879999999999995,"5min":14.955,"15min":14.955},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.3129999999999999,"5min":1.7829999999999999,"15min":1.861},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.585,"5min":10.579000000000001,"15min":10.579000000000001},"last":0.26900000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.667,"15min":1.6479999999999999},"min":{"1min":0.377,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.3819999999999997,"5min":11.051,"15min":11.051},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.736,"15min":1.7030000000000001},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":5.3959999999999999,"5min":11.036,"15min":11.036},"last":0.38200000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":20.216000000000001,"5min":4.9829999999999997,"15min":2.444},"min":{"1min":0.45100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":270.767,"5min":270.767,"15min":270.767},"last":10.797000000000001},{"interface":"front","average":{"1min":19.672000000000001,"5min":5.4089999999999998,"15min":3.032},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":270.274,"5min":270.274,"15min":270.274},"last":0.86599999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.39100000000000001,"15min":0.38700000000000001},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":0.41599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.624,"15min":0.65000000000000002},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.5409999999999999,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.46800000000000003,"15min":0.45800000000000002},"min":{"1min":0.35299999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.0920000000000001,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.34200000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:05:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52200000000000002,"15min":0.52400000000000002},"min":{"1min":0.36199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.48799999999999999,"15min":0.47599999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.49199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757985,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4426320,"kb_used_data":2941192,"kb_used_omap":0,"kb_used_meta":1485120,"kb_avail":89318832,"statfs":{"total":95995035648,"available":91462483968,"internally_reserved":0,"allocated":3011780608,"data_stored":3010091168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1520762880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":3.4119999999999999,"15min":2.137},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.0309999999999997,"5min":103.745,"15min":106.465},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.5880000000000001,"5min":3.71,"15min":2.3090000000000002},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.245},"max":{"1min":10.882999999999999,"5min":103.872,"15min":106.54600000000001},"last":0.40799999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":5.0410000000000004,"15min":3.556},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.23200000000000001},"max":{"1min":6.0529999999999999,"5min":262.26900000000001,"15min":262.26900000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":2.7709999999999999,"5min":5.0860000000000003,"15min":3.7040000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26900000000000002},"max":{"1min":10.462,"5min":262.291,"15min":262.291},"last":3.3769999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:05:39 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":2.6499999999999999,"15min":1.875},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":8.1340000000000003,"5min":106.238,"15min":106.238},"last":0.497},{"interface":"front","average":{"1min":2.9550000000000001,"5min":2.7109999999999999,"15min":2.1269999999999998},"min":{"1min":0.40500000000000003,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":8.4390000000000001,"5min":104.31699999999999,"15min":104.31699999999999},"last":3.4089999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":7.8579999999999997,"5min":7.0819999999999999,"15min":3.524},"min":{"1min":0.375,"5min":0.318,"15min":0.318},"max":{"1min":121.40600000000001,"5min":125.824,"15min":125.824},"last":0.99399999999999999},{"interface":"front","average":{"1min":7.9930000000000003,"5min":6.9649999999999999,"15min":3.4710000000000001},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":121.53700000000001,"5min":125.761,"15min":125.761},"last":0.58999999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63900000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.623,"5min":0.873,"15min":0.873},"last":0.42299999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.67500000000000004,"15min":0.79700000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":11.093,"15min":11.093},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.442,"5min":0.61899999999999999,"15min":0.88800000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.69699999999999995,"5min":5.9329999999999998,"15min":13.083},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.68999999999999995,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.948,"5min":8.3520000000000003,"15min":9.6809999999999992},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.89200000000000002,"15min":1.0589999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":0.90900000000000003,"5min":11.791,"15min":15.461},"last":0.66900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":570097664,"data_stored":570097664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":840302592,"data_stored":840302592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2537095168,"data_stored":2537095168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1055850496,"data_stored":1055850496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2248151040,"data_stored":2248146968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3003518976,"data_stored":3003514904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":627576832,"data_stored":627572760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1518600192,"data_stored":1518600192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:06:29.372 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:06:29.372 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:29.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:29.475 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:29.502 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:29.780 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:29.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:29.840 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:06:29.884 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:29.904 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:29.914 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:29.964 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:29.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:30.069 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:30.098 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:30.150 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:30.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:30.244 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:30.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:30.400 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:30.893 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:30.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:31.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:31.017 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:06:31.042 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:31.085 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:31.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:31.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:30 smithi029 ceph-mon[223858]: pgmap v2165: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 293 op/s; 31706/87495 objects degraded (36.237%); 3731/87495 objects misplaced (4.264%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2910744745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:06:31.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:06:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:30 smithi029 ceph-mon[216903]: pgmap v2165: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 293 op/s; 31706/87495 objects degraded (36.237%); 3731/87495 objects misplaced (4.264%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2910744745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:06:31.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:06:31.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:31.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:30 smithi177 ceph-mon[195797]: pgmap v2165: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 293 op/s; 31706/87495 objects degraded (36.237%); 3731/87495 objects misplaced (4.264%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:31.211 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2910744745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:06:31.212 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:06:31.214 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:31.229 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:31.274 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:31.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:31.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:31.523 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:32.200 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:06:32.200 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: osdmap e670: 8 total, 8 up, 7 in 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.224 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 31706/87495 objects degraded (36.237%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: osdmap e670: 8 total, 8 up, 7 in 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 31706/87495 objects degraded (36.237%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:32.437 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:32.437 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.437 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:06:32.437 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: osdmap e670: 8 total, 8 up, 7 in 2023-10-09T05:06:32.437 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.437 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: failed to encode map e670 with expected crc 2023-10-09T05:06:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 31706/87495 objects degraded (36.237%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:32.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:06:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:06:32] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-10-09T05:06:33.129 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:06:33.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:33 smithi029 ceph-mon[223858]: pgmap v2167: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 31088/86745 objects degraded (35.838%); 3554/86745 objects misplaced (4.097%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:06:33.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:33 smithi029 ceph-mon[216903]: pgmap v2167: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 31088/86745 objects degraded (35.838%); 3554/86745 objects misplaced (4.097%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:06:33.513 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:33.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:33 smithi177 ceph-mon[195797]: pgmap v2167: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 31088/86745 objects degraded (35.838%); 3554/86745 objects misplaced (4.097%); 8.3 MiB/s, 133 objects/s recovering 2023-10-09T05:06:33.943 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:33.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:34.066 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:34.096 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:34.200 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:34.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:34.304 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:34.332 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:34.376 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:34.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:34.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:34.506 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:34.548 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:34.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:34.652 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:34.705 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:35.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:35 smithi029 ceph-mon[216903]: pgmap v2168: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 30663/84141 objects degraded (36.442%); 3364/84141 objects misplaced (3.998%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T05:06:35.125 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:35 smithi177 ceph-mon[195797]: pgmap v2168: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 30663/84141 objects degraded (36.442%); 3364/84141 objects misplaced (3.998%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T05:06:35.136 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:35.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:35.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:35.244 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:35.278 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:35.333 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:35.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:35.424 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:06:35.424 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:06:35.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:35 smithi029 ceph-mon[223858]: pgmap v2168: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 30663/84141 objects degraded (36.442%); 3364/84141 objects misplaced (3.998%); 8.2 MiB/s, 130 objects/s recovering 2023-10-09T05:06:35.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:35.468 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:35.630 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:35.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:35.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:35.736 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:35.772 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:35.829 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:35.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:35.901 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:06:35.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:35.960 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:36.208 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:06:36.244 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:06:36.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:36.267 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 30663/84141 objects degraded (36.442%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:36.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:36.321 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:06:36.341 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:06:36.369 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:36.387 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:06:36.397 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:36.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 30663/84141 objects degraded (36.442%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:36.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 30663/84141 objects degraded (36.442%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:36.430 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:06:36.446 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:36.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:36.501 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:06:36.541 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:06:36.553 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:36.580 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:06:36.585 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:36.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.601 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.602 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.602 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.602 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.602 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:06:36.639 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:36.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:36.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:36.729 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:36.746 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:36.776 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:36.874 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:36.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:36.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:36.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:37.034 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:37.053 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:06:37.053 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29934/82965 objects degraded (36.080%), 24 pgs degraded, 16 pgs undersized","count":40},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2933,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":670,"num_osds":8,"num_up_osds":8,"osd_up_since":1696827803,"num_in_osds":7,"osd_in_since":1696827750,"num_remapped_pgs":16},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":93},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":14},{"state_name":"active+recovery_wait+degraded","count":10},{"state_name":"active+recovering+undersized+remapped","count":2}],"num_pgs":119,"num_pools":5,"num_objects":27655,"data_bytes":1811355576,"bytes_used":18214334464,"bytes_avail":653750915072,"bytes_total":671965249536,"degraded_objects":29934,"degraded_total":82965,"degraded_ratio":0.3608027481468089,"misplaced_objects":3163,"misplaced_total":82965,"misplaced_ratio":0.038124510335683724,"recovering_objects_per_sec":131,"recovering_bytes_per_sec":8626807,"recovering_keys_per_sec":0,"num_objects_recovered":1317,"num_bytes_recovered":86310912,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":262},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":36,"modified":"2023-10-09T05:04:23.723180+0000","services":{}},"progress_events":{"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8039e601-7bd2-4b88-bccf-3b879c49d7d2":{"message":"Global Recovery Event (12m)\n [=====================.......] (remaining: 3m)","progress":0.78151261806488037,"add_to_ceph_s":true}}} 2023-10-09T05:06:37.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:37 smithi177 ceph-mon[195797]: pgmap v2169: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 29934/82965 objects degraded (36.080%); 3163/82965 objects misplaced (3.812%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:37.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2645560557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:37.196 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:37 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1047555852' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:06:37.380 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:37.390 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:37 smithi029 ceph-mon[216903]: pgmap v2169: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 29934/82965 objects degraded (36.080%); 3163/82965 objects misplaced (3.812%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:37.391 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2645560557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:37.391 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:37 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1047555852' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:06:37.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:37 smithi029 ceph-mon[223858]: pgmap v2169: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 29934/82965 objects degraded (36.080%); 3163/82965 objects misplaced (3.812%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:37.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2645560557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:37.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:37 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1047555852' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:06:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:37.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:37.493 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:37.521 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:37.562 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:37.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:37.665 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:37.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:37.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:37.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:37.854 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:37.880 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:38.091 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:38.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:38.199 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:38.227 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:38.296 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:38.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:38.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:38.402 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:38.430 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:38.632 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:06:38.632 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:06:38.632 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:06:38.632 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:06:38.883 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:38.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:38.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:39.092 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:39.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:39 smithi029 ceph-mon[216903]: pgmap v2170: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 29796/81648 objects degraded (36.493%); 3028/81648 objects misplaced (3.709%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:39.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:39 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:39 smithi029 ceph-mon[223858]: pgmap v2170: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 29796/81648 objects degraded (36.493%); 3028/81648 objects misplaced (3.709%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:39.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:39 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:39.346 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:39.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:39.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:39.453 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:39.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:39 smithi177 ceph-mon[195797]: pgmap v2170: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 264 op/s; 29796/81648 objects degraded (36.493%); 3028/81648 objects misplaced (3.709%); 8.2 MiB/s, 131 objects/s recovering 2023-10-09T05:06:39.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:39.479 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:39 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:39.482 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:39.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:06:39.585 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:39.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:39.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:39.691 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:39.720 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:39.778 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:39.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:39.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:39.888 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:39.922 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:39.970 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:39.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:40.027 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:40.077 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:40.107 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:40.169 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:40.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:40.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:40.273 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:40.306 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:40.464 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:40.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:40.506 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:40.549 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:06:40.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:06:40.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:40.730 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:40.943 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:40.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:41.052 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:41.180 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:41.224 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[216903]: pgmap v2171: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 29296/79458 objects degraded (36.870%); 3028/79458 objects misplaced (3.811%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:06:41.224 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:06:41.224 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:06:41.224 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:41.225 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 29296/79458 objects degraded (36.870%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:41.225 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[223858]: pgmap v2171: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 29296/79458 objects degraded (36.870%); 3028/79458 objects misplaced (3.811%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:06:41.225 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:06:41.225 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:06:41.225 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:41.225 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 29296/79458 objects degraded (36.870%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:41.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:06:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:41 smithi177 ceph-mon[195797]: pgmap v2171: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 29296/79458 objects degraded (36.870%); 3028/79458 objects misplaced (3.811%); 8.3 MiB/s, 132 objects/s recovering 2023-10-09T05:06:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:06:41.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:06:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:06:41.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 29296/79458 objects degraded (36.870%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:41.644 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:41.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:41.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:41.765 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:41.795 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:06:41.795 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:06:41.795 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:06:41.795 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:06:41.795 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:06:41.803 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:41.854 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:41.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:41.958 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:41.992 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:42.045 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:42.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:42.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:42.173 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:42.215 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:42.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:42.319 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:42.346 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:42.715 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:06:42.911 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:42.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:42.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:43.021 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:43.098 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:43.118 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:43.155 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:06:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:06:42] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-10-09T05:06:43.161 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:43.333 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:43.377 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:43.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:43.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:43 smithi029 ceph-mon[216903]: pgmap v2172: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 247 op/s; 28672/78654 objects degraded (36.453%); 2833/78654 objects misplaced (3.602%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:06:43.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:43 smithi029 ceph-mon[223858]: pgmap v2172: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 247 op/s; 28672/78654 objects degraded (36.453%); 2833/78654 objects misplaced (3.602%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:06:43.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:43.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:43 smithi177 ceph-mon[195797]: pgmap v2172: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 247 op/s; 28672/78654 objects degraded (36.453%); 2833/78654 objects misplaced (3.602%); 7.6 MiB/s, 122 objects/s recovering 2023-10-09T05:06:43.581 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:43.870 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:43.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:43.982 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:44.021 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:44.079 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:44.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:44.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:44.183 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:44.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:44 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:44.282 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:44.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:44 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:44.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:44 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:44.951 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:91970 14.4 14:21827fbb:::benchmark_data_smithi029_550670_object14359:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e576)", 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:43.424517+0000", 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 301.52357412499998, 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6851607230000001, 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.952 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 91970 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424517+0000", 2023-10-09T05:06:44.953 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424517+0000", 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424516+0000", 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:44.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424523+0000", 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4070000000000004e-06 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424535+0000", 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1990999999999999e-05 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424543+0000", 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7840000000000004e-06 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.439331+0000", 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014787692999999999 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:44.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.439342+0000", 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0967e-05 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:50.107237+0000", 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6678946870000004 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:50.109678+0000", 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024416030000000001 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e576,539 14:33039425:::benchmark_data_smithi029_550670_object2899:head,512'5047)", 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 300.509581486, 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0750518820000003, 2023-10-09T05:06:44.958 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438513+0000", 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7309999999999999e-06 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438521+0000", 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2729999999999997e-06 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438527+0000", 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5690000000000001e-06 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513501+0000", 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0749741339999996 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513511+0000", 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7140000000000005e-06 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513562+0000", 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0744000000000001e-05 2023-10-09T05:06:44.962 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e577,539 14:32986eef:::benchmark_data_smithi029_550670_object2913:head,512'5048)", 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 300.50931615000002, 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0749084590000004, 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.964 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438778+0000", 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9220000000000001e-06 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438799+0000", 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0863999999999999e-05 2023-10-09T05:06:44.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438804+0000", 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.553e-06 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513642+0000", 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0748375110000001 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513648+0000", 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4370000000000004e-06 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513684+0000", 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5435000000000002e-05 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e578,539 14:31dc1b0e:::benchmark_data_smithi029_550670_object2955:head,514'5049)", 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:45.494277+0000", 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 299.45381492799999, 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0194656899999996, 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494277+0000", 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494277+0000", 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494276+0000", 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494283+0000", 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3590000000000002e-06 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494299+0000", 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5611e-05 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494307+0000", 2023-10-09T05:06:44.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3650000000000004e-06 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513707+0000", 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0193997130000003 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:44.971 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513712+0000", 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7350000000000001e-06 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513742+0000", 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9881000000000001e-05 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.972 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e579,539 14:3a70dd65:::benchmark_data_smithi029_550670_object2985:head,523'5050)", 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:46.524601+0000", 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 298.42349038700002, 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.989196808, 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524601+0000", 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524601+0000", 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524600+0000", 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524610+0000", 2023-10-09T05:06:44.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7880000000000001e-06 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524623+0000", 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2352e-05 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524645+0000", 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2943e-05 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513763+0000", 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9891175649999999 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513770+0000", 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8299999999999998e-06 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513798+0000", 2023-10-09T05:06:44.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8070000000000001e-05 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e580,539 14:35c61ae4:::benchmark_data_smithi029_550670_object2997:head,523'5051)", 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:48.499945+0000", 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 296.44814671699999, 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0139077599999999, 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:06:44.977 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499945+0000", 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499945+0000", 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499944+0000", 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499953+0000", 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2609999999999999e-06 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499967+0000", 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4497e-05 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499976+0000", 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4300000000000006e-06 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513821+0000", 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.01384556 2023-10-09T05:06:44.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513829+0000", 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.25e-06 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513853+0000", 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3873e-05 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.981 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92055 14.1c 14:3e5947b2:::benchmark_data_smithi029_550670_object14444:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473836+0000", 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47425579700001, 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7604524719999999, 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:44.982 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92055 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473836+0000", 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473836+0000", 2023-10-09T05:06:44.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473834+0000", 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473844+0000", 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6069999999999997e-06 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473859+0000", 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5299e-05 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473867+0000", 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7039999999999994e-06 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473922+0000", 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5556000000000001e-05 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473930+0000", 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0180000000000001e-06 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.202764+0000", 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7288333829999996 2023-10-09T05:06:44.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.206360+0000", 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003595844 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.230795+0000", 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024435351000000001 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.234288+0000", 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003493229 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92069 14.c 14:365687b7:::benchmark_data_smithi029_550670_object14458:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47421248399999, 2023-10-09T05:06:44.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6586282880000001, 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92069 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:06:44.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473882+0000", 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1420000000000002e-06 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473893+0000", 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1647999999999999e-05 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473897+0000", 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6090000000000001e-06 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473947+0000", 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9599999999999999e-05 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:44.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473955+0000", 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0809999999999994e-06 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.078473+0000", 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6045187260000002 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.080872+0000", 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023986160000000001 2023-10-09T05:06:44.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.130299+0000", 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.049426781000000003 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.132507+0000", 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022084539999999999 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.994 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92089 14.1c 14:3c0d3de8:::benchmark_data_smithi029_550670_object14478:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.474180158, 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1017356180000002, 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:44.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92089 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:06:44.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473914+0000", 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3110000000000001e-06 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473921+0000", 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5810000000000002e-06 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473929+0000", 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5390000000000007e-06 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473954+0000", 2023-10-09T05:06:44.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4389999999999999e-05 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473958+0000", 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0969999999999999e-06 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.206952+0000", 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7329936870000004 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:44.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.209989+0000", 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00303732 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.243365+0000", 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.033375984999999997 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.246468+0000", 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031034909999999999 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.543790+0000", 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29732161899999998 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.547548+0000", 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037581609999999999 2023-10-09T05:06:45.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.554854+0000", 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073057 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.560593+0000", 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057395099999999998 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.564110+0000", 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035166619999999998 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.565379+0000", 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00126932 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.567433+0000", 2023-10-09T05:06:45.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020534860000000002 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.575647+0000", 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082141650000000007 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92094 14.c 14:37d5fb26:::benchmark_data_smithi029_550670_object14483:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473941+0000", 2023-10-09T05:06:45.004 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.474150944, 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9777295480000001, 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92094 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473941+0000", 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473941+0000", 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473940+0000", 2023-10-09T05:06:45.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473943+0000", 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.481e-06 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473949+0000", 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.964e-06 2023-10-09T05:06:45.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473953+0000", 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5360000000000003e-06 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473980+0000", 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6869e-05 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473986+0000", 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1099999999999999e-06 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.101391+0000", 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6274046120000003 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.104333+0000", 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002942269 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.144237+0000", 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.039903921000000002 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.147577+0000", 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033400230000000001 2023-10-09T05:06:45.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.377144+0000", 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.22956679399999999 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.379900+0000", 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027565179999999999 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.385303+0000", 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054023279999999996 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.441018+0000", 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.055715573999999997 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.012 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.442356+0000", 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013380620000000001 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.443438+0000", 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010813909999999999 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.445000+0000", 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015619189999999999 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.451670+0000", 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066705430000000001 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92107 14.1c 14:3ef3cb94:::benchmark_data_smithi029_550670_object14496:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:06:45.014 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.474134144, 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3570599840000002, 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92107 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473960+0000", 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.818e-06 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473966+0000", 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1890000000000004e-06 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473970+0000", 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.827e-06 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473985+0000", 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4782e-05 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473988+0000", 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1109999999999999e-06 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.246758+0000", 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7727702189999999 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.249559+0000", 2023-10-09T05:06:45.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002801347 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.266902+0000", 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017342453000000001 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.269999+0000", 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030973429999999998 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.635209+0000", 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36521015099999998 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.637768+0000", 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002558383 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.643047+0000", 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052789719999999998 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.673755+0000", 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030708092999999999 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.674062+0000", 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030731299999999998 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.823732+0000", 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14966990299999999 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.825231+0000", 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014994979999999999 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.831017+0000", 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005785887 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92161 14.c 14:309adb2f:::benchmark_data_smithi029_550670_object14550:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473974+0000", 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47411775200001, 2023-10-09T05:06:45.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9877311280000001, 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92161 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473974+0000", 2023-10-09T05:06:45.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473974+0000", 2023-10-09T05:06:45.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473973+0000", 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473976+0000", 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2730000000000001e-06 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473982+0000", 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8470000000000001e-06 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473985+0000", 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9009999999999996e-06 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.029 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474021+0000", 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5642999999999997e-05 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474031+0000", 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0321e-05 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.104642+0000", 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6306100810000004 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.108462+0000", 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038201300000000001 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.160022+0000", 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.051560828000000003 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.163297+0000", 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032748669999999999 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.403246+0000", 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.23994837599999999 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.406013+0000", 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027670709999999999 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.411554+0000", 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055412739999999997 2023-10-09T05:06:45.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.441330+0000", 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029776054 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.452572+0000", 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011241660000000001 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.453779+0000", 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001206801 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.455288+0000", 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001509203 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.461705+0000", 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064171209999999996 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92165 14.c 14:3175512b:::benchmark_data_smithi029_550670_object14554:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473998+0000", 2023-10-09T05:06:45.036 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47409390000001, 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9992911180000004, 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92165 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.037 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473998+0000", 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473998+0000", 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.038 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473997+0000", 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474000+0000", 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3920000000000001e-06 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474006+0000", 2023-10-09T05:06:45.039 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9970000000000004e-06 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474010+0000", 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8369999999999999e-06 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474046+0000", 2023-10-09T05:06:45.040 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6656000000000003e-05 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474049+0000", 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.207e-06 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.163697+0000", 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6896473920000004 2023-10-09T05:06:45.041 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.166772+0000", 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030751329999999999 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.183820+0000", 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017048447000000001 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.042 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.187210+0000", 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033900940000000002 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.428328+0000", 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.241117529 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.043 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.431210+0000", 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028819100000000001 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.437276+0000", 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060659659999999999 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.452887+0000", 2023-10-09T05:06:45.044 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015610939000000001 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.462618+0000", 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097308080000000005 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.463787+0000", 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001168955 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.045 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.465742+0000", 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001955156 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.473289+0000", 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0075470160000000001 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.046 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92174 14.c 14:376305e3:::benchmark_data_smithi029_550670_object14563:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47406727500001, 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0689501889999997, 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.047 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92174 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.048 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474026+0000", 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3549999999999999e-06 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.049 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474032+0000", 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8200000000000002e-06 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474036+0000", 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4240000000000002e-06 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.050 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474069+0000", 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3745000000000003e-05 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474075+0000", 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6489999999999996e-06 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.051 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.167560+0000", 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6934853460000001 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.170840+0000", 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032801850000000001 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.052 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.198844+0000", 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028004037999999998 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.202080+0000", 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032359670000000002 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.503990+0000", 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30190954800000003 2023-10-09T05:06:45.053 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.506993+0000", 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030031099999999998 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.512309+0000", 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053163180000000004 2023-10-09T05:06:45.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.534225+0000", 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021915537999999998 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.535117+0000", 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000892 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.535405+0000", 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028789000000000001 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.536826+0000", 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014215880000000001 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.542974+0000", 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061479940000000004 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92180 14.1c 14:3ca86941:::benchmark_data_smithi029_550670_object14569:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:06:45.057 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.474041212, 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3662740299999996, 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92180 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474053+0000", 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.768e-06 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474059+0000", 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4579999999999998e-06 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474065+0000", 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6949999999999997e-06 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.061 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474084+0000", 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9241000000000001e-05 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474087+0000", 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0939999999999999e-06 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.249878+0000", 2023-10-09T05:06:45.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7757908350000005 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.252725+0000", 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028465470000000001 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.281495+0000", 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028770463999999999 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.284552+0000", 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030568209999999999 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.660798+0000", 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37624634699999998 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.663753+0000", 2023-10-09T05:06:45.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002954852 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.669646+0000", 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058927800000000002 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.699397+0000", 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029751418000000002 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.831873+0000", 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.132475605 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.833045+0000", 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001172137 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.834456+0000", 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014105350000000001 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.840324+0000", 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058687949999999996 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92182 14.1c 14:3fd3171a:::benchmark_data_smithi029_550670_object14571:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.067 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47401952300001, 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3752296690000003, 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92182 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474074+0000", 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2910000000000002e-06 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474079+0000", 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4280000000000004e-06 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474083+0000", 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6009999999999999e-06 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474100+0000", 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7336000000000001e-05 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474103+0000", 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.853e-06 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.253164+0000", 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7790607170000001 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.256107+0000", 2023-10-09T05:06:45.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029431840000000002 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.295790+0000", 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.039682679999999998 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.298786+0000", 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029965109999999999 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.687348+0000", 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38856141199999999 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.690035+0000", 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026874889999999999 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.695938+0000", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059025290000000001 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.722603+0000", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026665728999999999 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.841224+0000", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.118620266 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.842115+0000", 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00089085200000000003 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.843556+0000", 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014413410000000001 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.849302+0000", 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057457899999999998 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.076 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92208 14.1c 14:3948b646:::benchmark_data_smithi029_550670_object14597:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47400320400001, 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3847670389999998, 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92208 2023-10-09T05:06:45.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474094+0000", 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8320000000000002e-06 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474100+0000", 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5479999999999998e-06 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474104+0000", 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6770000000000001e-06 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474124+0000", 2023-10-09T05:06:45.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0072e-05 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474127+0000", 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.869e-06 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.299950+0000", 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8258230119999999 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.302809+0000", 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028591889999999998 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.320588+0000", 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017778321999999999 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.323603+0000", 2023-10-09T05:06:45.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030157880000000001 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.711636+0000", 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.388033138 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.714179+0000", 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002542248 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.719284+0000", 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051056419999999996 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.746816+0000", 2023-10-09T05:06:45.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.027531282000000001 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.850167+0000", 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.103351006 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.851177+0000", 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010101140000000001 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.852656+0000", 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001479314 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.858855+0000", 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006199296 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92211 14.1c 14:3ce01d9d:::benchmark_data_smithi029_550670_object14600:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474112+0000", 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.473979719, 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4040441320000001, 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.089 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.089 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92211 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474112+0000", 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474112+0000", 2023-10-09T05:06:45.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474111+0000", 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474114+0000", 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2230000000000001e-06 2023-10-09T05:06:45.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474119+0000", 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2970000000000003e-06 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474122+0000", 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4419999999999998e-06 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.092 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474142+0000", 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9797e-05 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9299999999999999e-06 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.303237+0000", 2023-10-09T05:06:45.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8290917640000002 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.306169+0000", 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029325200000000001 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.349452+0000", 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.043282893000000003 2023-10-09T05:06:45.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.352483+0000", 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003030516 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.759382+0000", 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40689936700000001 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.761976+0000", 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025939309999999998 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.767278+0000", 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005301516 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.798076+0000", 2023-10-09T05:06:45.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030798012999999999 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.869441+0000", 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.071365108999999996 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.870336+0000", 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00089563100000000003 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.871911+0000", 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015744979999999999 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.878156+0000", 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062450029999999998 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.098 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92219 14.1c 14:3b1c3ae5:::benchmark_data_smithi029_550670_object14608:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474128+0000", 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.47396377600001, 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.394299921, 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.099 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92219 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474128+0000", 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474128+0000", 2023-10-09T05:06:45.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474127+0000", 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474129+0000", 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0320000000000002e-06 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474136+0000", 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1340000000000001e-06 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474139+0000", 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.332e-06 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.102 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474168+0000", 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9552000000000001e-05 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474172+0000", 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7910000000000002e-06 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.307293+0000", 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8331210049999997 2023-10-09T05:06:45.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.310098+0000", 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002804993 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.334589+0000", 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024490607000000001 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.337720+0000", 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031307420000000002 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.735191+0000", 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39747148799999998 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.737741+0000", 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025500929999999998 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.742871+0000", 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051294360000000002 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.770928+0000", 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028057289999999999 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.859794+0000", 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.088865880999999994 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.860794+0000", 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001000623 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.862284+0000", 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014899030000000001 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.868428+0000", 2023-10-09T05:06:45.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006143328 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92256 14.c 14:30b496db:::benchmark_data_smithi029_550670_object14645:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 221.473946668, 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1024558200000003, 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:06:45.109 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92256 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:06:45.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474147+0000", 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3259999999999998e-06 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474152+0000", 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3909999999999997e-06 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474156+0000", 2023-10-09T05:06:45.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0709999999999996e-06 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474173+0000", 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.628e-05 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474176+0000", 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9330000000000001e-06 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.202517+0000", 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7283419310000001 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.205529+0000", 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030117899999999999 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.234712+0000", 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029183121999999999 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.238172+0000", 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003459419 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.543400+0000", 2023-10-09T05:06:45.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30522812399999999 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.547146+0000", 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003746217 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.554862+0000", 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0077156530000000003 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.560664+0000", 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058017729999999997 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.561083+0000", 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00041973100000000001 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:06:45.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.565647+0000", 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045636569999999996 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.567799+0000", 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021523919999999999 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.576601+0000", 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0088013180000000007 2023-10-09T05:06:45.118 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.119 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.119 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.119 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:06:45.119 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:06:45.119 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:06:45.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:45 smithi177 ceph-mon[195797]: pgmap v2173: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 28175/75519 objects degraded (37.308%); 2644/75519 objects misplaced (3.501%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:06:45.473 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:45 smithi029 ceph-mon[223858]: pgmap v2173: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 28175/75519 objects degraded (37.308%); 2644/75519 objects misplaced (3.501%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:06:45.473 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:45 smithi029 ceph-mon[216903]: pgmap v2173: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 311 op/s; 28175/75519 objects degraded (37.308%); 2644/75519 objects misplaced (3.501%); 8.8 MiB/s, 141 objects/s recovering 2023-10-09T05:06:45.778 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:45.781 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:06:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:45.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:45.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:46.012 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:46.153 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:46.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:46.179 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:46.271 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:46.300 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:46.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:46.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:46.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 28175/75519 objects degraded (37.308%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:46.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 28175/75519 objects degraded (37.308%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:46.507 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:46.533 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 28175/75519 objects degraded (37.308%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:46.537 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:46.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:46.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:46.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:46.776 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:46.818 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:46.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:46.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:46.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:46.951 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:46.995 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:46.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:46.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:47.100 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:47.202 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:47.223 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:47 smithi177 ceph-mon[195797]: pgmap v2174: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 27272/74076 objects degraded (36.816%); 2452/74076 objects misplaced (3.310%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T05:06:47.223 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/376213508' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:47.330 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:06:47.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:47 smithi029 ceph-mon[216903]: pgmap v2174: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 27272/74076 objects degraded (36.816%); 2452/74076 objects misplaced (3.310%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T05:06:47.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/376213508' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:47.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:47 smithi029 ceph-mon[223858]: pgmap v2174: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 279 op/s; 27272/74076 objects degraded (36.816%); 2452/74076 objects misplaced (3.310%); 9.3 MiB/s, 148 objects/s recovering 2023-10-09T05:06:47.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/376213508' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:06:47.898 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:47.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:47.981 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:06:47.981 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:06:47.981 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:06:47.981 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:06:48.004 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:48.040 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:48 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:48 smithi177 ceph-mon[195797]: failed to encode map e671 with expected crc 2023-10-09T05:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:48 smithi177 ceph-mon[195797]: failed to encode map e671 with expected crc 2023-10-09T05:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/376213508' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:06:48.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:48 smithi177 ceph-mon[195797]: osdmap e671: 8 total, 8 up, 7 in 2023-10-09T05:06:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:06:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[223858]: failed to encode map e671 with expected crc 2023-10-09T05:06:48.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[223858]: failed to encode map e671 with expected crc 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/376213508' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[223858]: osdmap e671: 8 total, 8 up, 7 in 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[216903]: failed to encode map e671 with expected crc 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[216903]: failed to encode map e671 with expected crc 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/376213508' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:06:48.423 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:48 smithi029 ceph-mon[216903]: osdmap e671: 8 total, 8 up, 7 in 2023-10-09T05:06:48.641 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:48.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:48.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:48.786 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:06:48.790 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:48.853 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:48.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:48.966 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:06:48.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:48.984 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:06:49.063 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:49.190 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:49.276 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:49 smithi029 ceph-mon[216903]: pgmap v2176: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 27134/72642 objects degraded (37.353%); 2210/72642 objects misplaced (3.042%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T05:06:49.547 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:49 smithi177 ceph-mon[195797]: pgmap v2176: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 27134/72642 objects degraded (37.353%); 2210/72642 objects misplaced (3.042%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T05:06:49.623 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:49.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:49.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:49.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:49 smithi029 ceph-mon[223858]: pgmap v2176: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 27134/72642 objects degraded (37.353%); 2210/72642 objects misplaced (3.042%); 9.4 MiB/s, 150 objects/s recovering 2023-10-09T05:06:49.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:49.759 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:49.804 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:49.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:49.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:49.908 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:49.936 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:50.541 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:50.545 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:50 smithi029 ceph-mon[223858]: from='client.47609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:50.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:50 smithi029 ceph-mon[216903]: from='client.47609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:50.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:50.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:50.647 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:06:50.675 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:50 smithi177 ceph-mon[195797]: from='client.47609 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:06:50.678 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2176,"stamp":"2023-10-09T05:06:47.875707+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1585846200,"num_objects":24214,"num_object_clones":0,"num_object_copies":72642,"num_objects_missing_on_primary":8164,"num_objects_missing":8164,"num_objects_degraded":27134,"num_objects_misplaced":2210,"num_objects_unfound":0,"num_objects_dirty":24214,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":131328,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83785,"num_bytes_recovered":5082286734,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":211549,"ondisk_log_size":211549,"up":357,"acting":341,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17390060,"kb_used_data":8989684,"kb_used_omap":34,"kb_used_meta":8400157,"kb_avail":638826004,"statfs":{"total":671965249536,"available":654157828096,"internally_reserved":0,"allocated":9205436416,"data_stored":9189159634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8601761162},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,2,0,3,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-196739072,"num_objects":-3002,"num_object_clones":0,"num_object_copies":-9006,"num_objects_missing_on_primary":-1128,"num_objects_missing":-1128,"num_objects_degraded":-2662,"num_objects_misplaced":-818,"num_objects_unfound":0,"num_objects_dirty":-3002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3002,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1507,"num_bytes_recovered":98762752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008220"},"pg_stats":[{"pgid":"14.23","version":"670'6029","reported_seq":12657,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:43.686230+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:06:43.686230+0000","last_peered":"2023-10-09T05:06:43.686230+0000","last_clean":"2023-10-09T05:06:43.686230+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:06:43.686230+0000","last_undegraded":"2023-10-09T05:06:43.686230+0000","last_fullsized":"2023-10-09T05:06:43.686230+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6029,"log_dups_size":0,"ondisk_log_size":6029,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25362432,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2045,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"670'5965","reported_seq":13516,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:43.606062+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:06:43.606062+0000","last_peered":"2023-10-09T05:06:43.606062+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:06:43.606062+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5965,"log_dups_size":0,"ondisk_log_size":5965,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24739840,"num_objects":378,"num_object_clones":0,"num_object_copies":1134,"num_objects_missing_on_primary":351,"num_objects_missing":351,"num_objects_degraded":898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2030,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1105,"num_bytes_recovered":70172672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":43},{"shards":"3,4,5,6,7","objects":88},{"shards":"3,4,5,7","objects":1},{"shards":"4,5,7","objects":193}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"670'6050","reported_seq":16751,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:43.281940+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:06:43.281940+0000","last_peered":"2023-10-09T05:06:43.281940+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:06:43.281940+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:06:43.281940+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23789568,"num_objects":363,"num_object_clones":0,"num_object_copies":1089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2067,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":363}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"670'6041","reported_seq":8293,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:47.071879+0000","last_change":"2023-10-09T05:05:26.963137+0000","last_active":"2023-10-09T05:06:47.071879+0000","last_peered":"2023-10-09T05:06:47.071879+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:06:47.071879+0000","last_undegraded":"2023-10-09T05:05:26.962565+0000","last_fullsized":"2023-10-09T05:06:47.071879+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25116672,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1186,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2048,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":606,"num_bytes_recovered":39731200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":334},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"670'6623","reported_seq":20188,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223986+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:06:47.223986+0000","last_peered":"2023-10-09T05:06:47.223986+0000","last_clean":"2023-10-09T05:06:47.223986+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:06:47.223986+0000","last_undegraded":"2023-10-09T05:06:47.223986+0000","last_fullsized":"2023-10-09T05:06:47.223986+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6623,"log_dups_size":0,"ondisk_log_size":6623,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52789248,"num_objects":806,"num_object_clones":0,"num_object_copies":2418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4082,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":172,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073990+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:06:17.073990+0000","last_peered":"2023-10-09T05:06:17.073990+0000","last_clean":"2023-10-09T05:06:17.073990+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:06:17.073990+0000","last_undegraded":"2023-10-09T05:06:17.073990+0000","last_fullsized":"2023-10-09T05:06:17.073990+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":317,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079014+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:06:17.079014+0000","last_peered":"2023-10-09T05:06:17.079014+0000","last_clean":"2023-10-09T05:06:17.079014+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:06:17.079014+0000","last_undegraded":"2023-10-09T05:06:17.079014+0000","last_fullsized":"2023-10-09T05:06:17.079014+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"670'6589","reported_seq":9819,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:43.328559+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:06:43.328559+0000","last_peered":"2023-10-09T05:06:43.328559+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:06:43.328559+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:06:43.328559+0000","mapping_epoch":638,"log_start":"444'717","ondisk_log_start":"444'717","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5872,"log_dups_size":0,"ondisk_log_size":5872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53116928,"num_objects":811,"num_object_clones":0,"num_object_copies":2433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4050,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":811}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.994408+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:06:30.994408+0000","last_peered":"2023-10-09T05:06:30.994408+0000","last_clean":"2023-10-09T05:06:30.994408+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:06:30.994408+0000","last_undegraded":"2023-10-09T05:06:30.994408+0000","last_fullsized":"2023-10-09T05:06:30.994408+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":183,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079384+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:06:17.079384+0000","last_peered":"2023-10-09T05:06:17.079384+0000","last_clean":"2023-10-09T05:06:17.079384+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:06:17.079384+0000","last_undegraded":"2023-10-09T05:06:17.079384+0000","last_fullsized":"2023-10-09T05:06:17.079384+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"670'6637","reported_seq":18298,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:44.083374+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:06:44.083374+0000","last_peered":"2023-10-09T05:06:44.083374+0000","last_clean":"2023-10-09T05:06:44.083374+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:06:44.083374+0000","last_undegraded":"2023-10-09T05:06:44.083374+0000","last_fullsized":"2023-10-09T05:06:44.083374+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6391,"log_dups_size":0,"ondisk_log_size":6391,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50069504,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4097,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.080934+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:06:17.080934+0000","last_peered":"2023-10-09T05:06:17.080934+0000","last_clean":"2023-10-09T05:06:17.080934+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:06:17.080934+0000","last_undegraded":"2023-10-09T05:06:17.080934+0000","last_fullsized":"2023-10-09T05:06:17.080934+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079606+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:06:17.079606+0000","last_peered":"2023-10-09T05:06:17.079606+0000","last_clean":"2023-10-09T05:06:17.079606+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:06:17.079606+0000","last_undegraded":"2023-10-09T05:06:17.079606+0000","last_fullsized":"2023-10-09T05:06:17.079606+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"670'6865","reported_seq":24224,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.173652+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:06:47.173652+0000","last_peered":"2023-10-09T05:06:47.173652+0000","last_clean":"2023-10-09T05:06:47.173652+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:06:47.173652+0000","last_undegraded":"2023-10-09T05:06:47.173652+0000","last_fullsized":"2023-10-09T05:06:47.173652+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4044,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":45,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.239988+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:06:31.239988+0000","last_peered":"2023-10-09T05:06:31.239988+0000","last_clean":"2023-10-09T05:06:31.239988+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:06:31.239988+0000","last_undegraded":"2023-10-09T05:06:31.239988+0000","last_fullsized":"2023-10-09T05:06:31.239988+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":291,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.080848+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:06:17.080848+0000","last_peered":"2023-10-09T05:06:17.080848+0000","last_clean":"2023-10-09T05:06:17.080848+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:06:17.080848+0000","last_undegraded":"2023-10-09T05:06:17.080848+0000","last_fullsized":"2023-10-09T05:06:17.080848+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"670'7164","reported_seq":16101,"reported_epoch":670,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:06:44.878256+0000","last_change":"2023-10-09T05:05:26.975807+0000","last_active":"2023-10-09T05:06:44.878256+0000","last_peered":"2023-10-09T05:06:44.878256+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:06:44.878256+0000","last_undegraded":"2023-10-09T05:06:44.878256+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50855936,"num_objects":776,"num_object_clones":0,"num_object_copies":2328,"num_objects_missing_on_primary":1054,"num_objects_missing":1054,"num_objects_degraded":0,"num_objects_misplaced":1954,"num_objects_unfound":0,"num_objects_dirty":776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4204,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2100,"num_bytes_recovered":137560064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":106},{"shards":"0,2,4,7","objects":1},{"shards":"2,4,7","objects":614},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":59,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.239641+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:06:31.239641+0000","last_peered":"2023-10-09T05:06:31.239641+0000","last_clean":"2023-10-09T05:06:31.239641+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:06:31.239641+0000","last_undegraded":"2023-10-09T05:06:31.239641+0000","last_fullsized":"2023-10-09T05:06:31.239641+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":281,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.087707+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:06:17.087707+0000","last_peered":"2023-10-09T05:06:17.087707+0000","last_clean":"2023-10-09T05:06:17.087707+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:06:17.087707+0000","last_undegraded":"2023-10-09T05:06:17.087707+0000","last_fullsized":"2023-10-09T05:06:17.087707+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"671'6839","reported_seq":16091,"reported_epoch":671,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:06:47.780666+0000","last_change":"2023-10-09T05:05:00.137617+0000","last_active":"2023-10-09T05:06:47.780666+0000","last_peered":"2023-10-09T05:06:47.780666+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:06:47.780666+0000","last_undegraded":"2023-10-09T05:06:47.780666+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5303,"log_dups_size":0,"ondisk_log_size":5303,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50069504,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":132,"num_objects_missing":132,"num_objects_degraded":0,"num_objects_misplaced":256,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4230,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2342,"num_bytes_recovered":153485312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":5},{"shards":"1,2,3,5","objects":1},{"shards":"1,3","objects":633},{"shards":"2,3,5","objects":125}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":344,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:30.995908+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:06:30.995908+0000","last_peered":"2023-10-09T05:06:30.995908+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:06:30.995908+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:06:30.995908+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":95,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.996590+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:06:30.996590+0000","last_peered":"2023-10-09T05:06:30.996590+0000","last_clean":"2023-10-09T05:06:30.996590+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:06:30.996590+0000","last_undegraded":"2023-10-09T05:06:30.996590+0000","last_fullsized":"2023-10-09T05:06:30.996590+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"671'6666","reported_seq":18743,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223790+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:06:47.223790+0000","last_peered":"2023-10-09T05:06:47.223790+0000","last_clean":"2023-10-09T05:06:47.223790+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:06:47.223790+0000","last_undegraded":"2023-10-09T05:06:47.223790+0000","last_fullsized":"2023-10-09T05:06:47.223790+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6666,"log_dups_size":0,"ondisk_log_size":6666,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49905664,"num_objects":762,"num_object_clones":0,"num_object_copies":2286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4128,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":125,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074919+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:06:17.074919+0000","last_peered":"2023-10-09T05:06:17.074919+0000","last_clean":"2023-10-09T05:06:17.074919+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:06:17.074919+0000","last_undegraded":"2023-10-09T05:06:17.074919+0000","last_fullsized":"2023-10-09T05:06:17.074919+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":140,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074610+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:06:17.074610+0000","last_peered":"2023-10-09T05:06:17.074610+0000","last_clean":"2023-10-09T05:06:17.074610+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:06:17.074610+0000","last_undegraded":"2023-10-09T05:06:17.074610+0000","last_fullsized":"2023-10-09T05:06:17.074610+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"670'6859","reported_seq":19903,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:43.526913+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:06:43.526913+0000","last_peered":"2023-10-09T05:06:43.526913+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:06:43.526913+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51937280,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1317,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4000,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3538,"num_bytes_recovered":227278848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":289}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":146,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074537+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:06:17.074537+0000","last_peered":"2023-10-09T05:06:17.074537+0000","last_clean":"2023-10-09T05:06:17.074537+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:06:17.074537+0000","last_undegraded":"2023-10-09T05:06:17.074537+0000","last_fullsized":"2023-10-09T05:06:17.074537+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.003185+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:06:31.003185+0000","last_peered":"2023-10-09T05:06:31.003185+0000","last_clean":"2023-10-09T05:06:31.003185+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:06:31.003185+0000","last_undegraded":"2023-10-09T05:06:31.003185+0000","last_fullsized":"2023-10-09T05:06:31.003185+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"671'6953","reported_seq":14260,"reported_epoch":671,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:47.803089+0000","last_change":"2023-10-09T05:03:57.525846+0000","last_active":"2023-10-09T05:06:47.803089+0000","last_peered":"2023-10-09T05:06:47.803089+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:06:47.803089+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42926080,"num_objects":655,"num_object_clones":0,"num_object_copies":1965,"num_objects_missing_on_primary":757,"num_objects_missing":757,"num_objects_degraded":1491,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4124,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2056,"num_bytes_recovered":134709248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,5,7","objects":2},{"shards":"1,7","objects":-38},{"shards":"2,5,7","objects":316},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":174,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.996106+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:06:30.996106+0000","last_peered":"2023-10-09T05:06:30.996106+0000","last_clean":"2023-10-09T05:06:30.996106+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:06:30.996106+0000","last_undegraded":"2023-10-09T05:06:30.996106+0000","last_fullsized":"2023-10-09T05:06:30.996106+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":49,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.239337+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:06:31.239337+0000","last_peered":"2023-10-09T05:06:31.239337+0000","last_clean":"2023-10-09T05:06:31.239337+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:06:31.239337+0000","last_undegraded":"2023-10-09T05:06:31.239337+0000","last_fullsized":"2023-10-09T05:06:31.239337+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"670'6977","reported_seq":15698,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.013289+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:06:44.013289+0000","last_peered":"2023-10-09T05:06:44.013289+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:06:44.013289+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5214,"log_dups_size":0,"ondisk_log_size":5214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52232192,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2178,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4078,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2367,"num_bytes_recovered":142147584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":715},{"shards":"4,6","objects":34}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073959+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:06:17.073959+0000","last_peered":"2023-10-09T05:06:17.073959+0000","last_clean":"2023-10-09T05:06:17.073959+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:06:17.073959+0000","last_undegraded":"2023-10-09T05:06:17.073959+0000","last_fullsized":"2023-10-09T05:06:17.073959+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":308,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.012765+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:06:31.012765+0000","last_peered":"2023-10-09T05:06:31.012765+0000","last_clean":"2023-10-09T05:06:31.012765+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:06:31.012765+0000","last_undegraded":"2023-10-09T05:06:31.012765+0000","last_fullsized":"2023-10-09T05:06:31.012765+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"670'6914","reported_seq":13796,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:43.551121+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:06:43.551121+0000","last_peered":"2023-10-09T05:06:43.551121+0000","last_clean":"2023-10-09T05:06:43.551121+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:06:43.551121+0000","last_undegraded":"2023-10-09T05:06:43.551121+0000","last_fullsized":"2023-10-09T05:06:43.551121+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4621,"log_dups_size":0,"ondisk_log_size":4621,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49250316,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4052,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":304,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076863+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:06:17.076863+0000","last_peered":"2023-10-09T05:06:17.076863+0000","last_clean":"2023-10-09T05:06:17.076863+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:06:17.076863+0000","last_undegraded":"2023-10-09T05:06:17.076863+0000","last_fullsized":"2023-10-09T05:06:17.076863+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":165,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.084453+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:06:17.084453+0000","last_peered":"2023-10-09T05:06:17.084453+0000","last_clean":"2023-10-09T05:06:17.084453+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:06:17.084453+0000","last_undegraded":"2023-10-09T05:06:17.084453+0000","last_fullsized":"2023-10-09T05:06:17.084453+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"670'5969","reported_seq":14847,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:43.631205+0000","last_change":"2023-10-09T05:05:29.132074+0000","last_active":"2023-10-09T05:06:43.631205+0000","last_peered":"2023-10-09T05:06:43.631205+0000","last_clean":"2023-10-09T05:06:43.631205+0000","last_became_active":"2023-10-09T05:05:29.125934+0000","last_became_peered":"2023-10-09T05:05:29.125934+0000","last_unstale":"2023-10-09T05:06:43.631205+0000","last_undegraded":"2023-10-09T05:06:43.631205+0000","last_fullsized":"2023-10-09T05:06:43.631205+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5628,"log_dups_size":0,"ondisk_log_size":5628,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24477696,"num_objects":373,"num_object_clones":0,"num_object_copies":1119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2034,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":199,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074746+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:06:17.074746+0000","last_peered":"2023-10-09T05:06:17.074746+0000","last_clean":"2023-10-09T05:06:17.074746+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:06:17.074746+0000","last_undegraded":"2023-10-09T05:06:17.074746+0000","last_fullsized":"2023-10-09T05:06:17.074746+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":59,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.992609+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:06:30.992609+0000","last_peered":"2023-10-09T05:06:30.992609+0000","last_clean":"2023-10-09T05:06:30.992609+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:06:30.992609+0000","last_undegraded":"2023-10-09T05:06:30.992609+0000","last_fullsized":"2023-10-09T05:06:30.992609+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"670'6031","reported_seq":11007,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.915460+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:06:44.915460+0000","last_peered":"2023-10-09T05:06:44.915460+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:06:44.915460+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6031,"log_dups_size":0,"ondisk_log_size":6031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25231360,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":561,"num_objects_missing":561,"num_objects_degraded":1104,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2047,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":837,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"0,2,3,5,7","objects":1},{"shards":"0,2,5,7","objects":1},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":163},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":27,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:17.074022+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:06:17.074022+0000","last_peered":"2023-10-09T05:06:17.074022+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:06:17.074022+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:06:17.074022+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":156,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073780+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:06:17.073780+0000","last_peered":"2023-10-09T05:06:17.073780+0000","last_clean":"2023-10-09T05:06:17.073780+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:06:17.073780+0000","last_undegraded":"2023-10-09T05:06:17.073780+0000","last_fullsized":"2023-10-09T05:06:17.073780+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"670'6082","reported_seq":10206,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:46.591857+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:06:46.591857+0000","last_peered":"2023-10-09T05:06:46.591857+0000","last_clean":"2023-10-09T05:06:46.591857+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:06:46.591857+0000","last_undegraded":"2023-10-09T05:06:46.591857+0000","last_fullsized":"2023-10-09T05:06:46.591857+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22429696,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2089,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":53,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.239250+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:06:31.239250+0000","last_peered":"2023-10-09T05:06:31.239250+0000","last_clean":"2023-10-09T05:06:31.239250+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:06:31.239250+0000","last_undegraded":"2023-10-09T05:06:31.239250+0000","last_fullsized":"2023-10-09T05:06:31.239250+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":216,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076830+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:06:17.076830+0000","last_peered":"2023-10-09T05:06:17.076830+0000","last_clean":"2023-10-09T05:06:17.076830+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:06:17.076830+0000","last_undegraded":"2023-10-09T05:06:17.076830+0000","last_fullsized":"2023-10-09T05:06:17.076830+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"670'6035","reported_seq":16736,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:42.805454+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:06:42.805454+0000","last_peered":"2023-10-09T05:06:42.805454+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:06:42.805454+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:06:42.805454+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6035,"log_dups_size":0,"ondisk_log_size":6035,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24772608,"num_objects":378,"num_object_clones":0,"num_object_copies":1134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2051,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":378}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":731,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.234514+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:06:31.234514+0000","last_peered":"2023-10-09T05:06:31.234514+0000","last_clean":"2023-10-09T05:06:31.234514+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:06:31.234514+0000","last_undegraded":"2023-10-09T05:06:31.234514+0000","last_fullsized":"2023-10-09T05:06:31.234514+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.339987+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:06:47.339987+0000","last_peered":"2023-10-09T05:06:47.339987+0000","last_clean":"2023-10-09T05:06:47.339987+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:06:47.339987+0000","last_undegraded":"2023-10-09T05:06:47.339987+0000","last_fullsized":"2023-10-09T05:06:47.339987+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1258,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076987+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:06:17.076987+0000","last_peered":"2023-10-09T05:06:17.076987+0000","last_clean":"2023-10-09T05:06:17.076987+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:06:17.076987+0000","last_undegraded":"2023-10-09T05:06:17.076987+0000","last_fullsized":"2023-10-09T05:06:17.076987+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"670'6392","reported_seq":13166,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.935421+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:06:44.935421+0000","last_peered":"2023-10-09T05:06:44.935421+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:06:44.935421+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51150848,"num_objects":780,"num_object_clones":0,"num_object_copies":2340,"num_objects_missing_on_primary":1092,"num_objects_missing":1092,"num_objects_degraded":2265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3935,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1578,"num_bytes_recovered":103415808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36},{"shards":"0,2,3,4,5","objects":2},{"shards":"0,2,4,5","objects":1},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":345}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":274,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.080111+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:06:17.080111+0000","last_peered":"2023-10-09T05:06:17.080111+0000","last_clean":"2023-10-09T05:06:17.080111+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:06:17.080111+0000","last_undegraded":"2023-10-09T05:06:17.080111+0000","last_fullsized":"2023-10-09T05:06:17.080111+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074905+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:06:17.074905+0000","last_peered":"2023-10-09T05:06:17.074905+0000","last_clean":"2023-10-09T05:06:17.074905+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:06:17.074905+0000","last_undegraded":"2023-10-09T05:06:17.074905+0000","last_fullsized":"2023-10-09T05:06:17.074905+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1247,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076776+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:06:17.076776+0000","last_peered":"2023-10-09T05:06:17.076776+0000","last_clean":"2023-10-09T05:06:17.076776+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:06:17.076776+0000","last_undegraded":"2023-10-09T05:06:17.076776+0000","last_fullsized":"2023-10-09T05:06:17.076776+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"670'6673","reported_seq":20283,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.191266+0000","last_change":"2023-10-09T05:06:08.182013+0000","last_active":"2023-10-09T05:06:47.191266+0000","last_peered":"2023-10-09T05:06:47.191266+0000","last_clean":"2023-10-09T05:06:47.191266+0000","last_became_active":"2023-10-09T05:06:08.171542+0000","last_became_peered":"2023-10-09T05:06:08.171542+0000","last_unstale":"2023-10-09T05:06:47.191266+0000","last_undegraded":"2023-10-09T05:06:47.191266+0000","last_fullsized":"2023-10-09T05:06:47.191266+0000","mapping_epoch":667,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49512448,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4131,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4591,"num_bytes_recovered":223150080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":213,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076758+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:06:17.076758+0000","last_peered":"2023-10-09T05:06:17.076758+0000","last_clean":"2023-10-09T05:06:17.076758+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:06:17.076758+0000","last_undegraded":"2023-10-09T05:06:17.076758+0000","last_fullsized":"2023-10-09T05:06:17.076758+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":28,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074006+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:06:17.074006+0000","last_peered":"2023-10-09T05:06:17.074006+0000","last_clean":"2023-10-09T05:06:17.074006+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:06:17.074006+0000","last_undegraded":"2023-10-09T05:06:17.074006+0000","last_fullsized":"2023-10-09T05:06:17.074006+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"670'6631","reported_seq":12574,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:43.779250+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:06:43.779250+0000","last_peered":"2023-10-09T05:06:43.779250+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:06:43.779250+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"444'823","ondisk_log_start":"444'823","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50364416,"num_objects":768,"num_object_clones":0,"num_object_copies":2304,"num_objects_missing_on_primary":374,"num_objects_missing":374,"num_objects_degraded":1136,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4091,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1661,"num_bytes_recovered":108888064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":365},{"shards":"2,4,7","objects":374},{"shards":"3,7","objects":29}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":142,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073859+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:06:17.073859+0000","last_peered":"2023-10-09T05:06:17.073859+0000","last_clean":"2023-10-09T05:06:17.073859+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:06:17.073859+0000","last_undegraded":"2023-10-09T05:06:17.073859+0000","last_fullsized":"2023-10-09T05:06:17.073859+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":143,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074344+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:06:17.074344+0000","last_peered":"2023-10-09T05:06:17.074344+0000","last_clean":"2023-10-09T05:06:17.074344+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:06:17.074344+0000","last_undegraded":"2023-10-09T05:06:17.074344+0000","last_fullsized":"2023-10-09T05:06:17.074344+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1267,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.077180+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:06:17.077180+0000","last_peered":"2023-10-09T05:06:17.077180+0000","last_clean":"2023-10-09T05:06:17.077180+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:06:17.077180+0000","last_undegraded":"2023-10-09T05:06:17.077180+0000","last_fullsized":"2023-10-09T05:06:17.077180+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"670'6661","reported_seq":20479,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:43.749817+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:06:43.749817+0000","last_peered":"2023-10-09T05:06:43.749817+0000","last_clean":"2023-10-09T05:06:43.749817+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:06:43.749817+0000","last_undegraded":"2023-10-09T05:06:43.749817+0000","last_fullsized":"2023-10-09T05:06:43.749817+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48496640,"num_objects":740,"num_object_clones":0,"num_object_copies":2220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4121,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":189,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079215+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:06:17.079215+0000","last_peered":"2023-10-09T05:06:17.079215+0000","last_clean":"2023-10-09T05:06:17.079215+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:06:17.079215+0000","last_undegraded":"2023-10-09T05:06:17.079215+0000","last_fullsized":"2023-10-09T05:06:17.079215+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":98,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.014270+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:06:31.014270+0000","last_peered":"2023-10-09T05:06:31.014270+0000","last_clean":"2023-10-09T05:06:31.014270+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:06:31.014270+0000","last_undegraded":"2023-10-09T05:06:31.014270+0000","last_fullsized":"2023-10-09T05:06:31.014270+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1273,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079526+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:06:17.079526+0000","last_peered":"2023-10-09T05:06:17.079526+0000","last_clean":"2023-10-09T05:06:17.079526+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:06:17.079526+0000","last_undegraded":"2023-10-09T05:06:17.079526+0000","last_fullsized":"2023-10-09T05:06:17.079526+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"670'6858","reported_seq":13127,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.931202+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:06:44.931202+0000","last_peered":"2023-10-09T05:06:44.931202+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:06:44.931202+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"444'638","ondisk_log_start":"444'638","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6220,"log_dups_size":0,"ondisk_log_size":6220,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48824320,"num_objects":745,"num_object_clones":0,"num_object_copies":2235,"num_objects_missing_on_primary":1156,"num_objects_missing":1156,"num_objects_degraded":3276,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4249,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1753,"num_bytes_recovered":114884608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,2,5,7","objects":1},{"shards":"0,3","objects":-933},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":364}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":87,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:37.816559+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:06:37.816559+0000","last_peered":"2023-10-09T05:06:37.816559+0000","last_clean":"2023-10-09T05:06:37.816559+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:06:37.816559+0000","last_undegraded":"2023-10-09T05:06:37.816559+0000","last_fullsized":"2023-10-09T05:06:37.816559+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":49,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.239414+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:06:31.239414+0000","last_peered":"2023-10-09T05:06:31.239414+0000","last_clean":"2023-10-09T05:06:31.239414+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:06:31.239414+0000","last_undegraded":"2023-10-09T05:06:31.239414+0000","last_fullsized":"2023-10-09T05:06:31.239414+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:32.812308+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:06:32.812308+0000","last_peered":"2023-10-09T05:06:32.812308+0000","last_clean":"2023-10-09T05:06:32.812308+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:06:32.812308+0000","last_undegraded":"2023-10-09T05:06:32.812308+0000","last_fullsized":"2023-10-09T05:06:32.812308+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1256,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076906+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:06:17.076906+0000","last_peered":"2023-10-09T05:06:17.076906+0000","last_clean":"2023-10-09T05:06:17.076906+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:06:17.076906+0000","last_undegraded":"2023-10-09T05:06:17.076906+0000","last_fullsized":"2023-10-09T05:06:17.076906+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"670'6894","reported_seq":20009,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:43.715716+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:06:43.715716+0000","last_peered":"2023-10-09T05:06:43.715716+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:06:43.715716+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5213,"log_dups_size":0,"ondisk_log_size":5213,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49643520,"num_objects":757,"num_object_clones":0,"num_object_copies":2271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1324,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4034,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3573,"num_bytes_recovered":229572608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":275}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":755,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.234775+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:06:31.234775+0000","last_peered":"2023-10-09T05:06:31.234775+0000","last_clean":"2023-10-09T05:06:31.234775+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:06:31.234775+0000","last_undegraded":"2023-10-09T05:06:31.234775+0000","last_fullsized":"2023-10-09T05:06:31.234775+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073750+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:06:17.073750+0000","last_peered":"2023-10-09T05:06:17.073750+0000","last_clean":"2023-10-09T05:06:17.073750+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:06:17.073750+0000","last_undegraded":"2023-10-09T05:06:17.073750+0000","last_fullsized":"2023-10-09T05:06:17.073750+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:37.816533+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:06:37.816533+0000","last_peered":"2023-10-09T05:06:37.816533+0000","last_clean":"2023-10-09T05:06:37.816533+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:06:37.816533+0000","last_undegraded":"2023-10-09T05:06:37.816533+0000","last_fullsized":"2023-10-09T05:06:37.816533+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1267,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076916+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:06:17.076916+0000","last_peered":"2023-10-09T05:06:17.076916+0000","last_clean":"2023-10-09T05:06:17.076916+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:06:17.076916+0000","last_undegraded":"2023-10-09T05:06:17.076916+0000","last_fullsized":"2023-10-09T05:06:17.076916+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"670'6896","reported_seq":12679,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:43.728986+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:06:43.728986+0000","last_peered":"2023-10-09T05:06:43.728986+0000","last_clean":"2023-10-09T05:06:43.728986+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:06:43.728986+0000","last_undegraded":"2023-10-09T05:06:43.728986+0000","last_fullsized":"2023-10-09T05:06:43.728986+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4603,"log_dups_size":0,"ondisk_log_size":4603,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50429964,"num_objects":770,"num_object_clones":0,"num_object_copies":2310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4034,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":165,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079497+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:06:17.079497+0000","last_peered":"2023-10-09T05:06:17.079497+0000","last_clean":"2023-10-09T05:06:17.079497+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:06:17.079497+0000","last_undegraded":"2023-10-09T05:06:17.079497+0000","last_fullsized":"2023-10-09T05:06:17.079497+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.997385+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:06:30.997385+0000","last_peered":"2023-10-09T05:06:30.997385+0000","last_clean":"2023-10-09T05:06:30.997385+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:06:30.997385+0000","last_undegraded":"2023-10-09T05:06:30.997385+0000","last_fullsized":"2023-10-09T05:06:30.997385+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:42.822201+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:06:42.822201+0000","last_peered":"2023-10-09T05:06:42.822201+0000","last_clean":"2023-10-09T05:06:42.822201+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:06:42.822201+0000","last_undegraded":"2023-10-09T05:06:42.822201+0000","last_fullsized":"2023-10-09T05:06:42.822201+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"670'6830","reported_seq":13736,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:43.783421+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:06:43.783421+0000","last_peered":"2023-10-09T05:06:43.783421+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:06:43.783421+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4703,"log_dups_size":0,"ondisk_log_size":4703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":427,"num_objects_missing":427,"num_objects_degraded":1509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4001,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1920,"num_bytes_recovered":125796352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,3,5,7","objects":1},{"shards":"2,5,7","objects":336},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":59,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.992853+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:06:30.992853+0000","last_peered":"2023-10-09T05:06:30.992853+0000","last_clean":"2023-10-09T05:06:30.992853+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:06:30.992853+0000","last_undegraded":"2023-10-09T05:06:30.992853+0000","last_fullsized":"2023-10-09T05:06:30.992853+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1273,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.076694+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:06:17.076694+0000","last_peered":"2023-10-09T05:06:17.076694+0000","last_clean":"2023-10-09T05:06:17.076694+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:06:17.076694+0000","last_undegraded":"2023-10-09T05:06:17.076694+0000","last_fullsized":"2023-10-09T05:06:17.076694+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":163,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.081470+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:06:17.081470+0000","last_peered":"2023-10-09T05:06:17.081470+0000","last_clean":"2023-10-09T05:06:17.081470+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:06:17.081470+0000","last_undegraded":"2023-10-09T05:06:17.081470+0000","last_fullsized":"2023-10-09T05:06:17.081470+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":190,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.078801+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:06:17.078801+0000","last_peered":"2023-10-09T05:06:17.078801+0000","last_clean":"2023-10-09T05:06:17.078801+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:06:17.078801+0000","last_undegraded":"2023-10-09T05:06:17.078801+0000","last_fullsized":"2023-10-09T05:06:17.078801+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1246,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.077412+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:06:17.077412+0000","last_peered":"2023-10-09T05:06:17.077412+0000","last_clean":"2023-10-09T05:06:17.077412+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:06:17.077412+0000","last_undegraded":"2023-10-09T05:06:17.077412+0000","last_fullsized":"2023-10-09T05:06:17.077412+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"670'7048","reported_seq":15994,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.104912+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:06:44.104912+0000","last_peered":"2023-10-09T05:06:44.104912+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:06:44.104912+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5285,"log_dups_size":0,"ondisk_log_size":5285,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47579136,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2254,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4149,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2429,"num_bytes_recovered":146866176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":-33},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":712}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":85,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.997467+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:06:30.997467+0000","last_peered":"2023-10-09T05:06:30.997467+0000","last_clean":"2023-10-09T05:06:30.997467+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:06:30.997467+0000","last_undegraded":"2023-10-09T05:06:30.997467+0000","last_fullsized":"2023-10-09T05:06:30.997467+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":28,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073482+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:06:17.073482+0000","last_peered":"2023-10-09T05:06:17.073482+0000","last_clean":"2023-10-09T05:06:17.073482+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:06:17.073482+0000","last_undegraded":"2023-10-09T05:06:17.073482+0000","last_fullsized":"2023-10-09T05:06:17.073482+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.082899+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:06:17.082899+0000","last_peered":"2023-10-09T05:06:17.082899+0000","last_clean":"2023-10-09T05:06:17.082899+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:06:17.082899+0000","last_undegraded":"2023-10-09T05:06:17.082899+0000","last_fullsized":"2023-10-09T05:06:17.082899+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1267,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.080082+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:06:17.080082+0000","last_peered":"2023-10-09T05:06:17.080082+0000","last_clean":"2023-10-09T05:06:17.080082+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:06:17.080082+0000","last_undegraded":"2023-10-09T05:06:17.080082+0000","last_fullsized":"2023-10-09T05:06:17.080082+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"670'6708","reported_seq":18747,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.181528+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:06:47.181528+0000","last_peered":"2023-10-09T05:06:47.181528+0000","last_clean":"2023-10-09T05:06:47.181528+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:06:47.181528+0000","last_undegraded":"2023-10-09T05:06:47.181528+0000","last_fullsized":"2023-10-09T05:06:47.181528+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47153152,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4169,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":135,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074297+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:06:17.074297+0000","last_peered":"2023-10-09T05:06:17.074297+0000","last_clean":"2023-10-09T05:06:17.074297+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:06:17.074297+0000","last_undegraded":"2023-10-09T05:06:17.074297+0000","last_fullsized":"2023-10-09T05:06:17.074297+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:37.816504+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:06:37.816504+0000","last_peered":"2023-10-09T05:06:37.816504+0000","last_clean":"2023-10-09T05:06:37.816504+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:06:37.816504+0000","last_undegraded":"2023-10-09T05:06:37.816504+0000","last_fullsized":"2023-10-09T05:06:37.816504+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":49,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.992851+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:06:30.992851+0000","last_peered":"2023-10-09T05:06:30.992851+0000","last_clean":"2023-10-09T05:06:30.992851+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:06:30.992851+0000","last_undegraded":"2023-10-09T05:06:30.992851+0000","last_fullsized":"2023-10-09T05:06:30.992851+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1270,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.078699+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:06:17.078699+0000","last_peered":"2023-10-09T05:06:17.078699+0000","last_clean":"2023-10-09T05:06:17.078699+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:06:17.078699+0000","last_undegraded":"2023-10-09T05:06:17.078699+0000","last_fullsized":"2023-10-09T05:06:17.078699+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"670'6831","reported_seq":24191,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.186946+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:06:47.186946+0000","last_peered":"2023-10-09T05:06:47.186946+0000","last_clean":"2023-10-09T05:06:47.186946+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:06:47.186946+0000","last_undegraded":"2023-10-09T05:06:47.186946+0000","last_fullsized":"2023-10-09T05:06:47.186946+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49872896,"num_objects":761,"num_object_clones":0,"num_object_copies":2283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4010,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":294,"reported_epoch":669,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:17.094784+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:06:17.094784+0000","last_peered":"2023-10-09T05:06:17.094784+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:06:17.094784+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:06:17.094784+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:37.816600+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:06:37.816600+0000","last_peered":"2023-10-09T05:06:37.816600+0000","last_clean":"2023-10-09T05:06:37.816600+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:06:37.816600+0000","last_undegraded":"2023-10-09T05:06:37.816600+0000","last_fullsized":"2023-10-09T05:06:37.816600+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":174,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079107+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:06:17.079107+0000","last_peered":"2023-10-09T05:06:17.079107+0000","last_clean":"2023-10-09T05:06:17.079107+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:06:17.079107+0000","last_undegraded":"2023-10-09T05:06:17.079107+0000","last_fullsized":"2023-10-09T05:06:17.079107+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1226,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.078652+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:06:17.078652+0000","last_peered":"2023-10-09T05:06:17.078652+0000","last_clean":"2023-10-09T05:06:17.078652+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:06:17.078652+0000","last_undegraded":"2023-10-09T05:06:17.078652+0000","last_fullsized":"2023-10-09T05:06:17.078652+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"670'7188","reported_seq":15803,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.901689+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:06:44.901689+0000","last_peered":"2023-10-09T05:06:44.901689+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:06:44.901689+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5763,"log_dups_size":0,"ondisk_log_size":5763,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49283072,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":1144,"num_objects_missing":1144,"num_objects_degraded":2736,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4228,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2019,"num_bytes_recovered":132317184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-2},{"shards":"0,2,4,7","objects":1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":682},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":186,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.079757+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:06:17.079757+0000","last_peered":"2023-10-09T05:06:17.079757+0000","last_clean":"2023-10-09T05:06:17.079757+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:06:17.079757+0000","last_undegraded":"2023-10-09T05:06:17.079757+0000","last_fullsized":"2023-10-09T05:06:17.079757+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":106,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:37.816582+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:06:37.816582+0000","last_peered":"2023-10-09T05:06:37.816582+0000","last_clean":"2023-10-09T05:06:37.816582+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:06:37.816582+0000","last_undegraded":"2023-10-09T05:06:37.816582+0000","last_fullsized":"2023-10-09T05:06:37.816582+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073702+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:06:17.073702+0000","last_peered":"2023-10-09T05:06:17.073702+0000","last_clean":"2023-10-09T05:06:17.073702+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:06:17.073702+0000","last_undegraded":"2023-10-09T05:06:17.073702+0000","last_fullsized":"2023-10-09T05:06:17.073702+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"670'6481","reported_seq":13619,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:44.915992+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:06:44.915992+0000","last_peered":"2023-10-09T05:06:44.915992+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:06:44.915992+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"444'237","ondisk_log_start":"444'237","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6244,"log_dups_size":0,"ondisk_log_size":6244,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45318144,"num_objects":692,"num_object_clones":0,"num_object_copies":2076,"num_objects_missing_on_primary":1115,"num_objects_missing":1115,"num_objects_degraded":2251,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4025,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1667,"num_bytes_recovered":109248512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-36},{"shards":"0,2,3","objects":2},{"shards":"0,2,3,4,5","objects":1},{"shards":"0,2,4,5","objects":1},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":339}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":135,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.074689+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:06:17.074689+0000","last_peered":"2023-10-09T05:06:17.074689+0000","last_clean":"2023-10-09T05:06:17.074689+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:06:17.074689+0000","last_undegraded":"2023-10-09T05:06:17.074689+0000","last_fullsized":"2023-10-09T05:06:17.074689+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":86,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:30.995163+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:06:30.995163+0000","last_peered":"2023-10-09T05:06:30.995163+0000","last_clean":"2023-10-09T05:06:30.995163+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:06:30.995163+0000","last_undegraded":"2023-10-09T05:06:30.995163+0000","last_fullsized":"2023-10-09T05:06:30.995163+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"670'6589","reported_seq":11433,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:43.547302+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:06:43.547302+0000","last_peered":"2023-10-09T05:06:43.547302+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:06:43.547302+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:06:43.547302+0000","mapping_epoch":597,"log_start":"444'656","ondisk_log_start":"444'656","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5933,"log_dups_size":0,"ondisk_log_size":5933,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52756480,"num_objects":805,"num_object_clones":0,"num_object_copies":2415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4054,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":805}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":806,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:31.239267+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:06:31.239267+0000","last_peered":"2023-10-09T05:06:31.239267+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:06:31.239267+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:06:31.239267+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":28,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073532+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:06:17.073532+0000","last_peered":"2023-10-09T05:06:17.073532+0000","last_clean":"2023-10-09T05:06:17.073532+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:06:17.073532+0000","last_undegraded":"2023-10-09T05:06:17.073532+0000","last_fullsized":"2023-10-09T05:06:17.073532+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"670'6658","reported_seq":11087,"reported_epoch":670,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:47.050383+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:06:47.050383+0000","last_peered":"2023-10-09T05:06:47.050383+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:06:47.050383+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6120,"log_dups_size":0,"ondisk_log_size":6120,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48267264,"num_objects":737,"num_object_clones":0,"num_object_copies":2211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2146,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4127,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1358,"num_bytes_recovered":88965120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":581},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":90}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":752,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.239742+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:06:31.239742+0000","last_peered":"2023-10-09T05:06:31.239742+0000","last_clean":"2023-10-09T05:06:31.239742+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:06:31.239742+0000","last_undegraded":"2023-10-09T05:06:31.239742+0000","last_fullsized":"2023-10-09T05:06:31.239742+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":28,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073422+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:06:17.073422+0000","last_peered":"2023-10-09T05:06:17.073422+0000","last_clean":"2023-10-09T05:06:17.073422+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:06:17.073422+0000","last_undegraded":"2023-10-09T05:06:17.073422+0000","last_fullsized":"2023-10-09T05:06:17.073422+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"670'6675","reported_seq":17485,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:43.736702+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:06:43.736702+0000","last_peered":"2023-10-09T05:06:43.736702+0000","last_clean":"2023-10-09T05:06:43.736702+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:06:43.736702+0000","last_undegraded":"2023-10-09T05:06:43.736702+0000","last_fullsized":"2023-10-09T05:06:43.736702+0000","mapping_epoch":638,"log_start":"444'947","ondisk_log_start":"444'947","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5728,"log_dups_size":0,"ondisk_log_size":5728,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47251456,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4143,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":291,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.012679+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:06:31.012679+0000","last_peered":"2023-10-09T05:06:31.012679+0000","last_clean":"2023-10-09T05:06:31.012679+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:06:31.012679+0000","last_undegraded":"2023-10-09T05:06:31.012679+0000","last_fullsized":"2023-10-09T05:06:31.012679+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":298,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.087789+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:06:17.087789+0000","last_peered":"2023-10-09T05:06:17.087789+0000","last_clean":"2023-10-09T05:06:17.087789+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:06:17.087789+0000","last_undegraded":"2023-10-09T05:06:17.087789+0000","last_fullsized":"2023-10-09T05:06:17.087789+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"670'6526","reported_seq":18137,"reported_epoch":670,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:43.562000+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:06:43.562000+0000","last_peered":"2023-10-09T05:06:43.562000+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:06:43.562000+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:06:43.562000+0000","mapping_epoch":597,"log_start":"444'725","ondisk_log_start":"444'725","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5801,"log_dups_size":0,"ondisk_log_size":5801,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51642368,"num_objects":788,"num_object_clones":0,"num_object_copies":2364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4027,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":788}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":51,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-10-09T05:06:31.240078+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:06:31.240078+0000","last_peered":"2023-10-09T05:06:31.240078+0000","last_clean":"2023-10-09T05:06:31.240078+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:06:31.240078+0000","last_undegraded":"2023-10-09T05:06:31.240078+0000","last_fullsized":"2023-10-09T05:06:31.240078+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":146,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-10-09T05:06:17.073489+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:06:17.073489+0000","last_peered":"2023-10-09T05:06:17.073489+0000","last_clean":"2023-10-09T05:06:17.073489+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:06:17.073489+0000","last_undegraded":"2023-10-09T05:06:17.073489+0000","last_fullsized":"2023-10-09T05:06:17.073489+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":1585250328,"num_objects":24190,"num_object_clones":0,"num_object_copies":72570,"num_objects_missing_on_primary":8163,"num_objects_missing":8163,"num_objects_degraded":27129,"num_objects_misplaced":2210,"num_objects_unfound":0,"num_objects_dirty":24190,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131028,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83616,"num_bytes_recovered":5075173376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11614433280,"data_stored":11614421064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":209723,"ondisk_log_size":209723,"up":108,"acting":92,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234713,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2914604,"kb_used_data":1405348,"kb_used_omap":0,"kb_used_meta":1509248,"kb_avail":90830548,"statfs":{"total":95995035648,"available":93010481152,"internally_reserved":0,"allocated":1439076352,"data_stored":1437371389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1545469952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:16 2023","interfaces":[{"interface":"back","average":{"1min":11.641999999999999,"5min":6.4889999999999999,"15min":2.593},"min":{"1min":0.25900000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":221.91800000000001,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.504},{"interface":"front","average":{"1min":11.976000000000001,"5min":6.7329999999999997,"15min":2.839},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":221.87799999999999,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.51800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:06:16 2023","interfaces":[{"interface":"back","average":{"1min":4.8289999999999997,"5min":4.2850000000000001,"15min":2.1459999999999999},"min":{"1min":0.32800000000000001,"5min":0.28899999999999998,"15min":0.25800000000000001},"max":{"1min":63.927999999999997,"5min":63.927999999999997,"15min":63.927999999999997},"last":179.01900000000001},{"interface":"front","average":{"1min":4.7889999999999997,"5min":4.0110000000000001,"15min":2.0990000000000002},"min":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":64.334000000000003,"5min":64.334000000000003,"15min":64.334000000000003},"last":179.05099999999999}]},{"osd":2,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":2.7069999999999999,"15min":3.8879999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.9269999999999996,"5min":64.480999999999995,"15min":207.96600000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.524,"5min":2.6469999999999998,"15min":4.0019999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":9.8989999999999991,"5min":55.134,"15min":207.97900000000001},"last":0.54200000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:06:12 2023","interfaces":[{"interface":"back","average":{"1min":15.56,"5min":8.3770000000000007,"15min":3.7829999999999999},"min":{"1min":0.36699999999999999,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":241.666,"5min":241.666,"15min":241.666},"last":261.57499999999999},{"interface":"front","average":{"1min":14.212,"5min":7.8179999999999996,"15min":3.7149999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":230.751,"5min":230.751,"15min":230.751},"last":261.17599999999999}]},{"osd":4,"last update":"Mon Oct 9 05:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.48099999999999998,"15min":0.47499999999999998},"min":{"1min":0.28000000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.54100000000000004,"5min":2.923,"15min":2.923},"last":0.44900000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":1.6399999999999999,"15min":1.7230000000000001},"min":{"1min":0.308,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":0.57399999999999995,"5min":57.064999999999998,"15min":131.84899999999999},"last":74.915000000000006},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.734,"15min":2.0139999999999998},"min":{"1min":0.29099999999999998,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":2.2210000000000001,"5min":53.225000000000001,"15min":135.74199999999999},"last":74.972999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:05:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":1.458,"15min":1.1699999999999999},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.88200000000000001,"5min":57.168999999999997,"15min":57.168999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":1.651,"15min":1.3129999999999999},"min":{"1min":0.374,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":2.2770000000000001,"5min":57.183999999999997,"15min":57.183999999999997},"last":0.49299999999999999}]}]},{"osd":6,"up_from":46,"seq":197568496345,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2032876,"kb_used_data":522276,"kb_used_omap":0,"kb_used_meta":1510592,"kb_avail":91712276,"statfs":{"total":95995035648,"available":93913370624,"internally_reserved":0,"allocated":534810624,"data_stored":533105894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1546846208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":6.9930000000000003,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":49.659999999999997,"5min":321.346,"15min":321.346},"last":4.75},{"interface":"front","average":{"1min":6.0620000000000003,"5min":5.2439999999999998,"15min":2.355},"min":{"1min":0.35799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":46.987000000000002,"5min":169.119,"15min":169.119},"last":0.377}]},{"osd":1,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":2.415,"15min":3.2170000000000001},"min":{"1min":0.39000000000000001,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":10.973000000000001,"5min":72.840999999999994,"15min":220.48400000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":2.2810000000000001,"5min":2.665,"15min":3.2000000000000002},"min":{"1min":0.34899999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":10.02,"5min":72.832999999999998,"15min":220.56200000000001},"last":5.0039999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:05:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.6799999999999999,"15min":2.0630000000000002},"min":{"1min":0.376,"5min":0.32900000000000001,"15min":0.25600000000000001},"max":{"1min":5.0090000000000003,"5min":23.943000000000001,"15min":133.535},"last":2.5379999999999998},{"interface":"front","average":{"1min":1.877,"5min":1.651,"15min":1.9850000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":4.9359999999999999,"5min":25.978999999999999,"15min":133.51900000000001},"last":3.2879999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":5.6349999999999998,"5min":3.3769999999999998,"15min":2.956},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":45.915999999999997,"5min":64.355999999999995,"15min":147.43600000000001},"last":2.7360000000000002},{"interface":"front","average":{"1min":6.0229999999999997,"5min":3.2040000000000002,"15min":3.0270000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":45.868000000000002,"5min":56.133000000000003,"15min":147.41999999999999},"last":2.613}]},{"osd":4,"last update":"Mon Oct 9 05:06:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.88900000000000001,"15min":0.78600000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.441,"5min":18.186,"15min":18.186},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.0329999999999999,"15min":1.0129999999999999},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.2400000000000002,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.41299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":1.006,"15min":0.99099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.499,"5min":25.934000000000001,"15min":25.934000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.83599999999999997,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.254},"max":{"1min":0.57699999999999996,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.39900000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.73899999999999999,"15min":1.052},"min":{"1min":0.314,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":7.2549999999999999,"5min":7.2549999999999999,"15min":36.204999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.99199999999999999,"15min":1.0269999999999999},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.32500000000000001},"max":{"1min":0.58999999999999997,"5min":20.041,"15min":20.041},"last":0.42499999999999999}]}]},{"osd":1,"up_from":118,"seq":506806141413,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2092264,"kb_used_data":748576,"kb_used_omap":4,"kb_used_meta":1343675,"kb_avail":91652888,"statfs":{"total":95995035648,"available":93852557312,"internally_reserved":0,"allocated":766541824,"data_stored":762961506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1375924110},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":4.2030000000000003,"5min":4.7080000000000002,"15min":2.254},"min":{"1min":0.215,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":27.922000000000001,"5min":100.247,"15min":100.247},"last":0.40300000000000002},{"interface":"front","average":{"1min":4.2709999999999999,"5min":5.1310000000000002,"15min":2.3740000000000001},"min":{"1min":0.32300000000000001,"5min":0.247,"15min":0.214},"max":{"1min":19.297000000000001,"5min":99.935000000000002,"15min":99.935000000000002},"last":0.318}]},{"osd":2,"last update":"Mon Oct 9 05:05:48 2023","interfaces":[{"interface":"back","average":{"1min":3.202,"5min":1.5429999999999999,"15min":2.5939999999999999},"min":{"1min":0.309,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.478,"5min":12.460000000000001,"15min":263.089},"last":0.33900000000000002},{"interface":"front","average":{"1min":3.335,"5min":1.9319999999999999,"15min":2.653},"min":{"1min":0.28100000000000003,"5min":0.252,"15min":0.222},"max":{"1min":10.385,"5min":13.933999999999999,"15min":254.50899999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:06:19 2023","interfaces":[{"interface":"back","average":{"1min":10.869999999999999,"5min":8.1379999999999999,"15min":3.9049999999999998},"min":{"1min":0.41699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":102.729,"5min":235.43600000000001,"15min":235.43600000000001},"last":0.439},{"interface":"front","average":{"1min":8.6829999999999998,"5min":7.2679999999999998,"15min":3.456},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":104.13500000000001,"5min":234.32499999999999,"15min":234.32499999999999},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:06:29 2023","interfaces":[{"interface":"back","average":{"1min":4.1589999999999998,"5min":2.0699999999999998,"15min":1.54},"min":{"1min":0.39600000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":19.358000000000001,"5min":19.358000000000001,"15min":19.358000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":3.1440000000000001,"5min":1.778,"15min":1.224},"min":{"1min":0.36299999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.832999999999998,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.97699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1479999999999997,"5min":1.889,"15min":1.4410000000000001},"min":{"1min":0.42199999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":19.382999999999999,"5min":19.382999999999999,"15min":19.382999999999999},"last":80.600999999999999},{"interface":"front","average":{"1min":3.1480000000000001,"5min":1.9630000000000001,"15min":1.47},"min":{"1min":0.41199999999999998,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":18.777999999999999,"5min":18.777999999999999,"15min":19.597999999999999},"last":80.390000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1539999999999999,"5min":1.974,"15min":1.4490000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":19.338000000000001,"5min":19.338000000000001,"15min":19.338000000000001},"last":78.311999999999998},{"interface":"front","average":{"1min":3.173,"5min":1.8420000000000001,"15min":1.4830000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":18.867000000000001,"5min":18.867000000000001,"15min":19.548999999999999},"last":78.278000000000006}]},{"osd":7,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":3.202,"5min":1.9550000000000001,"15min":1.681},"min":{"1min":0.51000000000000001,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":18.899999999999999,"5min":18.899999999999999,"15min":22.731999999999999},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.994,"5min":1.5629999999999999,"15min":1.488},"min":{"1min":0.5,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":10.345000000000001,"5min":11.903,"15min":22.765999999999998},"last":0.51400000000000001}]}]},{"osd":0,"up_from":91,"seq":390842024496,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667384,"kb_used_data":504752,"kb_used_omap":5,"kb_used_meta":1162490,"kb_avail":92077768,"statfs":{"total":95995035648,"available":94287634432,"internally_reserved":0,"allocated":516866048,"data_stored":513977046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1190390394},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,1,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:06:12 2023","interfaces":[{"interface":"back","average":{"1min":5.1909999999999998,"5min":3.673,"15min":2.5150000000000001},"min":{"1min":0.28299999999999997,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":19.042000000000002,"5min":53.176000000000002,"15min":155.411},"last":0.222},{"interface":"front","average":{"1min":5.5419999999999998,"5min":3.859,"15min":2.5430000000000001},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.657,"5min":53.372999999999998,"15min":154.75},"last":0.39100000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:06:33 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":2.4369999999999998,"15min":2.7090000000000001},"min":{"1min":0.29099999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":19.141999999999999,"5min":19.141999999999999,"15min":151.06399999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":4.7279999999999998,"5min":2.9020000000000001,"15min":2.9409999999999998},"min":{"1min":0.377,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":15.444000000000001,"5min":20.239000000000001,"15min":151.22399999999999},"last":1.645}]},{"osd":3,"last update":"Mon Oct 9 05:05:59 2023","interfaces":[{"interface":"back","average":{"1min":13.766999999999999,"5min":7.6890000000000001,"15min":4.4539999999999997},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":114.871,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.33400000000000002},{"interface":"front","average":{"1min":15.404999999999999,"5min":7.9009999999999998,"15min":4.6449999999999996},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":115.375,"5min":290.73899999999998,"15min":290.73899999999998},"last":0.29899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:06:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.0070000000000001,"15min":1.903},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.5170000000000003,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":57.360999999999997},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.7709999999999999,"15min":1.613},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.9399999999999995,"5min":11.81,"15min":11.81},"last":57.338000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.4300000000000002,"15min":1.494},"min":{"1min":0.42199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.51,"5min":16.888000000000002,"15min":16.888000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.0979999999999999,"15min":1.3460000000000001},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.5670000000000002,"5min":14.036,"15min":14.036},"last":0.48899999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":2.6629999999999998,"15min":1.6459999999999999},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.598000000000001,"5min":20.129999999999999,"15min":39.594999999999999},"last":54.802},{"interface":"front","average":{"1min":2.1150000000000002,"5min":2.6299999999999999,"15min":1.6599999999999999},"min":{"1min":0.36299999999999999,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":7.6020000000000003,"5min":47.241,"15min":47.241},"last":58.222999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.3460000000000001,"15min":1.524},"min":{"1min":0.441,"5min":0.41899999999999998,"15min":0.39400000000000002},"max":{"1min":7.6360000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.929,"5min":2.2120000000000002,"15min":1.508},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":9.9770000000000003,"5min":21.68,"15min":21.68},"last":0.51300000000000001}]}]},{"osd":2,"up_from":0,"seq":949187772775,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356468,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1901476,"kb_used_data":929884,"kb_used_omap":2,"kb_used_meta":971581,"kb_avail":91843676,"statfs":{"total":95995035648,"available":94047924224,"internally_reserved":0,"allocated":952201216,"data_stored":949209339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":994898997},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,2,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":5.5090000000000003,"5min":3.3820000000000001,"15min":1.5600000000000001},"min":{"1min":0.25700000000000001,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":51.145000000000003,"5min":108.631,"15min":108.631},"last":0.82599999999999996},{"interface":"front","average":{"1min":6.1710000000000003,"5min":3.6160000000000001,"15min":1.752},"min":{"1min":0.28599999999999998,"5min":0.247,"15min":0.247},"max":{"1min":51.191000000000003,"5min":110.012,"15min":110.012},"last":15.154}]},{"osd":1,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0190000000000001,"15min":1.391},"min":{"1min":0.26100000000000001,"5min":0.221,"15min":0.221},"max":{"1min":13.374000000000001,"5min":15.218,"15min":15.983000000000001},"last":6.2990000000000004},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.754,"15min":1.294},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.245},"max":{"1min":4.54,"5min":17.544,"15min":26.692},"last":6.2400000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":1.5669999999999999,"15min":2.371},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":11.996,"5min":11.996,"15min":212.065},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.8799999999999999,"5min":1.694,"15min":2.4079999999999999},"min":{"1min":0.32100000000000001,"5min":0.253,"15min":0.23499999999999999},"max":{"1min":6.2539999999999996,"5min":12.987,"15min":211.98400000000001},"last":3.6840000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8100000000000001,"15min":1.75},"min":{"1min":0.434,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.75,"5min":11.193,"15min":11.193},"last":3.7360000000000002},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.2270000000000001,"15min":1.0069999999999999},"min":{"1min":0.41799999999999998,"5min":0.309,"15min":0.309},"max":{"1min":4.3609999999999998,"5min":10.452,"15min":10.452},"last":6.2279999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.855,"15min":1.395},"min":{"1min":0.46000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.6040000000000001,"5min":57.804000000000002,"15min":57.804000000000002},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.95,"5min":2.1269999999999998,"15min":1.659},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":6.383,"5min":57.442,"15min":57.442},"last":3.7650000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.649,"15min":1.222},"min":{"1min":0.376,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":6.3659999999999997,"5min":11.252000000000001,"15min":11.252000000000001},"last":3.794},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.4319999999999999,"15min":1.327},"min":{"1min":0.441,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.988,"5min":14.731,"15min":24.916},"last":6.1769999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.5369999999999999,"15min":1.3580000000000001},"min":{"1min":0.47399999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.9470000000000001,"5min":14.784000000000001,"15min":25.004000000000001},"last":6.2640000000000002},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.4319999999999999,"15min":1.1120000000000001},"min":{"1min":0.40300000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.758,"5min":7.758,"15min":7.758},"last":0.89900000000000002}]}]},{"osd":4,"up_from":626,"seq":2688649527337,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2483252,"kb_used_data":2078060,"kb_used_omap":22,"kb_used_meta":405161,"kb_avail":91261900,"statfs":{"total":95995035648,"available":93452185600,"internally_reserved":0,"allocated":2127933440,"data_stored":2126220883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":414885709},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":9.2349999999999994,"5min":3.722,"15min":2.7679999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":130.34200000000001,"5min":130.34200000000001,"15min":130.34200000000001},"last":2.5059999999999998},{"interface":"front","average":{"1min":9.4740000000000002,"5min":4.306,"15min":3.5489999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":130.49700000000001,"5min":130.49700000000001,"15min":130.49700000000001},"last":1.8009999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":11.933999999999999,"5min":4.1520000000000001,"15min":2.8029999999999999},"min":{"1min":0.32300000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":182.32900000000001,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":11.624000000000001,"5min":3.7280000000000002,"15min":2.5099999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":182.41,"5min":182.41,"15min":182.41},"last":4.0789999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":1.825,"15min":1.7010000000000001},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.8639999999999999,"5min":11.051,"15min":11.051},"last":1.7370000000000001},{"interface":"front","average":{"1min":2.4300000000000002,"5min":1.8839999999999999,"15min":1.752},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.8810000000000002,"5min":11.036,"15min":11.036},"last":1.746}]},{"osd":3,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":5.1879999999999997,"15min":2.512},"min":{"1min":0.40300000000000002,"5min":0.318,"15min":0.318},"max":{"1min":10.797000000000001,"5min":270.767,"15min":270.767},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.2919999999999998,"5min":5.4989999999999997,"15min":3.0619999999999998},"min":{"1min":0.37,"5min":0.313,"15min":0.313},"max":{"1min":5.173,"5min":270.274,"15min":270.274},"last":6.1130000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.38,"15min":0.38400000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56499999999999995,"5min":1.304,"15min":1.304},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.438,"15min":0.44500000000000001},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.54300000000000004,"5min":1.504,"15min":1.504},"last":0.26500000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.57199999999999995,"15min":0.63200000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57499999999999996,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.46200000000000002,"15min":0.45600000000000002},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81899999999999995,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.372}]},{"osd":7,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.501,"15min":0.51700000000000002},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.58299999999999996,"5min":1.585,"15min":1.585},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.47899999999999998,"15min":0.47299999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.58799999999999997,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.312}]}]},{"osd":5,"up_from":38,"seq":163208757990,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4298204,"kb_used_data":2800788,"kb_used_omap":0,"kb_used_meta":1497408,"kb_avail":89446948,"statfs":{"total":95995035648,"available":91593674752,"internally_reserved":0,"allocated":2868006912,"data_stored":2866313577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1533345792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:23 2023","interfaces":[{"interface":"back","average":{"1min":7.2629999999999999,"5min":4.718,"15min":2.5369999999999999},"min":{"1min":0.27200000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":124.504,"5min":124.504,"15min":124.504},"last":4.6449999999999996},{"interface":"front","average":{"1min":7.468,"5min":5.0670000000000002,"15min":2.7229999999999999},"min":{"1min":0.254,"5min":0.246,"15min":0.245},"max":{"1min":124.678,"5min":124.678,"15min":124.678},"last":0.35799999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:06:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":5.2469999999999999,"15min":3.4630000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23200000000000001},"max":{"1min":25.411000000000001,"5min":262.26900000000001,"15min":262.26900000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.782,"5min":5.367,"15min":3.6019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27300000000000002,"15min":0.26900000000000002},"max":{"1min":25.303000000000001,"5min":262.291,"15min":262.291},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:06:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":1.653,"15min":1.919},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":9.6440000000000001,"5min":10.949999999999999,"15min":106.238},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.1280000000000001,"5min":1.851,"15min":2.1659999999999999},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.29999999999999999},"max":{"1min":4.8979999999999997,"5min":11.912000000000001,"15min":104.31699999999999},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":7.8579999999999997,"5min":7.0819999999999999,"15min":3.524},"min":{"1min":0.375,"5min":0.318,"15min":0.318},"max":{"1min":121.40600000000001,"5min":125.824,"15min":125.824},"last":0.40600000000000003},{"interface":"front","average":{"1min":7.9930000000000003,"5min":6.9649999999999999,"15min":3.4710000000000001},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":121.53700000000001,"5min":125.761,"15min":125.761},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:06:31 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.99,"5min":1.99,"15min":1.99},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.496,"5min":0.45800000000000002,"15min":0.46000000000000002},"min":{"1min":0.33600000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.54400000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.68600000000000005,"15min":0.73899999999999999},"min":{"1min":0.28499999999999998,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":2.0289999999999999,"5min":11.093,"15min":11.093},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.59899999999999998,"15min":0.81799999999999995},"min":{"1min":0.32000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.982,"5min":5.9329999999999998,"15min":13.083},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.68999999999999995,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.948,"5min":8.3520000000000003,"15min":9.6809999999999992},"last":0.436},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.89200000000000002,"15min":1.0589999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":0.90900000000000003,"5min":11.791,"15min":15.461},"last":1.599}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":504627200,"data_stored":504627200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":754581504,"data_stored":754581504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2491875328,"data_stored":2491875328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":936116224,"data_stored":936116224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2114654208,"data_stored":2114650136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2859470848,"data_stored":2859466776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":522063872,"data_stored":522059800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1431044096,"data_stored":1431044096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:06:50.687 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:50.825 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:06:50.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:50.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:06:50.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:51.038 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:51.043 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:06:51.050 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:06:51.050 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:06:51.050 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:06:51.050 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:06:51.051 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:06:51.077 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:06:51.138 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:06:51.173 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:06:51.178 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:51.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:51.218 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:06:51.268 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:06:51.288 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:51.309 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:51 smithi029 ceph-mon[216903]: pgmap v2177: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 26606/70308 objects degraded (37.842%); 2210/70308 objects misplaced (3.143%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:06:51.309 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/28613612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:51.309 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:51 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 26606/70308 objects degraded (37.842%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:51.313 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:06:51.363 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.376 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:06:51.398 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:51 smithi177 ceph-mon[195797]: pgmap v2177: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 26606/70308 objects degraded (37.842%); 2210/70308 objects misplaced (3.143%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:06:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/28613612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:51 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 26606/70308 objects degraded (37.842%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:51.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:51 smithi029 ceph-mon[223858]: pgmap v2177: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 304 op/s; 26606/70308 objects degraded (37.842%); 2210/70308 objects misplaced (3.143%); 9.5 MiB/s, 152 objects/s recovering 2023-10-09T05:06:51.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/28613612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:06:51.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:51 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 26606/70308 objects degraded (37.842%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:51.672 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:51.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:51.780 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:06:51.813 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:51.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:06:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:51.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:06:52.048 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:06:52.092 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:52.129 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:52.192 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:52.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:52.243 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:06:52.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:52.325 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:52.422 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:52.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:52.443 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:06:52.528 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:06:52.553 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:52.604 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:06:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:52.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:06:52.709 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:06:52.773 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:53.122 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:06:53.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:53.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:06:53.130 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:06:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:06:52] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.43.0" 2023-10-09T05:06:53.231 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:53.258 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:53.301 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:53.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:53.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:53.331 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:53.405 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:06:53.426 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:53 smithi029 ceph-mon[216903]: pgmap v2178: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 25925/69435 objects degraded (37.337%); 2028/69435 objects misplaced (2.921%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T05:06:53.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:53 smithi029 ceph-mon[223858]: pgmap v2178: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 25925/69435 objects degraded (37.337%); 2028/69435 objects misplaced (2.921%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T05:06:53.485 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:53 smithi177 ceph-mon[195797]: pgmap v2178: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 25925/69435 objects degraded (37.337%); 2028/69435 objects misplaced (2.921%); 9.7 MiB/s, 154 objects/s recovering 2023-10-09T05:06:53.687 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:06:53.693 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:06:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:53.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:06:53.811 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:53.951 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:54.282 INFO:teuthology.orchestra.run.smithi177.stdout:ERROR: (22) Invalid argument 2023-10-09T05:06:54.282 INFO:teuthology.orchestra.run.smithi177.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:06:54.367 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:54.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:54.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:54.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:54.549 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:55.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:06:55.245 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:06:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:55 smithi177 ceph-mon[195797]: pgmap v2179: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 25339/65751 objects degraded (38.538%); 1830/65751 objects misplaced (2.783%); 10 MiB/s, 161 objects/s recovering 2023-10-09T05:06:55.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:55 smithi029 ceph-mon[223858]: pgmap v2179: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 25339/65751 objects degraded (38.538%); 1830/65751 objects misplaced (2.783%); 10 MiB/s, 161 objects/s recovering 2023-10-09T05:06:55.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:55 smithi029 ceph-mon[216903]: pgmap v2179: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 325 op/s; 25339/65751 objects degraded (38.538%); 1830/65751 objects misplaced (2.783%); 10 MiB/s, 161 objects/s recovering 2023-10-09T05:06:55.913 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:55.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:55.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:56.024 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:56.054 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:56.108 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:56.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:56.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:56.213 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:56.241 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:56.270 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:56.293 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:56.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:56.372 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:56 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 25339/65751 objects degraded (38.538%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:56.372 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:56.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:56 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 25339/65751 objects degraded (38.538%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:56.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:06:56.517 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:06:56.523 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:56.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:56 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 25339/65751 objects degraded (38.538%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:06:56.799 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:06:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:56.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:06:56.909 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:06:56.938 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:56.989 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:06:56.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:56.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:06:57.098 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:57.212 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:57.605 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:57 smithi029 ceph-mon[216903]: pgmap v2180: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 331 op/s; 24381/64137 objects degraded (38.014%); 1617/64137 objects misplaced (2.521%); 10 MiB/s, 165 objects/s recovering 2023-10-09T05:06:57.605 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:57 smithi029 ceph-mon[223858]: pgmap v2180: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 331 op/s; 24381/64137 objects degraded (38.014%); 1617/64137 objects misplaced (2.521%); 10 MiB/s, 165 objects/s recovering 2023-10-09T05:06:57.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:57 smithi177 ceph-mon[195797]: pgmap v2180: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 331 op/s; 24381/64137 objects degraded (38.014%); 1617/64137 objects misplaced (2.521%); 10 MiB/s, 165 objects/s recovering 2023-10-09T05:06:58.388 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:06:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:06:58.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:06:58.497 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:06:58.527 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:06:58.539 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:06:58.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1800199203' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:58.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:58 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:58.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1800199203' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:58.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:58 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1800199203' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:58.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:58 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:06:59.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: pgmap v2181: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 24204/62403 objects degraded (38.787%); 1559/62403 objects misplaced (2.498%); 10 MiB/s, 190 objects/s recovering 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: failed to encode map e672 with expected crc 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: osdmap e672: 8 total, 8 up, 7 in 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: failed to encode map e672 with expected crc 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: pgmap v2181: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 24204/62403 objects degraded (38.787%); 1559/62403 objects misplaced (2.498%); 10 MiB/s, 190 objects/s recovering 2023-10-09T05:06:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: failed to encode map e672 with expected crc 2023-10-09T05:06:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:06:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: osdmap e672: 8 total, 8 up, 7 in 2023-10-09T05:06:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: failed to encode map e672 with expected crc 2023-10-09T05:06:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:06:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: pgmap v2181: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 24204/62403 objects degraded (38.787%); 1559/62403 objects misplaced (2.498%); 10 MiB/s, 190 objects/s recovering 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: failed to encode map e672 with expected crc 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: osdmap e672: 8 total, 8 up, 7 in 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: failed to encode map e672 with expected crc 2023-10-09T05:06:59.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:06:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:01.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:01 smithi029 ceph-mon[216903]: pgmap v2183: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 23455/59361 objects degraded (39.512%); 1559/59361 objects misplaced (2.626%); 11 MiB/s, 195 objects/s recovering 2023-10-09T05:07:01.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:01 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 23455/59361 objects degraded (39.512%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:01.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:01 smithi029 ceph-mon[223858]: pgmap v2183: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 23455/59361 objects degraded (39.512%); 1559/59361 objects misplaced (2.626%); 11 MiB/s, 195 objects/s recovering 2023-10-09T05:07:01.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:01 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 23455/59361 objects degraded (39.512%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:01 smithi177 ceph-mon[195797]: pgmap v2183: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 23455/59361 objects degraded (39.512%); 1559/59361 objects misplaced (2.626%); 11 MiB/s, 195 objects/s recovering 2023-10-09T05:07:01.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:01 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 23455/59361 objects degraded (39.512%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:02.124 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:02.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:02.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:02.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:02.352 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:03.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:07:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:07:02] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-10-09T05:07:03.627 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:03.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:03.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:03 smithi029 ceph-mon[216903]: pgmap v2184: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 22711/58461 objects degraded (38.848%); 1348/58461 objects misplaced (2.306%); 11 MiB/s, 197 objects/s recovering 2023-10-09T05:07:03.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:03 smithi029 ceph-mon[223858]: pgmap v2184: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 22711/58461 objects degraded (38.848%); 1348/58461 objects misplaced (2.306%); 11 MiB/s, 197 objects/s recovering 2023-10-09T05:07:03.742 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:03.772 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:03 smithi177 ceph-mon[195797]: pgmap v2184: 119 pgs: 2 active+recovering+undersized+remapped, 93 active+clean, 14 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 22711/58461 objects degraded (38.848%); 1348/58461 objects misplaced (2.306%); 11 MiB/s, 197 objects/s recovering 2023-10-09T05:07:03.775 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:03.827 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:03.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:03.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:03.931 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:03.967 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:04.019 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:04.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:04.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:04.127 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:04.248 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:04.590 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:04.601 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:07:04.602 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:07:04.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:04.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:04.642 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:07:04.655 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[216903]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[216903]: osdmap e673: 8 total, 8 up, 7 in 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[216903]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[216903]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[223858]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[223858]: osdmap e673: 8 total, 8 up, 7 in 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[223858]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:04 smithi029 ceph-mon[223858]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.714 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:04 smithi177 ceph-mon[195797]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:04 smithi177 ceph-mon[195797]: osdmap e673: 8 total, 8 up, 7 in 2023-10-09T05:07:04.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:04 smithi177 ceph-mon[195797]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:04 smithi177 ceph-mon[195797]: failed to encode map e673 with expected crc 2023-10-09T05:07:04.796 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:07:04.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:04.857 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:07:04.913 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:07:04.960 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:07:05.020 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:07:05.056 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:07:05.096 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:07:05.133 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:07:05.155 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.156 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.157 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.157 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:07:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: pgmap v2186: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 93 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 377 op/s; 20842/55068 objects degraded (37.848%); 2016/55068 objects misplaced (3.661%); 9.8 MiB/s, 214 objects/s recovering 2023-10-09T05:07:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: osdmap e674: 8 total, 8 up, 7 in 2023-10-09T05:07:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: from='client.37722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:05.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4104940874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:05.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:07:05.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: pgmap v2186: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 93 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 377 op/s; 20842/55068 objects degraded (37.848%); 2016/55068 objects misplaced (3.661%); 9.8 MiB/s, 214 objects/s recovering 2023-10-09T05:07:05.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: osdmap e674: 8 total, 8 up, 7 in 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: from='client.37722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4104940874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: pgmap v2186: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 93 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 377 op/s; 20842/55068 objects degraded (37.848%); 2016/55068 objects misplaced (3.661%); 9.8 MiB/s, 214 objects/s recovering 2023-10-09T05:07:05.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: osdmap e674: 8 total, 8 up, 7 in 2023-10-09T05:07:05.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: failed to encode map e674 with expected crc 2023-10-09T05:07:05.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: from='client.37722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:05.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4104940874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:06.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:06 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 20842/55068 objects degraded (37.848%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:06.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:06 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 20842/55068 objects degraded (37.848%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:06.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:06 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 20842/55068 objects degraded (37.848%), 24 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:07.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:07 smithi177 ceph-mon[195797]: pgmap v2188: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 19893/53577 objects degraded (37.130%); 1728/53577 objects misplaced (3.225%); 13 MiB/s, 238 objects/s recovering 2023-10-09T05:07:07.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:07 smithi029 ceph-mon[216903]: pgmap v2188: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 19893/53577 objects degraded (37.130%); 1728/53577 objects misplaced (3.225%); 13 MiB/s, 238 objects/s recovering 2023-10-09T05:07:07.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:07 smithi029 ceph-mon[223858]: pgmap v2188: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 19893/53577 objects degraded (37.130%); 1728/53577 objects misplaced (3.225%); 13 MiB/s, 238 objects/s recovering 2023-10-09T05:07:08.162 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:08.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:08.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:08.269 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:08.359 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:08.676 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:07:08.726 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2187,"stamp":"2023-10-09T05:07:04.441256+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1189943224,"num_objects":18173,"num_object_clones":0,"num_object_copies":54519,"num_objects_missing_on_primary":6403,"num_objects_missing":6403,"num_objects_degraded":20650,"num_objects_misplaced":2016,"num_objects_unfound":0,"num_objects_dirty":18173,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":137369,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87065,"num_bytes_recovered":5266442894,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":214489,"ondisk_log_size":214489,"up":357,"acting":341,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":483,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16705788,"kb_used_data":8245060,"kb_used_omap":34,"kb_used_meta":8460637,"kb_avail":639510276,"statfs":{"total":671965249536,"available":654858522624,"internally_reserved":0,"allocated":8442941440,"data_stored":8426586872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8663692682},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-172228608,"num_objects":-2628,"num_object_clones":0,"num_object_copies":-7884,"num_objects_missing_on_primary":-679,"num_objects_missing":-679,"num_objects_degraded":-3554,"num_objects_misplaced":457,"num_objects_unfound":0,"num_objects_dirty":-2628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2628,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1440,"num_bytes_recovered":80412672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.559938"},"pg_stats":[{"pgid":"14.23","version":"673'6136","reported_seq":12767,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:03.634946+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:07:03.634946+0000","last_peered":"2023-10-09T05:07:03.634946+0000","last_clean":"2023-10-09T05:07:03.634946+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:07:03.634946+0000","last_undegraded":"2023-10-09T05:07:03.634946+0000","last_fullsized":"2023-10-09T05:07:03.634946+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18350080,"num_objects":280,"num_object_clones":0,"num_object_copies":840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2152,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"673'6084","reported_seq":14117,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:03.832434+0000","last_change":"2023-10-09T05:03:28.642038+0000","last_active":"2023-10-09T05:07:03.832434+0000","last_peered":"2023-10-09T05:07:03.832434+0000","last_clean":"2023-10-09T04:54:25.089587+0000","last_became_active":"2023-10-09T05:03:28.599529+0000","last_became_peered":"2023-10-09T05:03:28.599529+0000","last_unstale":"2023-10-09T05:07:03.832434+0000","last_undegraded":"2023-10-09T05:03:26.709404+0000","last_fullsized":"2023-10-09T05:03:26.350390+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6084,"log_dups_size":0,"ondisk_log_size":6084,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16941056,"num_objects":259,"num_object_clones":0,"num_object_copies":777,"num_objects_missing_on_primary":232,"num_objects_missing":232,"num_objects_degraded":539,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2149,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1225,"num_bytes_recovered":78036992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[3,1],"avail_no_missing":["7"],"object_location_counts":[{"shards":"0,3,4,5,6,7","objects":53},{"shards":"1,3","objects":44},{"shards":"3,4,5,6,7","objects":88},{"shards":"4,5,7","objects":74}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.21","version":"673'6131","reported_seq":16835,"reported_epoch":673,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:03.539111+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:07:03.539111+0000","last_peered":"2023-10-09T05:07:03.539111+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:07:03.539111+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:07:03.539111+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6131,"log_dups_size":0,"ondisk_log_size":6131,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18481152,"num_objects":282,"num_object_clones":0,"num_object_copies":846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2148,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":282}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"672'6136","reported_seq":8674,"reported_epoch":672,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:02.245571+0000","last_change":"2023-10-09T05:05:26.963137+0000","last_active":"2023-10-09T05:07:02.245571+0000","last_peered":"2023-10-09T05:07:02.245571+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:07:02.245571+0000","last_undegraded":"2023-10-09T05:05:26.962565+0000","last_fullsized":"2023-10-09T05:07:02.245571+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5598,"log_dups_size":0,"ondisk_log_size":5598,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18890752,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2143,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":701,"num_bytes_recovered":45957120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,5,6","objects":18},{"shards":"2,4,5","objects":239},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":30}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"672'6809","reported_seq":20375,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:01.921802+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:07:01.921802+0000","last_peered":"2023-10-09T05:07:01.921802+0000","last_clean":"2023-10-09T05:07:01.921802+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:07:01.921802+0000","last_undegraded":"2023-10-09T05:07:01.921802+0000","last_fullsized":"2023-10-09T05:07:01.921802+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6809,"log_dups_size":0,"ondisk_log_size":6809,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40599552,"num_objects":620,"num_object_clones":0,"num_object_copies":1860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4268,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":175,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402602+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:06:58.402602+0000","last_peered":"2023-10-09T05:06:58.402602+0000","last_clean":"2023-10-09T05:06:58.402602+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:06:58.402602+0000","last_undegraded":"2023-10-09T05:06:58.402602+0000","last_fullsized":"2023-10-09T05:06:58.402602+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":320,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.401813+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:06:58.401813+0000","last_peered":"2023-10-09T05:06:58.401813+0000","last_clean":"2023-10-09T05:06:58.401813+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:06:58.401813+0000","last_undegraded":"2023-10-09T05:06:58.401813+0000","last_fullsized":"2023-10-09T05:06:58.401813+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"673'6832","reported_seq":10065,"reported_epoch":673,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:03.584391+0000","last_change":"2023-10-09T05:03:49.101895+0000","last_active":"2023-10-09T05:07:03.584391+0000","last_peered":"2023-10-09T05:07:03.584391+0000","last_clean":"2023-10-09T05:03:35.212052+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:07:03.584391+0000","last_undegraded":"2023-10-09T05:03:48.958199+0000","last_fullsized":"2023-10-09T05:07:03.584391+0000","mapping_epoch":638,"log_start":"444'1211","ondisk_log_start":"444'1211","created":532,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5621,"log_dups_size":0,"ondisk_log_size":5621,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37191680,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4293,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":964,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":["6","0","2","3","4"],"object_location_counts":[{"shards":"3,6,7","objects":568}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":96,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.339418+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:06:47.339418+0000","last_peered":"2023-10-09T05:06:47.339418+0000","last_clean":"2023-10-09T05:06:47.339418+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:06:47.339418+0000","last_undegraded":"2023-10-09T05:06:47.339418+0000","last_fullsized":"2023-10-09T05:06:47.339418+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":186,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402919+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:06:58.402919+0000","last_peered":"2023-10-09T05:06:58.402919+0000","last_clean":"2023-10-09T05:06:58.402919+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:06:58.402919+0000","last_undegraded":"2023-10-09T05:06:58.402919+0000","last_fullsized":"2023-10-09T05:06:58.402919+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"673'6832","reported_seq":18496,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:03.944716+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:07:03.944716+0000","last_peered":"2023-10-09T05:07:03.944716+0000","last_clean":"2023-10-09T05:07:03.944716+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:07:03.944716+0000","last_undegraded":"2023-10-09T05:07:03.944716+0000","last_fullsized":"2023-10-09T05:07:03.944716+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6586,"log_dups_size":0,"ondisk_log_size":6586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37289984,"num_objects":569,"num_object_clones":0,"num_object_copies":1707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4292,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":303,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402719+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:06:58.402719+0000","last_peered":"2023-10-09T05:06:58.402719+0000","last_clean":"2023-10-09T05:06:58.402719+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:06:58.402719+0000","last_undegraded":"2023-10-09T05:06:58.402719+0000","last_fullsized":"2023-10-09T05:06:58.402719+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":199,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405119+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:06:58.405119+0000","last_peered":"2023-10-09T05:06:58.405119+0000","last_clean":"2023-10-09T05:06:58.405119+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:06:58.405119+0000","last_undegraded":"2023-10-09T05:06:58.405119+0000","last_fullsized":"2023-10-09T05:06:58.405119+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"672'7026","reported_seq":24386,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:02.308697+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:07:02.308697+0000","last_peered":"2023-10-09T05:07:02.308697+0000","last_clean":"2023-10-09T05:07:02.308697+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:07:02.308697+0000","last_undegraded":"2023-10-09T05:07:02.308697+0000","last_fullsized":"2023-10-09T05:07:02.308697+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6626,"log_dups_size":0,"ondisk_log_size":6626,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4205,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":46,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223743+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:06:47.223743+0000","last_peered":"2023-10-09T05:06:47.223743+0000","last_clean":"2023-10-09T05:06:47.223743+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:06:47.223743+0000","last_undegraded":"2023-10-09T05:06:47.223743+0000","last_fullsized":"2023-10-09T05:06:47.223743+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":294,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402674+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:06:58.402674+0000","last_peered":"2023-10-09T05:06:58.402674+0000","last_clean":"2023-10-09T05:06:58.402674+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:06:58.402674+0000","last_undegraded":"2023-10-09T05:06:58.402674+0000","last_fullsized":"2023-10-09T05:06:58.402674+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"672'7339","reported_seq":17086,"reported_epoch":672,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-09T05:07:00.107809+0000","last_change":"2023-10-09T05:05:26.975807+0000","last_active":"2023-10-09T05:07:00.107809+0000","last_peered":"2023-10-09T05:07:00.107809+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:30.568117+0000","last_became_peered":"2023-10-09T05:03:30.568117+0000","last_unstale":"2023-10-09T05:07:00.107809+0000","last_undegraded":"2023-10-09T05:07:00.107809+0000","last_fullsized":"2023-10-09T05:03:26.334568+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5914,"log_dups_size":0,"ondisk_log_size":5914,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39387136,"num_objects":601,"num_object_clones":0,"num_object_copies":1803,"num_objects_missing_on_primary":852,"num_objects_missing":852,"num_objects_degraded":0,"num_objects_misplaced":1348,"num_objects_unfound":0,"num_objects_dirty":601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4379,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2302,"num_bytes_recovered":150798336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":133},{"shards":"0,2,4,7","objects":1},{"shards":"2,4,7","objects":412},{"shards":"2,6,7","objects":55}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":60,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223941+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:06:47.223941+0000","last_peered":"2023-10-09T05:06:47.223941+0000","last_clean":"2023-10-09T05:06:47.223941+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:06:47.223941+0000","last_undegraded":"2023-10-09T05:06:47.223941+0000","last_fullsized":"2023-10-09T05:06:47.223941+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":284,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402192+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:06:58.402192+0000","last_peered":"2023-10-09T05:06:58.402192+0000","last_clean":"2023-10-09T05:06:58.402192+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:06:58.402192+0000","last_undegraded":"2023-10-09T05:06:58.402192+0000","last_fullsized":"2023-10-09T05:06:58.402192+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"672'7009","reported_seq":17590,"reported_epoch":672,"state":"active+undersized+remapped","last_fresh":"2023-10-09T05:07:02.729316+0000","last_change":"2023-10-09T05:07:02.654605+0000","last_active":"2023-10-09T05:07:02.729316+0000","last_peered":"2023-10-09T05:07:02.729316+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.334098+0000","last_became_peered":"2023-10-09T05:02:36.334098+0000","last_unstale":"2023-10-09T05:07:02.729316+0000","last_undegraded":"2023-10-09T05:07:02.729316+0000","last_fullsized":"2023-10-09T05:02:31.929271+0000","mapping_epoch":603,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4400,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2943,"num_bytes_recovered":162070528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,3],"avail_no_missing":["1","0","2","3","7"],"object_location_counts":[{"shards":"1,3","objects":594}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":347,"reported_epoch":672,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:58.401712+0000","last_change":"2023-10-09T05:03:49.707422+0000","last_active":"2023-10-09T05:06:58.401712+0000","last_peered":"2023-10-09T05:06:58.401712+0000","last_clean":"2023-10-09T05:03:43.329185+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:06:58.401712+0000","last_undegraded":"2023-10-09T05:03:47.970436+0000","last_fullsized":"2023-10-09T05:06:58.401712+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":900,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3,4","objects":2}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":97,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.415799+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:06:58.415799+0000","last_peered":"2023-10-09T05:06:58.415799+0000","last_clean":"2023-10-09T05:06:58.415799+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:06:58.415799+0000","last_undegraded":"2023-10-09T05:06:58.415799+0000","last_fullsized":"2023-10-09T05:06:58.415799+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"672'6823","reported_seq":18901,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:02.340602+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:07:02.340602+0000","last_peered":"2023-10-09T05:07:02.340602+0000","last_clean":"2023-10-09T05:07:02.340602+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:07:02.340602+0000","last_undegraded":"2023-10-09T05:07:02.340602+0000","last_fullsized":"2023-10-09T05:07:02.340602+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39616512,"num_objects":605,"num_object_clones":0,"num_object_copies":1815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4285,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":128,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403669+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:06:58.403669+0000","last_peered":"2023-10-09T05:06:58.403669+0000","last_clean":"2023-10-09T05:06:58.403669+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:06:58.403669+0000","last_undegraded":"2023-10-09T05:06:58.403669+0000","last_fullsized":"2023-10-09T05:06:58.403669+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":143,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403003+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:06:58.403003+0000","last_peered":"2023-10-09T05:06:58.403003+0000","last_clean":"2023-10-09T05:06:58.403003+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:06:58.403003+0000","last_undegraded":"2023-10-09T05:06:58.403003+0000","last_fullsized":"2023-10-09T05:06:58.403003+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"673'7083","reported_seq":20578,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:03.742773+0000","last_change":"2023-10-09T05:04:21.999533+0000","last_active":"2023-10-09T05:07:03.742773+0000","last_peered":"2023-10-09T05:07:03.742773+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:07:03.742773+0000","last_undegraded":"2023-10-09T05:04:21.946357+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37257216,"num_objects":569,"num_object_clones":0,"num_object_copies":1707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4224,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3762,"num_bytes_recovered":241958912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":349},{"shards":"2,3,4,6","objects":91},{"shards":"2,4,6","objects":64},{"shards":"2,4,6,7","objects":65}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":149,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402912+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:06:58.402912+0000","last_peered":"2023-10-09T05:06:58.402912+0000","last_clean":"2023-10-09T05:06:58.402912+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:06:58.402912+0000","last_undegraded":"2023-10-09T05:06:58.402912+0000","last_fullsized":"2023-10-09T05:06:58.402912+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":257,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.413940+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:06:58.413940+0000","last_peered":"2023-10-09T05:06:58.413940+0000","last_clean":"2023-10-09T05:06:58.413940+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:06:58.413940+0000","last_undegraded":"2023-10-09T05:06:58.413940+0000","last_fullsized":"2023-10-09T05:06:58.413940+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"672'7124","reported_seq":14944,"reported_epoch":672,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:02.759151+0000","last_change":"2023-10-09T05:07:02.669571+0000","last_active":"2023-10-09T05:07:02.759151+0000","last_peered":"2023-10-09T05:07:02.759151+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:57.324788+0000","last_became_peered":"2023-10-09T05:03:57.324788+0000","last_unstale":"2023-10-09T05:07:02.759151+0000","last_undegraded":"2023-10-09T05:03:54.781032+0000","last_fullsized":"2023-10-09T05:03:52.999055+0000","mapping_epoch":642,"log_start":"445'2127","ondisk_log_start":"445'2127","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4997,"log_dups_size":0,"ondisk_log_size":4997,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":587,"num_objects_missing":587,"num_objects_degraded":1,"num_objects_misplaced":668,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4295,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2227,"num_bytes_recovered":145915904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[1,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"1,2,5,7","objects":1},{"shards":"1,7","objects":-38},{"shards":"2,5,7","objects":146},{"shards":"3,5,7","objects":374},{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403135+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:06:58.403135+0000","last_peered":"2023-10-09T05:06:58.403135+0000","last_clean":"2023-10-09T05:06:58.403135+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:06:58.403135+0000","last_undegraded":"2023-10-09T05:06:58.403135+0000","last_fullsized":"2023-10-09T05:06:58.403135+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":50,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223589+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:06:47.223589+0000","last_peered":"2023-10-09T05:06:47.223589+0000","last_clean":"2023-10-09T05:06:47.223589+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:06:47.223589+0000","last_undegraded":"2023-10-09T05:06:47.223589+0000","last_fullsized":"2023-10-09T05:06:47.223589+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"673'7214","reported_seq":16652,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:04.108249+0000","last_change":"2023-10-09T05:03:49.797096+0000","last_active":"2023-10-09T05:07:04.108249+0000","last_peered":"2023-10-09T05:07:04.108249+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:29.900739+0000","last_became_peered":"2023-10-09T05:03:29.900739+0000","last_unstale":"2023-10-09T05:07:04.108249+0000","last_undegraded":"2023-10-09T05:03:49.796038+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1702,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4315,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2605,"num_bytes_recovered":157745152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"1,2,4,5,6","objects":47},{"shards":"2,4,5","objects":477},{"shards":"4,6","objects":35}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":112,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402834+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:06:58.402834+0000","last_peered":"2023-10-09T05:06:58.402834+0000","last_clean":"2023-10-09T05:06:58.402834+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:06:58.402834+0000","last_undegraded":"2023-10-09T05:06:58.402834+0000","last_fullsized":"2023-10-09T05:06:58.402834+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":311,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403263+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:06:58.403263+0000","last_peered":"2023-10-09T05:06:58.403263+0000","last_clean":"2023-10-09T05:06:58.403263+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:06:58.403263+0000","last_undegraded":"2023-10-09T05:06:58.403263+0000","last_fullsized":"2023-10-09T05:06:58.403263+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"673'7122","reported_seq":14007,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:03.635104+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:07:03.635104+0000","last_peered":"2023-10-09T05:07:03.635104+0000","last_clean":"2023-10-09T05:07:03.635104+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:07:03.635104+0000","last_undegraded":"2023-10-09T05:07:03.635104+0000","last_fullsized":"2023-10-09T05:07:03.635104+0000","mapping_epoch":633,"log_start":"445'2293","ondisk_log_start":"445'2293","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4829,"log_dups_size":0,"ondisk_log_size":4829,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35618828,"num_objects":544,"num_object_clones":0,"num_object_copies":1632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":544,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4260,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":307,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404559+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:06:58.404559+0000","last_peered":"2023-10-09T05:06:58.404559+0000","last_clean":"2023-10-09T05:06:58.404559+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:06:58.404559+0000","last_undegraded":"2023-10-09T05:06:58.404559+0000","last_fullsized":"2023-10-09T05:06:58.404559+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":169,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403940+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:06:58.403940+0000","last_peered":"2023-10-09T05:06:58.403940+0000","last_clean":"2023-10-09T05:06:58.403940+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:06:58.403940+0000","last_undegraded":"2023-10-09T05:06:58.403940+0000","last_fullsized":"2023-10-09T05:06:58.403940+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"673'6071","reported_seq":14952,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:03.546511+0000","last_change":"2023-10-09T05:05:29.132074+0000","last_active":"2023-10-09T05:07:03.546511+0000","last_peered":"2023-10-09T05:07:03.546511+0000","last_clean":"2023-10-09T05:07:03.546511+0000","last_became_active":"2023-10-09T05:05:29.125934+0000","last_became_peered":"2023-10-09T05:05:29.125934+0000","last_unstale":"2023-10-09T05:07:03.546511+0000","last_undegraded":"2023-10-09T05:07:03.546511+0000","last_fullsized":"2023-10-09T05:07:03.546511+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17793024,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2136,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":202,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403146+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:06:58.403146+0000","last_peered":"2023-10-09T05:06:58.403146+0000","last_clean":"2023-10-09T05:06:58.403146+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:06:58.403146+0000","last_undegraded":"2023-10-09T05:06:58.403146+0000","last_fullsized":"2023-10-09T05:06:58.403146+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223063+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:06:47.223063+0000","last_peered":"2023-10-09T05:06:47.223063+0000","last_clean":"2023-10-09T05:06:47.223063+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:06:47.223063+0000","last_undegraded":"2023-10-09T05:06:47.223063+0000","last_fullsized":"2023-10-09T05:06:47.223063+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"672'6099","reported_seq":11345,"reported_epoch":672,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:06:58.945025+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:06:58.945025+0000","last_peered":"2023-10-09T05:06:58.945025+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:06:58.945025+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6099,"log_dups_size":0,"ondisk_log_size":6099,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20774912,"num_objects":317,"num_object_clones":0,"num_object_copies":951,"num_objects_missing_on_primary":495,"num_objects_missing":495,"num_objects_degraded":903,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2115,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":905,"num_bytes_recovered":59277312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":13},{"shards":"2,3,5,7","objects":185},{"shards":"2,5,7","objects":97},{"shards":"3,5,6","objects":22}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":30,"reported_epoch":672,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:58.402821+0000","last_change":"2023-10-09T05:04:20.893885+0000","last_active":"2023-10-09T05:06:58.402821+0000","last_peered":"2023-10-09T05:06:58.402821+0000","last_clean":"2023-10-09T04:58:52.921988+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:06:58.402821+0000","last_undegraded":"2023-10-09T05:04:20.867334+0000","last_fullsized":"2023-10-09T05:06:58.402821+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,3,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":159,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402733+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:06:58.402733+0000","last_peered":"2023-10-09T05:06:58.402733+0000","last_clean":"2023-10-09T05:06:58.402733+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:06:58.402733+0000","last_undegraded":"2023-10-09T05:06:58.402733+0000","last_fullsized":"2023-10-09T05:06:58.402733+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"672'6154","reported_seq":10279,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:01.719428+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:07:01.719428+0000","last_peered":"2023-10-09T05:07:01.719428+0000","last_clean":"2023-10-09T05:07:01.719428+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:07:01.719428+0000","last_undegraded":"2023-10-09T05:07:01.719428+0000","last_fullsized":"2023-10-09T05:07:01.719428+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5616,"log_dups_size":0,"ondisk_log_size":5616,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17711104,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2161,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223515+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:06:47.223515+0000","last_peered":"2023-10-09T05:06:47.223515+0000","last_clean":"2023-10-09T05:06:47.223515+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:06:47.223515+0000","last_undegraded":"2023-10-09T05:06:47.223515+0000","last_fullsized":"2023-10-09T05:06:47.223515+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":219,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404574+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:06:58.404574+0000","last_peered":"2023-10-09T05:06:58.404574+0000","last_clean":"2023-10-09T05:06:58.404574+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:06:58.404574+0000","last_undegraded":"2023-10-09T05:06:58.404574+0000","last_fullsized":"2023-10-09T05:06:58.404574+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"673'6140","reported_seq":16845,"reported_epoch":673,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:03.743542+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:07:03.743542+0000","last_peered":"2023-10-09T05:07:03.743542+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:07:03.743542+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:07:03.743542+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17891328,"num_objects":273,"num_object_clones":0,"num_object_copies":819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2156,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":273}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":732,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223557+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:06:47.223557+0000","last_peered":"2023-10-09T05:06:47.223557+0000","last_clean":"2023-10-09T05:06:47.223557+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:06:47.223557+0000","last_undegraded":"2023-10-09T05:06:47.223557+0000","last_fullsized":"2023-10-09T05:06:47.223557+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":108,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.831136+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:06:47.831136+0000","last_peered":"2023-10-09T05:06:47.831136+0000","last_clean":"2023-10-09T05:06:47.831136+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:06:47.831136+0000","last_undegraded":"2023-10-09T05:06:47.831136+0000","last_fullsized":"2023-10-09T05:06:47.831136+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1261,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405427+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:06:58.405427+0000","last_peered":"2023-10-09T05:06:58.405427+0000","last_clean":"2023-10-09T05:06:58.405427+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:06:58.405427+0000","last_undegraded":"2023-10-09T05:06:58.405427+0000","last_fullsized":"2023-10-09T05:06:58.405427+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"672'6540","reported_seq":13905,"reported_epoch":672,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:00.157408+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:07:00.157408+0000","last_peered":"2023-10-09T05:07:00.157408+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:07:00.157408+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"444'200","ondisk_log_start":"444'200","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6340,"log_dups_size":0,"ondisk_log_size":6340,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41451520,"num_objects":632,"num_object_clones":0,"num_object_copies":1896,"num_objects_missing_on_primary":944,"num_objects_missing":944,"num_objects_degraded":1823,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4083,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1726,"num_bytes_recovered":113115136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36},{"shards":"0,2,4,5","objects":3},{"shards":"2,3,5","objects":344},{"shards":"2,3,6","objects":52},{"shards":"2,4,5","objects":197}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":277,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402400+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:06:58.402400+0000","last_peered":"2023-10-09T05:06:58.402400+0000","last_clean":"2023-10-09T05:06:58.402400+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:06:58.402400+0000","last_undegraded":"2023-10-09T05:06:58.402400+0000","last_fullsized":"2023-10-09T05:06:58.402400+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":111,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403665+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:06:58.403665+0000","last_peered":"2023-10-09T05:06:58.403665+0000","last_clean":"2023-10-09T05:06:58.403665+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:06:58.403665+0000","last_undegraded":"2023-10-09T05:06:58.403665+0000","last_fullsized":"2023-10-09T05:06:58.403665+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1250,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404473+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:06:58.404473+0000","last_peered":"2023-10-09T05:06:58.404473+0000","last_clean":"2023-10-09T05:06:58.404473+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:06:58.404473+0000","last_undegraded":"2023-10-09T05:06:58.404473+0000","last_fullsized":"2023-10-09T05:06:58.404473+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"672'6839","reported_seq":20450,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:02.091264+0000","last_change":"2023-10-09T05:06:08.182013+0000","last_active":"2023-10-09T05:07:02.091264+0000","last_peered":"2023-10-09T05:07:02.091264+0000","last_clean":"2023-10-09T05:07:02.091264+0000","last_became_active":"2023-10-09T05:06:08.171542+0000","last_became_peered":"2023-10-09T05:06:08.171542+0000","last_unstale":"2023-10-09T05:07:02.091264+0000","last_undegraded":"2023-10-09T05:07:02.091264+0000","last_fullsized":"2023-10-09T05:07:02.091264+0000","mapping_epoch":667,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38633472,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4297,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4591,"num_bytes_recovered":223150080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":216,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404562+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:06:58.404562+0000","last_peered":"2023-10-09T05:06:58.404562+0000","last_clean":"2023-10-09T05:06:58.404562+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:06:58.404562+0000","last_undegraded":"2023-10-09T05:06:58.404562+0000","last_fullsized":"2023-10-09T05:06:58.404562+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402751+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:06:58.402751+0000","last_peered":"2023-10-09T05:06:58.402751+0000","last_clean":"2023-10-09T05:06:58.402751+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:06:58.402751+0000","last_undegraded":"2023-10-09T05:06:58.402751+0000","last_fullsized":"2023-10-09T05:06:58.402751+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"673'6854","reported_seq":13472,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:03.835608+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:07:03.835608+0000","last_peered":"2023-10-09T05:07:03.835608+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:07:03.835608+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"444'1272","ondisk_log_start":"444'1272","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5582,"log_dups_size":0,"ondisk_log_size":5582,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35749888,"num_objects":545,"num_object_clones":0,"num_object_copies":1635,"num_objects_missing_on_primary":150,"num_objects_missing":150,"num_objects_degraded":688,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4314,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1885,"num_bytes_recovered":123568128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,3,4,7","objects":365},{"shards":"2,4,7","objects":150},{"shards":"3,7","objects":30}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":145,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402809+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:06:58.402809+0000","last_peered":"2023-10-09T05:06:58.402809+0000","last_clean":"2023-10-09T05:06:58.402809+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:06:58.402809+0000","last_undegraded":"2023-10-09T05:06:58.402809+0000","last_fullsized":"2023-10-09T05:06:58.402809+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":146,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403212+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:06:58.403212+0000","last_peered":"2023-10-09T05:06:58.403212+0000","last_clean":"2023-10-09T05:06:58.403212+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:06:58.403212+0000","last_undegraded":"2023-10-09T05:06:58.403212+0000","last_fullsized":"2023-10-09T05:06:58.403212+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1270,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404682+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:06:58.404682+0000","last_peered":"2023-10-09T05:06:58.404682+0000","last_clean":"2023-10-09T05:06:58.404682+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:06:58.404682+0000","last_undegraded":"2023-10-09T05:06:58.404682+0000","last_fullsized":"2023-10-09T05:06:58.404682+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"673'6874","reported_seq":20695,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:04.096569+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:07:04.096569+0000","last_peered":"2023-10-09T05:07:04.096569+0000","last_clean":"2023-10-09T05:07:04.096569+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:07:04.096569+0000","last_undegraded":"2023-10-09T05:07:04.096569+0000","last_fullsized":"2023-10-09T05:07:04.096569+0000","mapping_epoch":644,"log_start":"444'246","ondisk_log_start":"444'246","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6628,"log_dups_size":0,"ondisk_log_size":6628,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34537472,"num_objects":527,"num_object_clones":0,"num_object_copies":1581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4334,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":192,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402129+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:06:58.402129+0000","last_peered":"2023-10-09T05:06:58.402129+0000","last_clean":"2023-10-09T05:06:58.402129+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:06:58.402129+0000","last_undegraded":"2023-10-09T05:06:58.402129+0000","last_fullsized":"2023-10-09T05:06:58.402129+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:57.835994+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:06:57.835994+0000","last_peered":"2023-10-09T05:06:57.835994+0000","last_clean":"2023-10-09T05:06:57.835994+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:06:57.835994+0000","last_undegraded":"2023-10-09T05:06:57.835994+0000","last_fullsized":"2023-10-09T05:06:57.835994+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1276,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405042+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:06:58.405042+0000","last_peered":"2023-10-09T05:06:58.405042+0000","last_clean":"2023-10-09T05:06:58.405042+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:06:58.405042+0000","last_undegraded":"2023-10-09T05:06:58.405042+0000","last_fullsized":"2023-10-09T05:06:58.405042+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"672'7036","reported_seq":14018,"reported_epoch":672,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:00.111053+0000","last_change":"2023-10-09T05:02:36.385814+0000","last_active":"2023-10-09T05:07:00.111053+0000","last_peered":"2023-10-09T05:07:00.111053+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:07:00.111053+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"444'991","ondisk_log_start":"444'991","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":979,"num_objects_missing":979,"num_objects_degraded":2743,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4427,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1931,"num_bytes_recovered":126550016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,3","objects":2},{"shards":"0,3","objects":-933},{"shards":"2,3,5","objects":404},{"shards":"2,5,7","objects":187}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.338875+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:06:47.338875+0000","last_peered":"2023-10-09T05:06:47.338875+0000","last_clean":"2023-10-09T05:06:47.338875+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:06:47.338875+0000","last_undegraded":"2023-10-09T05:06:47.338875+0000","last_fullsized":"2023-10-09T05:06:47.338875+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":50,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.224174+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:06:47.224174+0000","last_peered":"2023-10-09T05:06:47.224174+0000","last_clean":"2023-10-09T05:06:47.224174+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:06:47.224174+0000","last_undegraded":"2023-10-09T05:06:47.224174+0000","last_fullsized":"2023-10-09T05:06:47.224174+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":106,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.340540+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:06:47.340540+0000","last_peered":"2023-10-09T05:06:47.340540+0000","last_clean":"2023-10-09T05:06:47.340540+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:06:47.340540+0000","last_undegraded":"2023-10-09T05:06:47.340540+0000","last_fullsized":"2023-10-09T05:06:47.340540+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1259,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405382+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:06:58.405382+0000","last_peered":"2023-10-09T05:06:58.405382+0000","last_clean":"2023-10-09T05:06:58.405382+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:06:58.405382+0000","last_undegraded":"2023-10-09T05:06:58.405382+0000","last_fullsized":"2023-10-09T05:06:58.405382+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"673'7115","reported_seq":20675,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:03.742583+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:07:03.742583+0000","last_peered":"2023-10-09T05:07:03.742583+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:07:03.742583+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'1681","ondisk_log_start":"445'1681","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35160064,"num_objects":536,"num_object_clones":0,"num_object_copies":1608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1103,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4255,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3794,"num_bytes_recovered":244056064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":356},{"shards":"2,3,4,6","objects":79},{"shards":"2,4,6","objects":47},{"shards":"2,4,6,7","objects":54}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":756,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.224180+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:06:47.224180+0000","last_peered":"2023-10-09T05:06:47.224180+0000","last_clean":"2023-10-09T05:06:47.224180+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:06:47.224180+0000","last_undegraded":"2023-10-09T05:06:47.224180+0000","last_fullsized":"2023-10-09T05:06:47.224180+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":138,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402731+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:06:58.402731+0000","last_peered":"2023-10-09T05:06:58.402731+0000","last_clean":"2023-10-09T05:06:58.402731+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:06:58.402731+0000","last_undegraded":"2023-10-09T05:06:58.402731+0000","last_fullsized":"2023-10-09T05:06:58.402731+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":88,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.338800+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:06:47.338800+0000","last_peered":"2023-10-09T05:06:47.338800+0000","last_clean":"2023-10-09T05:06:47.338800+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:06:47.338800+0000","last_undegraded":"2023-10-09T05:06:47.338800+0000","last_fullsized":"2023-10-09T05:06:47.338800+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1270,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404723+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:06:58.404723+0000","last_peered":"2023-10-09T05:06:58.404723+0000","last_clean":"2023-10-09T05:06:58.404723+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:06:58.404723+0000","last_undegraded":"2023-10-09T05:06:58.404723+0000","last_fullsized":"2023-10-09T05:06:58.404723+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"673'7094","reported_seq":12880,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:03.528954+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:07:03.528954+0000","last_peered":"2023-10-09T05:07:03.528954+0000","last_clean":"2023-10-09T05:07:03.528954+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:07:03.528954+0000","last_undegraded":"2023-10-09T05:07:03.528954+0000","last_fullsized":"2023-10-09T05:07:03.528954+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4801,"log_dups_size":0,"ondisk_log_size":4801,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37453836,"num_objects":572,"num_object_clones":0,"num_object_copies":1716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4232,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":168,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.401503+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:06:58.401503+0000","last_peered":"2023-10-09T05:06:58.401503+0000","last_clean":"2023-10-09T05:06:58.401503+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:06:58.401503+0000","last_undegraded":"2023-10-09T05:06:58.401503+0000","last_fullsized":"2023-10-09T05:06:58.401503+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.339430+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:06:47.339430+0000","last_peered":"2023-10-09T05:06:47.339430+0000","last_clean":"2023-10-09T05:06:47.339430+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:06:47.339430+0000","last_undegraded":"2023-10-09T05:06:47.339430+0000","last_fullsized":"2023-10-09T05:06:47.339430+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.346362+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:06:47.346362+0000","last_peered":"2023-10-09T05:06:47.346362+0000","last_clean":"2023-10-09T05:06:47.346362+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:06:47.346362+0000","last_undegraded":"2023-10-09T05:06:47.346362+0000","last_fullsized":"2023-10-09T05:06:47.346362+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"673'7032","reported_seq":14546,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:03.523785+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:07:03.523785+0000","last_peered":"2023-10-09T05:07:03.523785+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:07:03.523785+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37748736,"num_objects":576,"num_object_clones":0,"num_object_copies":1728,"num_objects_missing_on_primary":226,"num_objects_missing":226,"num_objects_degraded":1106,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4203,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2122,"num_bytes_recovered":139034624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5,7","objects":135},{"shards":"3,5,7","objects":402},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":60,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223275+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:06:47.223275+0000","last_peered":"2023-10-09T05:06:47.223275+0000","last_clean":"2023-10-09T05:06:47.223275+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:06:47.223275+0000","last_undegraded":"2023-10-09T05:06:47.223275+0000","last_fullsized":"2023-10-09T05:06:47.223275+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1276,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.404383+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:06:58.404383+0000","last_peered":"2023-10-09T05:06:58.404383+0000","last_clean":"2023-10-09T05:06:58.404383+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:06:58.404383+0000","last_undegraded":"2023-10-09T05:06:58.404383+0000","last_fullsized":"2023-10-09T05:06:58.404383+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":166,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403219+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:06:58.403219+0000","last_peered":"2023-10-09T05:06:58.403219+0000","last_clean":"2023-10-09T05:06:58.403219+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:06:58.403219+0000","last_undegraded":"2023-10-09T05:06:58.403219+0000","last_fullsized":"2023-10-09T05:06:58.403219+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":193,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405432+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:06:58.405432+0000","last_peered":"2023-10-09T05:06:58.405432+0000","last_clean":"2023-10-09T05:06:58.405432+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:06:58.405432+0000","last_undegraded":"2023-10-09T05:06:58.405432+0000","last_fullsized":"2023-10-09T05:06:58.405432+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1249,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405049+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:06:58.405049+0000","last_peered":"2023-10-09T05:06:58.405049+0000","last_clean":"2023-10-09T05:06:58.405049+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:06:58.405049+0000","last_undegraded":"2023-10-09T05:06:58.405049+0000","last_fullsized":"2023-10-09T05:06:58.405049+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"673'7256","reported_seq":16826,"reported_epoch":673,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:03.936282+0000","last_change":"2023-10-09T05:05:00.102485+0000","last_active":"2023-10-09T05:07:03.936282+0000","last_peered":"2023-10-09T05:07:03.936282+0000","last_clean":"2023-10-09T04:54:44.585563+0000","last_became_active":"2023-10-09T05:03:30.583386+0000","last_became_peered":"2023-10-09T05:03:30.583386+0000","last_unstale":"2023-10-09T05:07:03.936282+0000","last_undegraded":"2023-10-09T05:05:00.101770+0000","last_fullsized":"2023-10-09T05:03:25.325268+0000","mapping_epoch":627,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33947648,"num_objects":518,"num_object_clones":0,"num_object_copies":1554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1840,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4357,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2636,"num_bytes_recovered":160432128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"0,1,2,4,5","objects":1},{"shards":"0,4","objects":-34},{"shards":"1,2,4,5,6","objects":46},{"shards":"2,4,5","objects":505}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":86,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.339510+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:06:47.339510+0000","last_peered":"2023-10-09T05:06:47.339510+0000","last_clean":"2023-10-09T05:06:47.339510+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:06:47.339510+0000","last_undegraded":"2023-10-09T05:06:47.339510+0000","last_fullsized":"2023-10-09T05:06:47.339510+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":31,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402440+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:06:58.402440+0000","last_peered":"2023-10-09T05:06:58.402440+0000","last_clean":"2023-10-09T05:06:58.402440+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:06:58.402440+0000","last_undegraded":"2023-10-09T05:06:58.402440+0000","last_fullsized":"2023-10-09T05:06:58.402440+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.350643+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:06:47.350643+0000","last_peered":"2023-10-09T05:06:47.350643+0000","last_clean":"2023-10-09T05:06:47.350643+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:06:47.350643+0000","last_undegraded":"2023-10-09T05:06:47.350643+0000","last_fullsized":"2023-10-09T05:06:47.350643+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1270,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.406070+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:06:58.406070+0000","last_peered":"2023-10-09T05:06:58.406070+0000","last_clean":"2023-10-09T05:06:58.406070+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:06:58.406070+0000","last_undegraded":"2023-10-09T05:06:58.406070+0000","last_fullsized":"2023-10-09T05:06:58.406070+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"672'6867","reported_seq":18907,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:02.040498+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:07:02.040498+0000","last_peered":"2023-10-09T05:07:02.040498+0000","last_clean":"2023-10-09T05:07:02.040498+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:07:02.040498+0000","last_undegraded":"2023-10-09T05:07:02.040498+0000","last_fullsized":"2023-10-09T05:07:02.040498+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36732928,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4328,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":138,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403265+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:06:58.403265+0000","last_peered":"2023-10-09T05:06:58.403265+0000","last_clean":"2023-10-09T05:06:58.403265+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:06:58.403265+0000","last_undegraded":"2023-10-09T05:06:58.403265+0000","last_fullsized":"2023-10-09T05:06:58.403265+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":97,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.338643+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:06:47.338643+0000","last_peered":"2023-10-09T05:06:47.338643+0000","last_clean":"2023-10-09T05:06:47.338643+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:06:47.338643+0000","last_undegraded":"2023-10-09T05:06:47.338643+0000","last_fullsized":"2023-10-09T05:06:47.338643+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223180+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:06:47.223180+0000","last_peered":"2023-10-09T05:06:47.223180+0000","last_clean":"2023-10-09T05:06:47.223180+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:06:47.223180+0000","last_undegraded":"2023-10-09T05:06:47.223180+0000","last_fullsized":"2023-10-09T05:06:47.223180+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1273,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405378+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:06:58.405378+0000","last_peered":"2023-10-09T05:06:58.405378+0000","last_clean":"2023-10-09T05:06:58.405378+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:06:58.405378+0000","last_undegraded":"2023-10-09T05:06:58.405378+0000","last_fullsized":"2023-10-09T05:06:58.405378+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"672'6978","reported_seq":24339,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:07:02.206417+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:07:02.206417+0000","last_peered":"2023-10-09T05:07:02.206417+0000","last_clean":"2023-10-09T05:07:02.206417+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:07:02.206417+0000","last_undegraded":"2023-10-09T05:07:02.206417+0000","last_fullsized":"2023-10-09T05:07:02.206417+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4157,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":297,"reported_epoch":672,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:58.405368+0000","last_change":"2023-10-09T05:03:50.388470+0000","last_active":"2023-10-09T05:06:58.405368+0000","last_peered":"2023-10-09T05:06:58.405368+0000","last_clean":"2023-10-09T05:03:43.331164+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:06:58.405368+0000","last_undegraded":"2023-10-09T05:03:50.386675+0000","last_fullsized":"2023-10-09T05:06:58.405368+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":586,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.339832+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:06:47.339832+0000","last_peered":"2023-10-09T05:06:47.339832+0000","last_clean":"2023-10-09T05:06:47.339832+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:06:47.339832+0000","last_undegraded":"2023-10-09T05:06:47.339832+0000","last_fullsized":"2023-10-09T05:06:47.339832+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":177,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402039+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:06:58.402039+0000","last_peered":"2023-10-09T05:06:58.402039+0000","last_clean":"2023-10-09T05:06:58.402039+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:06:58.402039+0000","last_undegraded":"2023-10-09T05:06:58.402039+0000","last_fullsized":"2023-10-09T05:06:58.402039+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1229,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405181+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:06:58.405181+0000","last_peered":"2023-10-09T05:06:58.405181+0000","last_clean":"2023-10-09T05:06:58.405181+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:06:58.405181+0000","last_undegraded":"2023-10-09T05:06:58.405181+0000","last_fullsized":"2023-10-09T05:06:58.405181+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"672'7357","reported_seq":16652,"reported_epoch":672,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:00.146254+0000","last_change":"2023-10-09T05:03:31.505225+0000","last_active":"2023-10-09T05:07:00.146254+0000","last_peered":"2023-10-09T05:07:00.146254+0000","last_clean":"2023-10-09T04:54:44.589694+0000","last_became_active":"2023-10-09T05:03:31.452854+0000","last_became_peered":"2023-10-09T05:03:31.452854+0000","last_unstale":"2023-10-09T05:07:00.146254+0000","last_undegraded":"2023-10-09T05:03:27.294053+0000","last_fullsized":"2023-10-09T05:03:26.356282+0000","mapping_epoch":628,"log_start":"445'1425","ondisk_log_start":"445'1425","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":975,"num_objects_missing":975,"num_objects_degraded":2227,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4397,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2189,"num_bytes_recovered":143458304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[0,1],"avail_no_missing":["2"],"object_location_counts":[{"shards":"0,1","objects":-1},{"shards":"0,2,6","objects":1},{"shards":"2,4,7","objects":513},{"shards":"2,6,7","objects":70}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":189,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.405305+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:06:58.405305+0000","last_peered":"2023-10-09T05:06:58.405305+0000","last_clean":"2023-10-09T05:06:58.405305+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:06:58.405305+0000","last_undegraded":"2023-10-09T05:06:58.405305+0000","last_fullsized":"2023-10-09T05:06:58.405305+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":107,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.339698+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:06:47.339698+0000","last_peered":"2023-10-09T05:06:47.339698+0000","last_clean":"2023-10-09T05:06:47.339698+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:06:47.339698+0000","last_undegraded":"2023-10-09T05:06:47.339698+0000","last_fullsized":"2023-10-09T05:06:47.339698+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":148,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402758+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:06:58.402758+0000","last_peered":"2023-10-09T05:06:58.402758+0000","last_clean":"2023-10-09T05:06:58.402758+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:06:58.402758+0000","last_undegraded":"2023-10-09T05:06:58.402758+0000","last_fullsized":"2023-10-09T05:06:58.402758+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"672'6636","reported_seq":14392,"reported_epoch":672,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:00.093612+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:07:00.093612+0000","last_peered":"2023-10-09T05:07:00.093612+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:07:00.093612+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6098,"log_dups_size":0,"ondisk_log_size":6098,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35160064,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":962,"num_objects_missing":962,"num_objects_degraded":1789,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4180,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1822,"num_bytes_recovered":119406592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":-36},{"shards":"0,2,3","objects":2},{"shards":"2,3,5","objects":322},{"shards":"2,3,6","objects":63},{"shards":"2,4,5","objects":186}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":138,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402819+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:06:58.402819+0000","last_peered":"2023-10-09T05:06:58.402819+0000","last_clean":"2023-10-09T05:06:58.402819+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:06:58.402819+0000","last_undegraded":"2023-10-09T05:06:58.402819+0000","last_fullsized":"2023-10-09T05:06:58.402819+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":87,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.338723+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:06:47.338723+0000","last_peered":"2023-10-09T05:06:47.338723+0000","last_clean":"2023-10-09T05:06:47.338723+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:06:47.338723+0000","last_undegraded":"2023-10-09T05:06:47.338723+0000","last_fullsized":"2023-10-09T05:06:47.338723+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"673'6811","reported_seq":11659,"reported_epoch":673,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:03.829192+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:07:03.829192+0000","last_peered":"2023-10-09T05:07:03.829192+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:07:03.829192+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:07:03.829192+0000","mapping_epoch":597,"log_start":"444'1121","ondisk_log_start":"444'1121","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4276,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":583}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":807,"reported_epoch":671,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:06:47.223672+0000","last_change":"2023-10-09T05:03:30.578701+0000","last_active":"2023-10-09T05:06:47.223672+0000","last_peered":"2023-10-09T05:06:47.223672+0000","last_clean":"2023-10-09T05:02:14.589670+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:06:47.223672+0000","last_undegraded":"2023-10-09T05:03:25.328387+0000","last_fullsized":"2023-10-09T05:06:47.223672+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":576,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402569+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:06:58.402569+0000","last_peered":"2023-10-09T05:06:58.402569+0000","last_clean":"2023-10-09T05:06:58.402569+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:06:58.402569+0000","last_undegraded":"2023-10-09T05:06:58.402569+0000","last_fullsized":"2023-10-09T05:06:58.402569+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"672'6813","reported_seq":11411,"reported_epoch":672,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:02.241002+0000","last_change":"2023-10-09T05:04:36.878705+0000","last_active":"2023-10-09T05:07:02.241002+0000","last_peered":"2023-10-09T05:07:02.241002+0000","last_clean":"2023-10-09T04:54:44.580875+0000","last_became_active":"2023-10-09T05:01:57.415623+0000","last_became_peered":"2023-10-09T05:01:57.415623+0000","last_unstale":"2023-10-09T05:07:02.241002+0000","last_undegraded":"2023-10-09T05:04:36.878290+0000","last_fullsized":"2023-10-09T05:01:57.023399+0000","mapping_epoch":586,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38109184,"num_objects":582,"num_object_clones":0,"num_object_copies":1746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2034,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4282,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1414,"num_bytes_recovered":92635136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,6],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"0,2,5","objects":4},{"shards":"2,4,5","objects":525},{"shards":"2,4,5,6","objects":1},{"shards":"2,5,6","objects":61},{"shards":"5,6","objects":-9}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":753,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223992+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:06:47.223992+0000","last_peered":"2023-10-09T05:06:47.223992+0000","last_clean":"2023-10-09T05:06:47.223992+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:06:47.223992+0000","last_undegraded":"2023-10-09T05:06:47.223992+0000","last_fullsized":"2023-10-09T05:06:47.223992+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402500+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:06:58.402500+0000","last_peered":"2023-10-09T05:06:58.402500+0000","last_clean":"2023-10-09T05:06:58.402500+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:06:58.402500+0000","last_undegraded":"2023-10-09T05:06:58.402500+0000","last_fullsized":"2023-10-09T05:06:58.402500+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"673'6882","reported_seq":17695,"reported_epoch":673,"state":"active+clean","last_fresh":"2023-10-09T05:07:03.520267+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:07:03.520267+0000","last_peered":"2023-10-09T05:07:03.520267+0000","last_clean":"2023-10-09T05:07:03.520267+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:07:03.520267+0000","last_undegraded":"2023-10-09T05:07:03.520267+0000","last_fullsized":"2023-10-09T05:07:03.520267+0000","mapping_epoch":638,"log_start":"445'1378","ondisk_log_start":"445'1378","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5504,"log_dups_size":0,"ondisk_log_size":5504,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33685504,"num_objects":514,"num_object_clones":0,"num_object_copies":1542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4350,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":294,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.403234+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:06:58.403234+0000","last_peered":"2023-10-09T05:06:58.403234+0000","last_clean":"2023-10-09T05:06:58.403234+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:06:58.403234+0000","last_undegraded":"2023-10-09T05:06:58.403234+0000","last_fullsized":"2023-10-09T05:06:58.403234+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":302,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402275+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:06:58.402275+0000","last_peered":"2023-10-09T05:06:58.402275+0000","last_clean":"2023-10-09T05:06:58.402275+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:06:58.402275+0000","last_undegraded":"2023-10-09T05:06:58.402275+0000","last_fullsized":"2023-10-09T05:06:58.402275+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"673'6720","reported_seq":18334,"reported_epoch":673,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:03.563397+0000","last_change":"2023-10-09T05:02:20.857473+0000","last_active":"2023-10-09T05:07:03.563397+0000","last_peered":"2023-10-09T05:07:03.563397+0000","last_clean":"2023-10-09T05:01:41.745948+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:07:03.563397+0000","last_undegraded":"2023-10-09T05:02:20.769604+0000","last_fullsized":"2023-10-09T05:07:03.563397+0000","mapping_epoch":597,"log_start":"444'1133","ondisk_log_start":"444'1133","created":532,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5587,"log_dups_size":0,"ondisk_log_size":5587,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4221,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2743,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"1,3,6","objects":594}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":52,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-10-09T05:06:47.223829+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:06:47.223829+0000","last_peered":"2023-10-09T05:06:47.223829+0000","last_clean":"2023-10-09T05:06:47.223829+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:06:47.223829+0000","last_undegraded":"2023-10-09T05:06:47.223829+0000","last_fullsized":"2023-10-09T05:06:47.223829+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":149,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-10-09T05:06:58.402579+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:06:58.402579+0000","last_peered":"2023-10-09T05:06:58.402579+0000","last_clean":"2023-10-09T05:06:58.402579+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:06:58.402579+0000","last_undegraded":"2023-10-09T05:06:58.402579+0000","last_fullsized":"2023-10-09T05:06:58.402579+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":1189347352,"num_objects":18149,"num_object_clones":0,"num_object_copies":54447,"num_objects_missing_on_primary":6402,"num_objects_missing":6402,"num_objects_degraded":20645,"num_objects_misplaced":2016,"num_objects_unfound":0,"num_objects_dirty":18149,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137069,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":86896,"num_bytes_recovered":5259329536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10830098432,"data_stored":10830086216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":212663,"ondisk_log_size":212663,"up":108,"acting":92,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":17620,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":17192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234716,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2856452,"kb_used_data":1343100,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":90888700,"statfs":{"total":95995035648,"available":93070028800,"internally_reserved":0,"allocated":1375334400,"data_stored":1373621647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:16 2023","interfaces":[{"interface":"back","average":{"1min":11.641999999999999,"5min":6.4889999999999999,"15min":2.593},"min":{"1min":0.25900000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":221.91800000000001,"5min":221.91800000000001,"15min":221.91800000000001},"last":1},{"interface":"front","average":{"1min":11.976000000000001,"5min":6.7329999999999997,"15min":2.839},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.26300000000000001},"max":{"1min":221.87799999999999,"5min":221.87799999999999,"15min":221.87799999999999},"last":1.665}]},{"osd":1,"last update":"Mon Oct 9 05:06:16 2023","interfaces":[{"interface":"back","average":{"1min":4.8289999999999997,"5min":4.2850000000000001,"15min":2.1459999999999999},"min":{"1min":0.32800000000000001,"5min":0.28899999999999998,"15min":0.25800000000000001},"max":{"1min":63.927999999999997,"5min":63.927999999999997,"15min":63.927999999999997},"last":5.6760000000000002},{"interface":"front","average":{"1min":4.7889999999999997,"5min":4.0110000000000001,"15min":2.0990000000000002},"min":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":64.334000000000003,"5min":64.334000000000003,"15min":64.334000000000003},"last":0.96799999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.7469999999999999,"15min":3.8620000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3470000000000004,"5min":64.480999999999995,"15min":207.96600000000001},"last":1.034},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.7090000000000001,"15min":3.9550000000000001},"min":{"1min":0.35199999999999998,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":6.3579999999999997,"5min":55.134,"15min":207.97900000000001},"last":1.0429999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:06:12 2023","interfaces":[{"interface":"back","average":{"1min":15.56,"5min":8.3770000000000007,"15min":3.7829999999999999},"min":{"1min":0.36699999999999999,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":241.666,"5min":241.666,"15min":241.666},"last":1.119},{"interface":"front","average":{"1min":14.212,"5min":7.8179999999999996,"15min":3.7149999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":230.751,"5min":230.751,"15min":230.751},"last":1.583}]},{"osd":4,"last update":"Mon Oct 9 05:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.28699999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.48099999999999998,"15min":0.47499999999999998},"min":{"1min":0.28000000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.54100000000000004,"5min":2.923,"15min":2.923},"last":0.5}]},{"osd":5,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3849999999999998,"5min":1.847,"15min":1.897},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":74.915000000000006,"5min":74.915000000000006,"15min":131.84899999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":4.4939999999999998,"5min":2,"15min":1.99},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":74.972999999999999,"5min":74.972999999999999,"15min":135.74199999999999},"last":0.39500000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.90000000000000002,"15min":1.0680000000000001},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":2.1829999999999998,"5min":10.183999999999999,"15min":57.168999999999997},"last":0.435},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.0860000000000001,"15min":1.218},"min":{"1min":0.32900000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":1.7,"5min":21.097000000000001,"15min":57.183999999999997},"last":1.107}]}]},{"osd":6,"up_from":46,"seq":197568496349,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1912644,"kb_used_data":397948,"kb_used_omap":0,"kb_used_meta":1514688,"kb_avail":91832508,"statfs":{"total":95995035648,"available":94036488192,"internally_reserved":0,"allocated":407498752,"data_stored":405786232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1551040512},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":6.4480000000000004,"5min":6.9930000000000003,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.218,"15min":0.218},"max":{"1min":49.659999999999997,"5min":321.346,"15min":321.346},"last":0.95799999999999996},{"interface":"front","average":{"1min":6.0620000000000003,"5min":5.2439999999999998,"15min":2.355},"min":{"1min":0.35799999999999998,"5min":0.255,"15min":0.255},"max":{"1min":46.987000000000002,"5min":169.119,"15min":169.119},"last":0.28199999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":2.415,"15min":3.2170000000000001},"min":{"1min":0.39000000000000001,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":10.973000000000001,"5min":72.840999999999994,"15min":220.48400000000001},"last":4.2370000000000001},{"interface":"front","average":{"1min":2.2810000000000001,"5min":2.665,"15min":3.2000000000000002},"min":{"1min":0.34899999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":10.02,"5min":72.832999999999998,"15min":220.56200000000001},"last":0.78500000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":1.925,"15min":2.101},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.25600000000000001},"max":{"1min":7.1159999999999997,"5min":23.943000000000001,"15min":133.535},"last":2.9249999999999998},{"interface":"front","average":{"1min":2.5030000000000001,"5min":1.907,"15min":2.036},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":8.0039999999999996,"5min":25.978999999999999,"15min":133.51900000000001},"last":2.9140000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":5.6349999999999998,"5min":3.3769999999999998,"15min":2.956},"min":{"1min":0.33700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":45.915999999999997,"5min":64.355999999999995,"15min":147.43600000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":6.0229999999999997,"5min":3.2040000000000002,"15min":3.0270000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":45.868000000000002,"5min":56.133000000000003,"15min":147.41999999999999},"last":1.1639999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:06:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.88900000000000001,"15min":0.78600000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.441,"5min":18.186,"15min":18.186},"last":0.378},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.0329999999999999,"15min":1.0129999999999999},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.2400000000000002,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.41299999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":1.006,"15min":0.99099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.499,"5min":25.934000000000001,"15min":25.934000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.83599999999999997,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.254},"max":{"1min":0.57699999999999996,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.44500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.86499999999999999,"15min":0.998},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":7.5039999999999996,"5min":7.5039999999999996,"15min":36.204999999999998},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.0760000000000001,"15min":0.98599999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.3399999999999999,"5min":20.041,"15min":20.041},"last":1.0840000000000001}]}]},{"osd":1,"up_from":118,"seq":506806141416,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2055944,"kb_used_data":679680,"kb_used_omap":4,"kb_used_meta":1376251,"kb_avail":91689208,"statfs":{"total":95995035648,"available":93889748992,"internally_reserved":0,"allocated":695992320,"data_stored":692396508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1409281934},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":4.2030000000000003,"5min":4.7080000000000002,"15min":2.254},"min":{"1min":0.215,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":27.922000000000001,"5min":100.247,"15min":100.247},"last":6.923},{"interface":"front","average":{"1min":4.2709999999999999,"5min":5.1310000000000002,"15min":2.3740000000000001},"min":{"1min":0.32300000000000001,"5min":0.247,"15min":0.214},"max":{"1min":19.297000000000001,"5min":99.935000000000002,"15min":99.935000000000002},"last":3.419}]},{"osd":2,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":1.8340000000000001,"15min":2.7120000000000002},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.393000000000001,"5min":12.460000000000001,"15min":263.089},"last":6.9800000000000004},{"interface":"front","average":{"1min":4.359,"5min":2.3860000000000001,"15min":2.8929999999999998},"min":{"1min":0.36399999999999999,"5min":0.252,"15min":0.222},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":254.50899999999999},"last":0.41699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:06:19 2023","interfaces":[{"interface":"back","average":{"1min":10.869999999999999,"5min":8.1379999999999999,"15min":3.9049999999999998},"min":{"1min":0.41699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":102.729,"5min":235.43600000000001,"15min":235.43600000000001},"last":7.0279999999999996},{"interface":"front","average":{"1min":8.6829999999999998,"5min":7.2679999999999998,"15min":3.456},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":104.13500000000001,"5min":234.32499999999999,"15min":234.32499999999999},"last":2.0630000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:06:29 2023","interfaces":[{"interface":"back","average":{"1min":4.1589999999999998,"5min":2.0699999999999998,"15min":1.54},"min":{"1min":0.39600000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":19.358000000000001,"5min":19.358000000000001,"15min":19.358000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":3.1440000000000001,"5min":1.778,"15min":1.224},"min":{"1min":0.36299999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.832999999999998,"5min":18.832999999999998,"15min":18.832999999999998},"last":2.4180000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1479999999999997,"5min":1.889,"15min":1.4410000000000001},"min":{"1min":0.42199999999999999,"5min":0.36199999999999999,"15min":0.34599999999999997},"max":{"1min":19.382999999999999,"5min":19.382999999999999,"15min":19.382999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.1480000000000001,"5min":1.9630000000000001,"15min":1.47},"min":{"1min":0.41199999999999998,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":18.777999999999999,"5min":18.777999999999999,"15min":19.597999999999999},"last":2.4550000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1539999999999999,"5min":1.974,"15min":1.4490000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":19.338000000000001,"5min":19.338000000000001,"15min":19.338000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.173,"5min":1.8420000000000001,"15min":1.4830000000000001},"min":{"1min":0.42699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":18.867000000000001,"5min":18.867000000000001,"15min":19.548999999999999},"last":2.3690000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:06:32 2023","interfaces":[{"interface":"back","average":{"1min":3.202,"5min":1.9550000000000001,"15min":1.681},"min":{"1min":0.51000000000000001,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":18.899999999999999,"5min":18.899999999999999,"15min":22.731999999999999},"last":2.4870000000000001},{"interface":"front","average":{"1min":1.994,"5min":1.5629999999999999,"15min":1.488},"min":{"1min":0.5,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":10.345000000000001,"5min":11.903,"15min":22.765999999999998},"last":7.0099999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024499,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1597076,"kb_used_data":434572,"kb_used_omap":5,"kb_used_meta":1162490,"kb_avail":92148076,"statfs":{"total":95995035648,"available":94359629824,"internally_reserved":0,"allocated":445001728,"data_stored":442101084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1190390394},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:06:12 2023","interfaces":[{"interface":"back","average":{"1min":5.1909999999999998,"5min":3.673,"15min":2.5150000000000001},"min":{"1min":0.28299999999999997,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":19.042000000000002,"5min":53.176000000000002,"15min":155.411},"last":3.0150000000000001},{"interface":"front","average":{"1min":5.5419999999999998,"5min":3.859,"15min":2.5430000000000001},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.657,"5min":53.372999999999998,"15min":154.75},"last":3.0569999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:06:33 2023","interfaces":[{"interface":"back","average":{"1min":3.4660000000000002,"5min":2.4369999999999998,"15min":2.7090000000000001},"min":{"1min":0.29099999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":19.141999999999999,"5min":19.141999999999999,"15min":151.06399999999999},"last":2.9289999999999998},{"interface":"front","average":{"1min":4.7279999999999998,"5min":2.9020000000000001,"15min":2.9409999999999998},"min":{"1min":0.377,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":15.444000000000001,"5min":20.239000000000001,"15min":151.22399999999999},"last":0.49099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:05:59 2023","interfaces":[{"interface":"back","average":{"1min":13.766999999999999,"5min":7.6890000000000001,"15min":4.4539999999999997},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":114.871,"5min":290.89100000000002,"15min":290.89100000000002},"last":2.9649999999999999},{"interface":"front","average":{"1min":15.404999999999999,"5min":7.9009999999999998,"15min":4.6449999999999996},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":115.375,"5min":290.73899999999998,"15min":290.73899999999998},"last":0.41899999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:06:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":2.0070000000000001,"15min":1.903},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.5170000000000003,"5min":9.5329999999999995,"15min":9.5329999999999995},"last":0.377},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.7709999999999999,"15min":1.613},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":9.9399999999999995,"5min":11.81,"15min":11.81},"last":2.8380000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":2.4300000000000002,"15min":1.494},"min":{"1min":0.42199999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.51,"5min":16.888000000000002,"15min":16.888000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.0979999999999999,"15min":1.3460000000000001},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.5670000000000002,"5min":14.036,"15min":14.036},"last":0.47099999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":2.6629999999999998,"15min":1.6459999999999999},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":15.598000000000001,"5min":20.129999999999999,"15min":39.594999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.1150000000000002,"5min":2.6299999999999999,"15min":1.6599999999999999},"min":{"1min":0.36299999999999999,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":7.6020000000000003,"5min":47.241,"15min":47.241},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:06:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":2.3460000000000001,"15min":1.524},"min":{"1min":0.441,"5min":0.41899999999999998,"15min":0.39400000000000002},"max":{"1min":7.6360000000000001,"5min":21.658000000000001,"15min":21.658000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.929,"5min":2.2120000000000002,"15min":1.508},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":9.9770000000000003,"5min":21.68,"15min":21.68},"last":2.8929999999999998}]}]},{"osd":2,"up_from":0,"seq":949187772778,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356472,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1785676,"kb_used_data":777988,"kb_used_omap":2,"kb_used_meta":1007677,"kb_avail":91959476,"statfs":{"total":95995035648,"available":94166503424,"internally_reserved":0,"allocated":796659712,"data_stored":793652269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1031861301},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":5.5090000000000003,"5min":3.3820000000000001,"15min":1.5600000000000001},"min":{"1min":0.25700000000000001,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":51.145000000000003,"5min":108.631,"15min":108.631},"last":1.28},{"interface":"front","average":{"1min":6.1710000000000003,"5min":3.6160000000000001,"15min":1.752},"min":{"1min":0.28599999999999998,"5min":0.247,"15min":0.247},"max":{"1min":51.191000000000003,"5min":110.012,"15min":110.012},"last":1.296}]},{"osd":1,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0190000000000001,"15min":1.391},"min":{"1min":0.26100000000000001,"5min":0.221,"15min":0.221},"max":{"1min":13.374000000000001,"5min":15.218,"15min":15.983000000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.754,"15min":1.294},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.245},"max":{"1min":4.54,"5min":17.544,"15min":26.692},"last":0.24199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":1.5669999999999999,"15min":2.371},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":11.996,"5min":11.996,"15min":212.065},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.8799999999999999,"5min":1.694,"15min":2.4079999999999999},"min":{"1min":0.32100000000000001,"5min":0.253,"15min":0.23499999999999999},"max":{"1min":6.2539999999999996,"5min":12.987,"15min":211.98400000000001},"last":0.28499999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.8100000000000001,"15min":1.75},"min":{"1min":0.434,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.75,"5min":11.193,"15min":11.193},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.2270000000000001,"15min":1.0069999999999999},"min":{"1min":0.41799999999999998,"5min":0.309,"15min":0.309},"max":{"1min":4.3609999999999998,"5min":10.452,"15min":10.452},"last":0.36499999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.855,"15min":1.395},"min":{"1min":0.46000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":3.6040000000000001,"5min":57.804000000000002,"15min":57.804000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.95,"5min":2.1269999999999998,"15min":1.659},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":6.383,"5min":57.442,"15min":57.442},"last":0.379}]},{"osd":6,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.649,"15min":1.222},"min":{"1min":0.376,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":6.3659999999999997,"5min":11.252000000000001,"15min":11.252000000000001},"last":0.436},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.4319999999999999,"15min":1.327},"min":{"1min":0.441,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.988,"5min":14.731,"15min":24.916},"last":0.39500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.5369999999999999,"15min":1.3580000000000001},"min":{"1min":0.47399999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":2.9470000000000001,"5min":14.784000000000001,"15min":25.004000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.4319999999999999,"15min":1.1120000000000001},"min":{"1min":0.40300000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.758,"5min":7.758,"15min":7.758},"last":0.47599999999999998}]}]},{"osd":4,"up_from":626,"seq":2688649527341,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2313752,"kb_used_data":1912656,"kb_used_omap":22,"kb_used_meta":401065,"kb_avail":91431400,"statfs":{"total":95995035648,"available":93625753600,"internally_reserved":0,"allocated":1958559744,"data_stored":1956835473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":410691405},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":9.2349999999999994,"5min":3.722,"15min":2.7679999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":130.34200000000001,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":9.4740000000000002,"5min":4.306,"15min":3.5489999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":130.49700000000001,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.307}]},{"osd":1,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":11.933999999999999,"5min":4.1520000000000001,"15min":2.8029999999999999},"min":{"1min":0.32300000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":182.32900000000001,"5min":182.32900000000001,"15min":182.32900000000001},"last":103.997},{"interface":"front","average":{"1min":11.624000000000001,"5min":3.7280000000000002,"15min":2.5099999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":182.41,"5min":182.41,"15min":182.41},"last":104.333}]},{"osd":2,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":1.825,"15min":1.7010000000000001},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.8639999999999999,"5min":11.051,"15min":11.051},"last":0.29099999999999998},{"interface":"front","average":{"1min":2.4300000000000002,"5min":1.8839999999999999,"15min":1.752},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.8810000000000002,"5min":11.036,"15min":11.036},"last":0.36199999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":2.2010000000000001,"5min":5.1879999999999997,"15min":2.512},"min":{"1min":0.40300000000000002,"5min":0.318,"15min":0.318},"max":{"1min":10.797000000000001,"5min":270.767,"15min":270.767},"last":109.53400000000001},{"interface":"front","average":{"1min":2.2919999999999998,"5min":5.4989999999999997,"15min":3.0619999999999998},"min":{"1min":0.37,"5min":0.313,"15min":0.313},"max":{"1min":5.173,"5min":270.274,"15min":270.274},"last":109.84}]},{"osd":5,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.38,"15min":0.38400000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56499999999999995,"5min":1.304,"15min":1.304},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.438,"15min":0.44500000000000001},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.54300000000000004,"5min":1.504,"15min":1.504},"last":0.39600000000000002}]},{"osd":6,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.57199999999999995,"15min":0.63200000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57499999999999996,"5min":4.7910000000000004,"15min":4.7910000000000004},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.46200000000000002,"15min":0.45600000000000002},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81899999999999995,"5min":2.0920000000000001,"15min":2.0920000000000001},"last":0.34699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:06:30 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.501,"15min":0.51700000000000002},"min":{"1min":0.33500000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.58299999999999996,"5min":1.585,"15min":1.585},"last":0.372},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.47899999999999998,"15min":0.47299999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.58799999999999997,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.38600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757993,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4184244,"kb_used_data":2699116,"kb_used_omap":0,"kb_used_meta":1485120,"kb_avail":89560908,"statfs":{"total":95995035648,"available":91710369792,"internally_reserved":0,"allocated":2763894784,"data_stored":2762193659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1520762880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:06:23 2023","interfaces":[{"interface":"back","average":{"1min":7.2629999999999999,"5min":4.718,"15min":2.5369999999999999},"min":{"1min":0.27200000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":124.504,"5min":124.504,"15min":124.504},"last":0.433},{"interface":"front","average":{"1min":7.468,"5min":5.0670000000000002,"15min":2.7229999999999999},"min":{"1min":0.254,"5min":0.246,"15min":0.245},"max":{"1min":124.678,"5min":124.678,"15min":124.678},"last":0.34499999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:06:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3999999999999999,"5min":5.2469999999999999,"15min":3.4630000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23200000000000001},"max":{"1min":25.411000000000001,"5min":262.26900000000001,"15min":262.26900000000001},"last":113.977},{"interface":"front","average":{"1min":2.782,"5min":5.367,"15min":3.6019999999999999},"min":{"1min":0.32200000000000001,"5min":0.27300000000000002,"15min":0.26900000000000002},"max":{"1min":25.303000000000001,"5min":262.291,"15min":262.291},"last":112.947}]},{"osd":2,"last update":"Mon Oct 9 05:06:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":1.653,"15min":1.919},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":9.6440000000000001,"5min":10.949999999999999,"15min":106.238},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.1280000000000001,"5min":1.851,"15min":2.1659999999999999},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.29999999999999999},"max":{"1min":4.8979999999999997,"5min":11.912000000000001,"15min":104.31699999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":7.8579999999999997,"5min":7.0819999999999999,"15min":3.524},"min":{"1min":0.375,"5min":0.318,"15min":0.318},"max":{"1min":121.40600000000001,"5min":125.824,"15min":125.824},"last":0.46300000000000002},{"interface":"front","average":{"1min":7.9930000000000003,"5min":6.9649999999999999,"15min":3.4710000000000001},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":121.53700000000001,"5min":125.761,"15min":125.761},"last":0.42599999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:06:31 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.99,"5min":1.99,"15min":1.99},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.45800000000000002,"15min":0.46000000000000002},"min":{"1min":0.33600000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.35799999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.68600000000000005,"15min":0.73899999999999999},"min":{"1min":0.28499999999999998,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":2.0289999999999999,"5min":11.093,"15min":11.093},"last":0.372},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.59899999999999998,"15min":0.81799999999999995},"min":{"1min":0.32000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.982,"5min":5.9329999999999998,"15min":13.083},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:06:05 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.68999999999999995,"15min":0.99099999999999999},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.948,"5min":8.3520000000000003,"15min":9.6809999999999992},"last":0.442},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.89200000000000002,"15min":1.0589999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":0.90900000000000003,"5min":11.791,"15min":15.461},"last":0.498}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":435159040,"data_stored":435159040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":684851200,"data_stored":684851200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2462384128,"data_stored":2462384128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":786366464,"data_stored":786366464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1946816512,"data_stored":1946812440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2755596288,"data_stored":2755592216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":391647232,"data_stored":391643160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1367277568,"data_stored":1367277568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:07:08.733 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:07:08.846 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:08.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:08.968 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:08.981 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:08.995 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:09.043 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:09.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:09.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:09.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:09.177 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:09.226 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:09.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:09.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:09.414 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:09.633 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:09 smithi029 ceph-mon[223858]: pgmap v2189: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 316 op/s; 19703/51765 objects degraded (38.062%); 1447/51765 objects misplaced (2.795%); 11 MiB/s, 200 objects/s recovering 2023-10-09T05:07:09.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:09 smithi029 ceph-mon[216903]: pgmap v2189: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 316 op/s; 19703/51765 objects degraded (38.062%); 1447/51765 objects misplaced (2.795%); 11 MiB/s, 200 objects/s recovering 2023-10-09T05:07:09.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:09 smithi177 ceph-mon[195797]: pgmap v2189: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 316 op/s; 19703/51765 objects degraded (38.062%); 1447/51765 objects misplaced (2.795%); 11 MiB/s, 200 objects/s recovering 2023-10-09T05:07:09.886 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:09.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:09.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:09.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:10.072 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:10.173 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:10.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:10.282 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:10.314 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:10.365 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:10.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:10.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:10.470 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:10.500 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:10.553 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:10.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:10.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:10.662 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:10.688 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:10.730 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:10.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:10.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:10.783 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:07:10.835 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:10.862 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:10.920 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:10.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:10.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:11.024 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:11.088 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:11.476 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:11.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:11.568 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:11.571 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:11.585 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:11.612 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:11.660 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:11.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:11.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:11.767 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:11 smithi177 ceph-mon[195797]: pgmap v2190: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1019 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 19091/48936 objects degraded (39.012%); 1447/48936 objects misplaced (2.957%); 11 MiB/s, 204 objects/s recovering 2023-10-09T05:07:11.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:11 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 19091/48936 objects degraded (39.012%), 24 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:11.831 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:11 smithi029 ceph-mon[216903]: pgmap v2190: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1019 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 19091/48936 objects degraded (39.012%); 1447/48936 objects misplaced (2.957%); 11 MiB/s, 204 objects/s recovering 2023-10-09T05:07:11.832 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:11 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 19091/48936 objects degraded (39.012%), 24 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:11.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:11 smithi029 ceph-mon[223858]: pgmap v2190: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 1019 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 19091/48936 objects degraded (39.012%); 1447/48936 objects misplaced (2.957%); 11 MiB/s, 204 objects/s recovering 2023-10-09T05:07:11.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:11 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 19091/48936 objects degraded (39.012%), 24 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:11.843 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:11.914 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:12.096 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:12.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:12.205 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:12.291 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:13.174 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:07:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:07:12] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-10-09T05:07:13.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:13 smithi177 ceph-mon[195797]: pgmap v2191: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 997 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 18258/47895 objects degraded (38.121%); 1192/47895 objects misplaced (2.489%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:07:13.810 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:13 smithi029 ceph-mon[216903]: pgmap v2191: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 997 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 18258/47895 objects degraded (38.121%); 1192/47895 objects misplaced (2.489%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:07:13.810 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:13 smithi029 ceph-mon[223858]: pgmap v2191: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 997 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 354 op/s; 18258/47895 objects degraded (38.121%); 1192/47895 objects misplaced (2.489%); 12 MiB/s, 192 objects/s recovering 2023-10-09T05:07:14.878 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:14.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:14.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:14.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:14.992 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:15.071 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:15.548 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:15 smithi029 ceph-mon[216903]: pgmap v2192: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 909 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 17694/43665 objects degraded (40.522%); 888/43665 objects misplaced (2.034%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:07:15.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:15 smithi029 ceph-mon[223858]: pgmap v2192: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 909 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 17694/43665 objects degraded (40.522%); 888/43665 objects misplaced (2.034%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:07:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:15 smithi177 ceph-mon[195797]: pgmap v2192: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 909 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 17694/43665 objects degraded (40.522%); 888/43665 objects misplaced (2.034%); 12 MiB/s, 195 objects/s recovering 2023-10-09T05:07:16.592 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:07:16.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1680028736' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:16.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:16 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:16 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 17694/43665 objects degraded (40.522%), 23 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1680028736' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:16 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:16 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 17694/43665 objects degraded (40.522%), 23 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1680028736' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:16 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:16 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 17694/43665 objects degraded (40.522%), 23 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:17.119 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:17.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:17.228 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:17.283 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[223858]: pgmap v2193: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 867 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 16405/41661 objects degraded (39.377%); 505/41661 objects misplaced (1.212%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:07:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[223858]: failed to encode map e675 with expected crc 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[223858]: osdmap e675: 8 total, 8 up, 7 in 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[223858]: failed to encode map e675 with expected crc 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[216903]: pgmap v2193: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 867 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 16405/41661 objects degraded (39.377%); 505/41661 objects misplaced (1.212%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[216903]: failed to encode map e675 with expected crc 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:07:17.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[216903]: osdmap e675: 8 total, 8 up, 7 in 2023-10-09T05:07:17.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:17 smithi029 ceph-mon[216903]: failed to encode map e675 with expected crc 2023-10-09T05:07:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:17 smithi177 ceph-mon[195797]: pgmap v2193: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 867 MiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 16405/41661 objects degraded (39.377%); 505/41661 objects misplaced (1.212%); 13 MiB/s, 215 objects/s recovering 2023-10-09T05:07:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:17 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:07:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:17 smithi177 ceph-mon[195797]: failed to encode map e675 with expected crc 2023-10-09T05:07:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:17 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:07:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:17 smithi177 ceph-mon[195797]: osdmap e675: 8 total, 8 up, 7 in 2023-10-09T05:07:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:17 smithi177 ceph-mon[195797]: failed to encode map e675 with expected crc 2023-10-09T05:07:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[223858]: failed to encode map e676 with expected crc 2023-10-09T05:07:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[223858]: failed to encode map e676 with expected crc 2023-10-09T05:07:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[223858]: osdmap e676: 8 total, 8 up, 7 in 2023-10-09T05:07:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[223858]: failed to encode map e676 with expected crc 2023-10-09T05:07:18.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[216903]: failed to encode map e676 with expected crc 2023-10-09T05:07:18.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[216903]: failed to encode map e676 with expected crc 2023-10-09T05:07:18.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[216903]: osdmap e676: 8 total, 8 up, 7 in 2023-10-09T05:07:18.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:18 smithi029 ceph-mon[216903]: failed to encode map e676 with expected crc 2023-10-09T05:07:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:18 smithi177 ceph-mon[195797]: failed to encode map e676 with expected crc 2023-10-09T05:07:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:18 smithi177 ceph-mon[195797]: failed to encode map e676 with expected crc 2023-10-09T05:07:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:18 smithi177 ceph-mon[195797]: osdmap e676: 8 total, 8 up, 7 in 2023-10-09T05:07:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:18 smithi177 ceph-mon[195797]: failed to encode map e676 with expected crc 2023-10-09T05:07:19.202 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:19.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:19.308 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:19.335 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:19.379 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:19.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:19.486 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:19.521 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:19.590 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:19.596 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:19 smithi177 ceph-mon[195797]: pgmap v2196: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 813 MiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 16143/39075 objects degraded (41.313%); 505/39075 objects misplaced (1.292%); 13 MiB/s, 208 objects/s recovering 2023-10-09T05:07:19.596 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:19 smithi177 ceph-mon[195797]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.596 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:19 smithi177 ceph-mon[195797]: osdmap e677: 8 total, 8 up, 7 in 2023-10-09T05:07:19.596 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:19 smithi177 ceph-mon[195797]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.596 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:19 smithi177 ceph-mon[195797]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:19.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:19.698 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:19.724 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:19.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:19.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:19.894 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[216903]: pgmap v2196: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 813 MiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 16143/39075 objects degraded (41.313%); 505/39075 objects misplaced (1.292%); 13 MiB/s, 208 objects/s recovering 2023-10-09T05:07:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[216903]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[216903]: osdmap e677: 8 total, 8 up, 7 in 2023-10-09T05:07:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[216903]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[216903]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[223858]: pgmap v2196: 119 pgs: 2 active+recovering+undersized+remapped, 94 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 10 active+recovery_wait+degraded; 813 MiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 16143/39075 objects degraded (41.313%); 505/39075 objects misplaced (1.292%); 13 MiB/s, 208 objects/s recovering 2023-10-09T05:07:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[223858]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[223858]: osdmap e677: 8 total, 8 up, 7 in 2023-10-09T05:07:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[223858]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:19 smithi029 ceph-mon[223858]: failed to encode map e677 with expected crc 2023-10-09T05:07:19.948 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:21.184 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:21.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:21.294 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:21.324 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:21.384 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:21.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:21.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:21.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:21.501 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:07:21.501 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:07:21.501 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:07:21.501 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:07:21.501 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:07:21.574 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:21.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:21 smithi029 ceph-mon[216903]: pgmap v2198: 119 pgs: 1 active+recovering+undersized+remapped, 96 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded; 708 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 15597/34008 objects degraded (45.863%); 422/34008 objects misplaced (1.241%); 11 MiB/s, 265 objects/s recovering 2023-10-09T05:07:21.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:21 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 15597/34008 objects degraded (45.863%), 22 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:21.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:21 smithi029 ceph-mon[223858]: pgmap v2198: 119 pgs: 1 active+recovering+undersized+remapped, 96 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded; 708 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 15597/34008 objects degraded (45.863%); 422/34008 objects misplaced (1.241%); 11 MiB/s, 265 objects/s recovering 2023-10-09T05:07:21.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:21 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 15597/34008 objects degraded (45.863%), 22 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:21 smithi177 ceph-mon[195797]: pgmap v2198: 119 pgs: 1 active+recovering+undersized+remapped, 96 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded; 708 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 15597/34008 objects degraded (45.863%); 422/34008 objects misplaced (1.241%); 11 MiB/s, 265 objects/s recovering 2023-10-09T05:07:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:21 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 15597/34008 objects degraded (45.863%), 22 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:22.819 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:07:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:07:22] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.43.0" 2023-10-09T05:07:23.354 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:07:23.573 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:23.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:23.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[223858]: pgmap v2199: 119 pgs: 1 active+recovering+undersized+remapped, 96 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded; 676 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 14671/32478 objects degraded (45.172%); 11 MiB/s, 286 objects/s recovering 2023-10-09T05:07:23.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[223858]: failed to encode map e678 with expected crc 2023-10-09T05:07:23.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[223858]: osdmap e678: 8 total, 8 up, 7 in 2023-10-09T05:07:23.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[223858]: failed to encode map e678 with expected crc 2023-10-09T05:07:23.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[223858]: failed to encode map e678 with expected crc 2023-10-09T05:07:23.679 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:23.715 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:23.775 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:23.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:23.838 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:07:23.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[216903]: pgmap v2199: 119 pgs: 1 active+recovering+undersized+remapped, 96 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded; 676 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 14671/32478 objects degraded (45.172%); 11 MiB/s, 286 objects/s recovering 2023-10-09T05:07:23.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[216903]: failed to encode map e678 with expected crc 2023-10-09T05:07:23.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[216903]: osdmap e678: 8 total, 8 up, 7 in 2023-10-09T05:07:23.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[216903]: failed to encode map e678 with expected crc 2023-10-09T05:07:23.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:23 smithi029 ceph-mon[216903]: failed to encode map e678 with expected crc 2023-10-09T05:07:23.884 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:23.982 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:24.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:23 smithi177 ceph-mon[195797]: pgmap v2199: 119 pgs: 1 active+recovering+undersized+remapped, 96 active+clean, 13 active+recovery_wait+undersized+degraded+remapped, 9 active+recovery_wait+degraded; 676 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 14671/32478 objects degraded (45.172%); 11 MiB/s, 286 objects/s recovering 2023-10-09T05:07:24.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:23 smithi177 ceph-mon[195797]: failed to encode map e678 with expected crc 2023-10-09T05:07:24.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:23 smithi177 ceph-mon[195797]: osdmap e678: 8 total, 8 up, 7 in 2023-10-09T05:07:24.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:23 smithi177 ceph-mon[195797]: failed to encode map e678 with expected crc 2023-10-09T05:07:24.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:23 smithi177 ceph-mon[195797]: failed to encode map e678 with expected crc 2023-10-09T05:07:24.222 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:24.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:24.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:24.274 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:07:24.310 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:07:24.311 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:24.328 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:24.367 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:07:24.401 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:24.424 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:07:24.475 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:07:24.475 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 14671/32478 objects degraded (45.172%), 22 pgs degraded, 14 pgs undersized","count":36},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2981,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":679,"num_osds":8,"num_up_osds":8,"osd_up_since":1696827803,"num_in_osds":7,"osd_in_since":1696827750,"num_remapped_pgs":12},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":13},{"state_name":"active+recovery_wait+degraded","count":9},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":10826,"data_bytes":708450232,"bytes_used":16120254464,"bytes_avail":655844995072,"bytes_total":671965249536,"degraded_objects":14671,"degraded_total":32478,"degraded_ratio":0.45172116509637289,"recovering_objects_per_sec":286,"recovering_bytes_per_sec":11266589,"recovering_keys_per_sec":0,"num_objects_recovered":1717,"num_bytes_recovered":67633152,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":509},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":36,"modified":"2023-10-09T05:04:23.723180+0000","services":{}},"progress_events":{"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true},"8039e601-7bd2-4b88-bccf-3b879c49d7d2":{"message":"Global Recovery Event (13m)\n [======================......] (remaining: 3m)","progress":0.78991597890853882,"add_to_ceph_s":true}}} 2023-10-09T05:07:24.483 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:07:24.520 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:07:24.547 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:24.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:24.557 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:07:24.592 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:07:24.596 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.596 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.596 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.596 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.596 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.596 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.597 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.597 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:07:24.653 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:24.690 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:24.750 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:24.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:24.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[216903]: failed to encode map e679 with expected crc 2023-10-09T05:07:24.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[216903]: osdmap e679: 8 total, 8 up, 7 in 2023-10-09T05:07:24.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[216903]: failed to encode map e679 with expected crc 2023-10-09T05:07:24.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[216903]: failed to encode map e679 with expected crc 2023-10-09T05:07:24.852 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4049615009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2234466283' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[223858]: failed to encode map e679 with expected crc 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[223858]: osdmap e679: 8 total, 8 up, 7 in 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[223858]: failed to encode map e679 with expected crc 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[223858]: failed to encode map e679 with expected crc 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4049615009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:24.853 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2234466283' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:07:24.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:24.932 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:24 smithi177 ceph-mon[195797]: failed to encode map e679 with expected crc 2023-10-09T05:07:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:24 smithi177 ceph-mon[195797]: osdmap e679: 8 total, 8 up, 7 in 2023-10-09T05:07:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:24 smithi177 ceph-mon[195797]: failed to encode map e679 with expected crc 2023-10-09T05:07:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:24 smithi177 ceph-mon[195797]: failed to encode map e679 with expected crc 2023-10-09T05:07:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4049615009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:25.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2234466283' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:07:25.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[216903]: pgmap v2202: 119 pgs: 2 peering, 98 active+clean, 12 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 540 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 14043/25974 objects degraded (54.066%); 12 MiB/s, 393 objects/s recovering 2023-10-09T05:07:25.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[216903]: failed to encode map e680 with expected crc 2023-10-09T05:07:25.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[216903]: osdmap e680: 8 total, 8 up, 7 in 2023-10-09T05:07:25.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[216903]: failed to encode map e680 with expected crc 2023-10-09T05:07:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[223858]: pgmap v2202: 119 pgs: 2 peering, 98 active+clean, 12 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 540 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 14043/25974 objects degraded (54.066%); 12 MiB/s, 393 objects/s recovering 2023-10-09T05:07:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[223858]: failed to encode map e680 with expected crc 2023-10-09T05:07:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[223858]: osdmap e680: 8 total, 8 up, 7 in 2023-10-09T05:07:25.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:25 smithi029 ceph-mon[223858]: failed to encode map e680 with expected crc 2023-10-09T05:07:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:25 smithi177 ceph-mon[195797]: pgmap v2202: 119 pgs: 2 peering, 98 active+clean, 12 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 540 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 14043/25974 objects degraded (54.066%); 12 MiB/s, 393 objects/s recovering 2023-10-09T05:07:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:25 smithi177 ceph-mon[195797]: failed to encode map e680 with expected crc 2023-10-09T05:07:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:25 smithi177 ceph-mon[195797]: osdmap e680: 8 total, 8 up, 7 in 2023-10-09T05:07:26.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:25 smithi177 ceph-mon[195797]: failed to encode map e680 with expected crc 2023-10-09T05:07:26.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:26 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 14043/25974 objects degraded (54.066%), 19 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:26.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:26 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 14043/25974 objects degraded (54.066%), 19 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:26 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 14043/25974 objects degraded (54.066%), 19 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:27 smithi029 ceph-mon[216903]: pgmap v2204: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 98 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 494 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 11706/23760 objects degraded (49.268%); 462/23760 objects misplaced (1.944%); 15 MiB/s, 351 objects/s recovering 2023-10-09T05:07:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:27 smithi029 ceph-mon[223858]: pgmap v2204: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 98 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 494 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 11706/23760 objects degraded (49.268%); 462/23760 objects misplaced (1.944%); 15 MiB/s, 351 objects/s recovering 2023-10-09T05:07:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:27 smithi177 ceph-mon[195797]: pgmap v2204: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 98 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 494 MiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 11706/23760 objects degraded (49.268%); 462/23760 objects misplaced (1.944%); 15 MiB/s, 351 objects/s recovering 2023-10-09T05:07:28.056 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:28.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:28.166 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:28.235 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:28.271 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-09T05:07:28.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:07:28.448 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:28.517 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:28.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:28.639 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:28.677 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:28.735 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:28.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:28.843 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:28.880 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:28.944 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:28.950 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:28.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:29.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:29.083 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:29.152 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:29.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:29.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:29.289 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:29.661 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:29 smithi029 ceph-mon[216903]: pgmap v2205: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 433 MiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 11440/20850 objects degraded (54.868%); 462/20850 objects misplaced (2.216%); 11 MiB/s, 251 objects/s recovering 2023-10-09T05:07:29.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:29 smithi029 ceph-mon[223858]: pgmap v2205: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 433 MiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 11440/20850 objects degraded (54.868%); 462/20850 objects misplaced (2.216%); 11 MiB/s, 251 objects/s recovering 2023-10-09T05:07:30.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:29 smithi177 ceph-mon[195797]: pgmap v2205: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 99 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 433 MiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 11440/20850 objects degraded (54.868%); 462/20850 objects misplaced (2.216%); 11 MiB/s, 251 objects/s recovering 2023-10-09T05:07:30.049 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:30.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:30.155 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:30.274 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:30.385 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:07:30.443 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:30.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:30.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:30.563 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:30.595 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:30.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:30.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:30.762 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:30.823 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:30.857 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:31.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:31 smithi029 ceph-mon[223858]: pgmap v2206: 119 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 318 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 10997/15303 objects degraded (71.862%); 462/15303 objects misplaced (3.019%); 9.4 MiB/s, 216 objects/s recovering 2023-10-09T05:07:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:31 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 10997/15303 objects degraded (71.862%), 18 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:31 smithi029 ceph-mon[216903]: pgmap v2206: 119 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 318 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 10997/15303 objects degraded (71.862%); 462/15303 objects misplaced (3.019%); 9.4 MiB/s, 216 objects/s recovering 2023-10-09T05:07:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:31 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 10997/15303 objects degraded (71.862%), 18 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:31 smithi177 ceph-mon[195797]: pgmap v2206: 119 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 318 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 10997/15303 objects degraded (71.862%); 462/15303 objects misplaced (3.019%); 9.4 MiB/s, 216 objects/s recovering 2023-10-09T05:07:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:31 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 10997/15303 objects degraded (71.862%), 18 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:32.709 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:07:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/184184180' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:07:33.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/184184180' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:07:33.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/184184180' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:07:33.173 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:07:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:07:32] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.43.0" 2023-10-09T05:07:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: pgmap v2207: 119 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 288 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 10225/13866 objects degraded (73.742%); 1/13866 objects misplaced (0.007%); 13 MiB/s, 242 objects/s recovering 2023-10-09T05:07:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:07:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/184184180' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:07:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: osdmap e681: 8 total, 8 up, 7 in 2023-10-09T05:07:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:33 smithi177 ceph-mon[195797]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.067 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: pgmap v2207: 119 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 288 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 10225/13866 objects degraded (73.742%); 1/13866 objects misplaced (0.007%); 13 MiB/s, 242 objects/s recovering 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/184184180' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: osdmap e681: 8 total, 8 up, 7 in 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[216903]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: pgmap v2207: 119 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 11 active+recovery_wait+undersized+degraded+remapped, 7 active+recovery_wait+degraded; 288 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 10225/13866 objects degraded (73.742%); 1/13866 objects misplaced (0.007%); 13 MiB/s, 242 objects/s recovering 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/184184180' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: osdmap e681: 8 total, 8 up, 7 in 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:33 smithi029 ceph-mon[223858]: failed to encode map e681 with expected crc 2023-10-09T05:07:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:34.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:34.172 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:34.235 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:34.343 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:34.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:34.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:34.452 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:34.543 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:34.835 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:34.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:34.953 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:34.981 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[216903]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.981 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[216903]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.981 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[216903]: osdmap e682: 8 total, 8 up, 7 in 2023-10-09T05:07:34.981 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[216903]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.981 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[216903]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.981 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[216903]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[223858]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[223858]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[223858]: osdmap e682: 8 total, 8 up, 7 in 2023-10-09T05:07:34.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[223858]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[223858]: failed to encode map e682 with expected crc 2023-10-09T05:07:34.985 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:34 smithi029 ceph-mon[223858]: failed to encode map e682 with expected crc 2023-10-09T05:07:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:34 smithi177 ceph-mon[195797]: failed to encode map e682 with expected crc 2023-10-09T05:07:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:34 smithi177 ceph-mon[195797]: failed to encode map e682 with expected crc 2023-10-09T05:07:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:34 smithi177 ceph-mon[195797]: osdmap e682: 8 total, 8 up, 7 in 2023-10-09T05:07:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:34 smithi177 ceph-mon[195797]: failed to encode map e682 with expected crc 2023-10-09T05:07:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:34 smithi177 ceph-mon[195797]: failed to encode map e682 with expected crc 2023-10-09T05:07:35.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:34 smithi177 ceph-mon[195797]: failed to encode map e682 with expected crc 2023-10-09T05:07:35.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 77608 objects 2023-10-09T05:07:35.041 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :709.796 2023-10-09T05:07:35.052 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:35.170 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:35.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:35.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:35.242 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:35.275 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:35.307 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:35.563 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:35.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:35.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:35.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:35.701 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:35 smithi029 ceph-mon[216903]: pgmap v2210: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 101 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded; 120 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 8689/5802 objects degraded (149.759%); 53/5802 objects misplaced (0.913%); 10 MiB/s, 223 objects/s recovering 2023-10-09T05:07:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:35 smithi029 ceph-mon[223858]: pgmap v2210: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 101 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded; 120 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 8689/5802 objects degraded (149.759%); 53/5802 objects misplaced (0.913%); 10 MiB/s, 223 objects/s recovering 2023-10-09T05:07:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:35 smithi177 ceph-mon[195797]: pgmap v2210: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 101 active+clean, 10 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded; 120 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 8689/5802 objects degraded (149.759%); 53/5802 objects misplaced (0.913%); 10 MiB/s, 223 objects/s recovering 2023-10-09T05:07:36.256 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:07:36.256 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:07:36.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:07:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:36 smithi177 ceph-mon[195797]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:36 smithi177 ceph-mon[195797]: osdmap e683: 8 total, 8 up, 7 in 2023-10-09T05:07:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:36 smithi177 ceph-mon[195797]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:36 smithi177 ceph-mon[195797]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:36 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 8689/5802 objects degraded (149.759%), 17 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:37.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[216903]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[216903]: osdmap e683: 8 total, 8 up, 7 in 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[216903]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[216903]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 8689/5802 objects degraded (149.759%), 17 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[223858]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[223858]: osdmap e683: 8 total, 8 up, 7 in 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[223858]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[223858]: failed to encode map e683 with expected crc 2023-10-09T05:07:37.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:36 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 8689/5802 objects degraded (149.759%), 17 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:37 smithi177 ceph-mon[195797]: pgmap v2212: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 102 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded; 74 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 7150/3606 objects degraded (198.281%); 53/3606 objects misplaced (1.470%); 16 MiB/s, 392 objects/s recovering 2023-10-09T05:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:37 smithi177 ceph-mon[195797]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:37 smithi177 ceph-mon[195797]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:37 smithi177 ceph-mon[195797]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:37 smithi177 ceph-mon[195797]: osdmap e684: 8 total, 8 up, 7 in 2023-10-09T05:07:38.068 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:38.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[223858]: pgmap v2212: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 102 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded; 74 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 7150/3606 objects degraded (198.281%); 53/3606 objects misplaced (1.470%); 16 MiB/s, 392 objects/s recovering 2023-10-09T05:07:38.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[223858]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.070 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[223858]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[223858]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[223858]: osdmap e684: 8 total, 8 up, 7 in 2023-10-09T05:07:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:38.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:38.074 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[216903]: pgmap v2212: 119 pgs: 1 peering, 2 active+recovering+undersized+degraded+remapped, 102 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 5 active+recovery_wait+degraded; 74 MiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 7150/3606 objects degraded (198.281%); 53/3606 objects misplaced (1.470%); 16 MiB/s, 392 objects/s recovering 2023-10-09T05:07:38.074 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[216903]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.074 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[216903]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.074 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[216903]: failed to encode map e684 with expected crc 2023-10-09T05:07:38.074 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:37 smithi029 ceph-mon[216903]: osdmap e684: 8 total, 8 up, 7 in 2023-10-09T05:07:38.174 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:38.200 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:38.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:38.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:38.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:38.382 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:38 smithi177 ceph-mon[195797]: failed to encode map e685 with expected crc 2023-10-09T05:07:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:38 smithi177 ceph-mon[195797]: failed to encode map e685 with expected crc 2023-10-09T05:07:39.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:38 smithi177 ceph-mon[195797]: osdmap e685: 8 total, 8 up, 7 in 2023-10-09T05:07:39.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:38 smithi029 ceph-mon[216903]: failed to encode map e685 with expected crc 2023-10-09T05:07:39.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:38 smithi029 ceph-mon[216903]: failed to encode map e685 with expected crc 2023-10-09T05:07:39.143 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:38 smithi029 ceph-mon[216903]: osdmap e685: 8 total, 8 up, 7 in 2023-10-09T05:07:39.143 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:38 smithi029 ceph-mon[223858]: failed to encode map e685 with expected crc 2023-10-09T05:07:39.143 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:38 smithi029 ceph-mon[223858]: failed to encode map e685 with expected crc 2023-10-09T05:07:39.143 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:38 smithi029 ceph-mon[223858]: osdmap e685: 8 total, 8 up, 7 in 2023-10-09T05:07:39.447 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:39.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:39.552 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:39.586 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:39.720 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:39.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:39.736 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:07:39.826 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:39.854 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:39.901 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:39.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:39.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[216903]: pgmap v2215: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 105 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded; 40 MiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 6625/1983 objects degraded (334.090%); 12 MiB/s, 423 objects/s recovering 2023-10-09T05:07:39.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[216903]: failed to encode map e686 with expected crc 2023-10-09T05:07:39.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[216903]: failed to encode map e686 with expected crc 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[216903]: osdmap e686: 8 total, 8 up, 7 in 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[216903]: failed to encode map e686 with expected crc 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[223858]: pgmap v2215: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 105 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded; 40 MiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 6625/1983 objects degraded (334.090%); 12 MiB/s, 423 objects/s recovering 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[223858]: failed to encode map e686 with expected crc 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[223858]: failed to encode map e686 with expected crc 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[223858]: osdmap e686: 8 total, 8 up, 7 in 2023-10-09T05:07:39.978 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:39 smithi029 ceph-mon[223858]: failed to encode map e686 with expected crc 2023-10-09T05:07:40.008 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:39 smithi177 ceph-mon[195797]: pgmap v2215: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 105 active+clean, 9 active+recovery_wait+undersized+degraded+remapped, 4 active+recovery_wait+degraded; 40 MiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 6625/1983 objects degraded (334.090%); 12 MiB/s, 423 objects/s recovering 2023-10-09T05:07:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:39 smithi177 ceph-mon[195797]: failed to encode map e686 with expected crc 2023-10-09T05:07:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:39 smithi177 ceph-mon[195797]: failed to encode map e686 with expected crc 2023-10-09T05:07:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:39 smithi177 ceph-mon[195797]: osdmap e686: 8 total, 8 up, 7 in 2023-10-09T05:07:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:39 smithi177 ceph-mon[195797]: failed to encode map e686 with expected crc 2023-10-09T05:07:40.040 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:40.154 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:07:40.288 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-10-09T05:07:40.289 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-10-09T05:07:40.968 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:40.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:40.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: failed to encode map e686 with expected crc 2023-10-09T05:07:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: osdmap e687: 8 total, 8 up, 7 in 2023-10-09T05:07:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:40 smithi177 ceph-mon[195797]: pgmap v2218: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 4156/1050 objects degraded (395.810%); 1.5 MiB/s, 372 objects/s recovering 2023-10-09T05:07:41.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: failed to encode map e686 with expected crc 2023-10-09T05:07:41.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: osdmap e687: 8 total, 8 up, 7 in 2023-10-09T05:07:41.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[216903]: pgmap v2218: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 4156/1050 objects degraded (395.810%); 1.5 MiB/s, 372 objects/s recovering 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: failed to encode map e686 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: osdmap e687: 8 total, 8 up, 7 in 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: failed to encode map e687 with expected crc 2023-10-09T05:07:41.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:40 smithi029 ceph-mon[223858]: pgmap v2218: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 262 op/s; 4156/1050 objects degraded (395.810%); 1.5 MiB/s, 372 objects/s recovering 2023-10-09T05:07:41.073 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:41.100 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:41.196 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:41.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:41.242 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:41.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:41.370 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:41.468 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:41.549 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:41.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:41.643 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:07:41.645 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:07:41.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:41.700 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:41.835 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:07:41.855 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:07:41.879 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:07:41.901 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:07:41.920 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:07:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: osdmap e688: 8 total, 8 up, 7 in 2023-10-09T05:07:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 4156/1050 objects degraded (395.810%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[216903]: from='client.37755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: osdmap e688: 8 total, 8 up, 7 in 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: failed to encode map e688 with expected crc 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 4156/1050 objects degraded (395.810%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:41.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:41.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:41 smithi029 ceph-mon[223858]: from='client.37755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:41.938 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:07:41.957 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:07:41.988 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.990 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:07:41.991 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:07:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: osdmap e688: 8 total, 8 up, 7 in 2023-10-09T05:07:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: failed to encode map e688 with expected crc 2023-10-09T05:07:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: failed to encode map e688 with expected crc 2023-10-09T05:07:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: failed to encode map e688 with expected crc 2023-10-09T05:07:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: failed to encode map e688 with expected crc 2023-10-09T05:07:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: failed to encode map e688 with expected crc 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: failed to encode map e688 with expected crc 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 4156/1050 objects degraded (395.810%), 11 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:42.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:41 smithi177 ceph-mon[195797]: from='client.37755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:42.943 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:07:42.944 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:07:42.944 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:07:42.944 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:07:42.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: 14.1a scrub starts 2023-10-09T05:07:42.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: 14.5 scrub starts 2023-10-09T05:07:42.971 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: failed to encode map e689 with expected crc 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: failed to encode map e689 with expected crc 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: osdmap e689: 8 total, 8 up, 7 in 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: failed to encode map e689 with expected crc 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: failed to encode map e689 with expected crc 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/632578959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: pgmap v2221: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 4156/1050 objects degraded (395.810%); 1024 KiB/s, 213 objects/s recovering 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: failed to encode map e689 with expected crc 2023-10-09T05:07:42.972 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:42 smithi177 ceph-mon[195797]: osdmap e690: 8 total, 8 up, 7 in 2023-10-09T05:07:43.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: 14.1a scrub starts 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: 14.5 scrub starts 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: osdmap e689: 8 total, 8 up, 7 in 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/632578959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: pgmap v2221: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 4156/1050 objects degraded (395.810%); 1024 KiB/s, 213 objects/s recovering 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[223858]: osdmap e690: 8 total, 8 up, 7 in 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:07:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:07:42] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.43.0" 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: 14.1a scrub starts 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: 14.5 scrub starts 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: osdmap e689: 8 total, 8 up, 7 in 2023-10-09T05:07:43.014 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/632578959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:43.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: pgmap v2221: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 108 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 4156/1050 objects degraded (395.810%); 1024 KiB/s, 213 objects/s recovering 2023-10-09T05:07:43.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: failed to encode map e689 with expected crc 2023-10-09T05:07:43.015 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:42 smithi029 ceph-mon[216903]: osdmap e690: 8 total, 8 up, 7 in 2023-10-09T05:07:43.358 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:43.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:43.463 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:43.476 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2219,"stamp":"2023-10-09T05:07:40.742526+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":22075284,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":1042,"num_objects_missing":1042,"num_objects_degraded":4156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":501,"num_read_kb":925,"num_write":155518,"num_write_kb":4969358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96831,"num_bytes_recovered":5683710696,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":219823,"ondisk_log_size":219823,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":477,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13915488,"kb_used_data":5347816,"kb_used_omap":34,"kb_used_meta":8567581,"kb_avail":642300576,"statfs":{"total":671965249536,"available":657715789824,"internally_reserved":0,"allocated":5476163584,"data_stored":5459387386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8773203338},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-55722020,"num_objects":-852,"num_object_clones":0,"num_object_copies":-2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2994,"num_objects_misplaced":-53,"num_objects_unfound":0,"num_objects_dirty":-852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1051,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1489,"num_bytes_recovered":6094848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.025120"},"pg_stats":[{"pgid":"14.23","version":"682'6398","reported_seq":13043,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725640+0000","last_change":"2023-10-09T05:05:01.796020+0000","last_active":"2023-10-09T05:07:37.725640+0000","last_peered":"2023-10-09T05:07:37.725640+0000","last_clean":"2023-10-09T05:07:37.725640+0000","last_became_active":"2023-10-09T05:05:01.790795+0000","last_became_peered":"2023-10-09T05:05:01.790795+0000","last_unstale":"2023-10-09T05:07:37.725640+0000","last_undegraded":"2023-10-09T05:07:37.725640+0000","last_fullsized":"2023-10-09T05:07:37.725640+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2414,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1402,"num_bytes_recovered":78610432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.22","version":"682'6368","reported_seq":15347,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727577+0000","last_change":"2023-10-09T05:07:24.640896+0000","last_active":"2023-10-09T05:07:37.727577+0000","last_peered":"2023-10-09T05:07:37.727577+0000","last_clean":"2023-10-09T05:07:37.727577+0000","last_became_active":"2023-10-09T05:07:24.627714+0000","last_became_peered":"2023-10-09T05:07:24.627714+0000","last_unstale":"2023-10-09T05:07:37.727577+0000","last_undegraded":"2023-10-09T05:07:37.727577+0000","last_fullsized":"2023-10-09T05:07:37.727577+0000","mapping_epoch":679,"log_start":"444'341","ondisk_log_start":"444'341","created":613,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2433,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1606,"num_bytes_recovered":92127232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.21","version":"682'6410","reported_seq":17128,"reported_epoch":685,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:37.726387+0000","last_change":"2023-10-09T05:02:59.009601+0000","last_active":"2023-10-09T05:07:37.726387+0000","last_peered":"2023-10-09T05:07:37.726387+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.991366+0000","last_became_peered":"2023-10-09T05:02:58.991366+0000","last_unstale":"2023-10-09T05:07:37.726387+0000","last_undegraded":"2023-10-09T05:02:58.967655+0000","last_fullsized":"2023-10-09T05:07:37.726387+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":613,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6410,"log_dups_size":0,"ondisk_log_size":6410,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2427,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":3}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.20","version":"682'6406","reported_seq":10473,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.586936+0000","last_change":"2023-10-09T05:07:36.585949+0000","last_active":"2023-10-09T05:07:36.586936+0000","last_peered":"2023-10-09T05:07:36.586936+0000","last_clean":"2023-10-09T05:07:36.586936+0000","last_became_active":"2023-10-09T05:03:26.458540+0000","last_became_peered":"2023-10-09T05:03:26.458540+0000","last_unstale":"2023-10-09T05:07:36.586936+0000","last_undegraded":"2023-10-09T05:07:36.586936+0000","last_fullsized":"2023-10-09T05:07:36.586936+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":613,"last_epoch_clean":683,"parent":"0.0","parent_split_bits":6,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5868,"log_dups_size":0,"ondisk_log_size":5868,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1196032,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2413,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":63651840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.17","version":"682'7432","reported_seq":21010,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713082+0000","last_change":"2023-10-09T05:04:38.062806+0000","last_active":"2023-10-09T05:07:35.713082+0000","last_peered":"2023-10-09T05:07:35.713082+0000","last_clean":"2023-10-09T05:07:35.713082+0000","last_became_active":"2023-10-09T05:04:38.052776+0000","last_became_peered":"2023-10-09T05:04:38.052776+0000","last_unstale":"2023-10-09T05:07:35.713082+0000","last_undegraded":"2023-10-09T05:07:35.713082+0000","last_fullsized":"2023-10-09T05:07:35.713082+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4891,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4585,"num_bytes_recovered":259981312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":188,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725972+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:07:37.725972+0000","last_peered":"2023-10-09T05:07:37.725972+0000","last_clean":"2023-10-09T05:07:37.725972+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:07:37.725972+0000","last_undegraded":"2023-10-09T05:07:37.725972+0000","last_fullsized":"2023-10-09T05:07:37.725972+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":334,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726308+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:07:37.726308+0000","last_peered":"2023-10-09T05:07:37.726308+0000","last_clean":"2023-10-09T05:07:37.726308+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:07:37.726308+0000","last_undegraded":"2023-10-09T05:07:37.726308+0000","last_fullsized":"2023-10-09T05:07:37.726308+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"682'7383","reported_seq":10727,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728896+0000","last_change":"2023-10-09T05:07:16.717512+0000","last_active":"2023-10-09T05:07:37.728896+0000","last_peered":"2023-10-09T05:07:37.728896+0000","last_clean":"2023-10-09T05:07:37.728896+0000","last_became_active":"2023-10-09T05:03:49.096000+0000","last_became_peered":"2023-10-09T05:03:49.096000+0000","last_unstale":"2023-10-09T05:07:37.728896+0000","last_undegraded":"2023-10-09T05:07:37.728896+0000","last_fullsized":"2023-10-09T05:07:37.728896+0000","mapping_epoch":638,"log_start":"445'2358","ondisk_log_start":"445'2358","created":532,"last_epoch_clean":675,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5025,"log_dups_size":0,"ondisk_log_size":5025,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1081344,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4844,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1011,"num_bytes_recovered":63143936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":110,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719067+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:07:36.719067+0000","last_peered":"2023-10-09T05:07:36.719067+0000","last_clean":"2023-10-09T05:07:36.719067+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:07:36.719067+0000","last_undegraded":"2023-10-09T05:07:36.719067+0000","last_fullsized":"2023-10-09T05:07:36.719067+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":199,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726456+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:07:37.726456+0000","last_peered":"2023-10-09T05:07:37.726456+0000","last_clean":"2023-10-09T05:07:37.726456+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:07:37.726456+0000","last_undegraded":"2023-10-09T05:07:37.726456+0000","last_fullsized":"2023-10-09T05:07:37.726456+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"682'7364","reported_seq":19041,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725879+0000","last_change":"2023-10-09T05:04:17.022647+0000","last_active":"2023-10-09T05:07:37.725879+0000","last_peered":"2023-10-09T05:07:37.725879+0000","last_clean":"2023-10-09T05:07:37.725879+0000","last_became_active":"2023-10-09T05:04:17.009823+0000","last_became_peered":"2023-10-09T05:04:17.009823+0000","last_unstale":"2023-10-09T05:07:37.725879+0000","last_undegraded":"2023-10-09T05:07:37.725879+0000","last_fullsized":"2023-10-09T05:07:37.725879+0000","mapping_epoch":649,"log_start":"444'246","ondisk_log_start":"444'246","created":532,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7118,"log_dups_size":0,"ondisk_log_size":7118,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4824,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3810,"num_bytes_recovered":213450752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":316,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726414+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:07:37.726414+0000","last_peered":"2023-10-09T05:07:37.726414+0000","last_clean":"2023-10-09T05:07:37.726414+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:07:37.726414+0000","last_undegraded":"2023-10-09T05:07:37.726414+0000","last_fullsized":"2023-10-09T05:07:37.726414+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728940+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:07:37.728940+0000","last_peered":"2023-10-09T05:07:37.728940+0000","last_clean":"2023-10-09T05:07:37.728940+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:07:37.728940+0000","last_undegraded":"2023-10-09T05:07:37.728940+0000","last_fullsized":"2023-10-09T05:07:37.728940+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1a","version":"682'7588","reported_seq":24960,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713288+0000","last_change":"2023-10-09T05:02:58.988821+0000","last_active":"2023-10-09T05:07:35.713288+0000","last_peered":"2023-10-09T05:07:35.713288+0000","last_clean":"2023-10-09T05:07:35.713288+0000","last_became_active":"2023-10-09T05:02:58.982173+0000","last_became_peered":"2023-10-09T05:02:58.982173+0000","last_unstale":"2023-10-09T05:07:35.713288+0000","last_undegraded":"2023-10-09T05:07:35.713288+0000","last_fullsized":"2023-10-09T05:07:35.713288+0000","mapping_epoch":613,"log_start":"444'400","ondisk_log_start":"444'400","created":613,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7188,"log_dups_size":0,"ondisk_log_size":7188,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4767,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.712847+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:07:35.712847+0000","last_peered":"2023-10-09T05:07:35.712847+0000","last_clean":"2023-10-09T05:07:35.712847+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:07:35.712847+0000","last_undegraded":"2023-10-09T05:07:35.712847+0000","last_fullsized":"2023-10-09T05:07:35.712847+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":307,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726322+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:07:37.726322+0000","last_peered":"2023-10-09T05:07:37.726322+0000","last_clean":"2023-10-09T05:07:37.726322+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:07:37.726322+0000","last_undegraded":"2023-10-09T05:07:37.726322+0000","last_fullsized":"2023-10-09T05:07:37.726322+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1b","version":"682'7938","reported_seq":20535,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719318+0000","last_change":"2023-10-09T05:07:23.644852+0000","last_active":"2023-10-09T05:07:36.719318+0000","last_peered":"2023-10-09T05:07:36.719318+0000","last_clean":"2023-10-09T05:07:36.719318+0000","last_became_active":"2023-10-09T05:07:23.623426+0000","last_became_peered":"2023-10-09T05:07:23.623426+0000","last_unstale":"2023-10-09T05:07:36.719318+0000","last_undegraded":"2023-10-09T05:07:36.719318+0000","last_fullsized":"2023-10-09T05:07:36.719318+0000","mapping_epoch":678,"log_start":"445'2396","ondisk_log_start":"445'2396","created":613,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5542,"log_dups_size":0,"ondisk_log_size":5542,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4978,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3237,"num_bytes_recovered":181469184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713263+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:07:35.713263+0000","last_peered":"2023-10-09T05:07:35.713263+0000","last_clean":"2023-10-09T05:07:35.713263+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:07:35.713263+0000","last_undegraded":"2023-10-09T05:07:35.713263+0000","last_fullsized":"2023-10-09T05:07:35.713263+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":297,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727283+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:07:37.727283+0000","last_peered":"2023-10-09T05:07:37.727283+0000","last_clean":"2023-10-09T05:07:37.727283+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:07:37.727283+0000","last_undegraded":"2023-10-09T05:07:37.727283+0000","last_fullsized":"2023-10-09T05:07:37.727283+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"682'7577","reported_seq":18180,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.718814+0000","last_change":"2023-10-09T05:07:04.498988+0000","last_active":"2023-10-09T05:07:36.718814+0000","last_peered":"2023-10-09T05:07:36.718814+0000","last_clean":"2023-10-09T05:07:36.718814+0000","last_became_active":"2023-10-09T05:07:04.477375+0000","last_became_peered":"2023-10-09T05:07:04.477375+0000","last_unstale":"2023-10-09T05:07:36.718814+0000","last_undegraded":"2023-10-09T05:07:36.718814+0000","last_fullsized":"2023-10-09T05:07:36.718814+0000","mapping_epoch":673,"log_start":"445'1536","ondisk_log_start":"445'1536","created":532,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1703936,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4968,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2943,"num_bytes_recovered":162070528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":368,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726276+0000","last_change":"2023-10-09T05:07:23.305378+0000","last_active":"2023-10-09T05:07:37.726276+0000","last_peered":"2023-10-09T05:07:37.726276+0000","last_clean":"2023-10-09T05:07:37.726276+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:07:37.726276+0000","last_undegraded":"2023-10-09T05:07:37.726276+0000","last_fullsized":"2023-10-09T05:07:37.726276+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":990,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":110,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719502+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:07:36.719502+0000","last_peered":"2023-10-09T05:07:36.719502+0000","last_clean":"2023-10-09T05:07:36.719502+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:07:36.719502+0000","last_undegraded":"2023-10-09T05:07:36.719502+0000","last_fullsized":"2023-10-09T05:07:36.719502+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.19","version":"682'7407","reported_seq":19497,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.712693+0000","last_change":"2023-10-09T05:03:24.507584+0000","last_active":"2023-10-09T05:07:35.712693+0000","last_peered":"2023-10-09T05:07:35.712693+0000","last_clean":"2023-10-09T05:07:35.712693+0000","last_became_active":"2023-10-09T05:03:24.375963+0000","last_became_peered":"2023-10-09T05:03:24.375963+0000","last_unstale":"2023-10-09T05:07:35.712693+0000","last_undegraded":"2023-10-09T05:07:35.712693+0000","last_fullsized":"2023-10-09T05:07:35.712693+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":532,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1343488,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4869,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2991,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":141,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728372+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:07:37.728372+0000","last_peered":"2023-10-09T05:07:37.728372+0000","last_clean":"2023-10-09T05:07:37.728372+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:07:37.728372+0000","last_undegraded":"2023-10-09T05:07:37.728372+0000","last_fullsized":"2023-10-09T05:07:37.728372+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":156,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725931+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:07:37.725931+0000","last_peered":"2023-10-09T05:07:37.725931+0000","last_clean":"2023-10-09T05:07:37.725931+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:07:37.725931+0000","last_undegraded":"2023-10-09T05:07:37.725931+0000","last_fullsized":"2023-10-09T05:07:37.725931+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.1e","version":"682'7658","reported_seq":22567,"reported_epoch":686,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:38.744981+0000","last_change":"2023-10-09T05:07:38.748322+0000","last_active":"2023-10-09T05:07:38.744981+0000","last_peered":"2023-10-09T05:07:38.744981+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982332+0000","last_became_peered":"2023-10-09T05:04:21.982332+0000","last_unstale":"2023-10-09T05:07:38.744981+0000","last_undegraded":"2023-10-09T05:07:38.744981+0000","last_fullsized":"2023-10-09T05:04:21.914502+0000","mapping_epoch":652,"log_start":"445'2310","ondisk_log_start":"445'2310","created":613,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4799,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4344,"num_bytes_recovered":256376832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":-1},{"shards":"4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":162,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725893+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:07:37.725893+0000","last_peered":"2023-10-09T05:07:37.725893+0000","last_clean":"2023-10-09T05:07:37.725893+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:07:37.725893+0000","last_undegraded":"2023-10-09T05:07:37.725893+0000","last_fullsized":"2023-10-09T05:07:37.725893+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":270,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728353+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:07:37.728353+0000","last_peered":"2023-10-09T05:07:37.728353+0000","last_clean":"2023-10-09T05:07:37.728353+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:07:37.728353+0000","last_undegraded":"2023-10-09T05:07:37.728353+0000","last_fullsized":"2023-10-09T05:07:37.728353+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1f","version":"682'7647","reported_seq":17645,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726474+0000","last_change":"2023-10-09T05:07:18.598407+0000","last_active":"2023-10-09T05:07:37.726474+0000","last_peered":"2023-10-09T05:07:37.726474+0000","last_clean":"2023-10-09T05:07:37.726474+0000","last_became_active":"2023-10-09T05:07:18.582264+0000","last_became_peered":"2023-10-09T05:07:18.582264+0000","last_unstale":"2023-10-09T05:07:37.726474+0000","last_undegraded":"2023-10-09T05:07:37.726474+0000","last_fullsized":"2023-10-09T05:07:37.726474+0000","mapping_epoch":676,"log_start":"445'2216","ondisk_log_start":"445'2216","created":613,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5431,"log_dups_size":0,"ondisk_log_size":5431,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4818,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3230,"num_bytes_recovered":180125696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":190,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725835+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:07:37.725835+0000","last_peered":"2023-10-09T05:07:37.725835+0000","last_clean":"2023-10-09T05:07:37.725835+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:07:37.725835+0000","last_undegraded":"2023-10-09T05:07:37.725835+0000","last_fullsized":"2023-10-09T05:07:37.725835+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":62,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713726+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:07:35.713726+0000","last_peered":"2023-10-09T05:07:35.713726+0000","last_clean":"2023-10-09T05:07:35.713726+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:07:35.713726+0000","last_undegraded":"2023-10-09T05:07:35.713726+0000","last_fullsized":"2023-10-09T05:07:35.713726+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.1c","version":"682'7740","reported_seq":19808,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.756460+0000","last_change":"2023-10-09T05:07:37.754116+0000","last_active":"2023-10-09T05:07:37.756460+0000","last_peered":"2023-10-09T05:07:37.756460+0000","last_clean":"2023-10-09T05:07:37.756460+0000","last_became_active":"2023-10-09T05:07:37.732427+0000","last_became_peered":"2023-10-09T05:07:37.732427+0000","last_unstale":"2023-10-09T05:07:37.756460+0000","last_undegraded":"2023-10-09T05:07:37.756460+0000","last_fullsized":"2023-10-09T05:07:37.756460+0000","mapping_epoch":684,"log_start":"445'1763","ondisk_log_start":"445'1763","created":613,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4841,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3508,"num_bytes_recovered":189071360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":125,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725779+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:07:37.725779+0000","last_peered":"2023-10-09T05:07:37.725779+0000","last_clean":"2023-10-09T05:07:37.725779+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:07:37.725779+0000","last_undegraded":"2023-10-09T05:07:37.725779+0000","last_fullsized":"2023-10-09T05:07:37.725779+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":324,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726721+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:07:37.726721+0000","last_peered":"2023-10-09T05:07:37.726721+0000","last_clean":"2023-10-09T05:07:37.726721+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:07:37.726721+0000","last_undegraded":"2023-10-09T05:07:37.726721+0000","last_fullsized":"2023-10-09T05:07:37.726721+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.1d","version":"682'7660","reported_seq":14558,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729079+0000","last_change":"2023-10-09T05:03:37.985128+0000","last_active":"2023-10-09T05:07:37.729079+0000","last_peered":"2023-10-09T05:07:37.729079+0000","last_clean":"2023-10-09T05:07:37.729079+0000","last_became_active":"2023-10-09T05:03:37.980555+0000","last_became_peered":"2023-10-09T05:03:37.980555+0000","last_unstale":"2023-10-09T05:07:37.729079+0000","last_undegraded":"2023-10-09T05:07:37.729079+0000","last_fullsized":"2023-10-09T05:07:37.729079+0000","mapping_epoch":633,"log_start":"445'2362","ondisk_log_start":"445'2362","created":613,"last_epoch_clean":634,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5298,"log_dups_size":0,"ondisk_log_size":5298,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425972,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4798,"num_write_kb":153697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2813,"num_bytes_recovered":150470656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":320,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727786+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:07:37.727786+0000","last_peered":"2023-10-09T05:07:37.727786+0000","last_clean":"2023-10-09T05:07:37.727786+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:07:37.727786+0000","last_undegraded":"2023-10-09T05:07:37.727786+0000","last_fullsized":"2023-10-09T05:07:37.727786+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729600+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:07:37.729600+0000","last_peered":"2023-10-09T05:07:37.729600+0000","last_clean":"2023-10-09T05:07:37.729600+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:07:37.729600+0000","last_undegraded":"2023-10-09T05:07:37.729600+0000","last_fullsized":"2023-10-09T05:07:37.729600+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"682'6338","reported_seq":15232,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728201+0000","last_change":"2023-10-09T05:05:29.132074+0000","last_active":"2023-10-09T05:07:37.728201+0000","last_peered":"2023-10-09T05:07:37.728201+0000","last_clean":"2023-10-09T05:07:37.728201+0000","last_became_active":"2023-10-09T05:05:29.125934+0000","last_became_peered":"2023-10-09T05:05:29.125934+0000","last_unstale":"2023-10-09T05:07:37.728201+0000","last_undegraded":"2023-10-09T05:07:37.728201+0000","last_fullsized":"2023-10-09T05:07:37.728201+0000","mapping_epoch":663,"log_start":"444'341","ondisk_log_start":"444'341","created":440,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5997,"log_dups_size":0,"ondisk_log_size":5997,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2403,"num_write_kb":77040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":90619904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":215,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726006+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:07:37.726006+0000","last_peered":"2023-10-09T05:07:37.726006+0000","last_clean":"2023-10-09T05:07:37.726006+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:07:37.726006+0000","last_undegraded":"2023-10-09T05:07:37.726006+0000","last_fullsized":"2023-10-09T05:07:37.726006+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.712537+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:07:35.712537+0000","last_peered":"2023-10-09T05:07:35.712537+0000","last_clean":"2023-10-09T05:07:35.712537+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:07:35.712537+0000","last_undegraded":"2023-10-09T05:07:35.712537+0000","last_fullsized":"2023-10-09T05:07:35.712537+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"682'6406","reported_seq":12672,"reported_epoch":682,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:34.934450+0000","last_change":"2023-10-09T05:03:00.260084+0000","last_active":"2023-10-09T05:07:34.934450+0000","last_peered":"2023-10-09T05:07:34.934450+0000","last_clean":"2023-10-09T04:54:25.075403+0000","last_became_active":"2023-10-09T05:03:00.123765+0000","last_became_peered":"2023-10-09T05:03:00.123765+0000","last_unstale":"2023-10-09T05:07:34.934450+0000","last_undegraded":"2023-10-09T05:03:00.112976+0000","last_fullsized":"2023-10-09T05:02:59.959383+0000","mapping_epoch":614,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6406,"log_dups_size":0,"ondisk_log_size":6406,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":191,"num_objects_missing":191,"num_objects_degraded":198,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2422,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1209,"num_bytes_recovered":79200256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"0,1","objects":10}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":49,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725856+0000","last_change":"2023-10-09T05:07:31.961439+0000","last_active":"2023-10-09T05:07:37.725856+0000","last_peered":"2023-10-09T05:07:37.725856+0000","last_clean":"2023-10-09T05:07:37.725856+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:07:37.725856+0000","last_undegraded":"2023-10-09T05:07:37.725856+0000","last_fullsized":"2023-10-09T05:07:37.725856+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726068+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:07:37.726068+0000","last_peered":"2023-10-09T05:07:37.726068+0000","last_clean":"2023-10-09T05:07:37.726068+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:07:37.726068+0000","last_undegraded":"2023-10-09T05:07:37.726068+0000","last_fullsized":"2023-10-09T05:07:37.726068+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"682'6433","reported_seq":10570,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.717096+0000","last_change":"2023-10-09T05:04:48.710530+0000","last_active":"2023-10-09T05:07:35.717096+0000","last_peered":"2023-10-09T05:07:35.717096+0000","last_clean":"2023-10-09T05:07:35.717096+0000","last_became_active":"2023-10-09T05:03:26.456970+0000","last_became_peered":"2023-10-09T05:03:26.456970+0000","last_unstale":"2023-10-09T05:07:35.717096+0000","last_undegraded":"2023-10-09T05:07:35.717096+0000","last_fullsized":"2023-10-09T05:07:35.717096+0000","mapping_epoch":628,"log_start":"444'538","ondisk_log_start":"444'538","created":440,"last_epoch_clean":659,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2440,"num_write_kb":77808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1237,"num_bytes_recovered":65421312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":66,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713651+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:07:35.713651+0000","last_peered":"2023-10-09T05:07:35.713651+0000","last_clean":"2023-10-09T05:07:35.713651+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:07:35.713651+0000","last_undegraded":"2023-10-09T05:07:35.713651+0000","last_fullsized":"2023-10-09T05:07:35.713651+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":232,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727999+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:07:37.727999+0000","last_peered":"2023-10-09T05:07:37.727999+0000","last_clean":"2023-10-09T05:07:37.727999+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:07:37.727999+0000","last_undegraded":"2023-10-09T05:07:37.727999+0000","last_fullsized":"2023-10-09T05:07:37.727999+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"682'6437","reported_seq":17155,"reported_epoch":685,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:37.727139+0000","last_change":"2023-10-09T05:02:58.965284+0000","last_active":"2023-10-09T05:07:37.727139+0000","last_peered":"2023-10-09T05:07:37.727139+0000","last_clean":"2023-10-09T05:01:53.681317+0000","last_became_active":"2023-10-09T05:02:58.961829+0000","last_became_peered":"2023-10-09T05:02:58.961829+0000","last_unstale":"2023-10-09T05:07:37.727139+0000","last_undegraded":"2023-10-09T05:02:58.946816+0000","last_fullsized":"2023-10-09T05:07:37.727139+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2453,"num_write_kb":77744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":103268352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":744,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713467+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:07:35.713467+0000","last_peered":"2023-10-09T05:07:35.713467+0000","last_clean":"2023-10-09T05:07:35.713467+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:07:35.713467+0000","last_undegraded":"2023-10-09T05:07:35.713467+0000","last_fullsized":"2023-10-09T05:07:35.713467+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719519+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:07:36.719519+0000","last_peered":"2023-10-09T05:07:36.719519+0000","last_clean":"2023-10-09T05:07:36.719519+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:07:36.719519+0000","last_undegraded":"2023-10-09T05:07:36.719519+0000","last_fullsized":"2023-10-09T05:07:36.719519+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1274,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728006+0000","last_change":"2023-10-09T05:03:51.194238+0000","last_active":"2023-10-09T05:07:37.728006+0000","last_peered":"2023-10-09T05:07:37.728006+0000","last_clean":"2023-10-09T05:07:37.728006+0000","last_became_active":"2023-10-09T05:03:51.193754+0000","last_became_peered":"2023-10-09T05:03:51.193754+0000","last_unstale":"2023-10-09T05:07:37.728006+0000","last_undegraded":"2023-10-09T05:07:37.728006+0000","last_fullsized":"2023-10-09T05:07:37.728006+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.6","version":"682'7136","reported_seq":16502,"reported_epoch":686,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:39.020033+0000","last_change":"2023-10-09T05:03:34.446752+0000","last_active":"2023-10-09T05:07:39.020033+0000","last_peered":"2023-10-09T05:07:39.020033+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.984745+0000","last_became_peered":"2023-10-09T05:03:30.984745+0000","last_unstale":"2023-10-09T05:07:39.020033+0000","last_undegraded":"2023-10-09T05:03:27.234787+0000","last_fullsized":"2023-10-09T05:03:26.336566+0000","mapping_epoch":628,"log_start":"445'1436","ondisk_log_start":"445'1436","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5700,"log_dups_size":0,"ondisk_log_size":5700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2392064,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":351,"num_objects_missing":351,"num_objects_degraded":434,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4679,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2322,"num_bytes_recovered":152174592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"0,1","objects":36}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":290,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725870+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:07:37.725870+0000","last_peered":"2023-10-09T05:07:37.725870+0000","last_clean":"2023-10-09T05:07:37.725870+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:07:37.725870+0000","last_undegraded":"2023-10-09T05:07:37.725870+0000","last_fullsized":"2023-10-09T05:07:37.725870+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":124,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728307+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:07:37.728307+0000","last_peered":"2023-10-09T05:07:37.728307+0000","last_clean":"2023-10-09T05:07:37.728307+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:07:37.728307+0000","last_undegraded":"2023-10-09T05:07:37.728307+0000","last_fullsized":"2023-10-09T05:07:37.728307+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1263,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727684+0000","last_change":"2023-10-09T05:03:47.991976+0000","last_active":"2023-10-09T05:07:37.727684+0000","last_peered":"2023-10-09T05:07:37.727684+0000","last_clean":"2023-10-09T05:07:37.727684+0000","last_became_active":"2023-10-09T05:03:47.991504+0000","last_became_peered":"2023-10-09T05:03:47.991504+0000","last_unstale":"2023-10-09T05:07:37.727684+0000","last_undegraded":"2023-10-09T05:07:37.727684+0000","last_fullsized":"2023-10-09T05:07:37.727684+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.7","version":"682'7425","reported_seq":21048,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713316+0000","last_change":"2023-10-09T05:06:08.182013+0000","last_active":"2023-10-09T05:07:35.713316+0000","last_peered":"2023-10-09T05:07:35.713316+0000","last_clean":"2023-10-09T05:07:35.713316+0000","last_became_active":"2023-10-09T05:06:08.171542+0000","last_became_peered":"2023-10-09T05:06:08.171542+0000","last_unstale":"2023-10-09T05:07:35.713316+0000","last_undegraded":"2023-10-09T05:07:35.713316+0000","last_fullsized":"2023-10-09T05:07:35.713316+0000","mapping_epoch":667,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":668,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7425,"log_dups_size":0,"ondisk_log_size":7425,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229376,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4883,"num_write_kb":156384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4591,"num_bytes_recovered":223150080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":229,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727939+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:07:37.727939+0000","last_peered":"2023-10-09T05:07:37.727939+0000","last_clean":"2023-10-09T05:07:37.727939+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:07:37.727939+0000","last_undegraded":"2023-10-09T05:07:37.727939+0000","last_fullsized":"2023-10-09T05:07:37.727939+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":44,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725608+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:07:37.725608+0000","last_peered":"2023-10-09T05:07:37.725608+0000","last_clean":"2023-10-09T05:07:37.725608+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:07:37.725608+0000","last_undegraded":"2023-10-09T05:07:37.725608+0000","last_fullsized":"2023-10-09T05:07:37.725608+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"682'7416","reported_seq":14499,"reported_epoch":685,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:37.727629+0000","last_change":"2023-10-09T05:03:56.875042+0000","last_active":"2023-10-09T05:07:37.727629+0000","last_peered":"2023-10-09T05:07:37.727629+0000","last_clean":"2023-10-09T04:54:25.124035+0000","last_became_active":"2023-10-09T05:03:54.644470+0000","last_became_peered":"2023-10-09T05:03:54.644470+0000","last_unstale":"2023-10-09T05:07:37.727629+0000","last_undegraded":"2023-10-09T05:03:51.149825+0000","last_fullsized":"2023-10-09T05:03:49.950388+0000","mapping_epoch":639,"log_start":"445'2355","ondisk_log_start":"445'2355","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5061,"log_dups_size":0,"ondisk_log_size":5061,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":387,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4876,"num_write_kb":155520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133464064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"3,7","objects":-364}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725694+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:07:37.725694+0000","last_peered":"2023-10-09T05:07:37.725694+0000","last_clean":"2023-10-09T05:07:37.725694+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:07:37.725694+0000","last_undegraded":"2023-10-09T05:07:37.725694+0000","last_fullsized":"2023-10-09T05:07:37.725694+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726142+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:07:37.726142+0000","last_peered":"2023-10-09T05:07:37.726142+0000","last_clean":"2023-10-09T05:07:37.726142+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:07:37.726142+0000","last_undegraded":"2023-10-09T05:07:37.726142+0000","last_fullsized":"2023-10-09T05:07:37.726142+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1283,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728122+0000","last_change":"2023-10-09T05:03:51.191373+0000","last_active":"2023-10-09T05:07:37.728122+0000","last_peered":"2023-10-09T05:07:37.728122+0000","last_clean":"2023-10-09T05:07:37.728122+0000","last_became_active":"2023-10-09T05:03:51.190815+0000","last_became_peered":"2023-10-09T05:03:51.190815+0000","last_unstale":"2023-10-09T05:07:37.728122+0000","last_undegraded":"2023-10-09T05:07:37.728122+0000","last_fullsized":"2023-10-09T05:07:37.728122+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"682'7438","reported_seq":21272,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725784+0000","last_change":"2023-10-09T05:03:57.209297+0000","last_active":"2023-10-09T05:07:37.725784+0000","last_peered":"2023-10-09T05:07:37.725784+0000","last_clean":"2023-10-09T05:07:37.725784+0000","last_became_active":"2023-10-09T05:03:57.198998+0000","last_became_peered":"2023-10-09T05:03:57.198998+0000","last_unstale":"2023-10-09T05:07:37.725784+0000","last_undegraded":"2023-10-09T05:07:37.725784+0000","last_fullsized":"2023-10-09T05:07:37.725784+0000","mapping_epoch":644,"log_start":"444'347","ondisk_log_start":"444'347","created":440,"last_epoch_clean":645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7091,"log_dups_size":0,"ondisk_log_size":7091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4898,"num_write_kb":155552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3896,"num_bytes_recovered":221970432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":205,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725806+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:07:37.725806+0000","last_peered":"2023-10-09T05:07:37.725806+0000","last_clean":"2023-10-09T05:07:37.725806+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:07:37.725806+0000","last_undegraded":"2023-10-09T05:07:37.725806+0000","last_fullsized":"2023-10-09T05:07:37.725806+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":113,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.720564+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:07:36.720564+0000","last_peered":"2023-10-09T05:07:36.720564+0000","last_clean":"2023-10-09T05:07:36.720564+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:07:36.720564+0000","last_undegraded":"2023-10-09T05:07:36.720564+0000","last_fullsized":"2023-10-09T05:07:36.720564+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1289,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728862+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:07:37.728862+0000","last_peered":"2023-10-09T05:07:37.728862+0000","last_clean":"2023-10-09T05:07:37.728862+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:07:37.728862+0000","last_undegraded":"2023-10-09T05:07:37.728862+0000","last_fullsized":"2023-10-09T05:07:37.728862+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.8","version":"682'7629","reported_seq":17322,"reported_epoch":682,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:35.307266+0000","last_change":"2023-10-09T05:07:31.969706+0000","last_active":"2023-10-09T05:07:35.307266+0000","last_peered":"2023-10-09T05:07:35.307266+0000","last_clean":"2023-10-09T04:54:25.113986+0000","last_became_active":"2023-10-09T05:02:36.346915+0000","last_became_peered":"2023-10-09T05:02:36.346915+0000","last_unstale":"2023-10-09T05:07:35.307266+0000","last_undegraded":"2023-10-09T05:02:33.720467+0000","last_fullsized":"2023-10-09T05:02:31.926256+0000","mapping_epoch":603,"log_start":"445'2146","ondisk_log_start":"445'2146","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5483,"log_dups_size":0,"ondisk_log_size":5483,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":18,"num_objects_missing":18,"num_objects_degraded":909,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5020,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2892,"num_bytes_recovered":165281792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,3],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,3","objects":-909}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":101,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719672+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:07:36.719672+0000","last_peered":"2023-10-09T05:07:36.719672+0000","last_clean":"2023-10-09T05:07:36.719672+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:07:36.719672+0000","last_undegraded":"2023-10-09T05:07:36.719672+0000","last_fullsized":"2023-10-09T05:07:36.719672+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":62,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713804+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:07:35.713804+0000","last_peered":"2023-10-09T05:07:35.713804+0000","last_clean":"2023-10-09T05:07:35.713804+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:07:35.713804+0000","last_undegraded":"2023-10-09T05:07:35.713804+0000","last_fullsized":"2023-10-09T05:07:35.713804+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":119,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719422+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:07:36.719422+0000","last_peered":"2023-10-09T05:07:36.719422+0000","last_clean":"2023-10-09T05:07:36.719422+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:07:36.719422+0000","last_undegraded":"2023-10-09T05:07:36.719422+0000","last_fullsized":"2023-10-09T05:07:36.719422+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1272,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727980+0000","last_change":"2023-10-09T05:03:51.194066+0000","last_active":"2023-10-09T05:07:37.727980+0000","last_peered":"2023-10-09T05:07:37.727980+0000","last_clean":"2023-10-09T05:07:37.727980+0000","last_became_active":"2023-10-09T05:03:51.193278+0000","last_became_peered":"2023-10-09T05:03:51.193278+0000","last_unstale":"2023-10-09T05:07:37.727980+0000","last_undegraded":"2023-10-09T05:07:37.727980+0000","last_fullsized":"2023-10-09T05:07:37.727980+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.e","version":"682'7645","reported_seq":21578,"reported_epoch":685,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:37.729247+0000","last_change":"2023-10-09T05:04:22.018585+0000","last_active":"2023-10-09T05:07:37.729247+0000","last_peered":"2023-10-09T05:07:37.729247+0000","last_clean":"2023-10-09T04:54:25.126594+0000","last_became_active":"2023-10-09T05:04:21.982240+0000","last_became_peered":"2023-10-09T05:04:21.982240+0000","last_unstale":"2023-10-09T05:07:37.729247+0000","last_undegraded":"2023-10-09T05:04:21.913654+0000","last_fullsized":"2023-10-09T05:04:21.877305+0000","mapping_epoch":652,"log_start":"445'2241","ondisk_log_start":"445'2241","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425984,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":923,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4785,"num_write_kb":153344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3974,"num_bytes_recovered":255852544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[6,1],"avail_no_missing":["6","0","2","7"],"object_location_counts":[{"shards":"1,6","objects":6}],"blocked_by":[],"up_primary":4,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":768,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713529+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:07:35.713529+0000","last_peered":"2023-10-09T05:07:35.713529+0000","last_clean":"2023-10-09T05:07:35.713529+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:07:35.713529+0000","last_undegraded":"2023-10-09T05:07:35.713529+0000","last_fullsized":"2023-10-09T05:07:35.713529+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725722+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:07:37.725722+0000","last_peered":"2023-10-09T05:07:37.725722+0000","last_clean":"2023-10-09T05:07:37.725722+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:07:37.725722+0000","last_undegraded":"2023-10-09T05:07:37.725722+0000","last_fullsized":"2023-10-09T05:07:37.725722+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719627+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:07:36.719627+0000","last_peered":"2023-10-09T05:07:36.719627+0000","last_clean":"2023-10-09T05:07:36.719627+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:07:36.719627+0000","last_undegraded":"2023-10-09T05:07:36.719627+0000","last_fullsized":"2023-10-09T05:07:36.719627+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1283,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.728066+0000","last_change":"2023-10-09T05:03:51.191180+0000","last_active":"2023-10-09T05:07:37.728066+0000","last_peered":"2023-10-09T05:07:37.728066+0000","last_clean":"2023-10-09T05:07:37.728066+0000","last_became_active":"2023-10-09T05:03:51.190347+0000","last_became_peered":"2023-10-09T05:03:51.190347+0000","last_unstale":"2023-10-09T05:07:37.728066+0000","last_undegraded":"2023-10-09T05:07:37.728066+0000","last_fullsized":"2023-10-09T05:07:37.728066+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.d","version":"682'7672","reported_seq":13471,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729579+0000","last_change":"2023-10-09T05:03:52.019068+0000","last_active":"2023-10-09T05:07:37.729579+0000","last_peered":"2023-10-09T05:07:37.729579+0000","last_clean":"2023-10-09T05:07:37.729579+0000","last_became_active":"2023-10-09T05:03:52.015065+0000","last_became_peered":"2023-10-09T05:03:52.015065+0000","last_unstale":"2023-10-09T05:07:37.729579+0000","last_undegraded":"2023-10-09T05:07:37.729579+0000","last_fullsized":"2023-10-09T05:07:37.729579+0000","mapping_epoch":641,"log_start":"445'2293","ondisk_log_start":"445'2293","created":440,"last_epoch_clean":642,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4810,"num_write_kb":153696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1983,"num_bytes_recovered":101318656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":181,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726616+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:07:37.726616+0000","last_peered":"2023-10-09T05:07:37.726616+0000","last_clean":"2023-10-09T05:07:37.726616+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:07:37.726616+0000","last_undegraded":"2023-10-09T05:07:37.726616+0000","last_fullsized":"2023-10-09T05:07:37.726616+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":99,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.718727+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:07:36.718727+0000","last_peered":"2023-10-09T05:07:36.718727+0000","last_clean":"2023-10-09T05:07:36.718727+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:07:36.718727+0000","last_undegraded":"2023-10-09T05:07:36.718727+0000","last_fullsized":"2023-10-09T05:07:36.718727+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.720489+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:07:36.720489+0000","last_peered":"2023-10-09T05:07:36.720489+0000","last_clean":"2023-10-09T05:07:36.720489+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:07:36.720489+0000","last_undegraded":"2023-10-09T05:07:36.720489+0000","last_fullsized":"2023-10-09T05:07:36.720489+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"682'7569","reported_seq":15501,"reported_epoch":685,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:37.729425+0000","last_change":"2023-10-09T05:03:55.849583+0000","last_active":"2023-10-09T05:07:37.729425+0000","last_peered":"2023-10-09T05:07:37.729425+0000","last_clean":"2023-10-09T04:54:25.147353+0000","last_became_active":"2023-10-09T05:03:55.719774+0000","last_became_peered":"2023-10-09T05:03:55.719774+0000","last_unstale":"2023-10-09T05:07:37.729425+0000","last_undegraded":"2023-10-09T05:03:53.564990+0000","last_fullsized":"2023-10-09T05:03:52.010584+0000","mapping_epoch":641,"log_start":"445'2127","ondisk_log_start":"445'2127","created":440,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":91,"num_objects_missing":91,"num_objects_degraded":836,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4740,"num_write_kb":152928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2257,"num_bytes_recovered":147881984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"3,7","objects":-363}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":72,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.712791+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:07:35.712791+0000","last_peered":"2023-10-09T05:07:35.712791+0000","last_clean":"2023-10-09T05:07:35.712791+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:07:35.712791+0000","last_undegraded":"2023-10-09T05:07:35.712791+0000","last_fullsized":"2023-10-09T05:07:35.712791+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1289,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727621+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:07:37.727621+0000","last_peered":"2023-10-09T05:07:37.727621+0000","last_clean":"2023-10-09T05:07:37.727621+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:07:37.727621+0000","last_undegraded":"2023-10-09T05:07:37.727621+0000","last_fullsized":"2023-10-09T05:07:37.727621+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":179,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725921+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:07:37.725921+0000","last_peered":"2023-10-09T05:07:37.725921+0000","last_clean":"2023-10-09T05:07:37.725921+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:07:37.725921+0000","last_undegraded":"2023-10-09T05:07:37.725921+0000","last_fullsized":"2023-10-09T05:07:37.725921+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":206,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729499+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:07:37.729499+0000","last_peered":"2023-10-09T05:07:37.729499+0000","last_clean":"2023-10-09T05:07:37.729499+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:07:37.729499+0000","last_undegraded":"2023-10-09T05:07:37.729499+0000","last_fullsized":"2023-10-09T05:07:37.729499+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1262,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727977+0000","last_change":"2023-10-09T05:03:51.192023+0000","last_active":"2023-10-09T05:07:37.727977+0000","last_peered":"2023-10-09T05:07:37.727977+0000","last_clean":"2023-10-09T05:07:37.727977+0000","last_became_active":"2023-10-09T05:03:51.191697+0000","last_became_peered":"2023-10-09T05:03:51.191697+0000","last_unstale":"2023-10-09T05:07:37.727977+0000","last_undegraded":"2023-10-09T05:07:37.727977+0000","last_fullsized":"2023-10-09T05:07:37.727977+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"682'7808","reported_seq":20214,"reported_epoch":686,"state":"active+clean","last_fresh":"2023-10-09T05:07:39.040671+0000","last_change":"2023-10-09T05:07:39.040671+0000","last_active":"2023-10-09T05:07:39.040671+0000","last_peered":"2023-10-09T05:07:39.040671+0000","last_clean":"2023-10-09T05:07:39.040671+0000","last_became_active":"2023-10-09T05:07:39.020728+0000","last_became_peered":"2023-10-09T05:07:39.020728+0000","last_unstale":"2023-10-09T05:07:39.040671+0000","last_undegraded":"2023-10-09T05:07:39.040671+0000","last_fullsized":"2023-10-09T05:07:39.040671+0000","mapping_epoch":685,"log_start":"445'1763","ondisk_log_start":"445'1763","created":440,"last_epoch_clean":686,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4909,"num_write_kb":156000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3187,"num_bytes_recovered":196542464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":99,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.718781+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:07:36.718781+0000","last_peered":"2023-10-09T05:07:36.718781+0000","last_clean":"2023-10-09T05:07:36.718781+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:07:36.718781+0000","last_undegraded":"2023-10-09T05:07:36.718781+0000","last_fullsized":"2023-10-09T05:07:36.718781+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":44,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725415+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:07:37.725415+0000","last_peered":"2023-10-09T05:07:37.725415+0000","last_clean":"2023-10-09T05:07:37.725415+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:07:37.725415+0000","last_undegraded":"2023-10-09T05:07:37.725415+0000","last_fullsized":"2023-10-09T05:07:37.725415+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":139,"reported_epoch":686,"state":"active+clean","last_fresh":"2023-10-09T05:07:39.019718+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:07:39.019718+0000","last_peered":"2023-10-09T05:07:39.019718+0000","last_clean":"2023-10-09T05:07:39.019718+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:07:39.019718+0000","last_undegraded":"2023-10-09T05:07:39.019718+0000","last_fullsized":"2023-10-09T05:07:39.019718+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1283,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729053+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:07:37.729053+0000","last_peered":"2023-10-09T05:07:37.729053+0000","last_clean":"2023-10-09T05:07:37.729053+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:07:37.729053+0000","last_undegraded":"2023-10-09T05:07:37.729053+0000","last_fullsized":"2023-10-09T05:07:37.729053+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.9","version":"682'7448","reported_seq":19500,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713014+0000","last_change":"2023-10-09T05:03:24.458780+0000","last_active":"2023-10-09T05:07:35.713014+0000","last_peered":"2023-10-09T05:07:35.713014+0000","last_clean":"2023-10-09T05:07:35.713014+0000","last_became_active":"2023-10-09T05:03:24.375411+0000","last_became_peered":"2023-10-09T05:03:24.375411+0000","last_unstale":"2023-10-09T05:07:35.713014+0000","last_undegraded":"2023-10-09T05:07:35.713014+0000","last_fullsized":"2023-10-09T05:07:35.713014+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":440,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7448,"log_dups_size":0,"ondisk_log_size":7448,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4909,"num_write_kb":156448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2978,"num_bytes_recovered":187006976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":151,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726138+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:07:37.726138+0000","last_peered":"2023-10-09T05:07:37.726138+0000","last_clean":"2023-10-09T05:07:37.726138+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:07:37.726138+0000","last_undegraded":"2023-10-09T05:07:37.726138+0000","last_fullsized":"2023-10-09T05:07:37.726138+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":110,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719536+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:07:36.719536+0000","last_peered":"2023-10-09T05:07:36.719536+0000","last_clean":"2023-10-09T05:07:36.719536+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:07:36.719536+0000","last_undegraded":"2023-10-09T05:07:36.719536+0000","last_fullsized":"2023-10-09T05:07:36.719536+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.714104+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:07:35.714104+0000","last_peered":"2023-10-09T05:07:35.714104+0000","last_clean":"2023-10-09T05:07:35.714104+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:07:35.714104+0000","last_undegraded":"2023-10-09T05:07:35.714104+0000","last_fullsized":"2023-10-09T05:07:35.714104+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1286,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729418+0000","last_change":"2023-10-09T05:03:51.197738+0000","last_active":"2023-10-09T05:07:37.729418+0000","last_peered":"2023-10-09T05:07:37.729418+0000","last_clean":"2023-10-09T05:07:37.729418+0000","last_became_active":"2023-10-09T05:03:51.197025+0000","last_became_peered":"2023-10-09T05:03:51.197025+0000","last_unstale":"2023-10-09T05:07:37.729418+0000","last_undegraded":"2023-10-09T05:07:37.729418+0000","last_fullsized":"2023-10-09T05:07:37.729418+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.a","version":"682'7596","reported_seq":24969,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713432+0000","last_change":"2023-10-09T05:02:58.968425+0000","last_active":"2023-10-09T05:07:35.713432+0000","last_peered":"2023-10-09T05:07:35.713432+0000","last_clean":"2023-10-09T05:07:35.713432+0000","last_became_active":"2023-10-09T05:02:58.962088+0000","last_became_peered":"2023-10-09T05:02:58.962088+0000","last_unstale":"2023-10-09T05:07:35.713432+0000","last_undegraded":"2023-10-09T05:07:35.713432+0000","last_fullsized":"2023-10-09T05:07:35.713432+0000","mapping_epoch":597,"log_start":"444'400","ondisk_log_start":"444'400","created":440,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":7196,"log_dups_size":0,"ondisk_log_size":7196,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4775,"num_write_kb":152672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3954,"num_bytes_recovered":253689856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":317,"reported_epoch":686,"state":"active+clean","last_fresh":"2023-10-09T05:07:38.739955+0000","last_change":"2023-10-09T05:07:23.309039+0000","last_active":"2023-10-09T05:07:38.739955+0000","last_peered":"2023-10-09T05:07:38.739955+0000","last_clean":"2023-10-09T05:07:38.739955+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:07:38.739955+0000","last_undegraded":"2023-10-09T05:07:38.739955+0000","last_fullsized":"2023-10-09T05:07:38.739955+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":116,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.721372+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:07:36.721372+0000","last_peered":"2023-10-09T05:07:36.721372+0000","last_clean":"2023-10-09T05:07:36.721372+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:07:36.721372+0000","last_undegraded":"2023-10-09T05:07:36.721372+0000","last_fullsized":"2023-10-09T05:07:36.721372+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":190,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725726+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:07:37.725726+0000","last_peered":"2023-10-09T05:07:37.725726+0000","last_clean":"2023-10-09T05:07:37.725726+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:07:37.725726+0000","last_undegraded":"2023-10-09T05:07:37.725726+0000","last_fullsized":"2023-10-09T05:07:37.725726+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1242,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729331+0000","last_change":"2023-10-09T05:05:01.746766+0000","last_active":"2023-10-09T05:07:37.729331+0000","last_peered":"2023-10-09T05:07:37.729331+0000","last_clean":"2023-10-09T05:07:37.729331+0000","last_became_active":"2023-10-09T05:05:01.746370+0000","last_became_peered":"2023-10-09T05:05:01.746370+0000","last_unstale":"2023-10-09T05:07:37.729331+0000","last_undegraded":"2023-10-09T05:07:37.729331+0000","last_fullsized":"2023-10-09T05:07:37.729331+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1793749,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.b","version":"682'7942","reported_seq":20530,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719741+0000","last_change":"2023-10-09T05:07:33.719293+0000","last_active":"2023-10-09T05:07:36.719741+0000","last_peered":"2023-10-09T05:07:36.719741+0000","last_clean":"2023-10-09T05:07:36.719741+0000","last_became_active":"2023-10-09T05:07:33.701427+0000","last_became_peered":"2023-10-09T05:07:33.701427+0000","last_unstale":"2023-10-09T05:07:36.719741+0000","last_undegraded":"2023-10-09T05:07:36.719741+0000","last_fullsized":"2023-10-09T05:07:36.719741+0000","mapping_epoch":681,"log_start":"445'2396","ondisk_log_start":"445'2396","created":440,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5546,"log_dups_size":0,"ondisk_log_size":5546,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4982,"num_write_kb":159360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3232,"num_bytes_recovered":181731328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":202,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.729128+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:07:37.729128+0000","last_peered":"2023-10-09T05:07:37.729128+0000","last_clean":"2023-10-09T05:07:37.729128+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:07:37.729128+0000","last_undegraded":"2023-10-09T05:07:37.729128+0000","last_fullsized":"2023-10-09T05:07:37.729128+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":120,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.721250+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:07:36.721250+0000","last_peered":"2023-10-09T05:07:36.721250+0000","last_clean":"2023-10-09T05:07:36.721250+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:07:36.721250+0000","last_undegraded":"2023-10-09T05:07:36.721250+0000","last_fullsized":"2023-10-09T05:07:36.721250+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":161,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725589+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:07:37.725589+0000","last_peered":"2023-10-09T05:07:37.725589+0000","last_clean":"2023-10-09T05:07:37.725589+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:07:37.725589+0000","last_undegraded":"2023-10-09T05:07:37.725589+0000","last_fullsized":"2023-10-09T05:07:37.725589+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"682'7209","reported_seq":16883,"reported_epoch":686,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-09T05:07:39.019991+0000","last_change":"2023-10-09T05:03:31.487914+0000","last_active":"2023-10-09T05:07:39.019991+0000","last_peered":"2023-10-09T05:07:39.019991+0000","last_clean":"2023-10-09T04:54:25.100137+0000","last_became_active":"2023-10-09T05:03:30.987864+0000","last_became_peered":"2023-10-09T05:03:30.987864+0000","last_unstale":"2023-10-09T05:07:39.019991+0000","last_undegraded":"2023-10-09T05:03:27.264036+0000","last_fullsized":"2023-10-09T05:03:26.332426+0000","mapping_epoch":628,"log_start":"445'1640","ondisk_log_start":"445'1640","created":532,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5569,"log_dups_size":0,"ondisk_log_size":5569,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":391,"num_objects_missing":391,"num_objects_degraded":459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4753,"num_write_kb":150912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2395,"num_bytes_recovered":156958720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[0,1],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":151,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725843+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:07:37.725843+0000","last_peered":"2023-10-09T05:07:37.725843+0000","last_clean":"2023-10-09T05:07:37.725843+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:07:37.725843+0000","last_undegraded":"2023-10-09T05:07:37.725843+0000","last_fullsized":"2023-10-09T05:07:37.725843+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":101,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.719582+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:07:36.719582+0000","last_peered":"2023-10-09T05:07:36.719582+0000","last_clean":"2023-10-09T05:07:36.719582+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:07:36.719582+0000","last_undegraded":"2023-10-09T05:07:36.719582+0000","last_fullsized":"2023-10-09T05:07:36.719582+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.11","version":"682'7373","reported_seq":12234,"reported_epoch":685,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-09T05:07:37.726848+0000","last_change":"2023-10-09T05:02:20.837052+0000","last_active":"2023-10-09T05:07:37.726848+0000","last_peered":"2023-10-09T05:07:37.726848+0000","last_clean":"2023-10-09T05:01:53.681135+0000","last_became_active":"2023-10-09T05:02:20.804002+0000","last_became_peered":"2023-10-09T05:02:20.804002+0000","last_unstale":"2023-10-09T05:07:37.726848+0000","last_undegraded":"2023-10-09T05:02:20.744438+0000","last_fullsized":"2023-10-09T05:07:37.726848+0000","mapping_epoch":597,"log_start":"445'2282","ondisk_log_start":"445'2282","created":532,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4838,"num_write_kb":155488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1107,"num_bytes_recovered":72515584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":["3","2","6"],"object_location_counts":[{"shards":"1,3,6","objects":21}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":825,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.713552+0000","last_change":"2023-10-09T05:07:31.951921+0000","last_active":"2023-10-09T05:07:35.713552+0000","last_peered":"2023-10-09T05:07:35.713552+0000","last_clean":"2023-10-09T05:07:35.713552+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:07:35.713552+0000","last_undegraded":"2023-10-09T05:07:35.713552+0000","last_fullsized":"2023-10-09T05:07:35.713552+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725529+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:07:37.725529+0000","last_peered":"2023-10-09T05:07:37.725529+0000","last_clean":"2023-10-09T05:07:37.725529+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:07:37.725529+0000","last_undegraded":"2023-10-09T05:07:37.725529+0000","last_fullsized":"2023-10-09T05:07:37.725529+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"682'7404","reported_seq":15114,"reported_epoch":684,"state":"active+clean","last_fresh":"2023-10-09T05:07:36.745178+0000","last_change":"2023-10-09T05:07:36.745178+0000","last_active":"2023-10-09T05:07:36.745178+0000","last_peered":"2023-10-09T05:07:36.745178+0000","last_clean":"2023-10-09T05:07:36.745178+0000","last_became_active":"2023-10-09T05:07:36.731672+0000","last_became_peered":"2023-10-09T05:07:36.731672+0000","last_unstale":"2023-10-09T05:07:36.745178+0000","last_undegraded":"2023-10-09T05:07:36.745178+0000","last_fullsized":"2023-10-09T05:07:36.745178+0000","mapping_epoch":683,"log_start":"444'538","ondisk_log_start":"444'538","created":532,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4873,"num_write_kb":155616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2313,"num_bytes_recovered":127303680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":765,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.714045+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:07:35.714045+0000","last_peered":"2023-10-09T05:07:35.714045+0000","last_clean":"2023-10-09T05:07:35.714045+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:07:35.714045+0000","last_undegraded":"2023-10-09T05:07:35.714045+0000","last_fullsized":"2023-10-09T05:07:35.714045+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":44,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725478+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:07:37.725478+0000","last_peered":"2023-10-09T05:07:37.725478+0000","last_clean":"2023-10-09T05:07:37.725478+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:07:37.725478+0000","last_undegraded":"2023-10-09T05:07:37.725478+0000","last_fullsized":"2023-10-09T05:07:37.725478+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"682'7424","reported_seq":18250,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726336+0000","last_change":"2023-10-09T05:03:49.113426+0000","last_active":"2023-10-09T05:07:37.726336+0000","last_peered":"2023-10-09T05:07:37.726336+0000","last_clean":"2023-10-09T05:07:37.726336+0000","last_became_active":"2023-10-09T05:03:49.107755+0000","last_became_peered":"2023-10-09T05:03:49.107755+0000","last_unstale":"2023-10-09T05:07:37.726336+0000","last_undegraded":"2023-10-09T05:07:37.726336+0000","last_fullsized":"2023-10-09T05:07:37.726336+0000","mapping_epoch":638,"log_start":"445'2449","ondisk_log_start":"445'2449","created":532,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":4975,"log_dups_size":0,"ondisk_log_size":4975,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4892,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2812,"num_bytes_recovered":160301056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":307,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.726661+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:07:37.726661+0000","last_peered":"2023-10-09T05:07:37.726661+0000","last_clean":"2023-10-09T05:07:37.726661+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:07:37.726661+0000","last_undegraded":"2023-10-09T05:07:37.726661+0000","last_fullsized":"2023-10-09T05:07:37.726661+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":315,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.727378+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:07:37.727378+0000","last_peered":"2023-10-09T05:07:37.727378+0000","last_clean":"2023-10-09T05:07:37.727378+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:07:37.727378+0000","last_undegraded":"2023-10-09T05:07:37.727378+0000","last_fullsized":"2023-10-09T05:07:37.727378+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"682'7293","reported_seq":18930,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.886251+0000","last_change":"2023-10-09T05:07:37.884526+0000","last_active":"2023-10-09T05:07:37.886251+0000","last_peered":"2023-10-09T05:07:37.886251+0000","last_clean":"2023-10-09T05:07:37.886251+0000","last_became_active":"2023-10-09T05:02:20.814021+0000","last_became_peered":"2023-10-09T05:02:20.814021+0000","last_unstale":"2023-10-09T05:07:37.886251+0000","last_undegraded":"2023-10-09T05:07:37.886251+0000","last_fullsized":"2023-10-09T05:07:37.886251+0000","mapping_epoch":597,"log_start":"445'2262","ondisk_log_start":"445'2262","created":532,"last_epoch_clean":685,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:53:31.951851+0000","last_clean_scrub_stamp":"2023-10-09T04:53:31.951851+0000","objects_scrubbed":0,"log_size":5031,"log_dups_size":0,"ondisk_log_size":5031,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4794,"num_write_kb":154080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2746,"num_bytes_recovered":166428672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":64,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-10-09T05:07:35.712937+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:07:35.712937+0000","last_peered":"2023-10-09T05:07:35.712937+0000","last_clean":"2023-10-09T05:07:35.712937+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:07:35.712937+0000","last_undegraded":"2023-10-09T05:07:35.712937+0000","last_fullsized":"2023-10-09T05:07:35.712937+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":162,"reported_epoch":685,"state":"active+clean","last_fresh":"2023-10-09T05:07:37.725468+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:07:37.725468+0000","last_peered":"2023-10-09T05:07:37.725468+0000","last_clean":"2023-10-09T05:07:37.725468+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:07:37.725468+0000","last_undegraded":"2023-10-09T05:07:37.725468+0000","last_fullsized":"2023-10-09T05:07:37.725468+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":36,"stat_sum":{"num_bytes":21479412,"num_objects":326,"num_object_clones":0,"num_object_copies":978,"num_objects_missing_on_primary":1042,"num_objects_missing":1042,"num_objects_degraded":4156,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":326,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":155218,"num_write_kb":4966913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96657,"num_bytes_recovered":5676597248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7578976256,"data_stored":7578976256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":217997,"ondisk_log_size":217997,"up":108,"acting":100,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":17710,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":16728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":7095738,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234723,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2497728,"kb_used_data":984376,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":91247424,"statfs":{"total":95995035648,"available":93437362176,"internally_reserved":0,"allocated":1008001024,"data_stored":1006244503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:18 2023","interfaces":[{"interface":"back","average":{"1min":13.488,"5min":8.5220000000000002,"15min":3.4199999999999999},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23599999999999999},"max":{"1min":160.38999999999999,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":14.443,"5min":8.7780000000000005,"15min":3.6429999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":160.47499999999999,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.28299999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:07:18 2023","interfaces":[{"interface":"back","average":{"1min":33.710000000000001,"5min":9.9939999999999998,"15min":4.1239999999999997},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":322.31999999999999,"5min":322.31999999999999,"15min":322.31999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":33.951999999999998,"5min":9.9659999999999993,"15min":4.0800000000000001},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":322.14800000000002,"5min":322.14800000000002,"15min":322.14800000000002},"last":2.4409999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.7469999999999999,"15min":3.8620000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3470000000000004,"5min":64.480999999999995,"15min":207.96600000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.7090000000000001,"15min":3.9550000000000001},"min":{"1min":0.35199999999999998,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":6.3579999999999997,"5min":55.134,"15min":207.97900000000001},"last":0.41399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":39.868000000000002,"5min":15.595000000000001,"15min":6.3470000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":349.863,"5min":349.863,"15min":349.863},"last":0.42499999999999999},{"interface":"front","average":{"1min":39.591000000000001,"5min":15.047000000000001,"15min":6.1660000000000004},"min":{"1min":0.32900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":349.67500000000001,"5min":349.67500000000001,"15min":349.67500000000001},"last":0.46000000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.38900000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.145,"5min":1.145,"15min":1.145},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.51600000000000001,"15min":0.48699999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.0609999999999999,"5min":3.0609999999999999,"15min":3.0609999999999999},"last":0.38200000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3849999999999998,"5min":1.847,"15min":1.897},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":74.915000000000006,"5min":74.915000000000006,"15min":131.84899999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":4.4939999999999998,"5min":2,"15min":1.99},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":74.972999999999999,"5min":74.972999999999999,"15min":135.74199999999999},"last":0.35199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.90000000000000002,"15min":1.0680000000000001},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":2.1829999999999998,"5min":10.183999999999999,"15min":57.168999999999997},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.0860000000000001,"15min":1.218},"min":{"1min":0.32900000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":1.7,"5min":21.097000000000001,"15min":57.183999999999997},"last":0.437}]}]},{"osd":6,"up_from":46,"seq":197568496356,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533268,"kb_used_data":22668,"kb_used_omap":0,"kb_used_meta":1510592,"kb_avail":92211884,"statfs":{"total":95995035648,"available":94424969216,"internally_reserved":0,"allocated":23212032,"data_stored":21459720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1546846208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":5.5570000000000004,"5min":4.1929999999999996,"15min":3.077},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":40.877000000000002,"5min":65.721999999999994,"15min":321.346},"last":0.28799999999999998},{"interface":"front","average":{"1min":5.8140000000000001,"5min":4.266,"15min":2.6280000000000001},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":40.887,"5min":65.585999999999999,"15min":169.119},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":4.601,"5min":2.1880000000000002,"15min":2.1850000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":43.604999999999997,"5min":43.604999999999997,"15min":90.278999999999996},"last":0.27500000000000002},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.4119999999999999,"15min":2.1640000000000001},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":44.697000000000003,"5min":44.697000000000003,"15min":90.414000000000001},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":1.925,"15min":2.101},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.25600000000000001},"max":{"1min":7.1159999999999997,"5min":23.943000000000001,"15min":133.535},"last":0.314},{"interface":"front","average":{"1min":2.5030000000000001,"5min":1.907,"15min":2.036},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":8.0039999999999996,"5min":25.978999999999999,"15min":133.51900000000001},"last":0.45700000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0990000000000002,"5min":3.7749999999999999,"15min":3.137},"min":{"1min":0.45900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":42.366,"5min":64.355999999999995,"15min":147.43600000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":4.4500000000000002,"5min":3.891,"15min":3.2080000000000002},"min":{"1min":0.41699999999999998,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":45.814999999999998,"5min":56.133000000000003,"15min":147.41999999999999},"last":0.498}]},{"osd":4,"last update":"Mon Oct 9 05:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.89700000000000002,"15min":0.78900000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3769999999999998,"5min":18.186,"15min":18.186},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0589999999999999,"15min":1.022},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.2469999999999999,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.432}]},{"osd":5,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.82399999999999995,"15min":0.94699999999999995},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.9539999999999997,"5min":7.9539999999999997,"15min":25.934000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91100000000000003,"15min":0.91500000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":5.2990000000000004,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.86499999999999999,"15min":0.998},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":7.5039999999999996,"5min":7.5039999999999996,"15min":36.204999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.0760000000000001,"15min":0.98599999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.3399999999999999,"5min":20.041,"15min":20.041},"last":0.52400000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141423,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1660740,"kb_used_data":244284,"kb_used_omap":4,"kb_used_meta":1416443,"kb_avail":92084412,"statfs":{"total":95995035648,"available":94294437888,"internally_reserved":0,"allocated":250146816,"data_stored":246461198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1450438542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9059999999999997,"5min":5.7039999999999997,"15min":2.5880000000000001},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":42.058999999999997,"5min":100.247,"15min":100.247},"last":0.23200000000000001},{"interface":"front","average":{"1min":31.283000000000001,"5min":11.256,"15min":4.4020000000000001},"min":{"1min":0.318,"5min":0.247,"15min":0.214},"max":{"1min":554.62,"5min":554.62,"15min":554.62},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":1.8340000000000001,"15min":2.7120000000000002},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.393000000000001,"5min":12.460000000000001,"15min":263.089},"last":0.30399999999999999},{"interface":"front","average":{"1min":4.359,"5min":2.3860000000000001,"15min":2.8929999999999998},"min":{"1min":0.36399999999999999,"5min":0.252,"15min":0.222},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":254.50899999999999},"last":0.26400000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:07:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":8.1950000000000003,"15min":3.9550000000000001},"min":{"1min":0.26400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.0279999999999996,"5min":235.43600000000001,"15min":235.43600000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":26.606999999999999,"5min":12.436999999999999,"15min":5.1699999999999999},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":554.84400000000005,"5min":554.84400000000005,"15min":554.84400000000005},"last":0.32900000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":25.523,"5min":6.9199999999999999,"15min":3.1560000000000001},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":554.44000000000005,"5min":554.44000000000005,"15min":554.44000000000005},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.409,"5min":1.871,"15min":1.2549999999999999},"min":{"1min":0.32600000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.5640000000000001,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.305,"5min":7.766,"15min":3.3719999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":554.54700000000003,"5min":554.54700000000003,"15min":554.54700000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":5.1310000000000002,"5min":2.6259999999999999,"15min":1.744},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":80.390000000000001,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.41999999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.236999999999998,"5min":7.7699999999999996,"15min":3.387},"min":{"1min":0.40400000000000003,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":554.36500000000001,"5min":554.36500000000001,"15min":554.36500000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":5.0069999999999997,"5min":2.5939999999999999,"15min":1.7350000000000001},"min":{"1min":0.41999999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":78.278000000000006,"5min":78.278000000000006,"15min":78.278000000000006},"last":0.46000000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.968,"15min":1.5900000000000001},"min":{"1min":0.48299999999999998,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.4850000000000003,"5min":18.899999999999999,"15min":19.664999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.589,"5min":1.6240000000000001,"15min":1.397},"min":{"1min":0.47299999999999998,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":7.0099999999999998,"5min":11.903,"15min":11.903},"last":0.5}]}]},{"osd":0,"up_from":91,"seq":390842024507,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1378344,"kb_used_data":154848,"kb_used_omap":5,"kb_used_meta":1223482,"kb_avail":92366808,"statfs":{"total":95995035648,"available":94583611392,"internally_reserved":0,"allocated":158564352,"data_stored":155573767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1252846202},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:07:13 2023","interfaces":[{"interface":"back","average":{"1min":3.395,"5min":3.6909999999999998,"15min":2.6059999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":17.73,"5min":53.176000000000002,"15min":155.411},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.7160000000000002,"5min":4.0359999999999996,"15min":2.6520000000000001},"min":{"1min":0.39100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":17.594000000000001,"5min":53.372999999999998,"15min":154.75},"last":5.4400000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:07:34 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.5990000000000002,"15min":2.8519999999999999},"min":{"1min":0.26500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":20.010999999999999,"5min":20.010999999999999,"15min":151.06399999999999},"last":0.375},{"interface":"front","average":{"1min":2.7349999999999999,"5min":3.2509999999999999,"15min":3.097},"min":{"1min":0.28199999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.364000000000001,"5min":20.239000000000001,"15min":151.22399999999999},"last":5.484}]},{"osd":3,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":7.7830000000000004,"15min":4.6100000000000003},"min":{"1min":0.307,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":19.096,"5min":290.89100000000002,"15min":290.89100000000002},"last":2.2959999999999998},{"interface":"front","average":{"1min":3.726,"5min":8.3650000000000002,"15min":4.8470000000000004},"min":{"1min":0.29299999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":15.554,"5min":290.73899999999998,"15min":290.73899999999998},"last":5.7489999999999997}]},{"osd":4,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9219999999999997,"5min":2.621,"15min":2.1080000000000001},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":57.360999999999997,"5min":57.360999999999997,"15min":57.360999999999997},"last":2.5569999999999999},{"interface":"front","average":{"1min":4.5979999999999999,"5min":2.3839999999999999,"15min":1.8180000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":57.338000000000001,"5min":57.338000000000001,"15min":57.338000000000001},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:07:23 2023","interfaces":[{"interface":"back","average":{"1min":8.4139999999999997,"5min":3.8599999999999999,"15min":1.9870000000000001},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":106.922,"5min":106.922,"15min":106.922},"last":5.5570000000000004},{"interface":"front","average":{"1min":8.3190000000000008,"5min":3.2080000000000002,"15min":1.8340000000000001},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":106.771,"5min":106.771,"15min":106.771},"last":2.714}]},{"osd":6,"last update":"Mon Oct 9 05:07:23 2023","interfaces":[{"interface":"back","average":{"1min":12.031000000000001,"5min":4.5830000000000002,"15min":2.3780000000000001},"min":{"1min":0.42099999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":112.208,"5min":112.208,"15min":112.208},"last":5.5220000000000002},{"interface":"front","average":{"1min":11.976000000000001,"5min":3.9820000000000002,"15min":2.391},"min":{"1min":0.372,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":110.46899999999999,"5min":110.46899999999999,"15min":110.46899999999999},"last":2.6819999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:07:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.0880000000000001,"15min":1.55},"min":{"1min":0.42899999999999999,"5min":0.41899999999999998,"15min":0.39400000000000002},"max":{"1min":13.787000000000001,"5min":13.787000000000001,"15min":21.658000000000001},"last":2.6499999999999999},{"interface":"front","average":{"1min":2.089,"5min":1.887,"15min":1.528},"min":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":6.9020000000000001,"5min":11.842000000000001,"15min":21.68},"last":0.498}]}]},{"osd":2,"up_from":0,"seq":949187772786,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356479,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212368,"kb_used_data":194824,"kb_used_omap":2,"kb_used_meta":1017533,"kb_avail":92532784,"statfs":{"total":95995035648,"available":94753570816,"internally_reserved":0,"allocated":199499776,"data_stored":196402545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1041953845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":7.4809999999999999,"5min":4.6970000000000001,"15min":2.0379999999999998},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":62.863,"5min":108.631,"15min":108.631},"last":0.26000000000000001},{"interface":"front","average":{"1min":7.7329999999999997,"5min":4.9349999999999996,"15min":2.2429999999999999},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":62.948,"5min":110.012,"15min":110.012},"last":0.31}]},{"osd":1,"last update":"Mon Oct 9 05:07:13 2023","interfaces":[{"interface":"back","average":{"1min":2.754,"5min":2.343,"15min":1.5449999999999999},"min":{"1min":0.25700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":13.103999999999999,"5min":15.218,"15min":15.983000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.1640000000000001,"15min":1.4690000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.164,"5min":17.544,"15min":26.692},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:06:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":1.5669999999999999,"15min":2.371},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":11.996,"5min":11.996,"15min":212.065},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.8799999999999999,"5min":1.694,"15min":2.4079999999999999},"min":{"1min":0.32100000000000001,"5min":0.253,"15min":0.23499999999999999},"max":{"1min":6.2539999999999996,"5min":12.987,"15min":211.98400000000001},"last":0.379}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.8,"15min":1.746},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9359999999999999,"5min":11.193,"15min":11.193},"last":0.748},{"interface":"front","average":{"1min":6.8650000000000002,"5min":2.4209999999999998,"15min":1.405},"min":{"1min":0.36499999999999999,"5min":0.309,"15min":0.309},"max":{"1min":62.683999999999997,"5min":62.683999999999997,"15min":62.683999999999997},"last":0.57899999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":6.4800000000000004,"5min":2.9929999999999999,"15min":1.6950000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":75.478999999999999,"5min":75.478999999999999,"15min":75.478999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":10.881,"5min":4.0910000000000002,"15min":2.2429999999999999},"min":{"1min":0.379,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":75.459000000000003,"5min":75.459000000000003,"15min":75.459000000000003},"last":0.69699999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":11.130000000000001,"5min":3.6459999999999999,"15min":1.9019999999999999},"min":{"1min":0.40799999999999997,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":79.093999999999994,"5min":79.093999999999994,"15min":79.093999999999994},"last":0.627},{"interface":"front","average":{"1min":11.366,"5min":3.4199999999999999,"15min":1.9950000000000001},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":78.555999999999997,"5min":78.555999999999997,"15min":78.555999999999997},"last":0.72599999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":6.226,"5min":2.4889999999999999,"15min":1.6699999999999999},"min":{"1min":0.42699999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":63.017000000000003,"5min":63.017000000000003,"15min":63.017000000000003},"last":0.67800000000000005},{"interface":"front","average":{"1min":6.4470000000000001,"5min":2.4740000000000002,"15min":1.4379999999999999},"min":{"1min":0.47599999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":62.808,"5min":62.808,"15min":62.808},"last":0.60599999999999998}]}]},{"osd":4,"up_from":626,"seq":2688649527348,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1830548,"kb_used_data":1437644,"kb_used_omap":22,"kb_used_meta":392873,"kb_avail":91914604,"statfs":{"total":95995035648,"available":94120554496,"internally_reserved":0,"allocated":1472147456,"data_stored":1470394411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":402302797},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":5.3360000000000003,"5min":4.3310000000000004,"15min":2.9710000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":70.512,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":5.9180000000000001,"5min":4.8550000000000004,"15min":3.7320000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":70.813999999999993,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.56000000000000005}]},{"osd":1,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4710000000000001,"5min":5.2199999999999998,"15min":3.1589999999999998},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":103.997,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":7.7279999999999998,"5min":4.8929999999999998,"15min":2.8980000000000001},"min":{"1min":0.34100000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.333,"5min":182.41,"15min":182.41},"last":0.94799999999999995}]},{"osd":2,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.907,"15min":1.728},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2210000000000001,"5min":11.051,"15min":11.051},"last":1.028},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.913,"15min":1.762},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.234,"5min":11.036,"15min":11.036},"last":0.97999999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":13.058,"5min":7.5650000000000004,"15min":3.3050000000000002},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.318},"max":{"1min":109.53400000000001,"5min":270.767,"15min":270.767},"last":1.375},{"interface":"front","average":{"1min":13.085000000000001,"5min":7.7469999999999999,"15min":3.8109999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":109.84,"5min":270.274,"15min":270.274},"last":1.004}]},{"osd":5,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":8.9659999999999993,"5min":2.0960000000000001,"15min":0.95599999999999996},"min":{"1min":0.25900000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":134.82900000000001,"5min":134.82900000000001,"15min":134.82900000000001},"last":2.4049999999999998},{"interface":"front","average":{"1min":9.0030000000000001,"5min":2.149,"15min":1.0149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":134.60400000000001,"5min":134.60400000000001,"15min":134.60400000000001},"last":0.61699999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":9.3529999999999998,"5min":2.3100000000000001,"15min":1.212},"min":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":139.77799999999999,"5min":139.77799999999999,"15min":139.77799999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":9.6219999999999999,"5min":2.2959999999999998,"15min":1.0669999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":140.78200000000001,"5min":140.78200000000001,"15min":140.78200000000001},"last":1.0609999999999999}]},{"osd":7,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.53200000000000003,"15min":0.49099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.036,"5min":3.036,"15min":3.036},"last":3.109}]}]},{"osd":5,"up_from":38,"seq":163208758000,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3802492,"kb_used_data":2309172,"kb_used_omap":0,"kb_used_meta":1493312,"kb_avail":89942660,"statfs":{"total":95995035648,"available":92101283840,"internally_reserved":0,"allocated":2364592128,"data_stored":2362851242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1529151488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":12.31,"5min":6.9690000000000003,"15min":3.2719999999999998},"min":{"1min":0.27800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":212.72800000000001,"5min":212.72800000000001,"15min":212.72800000000001},"last":0.20599999999999999},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.3040000000000003,"15min":3.4620000000000002},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.245},"max":{"1min":212.53999999999999,"5min":212.53999999999999,"15min":212.53999999999999},"last":0.245}]},{"osd":1,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":7.2240000000000002,"5min":3.8450000000000002,"15min":2.9129999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":113.977,"5min":121.504,"15min":262.26900000000001},"last":77.268000000000001},{"interface":"front","average":{"1min":6.9960000000000004,"5min":3.9409999999999998,"15min":2.9950000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":112.947,"5min":121.56,"15min":262.291},"last":77.287999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:06:42 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":1.653,"15min":1.919},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":9.6440000000000001,"5min":10.949999999999999,"15min":106.238},"last":0.32800000000000001},{"interface":"front","average":{"1min":2.1280000000000001,"5min":1.851,"15min":2.1659999999999999},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.29999999999999999},"max":{"1min":4.8979999999999997,"5min":11.912000000000001,"15min":104.31699999999999},"last":0.317}]},{"osd":3,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5759999999999996,"5min":7.8410000000000002,"15min":3.7989999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.318},"max":{"1min":75.635000000000005,"5min":125.824,"15min":125.824},"last":0.30099999999999999},{"interface":"front","average":{"1min":5.8579999999999997,"5min":7.8010000000000002,"15min":3.7770000000000001},"min":{"1min":0.38700000000000001,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":75.656999999999996,"5min":125.761,"15min":125.761},"last":0.28699999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.433},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.94399999999999995,"5min":1.99,"15min":1.99},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.45900000000000002,"15min":0.46000000000000002},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.165,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.26400000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:06:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.68600000000000005,"15min":0.73899999999999999},"min":{"1min":0.28499999999999998,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":2.0289999999999999,"5min":11.093,"15min":11.093},"last":0.378},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.59899999999999998,"15min":0.81799999999999995},"min":{"1min":0.32000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.982,"5min":5.9329999999999998,"15min":13.083},"last":0.41899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.68700000000000006,"15min":0.86699999999999999},"min":{"1min":0.309,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.093,"5min":8.3520000000000003,"15min":9.6809999999999992},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.90200000000000002,"15min":0.88500000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":2.0489999999999999,"5min":11.791,"15min":13.233000000000001},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":149422080,"data_stored":149422080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":235012096,"data_stored":235012096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2173894656,"data_stored":2173894656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":190840832,"data_stored":190840832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1456603136,"data_stored":1456603136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2357460992,"data_stored":2357460992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":15466496,"data_stored":15466496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1000275968,"data_stored":1000275968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:07:43.516 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:43.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:07:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:43 smithi177 ceph-mon[195797]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:43 smithi177 ceph-mon[195797]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:43 smithi177 ceph-mon[195797]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:43 smithi177 ceph-mon[195797]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:43 smithi177 ceph-mon[195797]: osdmap e691: 8 total, 8 up, 7 in 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[216903]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[216903]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[216903]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[216903]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[216903]: osdmap e691: 8 total, 8 up, 7 in 2023-10-09T05:07:44.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[223858]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[223858]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[223858]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[223858]: failed to encode map e690 with expected crc 2023-10-09T05:07:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:44.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:43 smithi029 ceph-mon[223858]: osdmap e691: 8 total, 8 up, 7 in 2023-10-09T05:07:44.514 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:07:44.577 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:44.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:44.681 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:44.714 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:44.767 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:44.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:44.872 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:44.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[216903]: 14.12 scrub starts 2023-10-09T05:07:44.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[216903]: failed to encode map e691 with expected crc 2023-10-09T05:07:44.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[216903]: failed to encode map e691 with expected crc 2023-10-09T05:07:44.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[216903]: pgmap v2224: 119 pgs: 1 activating, 1 peering, 2 active+recovering+undersized+degraded+remapped, 113 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 1389/1050 objects degraded (132.286%); 0 B/s, 489 objects/s recovering 2023-10-09T05:07:44.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[223858]: 14.12 scrub starts 2023-10-09T05:07:44.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[223858]: failed to encode map e691 with expected crc 2023-10-09T05:07:44.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[223858]: failed to encode map e691 with expected crc 2023-10-09T05:07:44.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:44 smithi029 ceph-mon[223858]: pgmap v2224: 119 pgs: 1 activating, 1 peering, 2 active+recovering+undersized+degraded+remapped, 113 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 1389/1050 objects degraded (132.286%); 0 B/s, 489 objects/s recovering 2023-10-09T05:07:44.906 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:44 smithi177 ceph-mon[195797]: 14.12 scrub starts 2023-10-09T05:07:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:44 smithi177 ceph-mon[195797]: failed to encode map e691 with expected crc 2023-10-09T05:07:45.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:44 smithi177 ceph-mon[195797]: failed to encode map e691 with expected crc 2023-10-09T05:07:45.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:44 smithi177 ceph-mon[195797]: pgmap v2224: 119 pgs: 1 activating, 1 peering, 2 active+recovering+undersized+degraded+remapped, 113 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 1389/1050 objects degraded (132.286%); 0 B/s, 489 objects/s recovering 2023-10-09T05:07:45.708 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:45.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: 14.18 scrub starts 2023-10-09T05:07:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: 14.a scrub starts 2023-10-09T05:07:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: 14.15 scrub starts 2023-10-09T05:07:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: failed to encode map e692 with expected crc 2023-10-09T05:07:45.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: failed to encode map e692 with expected crc 2023-10-09T05:07:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: osdmap e692: 8 total, 8 up, 7 in 2023-10-09T05:07:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: failed to encode map e692 with expected crc 2023-10-09T05:07:45.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:45 smithi177 ceph-mon[195797]: failed to encode map e692 with expected crc 2023-10-09T05:07:45.812 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:45.841 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:46.013 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:46.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:46.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:46.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: 14.18 scrub starts 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: 14.a scrub starts 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: 14.15 scrub starts 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: osdmap e692: 8 total, 8 up, 7 in 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.148 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[216903]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: 14.18 scrub starts 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: 14.a scrub starts 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: 14.15 scrub starts 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: osdmap e692: 8 total, 8 up, 7 in 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:45 smithi029 ceph-mon[223858]: failed to encode map e692 with expected crc 2023-10-09T05:07:46.151 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:46.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:07:46.662 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:46.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:46.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:46.731 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:07:46.731 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:07:46.731 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:07:46.731 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:07:46.731 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:07:46.772 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:46.806 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: 14.1b scrub starts 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: 14.1b scrub ok 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: 14.1d scrub starts 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: 14.1d scrub ok 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: osdmap e693: 8 total, 8 up, 7 in 2023-10-09T05:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: pgmap v2227: 119 pgs: 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 115 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 464/1050 objects degraded (44.190%); 0 B/s, 399 objects/s recovering 2023-10-09T05:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: 14.11 scrub starts 2023-10-09T05:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: 14.11 scrub ok 2023-10-09T05:07:47.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:46 smithi177 ceph-mon[195797]: Health check update: Degraded data redundancy: 1389/1050 objects degraded (132.286%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:47.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: 14.1b scrub starts 2023-10-09T05:07:47.065 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: 14.1b scrub ok 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: 14.1d scrub starts 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: 14.1d scrub ok 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: osdmap e693: 8 total, 8 up, 7 in 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: pgmap v2227: 119 pgs: 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 115 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 464/1050 objects degraded (44.190%); 0 B/s, 399 objects/s recovering 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: 14.11 scrub starts 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: 14.11 scrub ok 2023-10-09T05:07:47.066 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[216903]: Health check update: Degraded data redundancy: 1389/1050 objects degraded (132.286%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: 14.1b scrub starts 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: 14.1b scrub ok 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: 14.1d scrub starts 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: 14.1d scrub ok 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: osdmap e693: 8 total, 8 up, 7 in 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: failed to encode map e693 with expected crc 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: pgmap v2227: 119 pgs: 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 115 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 21 MiB data, 13 GiB used, 613 GiB / 626 GiB avail; 464/1050 objects degraded (44.190%); 0 B/s, 399 objects/s recovering 2023-10-09T05:07:47.067 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: 14.11 scrub starts 2023-10-09T05:07:47.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: 14.11 scrub ok 2023-10-09T05:07:47.068 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:46 smithi029 ceph-mon[223858]: Health check update: Degraded data redundancy: 1389/1050 objects degraded (132.286%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-09T05:07:47.157 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:47.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:47.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:47.260 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:07:47.298 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:47.365 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:07:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:47.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:07:47.392 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:47.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:47.499 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:47.774 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:07:47.799 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:07:47.804 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_9' removed 2023-10-09T05:07:47.903 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:47.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:47.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:48.009 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: failed to encode map e693 with expected crc 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: 14.1c scrub starts 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: 14.1c scrub ok 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: osdmap e694: 8 total, 8 up, 7 in 2023-10-09T05:07:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1702276307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3855828542' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:07:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:48.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:47 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:48.043 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:48.093 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:07:48.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:48.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:07:48.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: failed to encode map e693 with expected crc 2023-10-09T05:07:48.112 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: 14.1c scrub starts 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: 14.1c scrub ok 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: osdmap e694: 8 total, 8 up, 7 in 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1702276307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3855828542' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:48.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: failed to encode map e693 with expected crc 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: 14.1c scrub starts 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: 14.1c scrub ok 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: osdmap e694: 8 total, 8 up, 7 in 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: failed to encode map e694 with expected crc 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1702276307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:48.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:07:48.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3855828542' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-09T05:07:48.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:48.115 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:47 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:48.179 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:07:48.200 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:48.227 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:48.326 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:48.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:48.437 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:48.465 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:48.508 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:48.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:48.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:48.612 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:48.642 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:48.668 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:07:48.786 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:48.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:48.890 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:48.918 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: failed to encode map e694 with expected crc 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3855828542' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: osdmap e695: 8 total, 8 up, 7 in 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: pgmap v2230: 83 pgs: 83 active+clean; 582 KiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:07:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:48 smithi177 ceph-mon[195797]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: failed to encode map e694 with expected crc 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3855828542' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: osdmap e695: 8 total, 8 up, 7 in 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: pgmap v2230: 83 pgs: 83 active+clean; 582 KiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[216903]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: failed to encode map e694 with expected crc 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3855828542' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: osdmap e695: 8 total, 8 up, 7 in 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: pgmap v2230: 83 pgs: 83 active+clean; 582 KiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:48 smithi029 ceph-mon[223858]: failed to encode map e695 with expected crc 2023-10-09T05:07:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:49.173 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:49.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:49.274 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:49.303 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:49.349 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:49.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:49.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:49.454 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:49.482 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:49.526 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:49.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:49.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:49.630 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:49.659 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:49.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:49.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:49.862 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:07:49.862 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-09T05:07:49.862 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:07:49.866 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:07:49.866 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:07:49.866 DEBUG:teuthology.orchestra.run.smithi029:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:07:49.878 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93380 14.7 14.dfd5a7a7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.49465213100001, 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94592668400000002, 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.879 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93380 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381649+0000", 2023-10-09T05:07:49.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:07:49.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381660+0000", 2023-10-09T05:07:49.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1054e-05 2023-10-09T05:07:49.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381674+0000", 2023-10-09T05:07:49.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.385e-05 2023-10-09T05:07:49.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381683+0000", 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7490000000000004e-06 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381756+0000", 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3750000000000004e-05 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381767+0000", 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0208e-05 2023-10-09T05:07:49.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327567+0000", 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94579980799999996 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327577+0000", 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.08e-05 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93400 14.1c 14:3dc013b3:::benchmark_data_smithi029_550670_object15789:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.887 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.49459945199999, 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3919103770000003, 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93400 2023-10-09T05:07:49.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381706+0000", 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8140000000000001e-06 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381711+0000", 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.575e-06 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381715+0000", 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8240000000000001e-06 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381812+0000", 2023-10-09T05:07:49.893 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6555e-05 2023-10-09T05:07:49.893 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381822+0000", 2023-10-09T05:07:49.893 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0302000000000001e-05 2023-10-09T05:07:49.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.894 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.894 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.894 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.015600+0000", 2023-10-09T05:07:49.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6337784260000001 2023-10-09T05:07:49.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.017575+0000", 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019744390000000001 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.771430+0000", 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.75385499199999995 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.773614+0000", 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021838650000000001 2023-10-09T05:07:49.896 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93401 14.7 14.decc63e7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.49456397699998, 2023-10-09T05:07:49.897 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94601877700000003, 2023-10-09T05:07:49.898 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.898 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.898 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.898 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.898 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.898 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93401 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:07:49.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381738+0000", 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381741+0000", 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6749999999999998e-06 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381747+0000", 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5799999999999999e-06 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381751+0000", 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6689999999999997e-06 2023-10-09T05:07:49.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381798+0000", 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6303e-05 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381805+0000", 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0190000000000001e-06 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327746+0000", 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94594101799999997 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327758+0000", 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1878999999999999e-05 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.903 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93405 14.17 14.5711f157 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.49454442899997, 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94466796099999994, 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.904 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93405 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.905 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:07:49.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381761+0000", 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5280000000000002e-06 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.907 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381766+0000", 2023-10-09T05:07:49.908 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9130000000000002e-06 2023-10-09T05:07:49.908 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.908 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.908 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.908 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381769+0000", 2023-10-09T05:07:49.908 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.788e-06 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382138+0000", 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036829199999999999 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382146+0000", 2023-10-09T05:07:49.909 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8200000000000003e-06 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.326413+0000", 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94426679400000002 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.326426+0000", 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3084999999999999e-05 2023-10-09T05:07:49.910 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93407 14.5 14:a209006e:::benchmark_data_smithi029_550670_object15796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.49419546500002, 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2579248559999998, 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.911 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93407 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.912 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:07:49.913 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.914 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.914 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.914 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382114+0000", 2023-10-09T05:07:49.914 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2089999999999999e-06 2023-10-09T05:07:49.914 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.914 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.915 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.915 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382124+0000", 2023-10-09T05:07:49.915 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0268000000000001e-05 2023-10-09T05:07:49.915 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.915 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.915 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.916 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382134+0000", 2023-10-09T05:07:49.916 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:07:49.916 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.916 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.916 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.916 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382191+0000", 2023-10-09T05:07:49.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6889999999999999e-05 2023-10-09T05:07:49.917 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.917 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.917 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.917 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382201+0000", 2023-10-09T05:07:49.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0359e-05 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.374861+0000", 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9926602410000003 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.918 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:07:49.919 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.375679+0000", 2023-10-09T05:07:49.919 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00081763400000000003 2023-10-09T05:07:49.919 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.919 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.919 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.919 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.639295+0000", 2023-10-09T05:07:49.920 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.263616246 2023-10-09T05:07:49.920 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.920 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.920 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.920 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.640032+0000", 2023-10-09T05:07:49.920 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00073696500000000002 2023-10-09T05:07:49.921 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.921 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.921 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.921 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.921 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.921 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93418 14.1c 14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.49415266300002, 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3997008879999999, 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.922 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93418 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.923 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382153+0000", 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0680000000000001e-06 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.924 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382160+0000", 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9639999999999997e-06 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382165+0000", 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4569999999999996e-06 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382198+0000", 2023-10-09T05:07:49.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2703999999999997e-05 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382205+0000", 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.017835+0000", 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6356300770000001 2023-10-09T05:07:49.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.025741+0000", 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079060310000000009 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.751118+0000", 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72537742100000002 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.753397+0000", 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022792559999999999 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.757192+0000", 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037947889999999998 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.760499+0000", 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033070700000000001 2023-10-09T05:07:49.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.774353+0000", 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013853202 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.775194+0000", 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00084157500000000001 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.776410+0000", 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012155670000000001 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.781851+0000", 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054413259999999998 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93419 14.15 14:a8313c41:::benchmark_data_smithi029_550670_object15808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 265.47115698699997, 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2483047789999997, 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93419 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.931 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405145+0000", 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405149+0000", 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6789999999999996e-06 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405159+0000", 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8779999999999993e-06 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405163+0000", 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1359999999999999e-06 2023-10-09T05:07:49.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405203+0000", 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9718999999999997e-05 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405212+0000", 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2439999999999999e-06 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.935 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.625967+0000", 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2207548089999998 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.629287+0000", 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003320042 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.633107+0000", 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038200420000000001 2023-10-09T05:07:49.936 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.641198+0000", 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080906099999999998 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.646808+0000", 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056098529999999997 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.937 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.647586+0000", 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000778562 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.648854+0000", 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001267617 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.653451+0000", 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045966180000000002 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93432 14.c 14:374a58ab:::benchmark_data_smithi029_550670_object15821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 261.15144306299999, 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1048139219999999, 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.939 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93432 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.940 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724859+0000", 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724864+0000", 2023-10-09T05:07:49.941 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1170000000000003e-06 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724876+0000", 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1911e-05 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724881+0000", 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0379999999999999e-06 2023-10-09T05:07:49.942 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.774671+0000", 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0497900250000001 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.783271+0000", 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008600056 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.943 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.806956+0000", 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023684614999999999 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.809403+0000", 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002447467 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.813396+0000", 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039930909999999998 2023-10-09T05:07:49.944 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.816971+0000", 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00357516 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.822865+0000", 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058936639999999998 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.945 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.823152+0000", 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028665299999999999 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.824381+0000", 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012293689999999999 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.946 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.829674+0000", 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00529255 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93434 14.1c 14:38f7b3c2:::benchmark_data_smithi029_550670_object15823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 261.13513204200001, 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0950609299999998, 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.947 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93434 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.948 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741170+0000", 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.949 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741175+0000", 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1479999999999997e-06 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741186+0000", 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1286e-05 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741191+0000", 2023-10-09T05:07:49.950 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5639999999999998e-06 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.782108+0000", 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0409173940000001 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.790417+0000", 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083089150000000001 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.951 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.810827+0000", 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020409265999999999 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.812966+0000", 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002139448 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.816862+0000", 2023-10-09T05:07:49.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038954900000000002 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.822572+0000", 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057100429999999997 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.823625+0000", 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010534870000000001 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.830012+0000", 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063868370000000002 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.831216+0000", 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012044099999999999 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.836232+0000", 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005015345 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.1c 629/627 [PushReplyOp(14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head)])", 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.849210959, 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71326082199999996, 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.955 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.956 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027091+0000", 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027097+0000", 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5639999999999996e-06 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027107+0000", 2023-10-09T05:07:49.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.082e-05 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027113+0000", 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8239999999999996e-06 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.735833+0000", 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70872009899999999 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.740353+0000", 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045193639999999997 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.1c 629/627 [PushReplyOp(14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head)])", 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 259.84900883799997, 2023-10-09T05:07:49.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72351207900000003, 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027293+0000", 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027298+0000", 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9649999999999999e-06 2023-10-09T05:07:49.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027310+0000", 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1353e-05 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027316+0000", 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2890000000000003e-06 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.740750+0000", 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71343399900000004 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.750806+0000", 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010056034 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93889 14.c 14:34db2e8f:::benchmark_data_smithi029_550670_object16278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 232.169148839, 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.485937375, 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93889 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.964 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707152+0000", 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707158+0000", 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7729999999999996e-06 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707171+0000", 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2500000000000001e-05 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707176+0000", 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2159999999999997e-06 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806421+0000", 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.099245228000000005 2023-10-09T05:07:49.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.814839+0000", 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0084180560000000001 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.035337+0000", 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.220497577 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.038518+0000", 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031815480000000002 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.044091+0000", 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055728160000000004 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.181816+0000", 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.137724604 2023-10-09T05:07:49.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.182864+0000", 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010481100000000001 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.183975+0000", 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011108789999999999 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.185725+0000", 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00175004 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.193091+0000", 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073666629999999999 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.971 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93893 14.1c 14:3ca6af9b:::benchmark_data_smithi029_550670_object16282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 232.13718925500001, 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.309352242, 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93893 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.972 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739112+0000", 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739121+0000", 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2280000000000004e-06 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739139+0000", 2023-10-09T05:07:49.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7754e-05 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739149+0000", 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0462000000000001e-05 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806087+0000", 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.066937742999999994 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806189+0000", 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102457 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806269+0000", 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9692999999999997e-05 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.814900+0000", 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086310050000000006 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.830030+0000", 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015129956 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.047451+0000", 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.217420735 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.047488+0000", 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7416999999999998e-05 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.048466+0000", 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097746399999999994 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.978 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93904 14.c 14:33d8ac2a:::benchmark_data_smithi029_550670_object16293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 232.073364916, 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39957331299999999, 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93904 2023-10-09T05:07:49.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802936+0000", 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802945+0000", 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4160000000000004e-06 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802956+0000", 2023-10-09T05:07:49.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.117e-05 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802962+0000", 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0020000000000003e-06 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.815216+0000", 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012254335999999999 2023-10-09T05:07:49.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.823445+0000", 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082289100000000007 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.063574+0000", 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.240128587 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.066504+0000", 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029302120000000002 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.177873+0000", 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.111368644 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.193885+0000", 2023-10-09T05:07:49.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016012229999999999 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.194152+0000", 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026738700000000001 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.195130+0000", 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097748699999999997 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.196716+0000", 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001586212 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.202511+0000", 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057953019999999996 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.986 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97141 14.15 14:ae3b35de:::benchmark_data_smithi029_550670_object19530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 213.784982235, 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94747084699999995, 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97141 2023-10-09T05:07:49.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091319+0000", 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091327+0000", 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9740000000000002e-06 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091342+0000", 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4747e-05 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091351+0000", 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2089999999999994e-06 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091416+0000", 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4838999999999994e-05 2023-10-09T05:07:49.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091423+0000", 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8240000000000002e-06 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.018840+0000", 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92741732099999996 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.991 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.018983+0000", 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014266600000000001 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.019109+0000", 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012617699999999999 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022437+0000", 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033283459999999998 2023-10-09T05:07:49.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022749+0000", 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031135000000000001 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038718+0000", 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015969552000000001 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:49.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038764+0000", 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5419e-05 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038791+0000", 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7798e-05 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97180 14.15 14:ad3b1b35:::benchmark_data_smithi029_550670_object19569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 213.53918339699999, 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.705525913, 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97180 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:49.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:49.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337118+0000", 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337124+0000", 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1090000000000003e-06 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337138+0000", 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4467000000000001e-05 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337145+0000", 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0099999999999998e-06 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337214+0000", 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8980999999999996e-05 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337222+0000", 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7780000000000007e-06 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020444+0000", 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68322159100000002 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020544+0000", 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100602 2023-10-09T05:07:49.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020650+0000", 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105921 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022505+0000", 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018547959999999999 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022791+0000", 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028577600000000001 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042584+0000", 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019793139000000001 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042615+0000", 2023-10-09T05:07:50.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1220000000000003e-05 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042645+0000", 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0244999999999999e-05 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97245 14.15 14:ae966d39:::benchmark_data_smithi029_550670_object19634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:07:50.002 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 213.23401145, 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40766333999999999, 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97245 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:50.004 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642290+0000", 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642299+0000", 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9490000000000008e-06 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642312+0000", 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2159e-05 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.005 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642319+0000", 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3329999999999999e-06 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642389+0000", 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0358000000000006e-05 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.006 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642399+0000", 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2269999999999999e-06 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021495+0000", 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37909622700000001 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021614+0000", 2023-10-09T05:07:50.007 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011913600000000001 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021734+0000", 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119611 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022758+0000", 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010246599999999999 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.008 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.023321+0000", 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00056311399999999998 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049876+0000", 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026554853999999999 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.009 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.010 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:50.010 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049924+0000", 2023-10-09T05:07:50.010 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7633999999999998e-05 2023-10-09T05:07:50.010 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.010 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049955+0000", 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0854e-05 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.011 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:131600 14.1c 14:3ec2aceb:::benchmark_data_smithi029_550670_object53989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e670)", 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:46.742939+0000", 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 63.133363301999999, 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29314816599999999, 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:50.012 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131600 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742939+0000", 2023-10-09T05:07:50.013 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742939+0000", 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742938+0000", 2023-10-09T05:07:50.014 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742947+0000", 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6570000000000003e-06 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742967+0000", 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0922000000000001e-05 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.015 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742976+0000", 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6230000000000003e-06 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.743038+0000", 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1655999999999994e-05 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:50.016 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.743721+0000", 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00068335399999999995 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.753880+0000", 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010159385 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.753980+0000", 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9647000000000003e-05 2023-10-09T05:07:50.017 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.754306+0000", 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032565200000000001 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.757336+0000", 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003030516 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.018 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.757346+0000", 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0088000000000001e-05 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.035468+0000", 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.278121758 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:50.019 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.035515+0000", 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6464000000000001e-05 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.036088+0000", 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057302700000000004 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.020 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132518 14.c 14:363cf21e:::benchmark_data_smithi029_550670_object54907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.708129+0000", 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 60.168173449000001, 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30830919800000001, 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132518 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:50.021 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708129+0000", 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708129+0000", 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.022 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708128+0000", 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708135+0000", 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9009999999999997e-06 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708153+0000", 2023-10-09T05:07:50.023 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8155000000000002e-05 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708160+0000", 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4840000000000001e-06 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708199+0000", 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9623999999999999e-05 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.024 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.709034+0000", 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083480299999999996 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.009495+0000", 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30046083800000001 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:50.025 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.009588+0000", 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3477e-05 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.009884+0000", 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000295536 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.026 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.013458+0000", 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035743480000000002 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.013468+0000", 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0179e-05 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.015745+0000", 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022768760000000002 2023-10-09T05:07:50.027 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.015816+0000", 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.08e-05 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016438+0000", 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062235399999999998 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.028 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132520 14.c 14:353a445f:::benchmark_data_smithi029_550670_object54909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.713110+0000", 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 60.163192672999998, 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30387030700000001, 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132520 2023-10-09T05:07:50.029 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713110+0000", 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713110+0000", 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:07:50.030 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713109+0000", 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713114+0000", 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0300000000000001e-06 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:07:50.031 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713127+0000", 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2754e-05 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713132+0000", 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.412e-06 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713176+0000", 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3383999999999997e-05 2023-10-09T05:07:50.032 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713822+0000", 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000646614 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.012042+0000", 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29822033100000001 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.033 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.012112+0000", 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9851000000000006e-05 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.012298+0000", 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000185465 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.034 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.013478+0000", 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011797400000000001 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.014564+0000", 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001086683 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016498+0000", 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019336920000000001 2023-10-09T05:07:50.035 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016558+0000", 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0248999999999999e-05 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:07:50.036 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016980+0000", 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00042221000000000001 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:07:50.037 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:07:50.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:50.038 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: failed to encode map e695 with expected crc 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: failed to encode map e695 with expected crc 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: 1.8 scrub starts 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: 1.8 scrub ok 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 464/1050 objects degraded (44.190%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: osdmap e696: 8 total, 8 up, 7 in 2023-10-09T05:07:50.039 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:49 smithi177 ceph-mon[195797]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:07:50.040 INFO:teuthology.orchestra.run.smithi029.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:07:50.040 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: failed to encode map e695 with expected crc 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: failed to encode map e695 with expected crc 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: 1.8 scrub starts 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: 1.8 scrub ok 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 464/1050 objects degraded (44.190%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.043 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: osdmap e696: 8 total, 8 up, 7 in 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[223858]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: failed to encode map e695 with expected crc 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: failed to encode map e695 with expected crc 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: 1.8 scrub starts 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: 1.8 scrub ok 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 464/1050 objects degraded (44.190%), 2 pgs degraded, 2 pgs undersized) 2023-10-09T05:07:50.044 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T05:07:50.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-10-09T05:07:50.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: osdmap e696: 8 total, 8 up, 7 in 2023-10-09T05:07:50.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:49 smithi029 ceph-mon[216903]: failed to encode map e696 with expected crc 2023-10-09T05:07:50.070 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:50.183 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:07:50.183 DEBUG:teuthology.orchestra.run.smithi029:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:07:50.201 INFO:tasks.workunit:timeout=3h 2023-10-09T05:07:50.201 INFO:tasks.workunit:cleanup=True 2023-10-09T05:07:50.201 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:07:50.420 INFO:tasks.workunit.client.0.smithi029.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T05:07:50.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-09T05:07:50.796 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:50.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:50.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:50.903 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:50.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: osdmap e697: 8 total, 8 up, 7 in 2023-10-09T05:07:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[216903]: pgmap v2233: 83 pgs: 83 active+clean; 786 KiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: osdmap e697: 8 total, 8 up, 7 in 2023-10-09T05:07:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:50 smithi029 ceph-mon[223858]: pgmap v2233: 83 pgs: 83 active+clean; 786 KiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:50.928 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.928 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: osdmap e697: 8 total, 8 up, 7 in 2023-10-09T05:07:50.928 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: failed to encode map e697 with expected crc 2023-10-09T05:07:50.929 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:50 smithi177 ceph-mon[195797]: pgmap v2233: 83 pgs: 83 active+clean; 786 KiB data, 13 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:50.931 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:50.977 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:50.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:51.054 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:51.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:51.118 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:51.284 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:07:51.396 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:51.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:51.500 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:51.529 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:51.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: 1.0 scrub starts 2023-10-09T05:07:51.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: 1.0 scrub ok 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: failed to encode map e698 with expected crc 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: failed to encode map e698 with expected crc 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: osdmap e698: 8 total, 8 up, 7 in 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: failed to encode map e698 with expected crc 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[216903]: osdmap e699: 8 total, 8 up, 7 in 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: 1.0 scrub starts 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: 1.0 scrub ok 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: failed to encode map e698 with expected crc 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: failed to encode map e698 with expected crc 2023-10-09T05:07:51.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: osdmap e698: 8 total, 8 up, 7 in 2023-10-09T05:07:51.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: failed to encode map e698 with expected crc 2023-10-09T05:07:51.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:51 smithi029 ceph-mon[223858]: osdmap e699: 8 total, 8 up, 7 in 2023-10-09T05:07:52.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-09T05:07:52.082 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:52.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:52.188 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:52.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: 1.0 scrub starts 2023-10-09T05:07:52.215 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: 1.0 scrub ok 2023-10-09T05:07:52.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: failed to encode map e698 with expected crc 2023-10-09T05:07:52.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: failed to encode map e698 with expected crc 2023-10-09T05:07:52.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: osdmap e698: 8 total, 8 up, 7 in 2023-10-09T05:07:52.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: failed to encode map e698 with expected crc 2023-10-09T05:07:52.216 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:51 smithi177 ceph-mon[195797]: osdmap e699: 8 total, 8 up, 7 in 2023-10-09T05:07:52.218 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:52.265 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:52.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:52.355 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:07:52.357 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:07:52.371 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:52.398 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:52.439 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:52.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:52.545 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:52.578 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: failed to encode map e699 with expected crc 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: failed to encode map e699 with expected crc 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: failed to encode map e698 with expected crc 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: pgmap v2236: 83 pgs: 8 remapped+peering, 75 active+clean; 582 KiB data, 12 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: osdmap e700: 8 total, 8 up, 7 in 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: failed to encode map e700 with expected crc 2023-10-09T05:07:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[216903]: from='client.37779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:07:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:07:52] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: failed to encode map e699 with expected crc 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: failed to encode map e699 with expected crc 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: failed to encode map e698 with expected crc 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: pgmap v2236: 83 pgs: 8 remapped+peering, 75 active+clean; 582 KiB data, 12 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: osdmap e700: 8 total, 8 up, 7 in 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: failed to encode map e700 with expected crc 2023-10-09T05:07:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:52 smithi029 ceph-mon[223858]: from='client.37779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: failed to encode map e699 with expected crc 2023-10-09T05:07:53.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: failed to encode map e699 with expected crc 2023-10-09T05:07:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: failed to encode map e698 with expected crc 2023-10-09T05:07:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: pgmap v2236: 83 pgs: 8 remapped+peering, 75 active+clean; 582 KiB data, 12 GiB used, 613 GiB / 626 GiB avail 2023-10-09T05:07:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: osdmap e700: 8 total, 8 up, 7 in 2023-10-09T05:07:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: failed to encode map e700 with expected crc 2023-10-09T05:07:53.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:52 smithi177 ceph-mon[195797]: from='client.37779 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:07:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: osdmap e701: 8 total, 8 up, 7 in 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[216903]: failed to encode map e701 with expected crc 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: osdmap e701: 8 total, 8 up, 7 in 2023-10-09T05:07:54.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:53 smithi029 ceph-mon[223858]: failed to encode map e701 with expected crc 2023-10-09T05:07:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: failed to encode map e700 with expected crc 2023-10-09T05:07:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2023-10-09T05:07:54.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: osdmap e701: 8 total, 8 up, 7 in 2023-10-09T05:07:54.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:53 smithi177 ceph-mon[195797]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: 1.6 scrub starts 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: 1.6 scrub ok 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: pgmap v2239: 83 pgs: 3 peering, 2 remapped+peering, 78 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:55.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: 1.6 scrub starts 2023-10-09T05:07:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: 1.6 scrub ok 2023-10-09T05:07:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: pgmap v2239: 83 pgs: 3 peering, 2 remapped+peering, 78 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:55.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:54 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: failed to encode map e701 with expected crc 2023-10-09T05:07:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: 1.6 scrub starts 2023-10-09T05:07:55.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: 1.6 scrub ok 2023-10-09T05:07:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: pgmap v2239: 83 pgs: 3 peering, 2 remapped+peering, 78 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:55.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:54 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:55.490 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:55.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:55.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:55.504 INFO:tasks.workunit.client.0.smithi029.stderr:Updating files: 92% (10598/11517) Updating files: 93% (10711/11517) Updating files: 94% (10826/11517) Updating files: 95% (10942/11517) Updating files: 96% (11057/11517) Updating files: 97% (11172/11517) Updating files: 98% (11287/11517) Updating files: 99% (11402/11517) Updating files: 100% (11517/11517) Updating files: 100% (11517/11517), done. 2023-10-09T05:07:55.524 DEBUG:teuthology.orchestra.run.smithi029:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T05:07:55.596 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:07:55.629 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:55.677 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:07:55.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:55.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:07:55.711 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2237,"stamp":"2023-10-09T05:07:52.131795+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":171,"num_bytes_recovered":6195400,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1826,"ondisk_log_size":1826,"up":249,"acting":241,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":373,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13039952,"kb_used_data":4455896,"kb_used_omap":34,"kb_used_meta":8583965,"kb_avail":643176112,"statfs":{"total":671965249536,"available":658612338688,"internally_reserved":0,"allocated":4562837504,"data_stored":4545940259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8789980554},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-209018,"num_objects":-1,"num_object_clones":0,"num_object_copies":-3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":36686,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.340551"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":198,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.796034+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:07:47.796034+0000","last_peered":"2023-10-09T05:07:47.796034+0000","last_clean":"2023-10-09T05:07:47.796034+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:07:47.796034+0000","last_undegraded":"2023-10-09T05:07:47.796034+0000","last_fullsized":"2023-10-09T05:07:47.796034+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":343,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792493+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:07:46.792493+0000","last_peered":"2023-10-09T05:07:46.792493+0000","last_clean":"2023-10-09T05:07:46.792493+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:07:46.792493+0000","last_undegraded":"2023-10-09T05:07:46.792493+0000","last_fullsized":"2023-10-09T05:07:46.792493+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":120,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.784336+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:07:46.784336+0000","last_peered":"2023-10-09T05:07:46.784336+0000","last_clean":"2023-10-09T05:07:46.784336+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:07:46.784336+0000","last_undegraded":"2023-10-09T05:07:46.784336+0000","last_fullsized":"2023-10-09T05:07:46.784336+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":209,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792417+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:07:46.792417+0000","last_peered":"2023-10-09T05:07:46.792417+0000","last_clean":"2023-10-09T05:07:46.792417+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:07:46.792417+0000","last_undegraded":"2023-10-09T05:07:46.792417+0000","last_fullsized":"2023-10-09T05:07:46.792417+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":325,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791420+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:07:46.791420+0000","last_peered":"2023-10-09T05:07:46.791420+0000","last_clean":"2023-10-09T05:07:46.791420+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:07:46.791420+0000","last_undegraded":"2023-10-09T05:07:46.791420+0000","last_fullsized":"2023-10-09T05:07:46.791420+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":221,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789364+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:07:46.789364+0000","last_peered":"2023-10-09T05:07:46.789364+0000","last_clean":"2023-10-09T05:07:46.789364+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:07:46.789364+0000","last_undegraded":"2023-10-09T05:07:46.789364+0000","last_fullsized":"2023-10-09T05:07:46.789364+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":68,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268964+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:07:46.268964+0000","last_peered":"2023-10-09T05:07:46.268964+0000","last_clean":"2023-10-09T05:07:46.268964+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:07:46.268964+0000","last_undegraded":"2023-10-09T05:07:46.268964+0000","last_fullsized":"2023-10-09T05:07:46.268964+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":316,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791395+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:07:46.791395+0000","last_peered":"2023-10-09T05:07:46.791395+0000","last_clean":"2023-10-09T05:07:46.791395+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:07:46.791395+0000","last_undegraded":"2023-10-09T05:07:46.791395+0000","last_fullsized":"2023-10-09T05:07:46.791395+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":82,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268640+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:07:46.268640+0000","last_peered":"2023-10-09T05:07:46.268640+0000","last_clean":"2023-10-09T05:07:46.268640+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:07:46.268640+0000","last_undegraded":"2023-10-09T05:07:46.268640+0000","last_fullsized":"2023-10-09T05:07:46.268640+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":306,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791765+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:07:46.791765+0000","last_peered":"2023-10-09T05:07:46.791765+0000","last_clean":"2023-10-09T05:07:46.791765+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:07:46.791765+0000","last_undegraded":"2023-10-09T05:07:46.791765+0000","last_fullsized":"2023-10-09T05:07:46.791765+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":377,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792417+0000","last_change":"2023-10-09T05:07:23.305378+0000","last_active":"2023-10-09T05:07:46.792417+0000","last_peered":"2023-10-09T05:07:46.792417+0000","last_clean":"2023-10-09T05:07:46.792417+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:07:46.792417+0000","last_undegraded":"2023-10-09T05:07:46.792417+0000","last_fullsized":"2023-10-09T05:07:46.792417+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":990,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785363+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:07:46.785363+0000","last_peered":"2023-10-09T05:07:46.785363+0000","last_clean":"2023-10-09T05:07:46.785363+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:07:46.785363+0000","last_undegraded":"2023-10-09T05:07:46.785363+0000","last_fullsized":"2023-10-09T05:07:46.785363+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":151,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.796320+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:07:47.796320+0000","last_peered":"2023-10-09T05:07:47.796320+0000","last_clean":"2023-10-09T05:07:47.796320+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:07:47.796320+0000","last_undegraded":"2023-10-09T05:07:47.796320+0000","last_fullsized":"2023-10-09T05:07:47.796320+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":166,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795803+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:07:47.795803+0000","last_peered":"2023-10-09T05:07:47.795803+0000","last_clean":"2023-10-09T05:07:47.795803+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:07:47.795803+0000","last_undegraded":"2023-10-09T05:07:47.795803+0000","last_fullsized":"2023-10-09T05:07:47.795803+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":172,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795774+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:07:47.795774+0000","last_peered":"2023-10-09T05:07:47.795774+0000","last_clean":"2023-10-09T05:07:47.795774+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:07:47.795774+0000","last_undegraded":"2023-10-09T05:07:47.795774+0000","last_fullsized":"2023-10-09T05:07:47.795774+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":279,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792157+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:07:46.792157+0000","last_peered":"2023-10-09T05:07:46.792157+0000","last_clean":"2023-10-09T05:07:46.792157+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:07:46.792157+0000","last_undegraded":"2023-10-09T05:07:46.792157+0000","last_fullsized":"2023-10-09T05:07:46.792157+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":199,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791671+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:07:46.791671+0000","last_peered":"2023-10-09T05:07:46.791671+0000","last_clean":"2023-10-09T05:07:46.791671+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:07:46.791671+0000","last_undegraded":"2023-10-09T05:07:46.791671+0000","last_fullsized":"2023-10-09T05:07:46.791671+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268348+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:07:46.268348+0000","last_peered":"2023-10-09T05:07:46.268348+0000","last_clean":"2023-10-09T05:07:46.268348+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:07:46.268348+0000","last_undegraded":"2023-10-09T05:07:46.268348+0000","last_fullsized":"2023-10-09T05:07:46.268348+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":135,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795078+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:07:47.795078+0000","last_peered":"2023-10-09T05:07:47.795078+0000","last_clean":"2023-10-09T05:07:47.795078+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:07:47.795078+0000","last_undegraded":"2023-10-09T05:07:47.795078+0000","last_fullsized":"2023-10-09T05:07:47.795078+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":333,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792573+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:07:46.792573+0000","last_peered":"2023-10-09T05:07:46.792573+0000","last_clean":"2023-10-09T05:07:46.792573+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:07:46.792573+0000","last_undegraded":"2023-10-09T05:07:46.792573+0000","last_fullsized":"2023-10-09T05:07:46.792573+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":329,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789017+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:07:46.789017+0000","last_peered":"2023-10-09T05:07:46.789017+0000","last_clean":"2023-10-09T05:07:46.789017+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:07:46.789017+0000","last_undegraded":"2023-10-09T05:07:46.789017+0000","last_fullsized":"2023-10-09T05:07:46.789017+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":191,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792665+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:07:46.792665+0000","last_peered":"2023-10-09T05:07:46.792665+0000","last_clean":"2023-10-09T05:07:46.792665+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:07:46.792665+0000","last_undegraded":"2023-10-09T05:07:46.792665+0000","last_fullsized":"2023-10-09T05:07:46.792665+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":225,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.801473+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:07:47.801473+0000","last_peered":"2023-10-09T05:07:47.801473+0000","last_clean":"2023-10-09T05:07:47.801473+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:07:47.801473+0000","last_undegraded":"2023-10-09T05:07:47.801473+0000","last_fullsized":"2023-10-09T05:07:47.801473+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268043+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:07:46.268043+0000","last_peered":"2023-10-09T05:07:46.268043+0000","last_clean":"2023-10-09T05:07:46.268043+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:07:46.268043+0000","last_undegraded":"2023-10-09T05:07:46.268043+0000","last_fullsized":"2023-10-09T05:07:46.268043+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":59,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.801758+0000","last_change":"2023-10-09T05:07:31.961439+0000","last_active":"2023-10-09T05:07:47.801758+0000","last_peered":"2023-10-09T05:07:47.801758+0000","last_clean":"2023-10-09T05:07:47.801758+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:07:47.801758+0000","last_undegraded":"2023-10-09T05:07:47.801758+0000","last_fullsized":"2023-10-09T05:07:47.801758+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795007+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:07:47.795007+0000","last_peered":"2023-10-09T05:07:47.795007+0000","last_clean":"2023-10-09T05:07:47.795007+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:07:47.795007+0000","last_undegraded":"2023-10-09T05:07:47.795007+0000","last_fullsized":"2023-10-09T05:07:47.795007+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":76,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268313+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:07:46.268313+0000","last_peered":"2023-10-09T05:07:46.268313+0000","last_clean":"2023-10-09T05:07:46.268313+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:07:46.268313+0000","last_undegraded":"2023-10-09T05:07:46.268313+0000","last_fullsized":"2023-10-09T05:07:46.268313+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":241,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789443+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:07:46.789443+0000","last_peered":"2023-10-09T05:07:46.789443+0000","last_clean":"2023-10-09T05:07:46.789443+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:07:46.789443+0000","last_undegraded":"2023-10-09T05:07:46.789443+0000","last_fullsized":"2023-10-09T05:07:46.789443+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":754,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.269299+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:07:46.269299+0000","last_peered":"2023-10-09T05:07:46.269299+0000","last_clean":"2023-10-09T05:07:46.269299+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:07:46.269299+0000","last_undegraded":"2023-10-09T05:07:46.269299+0000","last_fullsized":"2023-10-09T05:07:46.269299+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":131,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785448+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:07:46.785448+0000","last_peered":"2023-10-09T05:07:46.785448+0000","last_clean":"2023-10-09T05:07:46.785448+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:07:46.785448+0000","last_undegraded":"2023-10-09T05:07:46.785448+0000","last_fullsized":"2023-10-09T05:07:46.785448+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1121,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.796657+0000","last_change":"2023-10-09T05:01:55.920874+0000","last_active":"2023-10-09T05:01:55.577469+0000","last_peered":"2023-10-09T05:01:53.546275+0000","last_clean":"2023-10-09T05:01:53.546275+0000","last_became_active":"2023-10-09T05:01:46.510931+0000","last_became_peered":"2023-10-09T05:01:46.510931+0000","last_unstale":"2023-10-09T05:07:49.796657+0000","last_undegraded":"2023-10-09T05:07:49.796657+0000","last_fullsized":"2023-10-09T05:07:49.796657+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":299,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791949+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:07:46.791949+0000","last_peered":"2023-10-09T05:07:46.791949+0000","last_clean":"2023-10-09T05:07:46.791949+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:07:46.791949+0000","last_undegraded":"2023-10-09T05:07:46.791949+0000","last_fullsized":"2023-10-09T05:07:46.791949+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":134,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.796247+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:07:47.796247+0000","last_peered":"2023-10-09T05:07:47.796247+0000","last_clean":"2023-10-09T05:07:47.796247+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:07:47.796247+0000","last_undegraded":"2023-10-09T05:07:47.796247+0000","last_fullsized":"2023-10-09T05:07:47.796247+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1115,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.796855+0000","last_change":"2023-10-09T05:01:55.592070+0000","last_active":"2023-10-09T05:01:55.591773+0000","last_peered":"2023-10-09T05:01:53.545265+0000","last_clean":"2023-10-09T04:59:25.875698+0000","last_became_active":"2023-10-09T05:01:44.508636+0000","last_became_peered":"2023-10-09T05:01:44.508636+0000","last_unstale":"2023-10-09T05:07:49.796855+0000","last_undegraded":"2023-10-09T05:07:49.796855+0000","last_fullsized":"2023-10-09T05:07:49.796855+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:48.624604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:48.624604+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":692046,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":238,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789189+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:07:46.789189+0000","last_peered":"2023-10-09T05:07:46.789189+0000","last_clean":"2023-10-09T05:07:46.789189+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:07:46.789189+0000","last_undegraded":"2023-10-09T05:07:46.789189+0000","last_fullsized":"2023-10-09T05:07:46.789189+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":54,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795838+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:07:47.795838+0000","last_peered":"2023-10-09T05:07:47.795838+0000","last_clean":"2023-10-09T05:07:47.795838+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:07:47.795838+0000","last_undegraded":"2023-10-09T05:07:47.795838+0000","last_fullsized":"2023-10-09T05:07:47.795838+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":168,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795059+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:07:47.795059+0000","last_peered":"2023-10-09T05:07:47.795059+0000","last_clean":"2023-10-09T05:07:47.795059+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:07:47.795059+0000","last_undegraded":"2023-10-09T05:07:47.795059+0000","last_fullsized":"2023-10-09T05:07:47.795059+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.803137+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:07:47.803137+0000","last_peered":"2023-10-09T05:07:47.803137+0000","last_clean":"2023-10-09T05:07:47.803137+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:07:47.803137+0000","last_undegraded":"2023-10-09T05:07:47.803137+0000","last_fullsized":"2023-10-09T05:07:47.803137+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1130,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.797736+0000","last_change":"2023-10-09T05:01:55.962280+0000","last_active":"2023-10-09T05:01:55.579643+0000","last_peered":"2023-10-09T05:01:53.546530+0000","last_clean":"2023-10-09T05:01:53.546530+0000","last_became_active":"2023-10-09T05:01:46.511168+0000","last_became_peered":"2023-10-09T05:01:46.511168+0000","last_unstale":"2023-10-09T05:07:49.797736+0000","last_undegraded":"2023-10-09T05:07:49.797736+0000","last_fullsized":"2023-10-09T05:07:49.797736+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":214,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791228+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:07:46.791228+0000","last_peered":"2023-10-09T05:07:46.791228+0000","last_clean":"2023-10-09T05:07:46.791228+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:07:46.791228+0000","last_undegraded":"2023-10-09T05:07:46.791228+0000","last_fullsized":"2023-10-09T05:07:46.791228+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785912+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:07:46.785912+0000","last_peered":"2023-10-09T05:07:46.785912+0000","last_clean":"2023-10-09T05:07:46.785912+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:07:46.785912+0000","last_undegraded":"2023-10-09T05:07:46.785912+0000","last_fullsized":"2023-10-09T05:07:46.785912+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1298,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789282+0000","last_change":"2023-10-09T05:03:51.195789+0000","last_active":"2023-10-09T05:07:46.789282+0000","last_peered":"2023-10-09T05:07:46.789282+0000","last_clean":"2023-10-09T05:07:46.789282+0000","last_became_active":"2023-10-09T05:03:51.195390+0000","last_became_peered":"2023-10-09T05:03:51.195390+0000","last_unstale":"2023-10-09T05:07:46.789282+0000","last_undegraded":"2023-10-09T05:07:46.789282+0000","last_fullsized":"2023-10-09T05:07:46.789282+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.786271+0000","last_change":"2023-10-09T05:02:19.778150+0000","last_active":"2023-10-09T05:07:46.786271+0000","last_peered":"2023-10-09T05:07:46.786271+0000","last_clean":"2023-10-09T05:07:46.786271+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:07:46.786271+0000","last_undegraded":"2023-10-09T05:07:46.786271+0000","last_fullsized":"2023-10-09T05:07:46.786271+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":72,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268404+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:07:46.268404+0000","last_peered":"2023-10-09T05:07:46.268404+0000","last_clean":"2023-10-09T05:07:46.268404+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:07:46.268404+0000","last_undegraded":"2023-10-09T05:07:46.268404+0000","last_fullsized":"2023-10-09T05:07:46.268404+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785287+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:07:46.785287+0000","last_peered":"2023-10-09T05:07:46.785287+0000","last_clean":"2023-10-09T05:07:46.785287+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:07:46.785287+0000","last_undegraded":"2023-10-09T05:07:46.785287+0000","last_fullsized":"2023-10-09T05:07:46.785287+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1119,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.796825+0000","last_change":"2023-10-09T05:01:55.922207+0000","last_active":"2023-10-09T05:01:55.579384+0000","last_peered":"2023-10-09T05:01:53.546201+0000","last_clean":"2023-10-09T05:01:53.546201+0000","last_became_active":"2023-10-09T05:01:46.511423+0000","last_became_peered":"2023-10-09T05:01:46.511423+0000","last_unstale":"2023-10-09T05:07:49.796825+0000","last_undegraded":"2023-10-09T05:07:49.796825+0000","last_fullsized":"2023-10-09T05:07:49.796825+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":778,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268776+0000","last_change":"2023-10-09T05:02:19.783230+0000","last_active":"2023-10-09T05:07:46.268776+0000","last_peered":"2023-10-09T05:07:46.268776+0000","last_clean":"2023-10-09T05:07:46.268776+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:07:46.268776+0000","last_undegraded":"2023-10-09T05:07:46.268776+0000","last_fullsized":"2023-10-09T05:07:46.268776+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":161,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795509+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:07:47.795509+0000","last_peered":"2023-10-09T05:07:47.795509+0000","last_clean":"2023-10-09T05:07:47.795509+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:07:47.795509+0000","last_undegraded":"2023-10-09T05:07:47.795509+0000","last_fullsized":"2023-10-09T05:07:47.795509+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.786195+0000","last_change":"2023-10-09T05:02:19.821742+0000","last_active":"2023-10-09T05:07:46.786195+0000","last_peered":"2023-10-09T05:07:46.786195+0000","last_clean":"2023-10-09T05:07:46.786195+0000","last_became_active":"2023-10-09T05:02:19.821396+0000","last_became_peered":"2023-10-09T05:02:19.821396+0000","last_unstale":"2023-10-09T05:07:46.786195+0000","last_undegraded":"2023-10-09T05:07:46.786195+0000","last_fullsized":"2023-10-09T05:07:46.786195+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:16:42.736772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1130,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.796936+0000","last_change":"2023-10-09T05:01:55.931072+0000","last_active":"2023-10-09T05:01:55.581264+0000","last_peered":"2023-10-09T05:01:53.546414+0000","last_clean":"2023-10-09T05:01:53.546414+0000","last_became_active":"2023-10-09T05:01:46.510926+0000","last_became_peered":"2023-10-09T05:01:46.510926+0000","last_unstale":"2023-10-09T05:07:49.796936+0000","last_undegraded":"2023-10-09T05:07:49.796936+0000","last_fullsized":"2023-10-09T05:07:49.796936+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":191,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:48.267347+0000","last_change":"2023-10-09T05:02:19.789563+0000","last_active":"2023-10-09T05:07:48.267347+0000","last_peered":"2023-10-09T05:07:48.267347+0000","last_clean":"2023-10-09T05:07:48.267347+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:07:48.267347+0000","last_undegraded":"2023-10-09T05:07:48.267347+0000","last_fullsized":"2023-10-09T05:07:48.267347+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.784147+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:07:46.784147+0000","last_peered":"2023-10-09T05:07:46.784147+0000","last_clean":"2023-10-09T05:07:46.784147+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:07:46.784147+0000","last_undegraded":"2023-10-09T05:07:46.784147+0000","last_fullsized":"2023-10-09T05:07:46.784147+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":112,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785862+0000","last_change":"2023-10-09T05:03:37.300874+0000","last_active":"2023-10-09T05:07:46.785862+0000","last_peered":"2023-10-09T05:07:46.785862+0000","last_clean":"2023-10-09T05:07:46.785862+0000","last_became_active":"2023-10-09T05:03:37.300591+0000","last_became_peered":"2023-10-09T05:03:37.300591+0000","last_unstale":"2023-10-09T05:07:46.785862+0000","last_undegraded":"2023-10-09T05:07:46.785862+0000","last_fullsized":"2023-10-09T05:07:46.785862+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:59.080120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":82,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268277+0000","last_change":"2023-10-09T05:03:47.985378+0000","last_active":"2023-10-09T05:07:46.268277+0000","last_peered":"2023-10-09T05:07:46.268277+0000","last_clean":"2023-10-09T05:07:46.268277+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:07:46.268277+0000","last_undegraded":"2023-10-09T05:07:46.268277+0000","last_fullsized":"2023-10-09T05:07:46.268277+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1298,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.788848+0000","last_change":"2023-10-09T05:03:51.189355+0000","last_active":"2023-10-09T05:07:46.788848+0000","last_peered":"2023-10-09T05:07:46.788848+0000","last_clean":"2023-10-09T05:07:46.788848+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:07:46.788848+0000","last_undegraded":"2023-10-09T05:07:46.788848+0000","last_fullsized":"2023-10-09T05:07:46.788848+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":188,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791673+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:07:46.791673+0000","last_peered":"2023-10-09T05:07:46.791673+0000","last_clean":"2023-10-09T05:07:46.791673+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:07:46.791673+0000","last_undegraded":"2023-10-09T05:07:46.791673+0000","last_fullsized":"2023-10-09T05:07:46.791673+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":215,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789506+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:07:46.789506+0000","last_peered":"2023-10-09T05:07:46.789506+0000","last_clean":"2023-10-09T05:07:46.789506+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:07:46.789506+0000","last_undegraded":"2023-10-09T05:07:46.789506+0000","last_fullsized":"2023-10-09T05:07:46.789506+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1135,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.796542+0000","last_change":"2023-10-09T05:02:00.592400+0000","last_active":"2023-10-09T05:01:52.542540+0000","last_peered":"2023-10-09T05:01:51.538182+0000","last_clean":"2023-10-09T05:01:35.372621+0000","last_became_active":"2023-10-09T05:01:51.537989+0000","last_became_peered":"2023-10-09T05:01:51.537989+0000","last_unstale":"2023-10-09T05:07:49.796542+0000","last_undegraded":"2023-10-09T05:07:49.796542+0000","last_fullsized":"2023-10-09T05:07:49.796542+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":109,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.784190+0000","last_change":"2023-10-09T05:02:19.756088+0000","last_active":"2023-10-09T05:07:46.784190+0000","last_peered":"2023-10-09T05:07:46.784190+0000","last_clean":"2023-10-09T05:07:46.784190+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:07:46.784190+0000","last_undegraded":"2023-10-09T05:07:46.784190+0000","last_fullsized":"2023-10-09T05:07:46.784190+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":54,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.794323+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:07:47.794323+0000","last_peered":"2023-10-09T05:07:47.794323+0000","last_clean":"2023-10-09T05:07:47.794323+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:07:47.794323+0000","last_undegraded":"2023-10-09T05:07:47.794323+0000","last_fullsized":"2023-10-09T05:07:47.794323+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":149,"reported_epoch":696,"state":"active+clean","last_fresh":"2023-10-09T05:07:48.948837+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:07:48.948837+0000","last_peered":"2023-10-09T05:07:48.948837+0000","last_clean":"2023-10-09T05:07:48.948837+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:07:48.948837+0000","last_undegraded":"2023-10-09T05:07:48.948837+0000","last_fullsized":"2023-10-09T05:07:48.948837+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1292,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789614+0000","last_change":"2023-10-09T05:03:51.190553+0000","last_active":"2023-10-09T05:07:46.789614+0000","last_peered":"2023-10-09T05:07:46.789614+0000","last_clean":"2023-10-09T05:07:46.789614+0000","last_became_active":"2023-10-09T05:03:51.190095+0000","last_became_peered":"2023-10-09T05:03:51.190095+0000","last_unstale":"2023-10-09T05:07:46.789614+0000","last_undegraded":"2023-10-09T05:07:46.789614+0000","last_fullsized":"2023-10-09T05:07:46.789614+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":161,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.800435+0000","last_change":"2023-10-09T05:03:24.374791+0000","last_active":"2023-10-09T05:07:47.800435+0000","last_peered":"2023-10-09T05:07:47.800435+0000","last_clean":"2023-10-09T05:07:47.800435+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:07:47.800435+0000","last_undegraded":"2023-10-09T05:07:47.800435+0000","last_fullsized":"2023-10-09T05:07:47.800435+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.786031+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:07:46.786031+0000","last_peered":"2023-10-09T05:07:46.786031+0000","last_clean":"2023-10-09T05:07:46.786031+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:07:46.786031+0000","last_undegraded":"2023-10-09T05:07:46.786031+0000","last_fullsized":"2023-10-09T05:07:46.786031+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":72,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268440+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:07:46.268440+0000","last_peered":"2023-10-09T05:07:46.268440+0000","last_clean":"2023-10-09T05:07:46.268440+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:07:46.268440+0000","last_undegraded":"2023-10-09T05:07:46.268440+0000","last_fullsized":"2023-10-09T05:07:46.268440+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1134,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.798427+0000","last_change":"2023-10-09T05:01:55.931032+0000","last_active":"2023-10-09T05:01:55.588891+0000","last_peered":"2023-10-09T05:01:53.547909+0000","last_clean":"2023-10-09T05:01:53.547909+0000","last_became_active":"2023-10-09T05:01:46.521753+0000","last_became_peered":"2023-10-09T05:01:46.521753+0000","last_unstale":"2023-10-09T05:07:49.798427+0000","last_undegraded":"2023-10-09T05:07:49.798427+0000","last_fullsized":"2023-10-09T05:07:49.798427+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":326,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792783+0000","last_change":"2023-10-09T05:07:23.309039+0000","last_active":"2023-10-09T05:07:46.792783+0000","last_peered":"2023-10-09T05:07:46.792783+0000","last_clean":"2023-10-09T05:07:46.792783+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:07:46.792783+0000","last_undegraded":"2023-10-09T05:07:46.792783+0000","last_fullsized":"2023-10-09T05:07:46.792783+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785920+0000","last_change":"2023-10-09T05:03:24.396124+0000","last_active":"2023-10-09T05:07:46.785920+0000","last_peered":"2023-10-09T05:07:46.785920+0000","last_clean":"2023-10-09T05:07:46.785920+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:07:46.785920+0000","last_undegraded":"2023-10-09T05:07:46.785920+0000","last_fullsized":"2023-10-09T05:07:46.785920+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":200,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791202+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:07:46.791202+0000","last_peered":"2023-10-09T05:07:46.791202+0000","last_clean":"2023-10-09T05:07:46.791202+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:07:46.791202+0000","last_undegraded":"2023-10-09T05:07:46.791202+0000","last_fullsized":"2023-10-09T05:07:46.791202+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"82'161","reported_seq":1098,"reported_epoch":697,"state":"remapped+peering","last_fresh":"2023-10-09T05:07:49.797055+0000","last_change":"2023-10-09T05:01:55.596045+0000","last_active":"2023-10-09T05:01:55.595783+0000","last_peered":"2023-10-09T05:01:53.549542+0000","last_clean":"2023-10-09T04:59:25.885390+0000","last_became_active":"2023-10-09T05:01:49.898674+0000","last_became_peered":"2023-10-09T05:01:49.898674+0000","last_unstale":"2023-10-09T05:07:49.797055+0000","last_undegraded":"2023-10-09T05:07:49.797055+0000","last_fullsized":"2023-10-09T05:07:49.797055+0000","mapping_epoch":697,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T04:03:57.976027+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1138389,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":211,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.789442+0000","last_change":"2023-10-09T05:03:47.997682+0000","last_active":"2023-10-09T05:07:46.789442+0000","last_peered":"2023-10-09T05:07:46.789442+0000","last_clean":"2023-10-09T05:07:46.789442+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:07:46.789442+0000","last_undegraded":"2023-10-09T05:07:46.789442+0000","last_fullsized":"2023-10-09T05:07:46.789442+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":130,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.785837+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:07:46.785837+0000","last_peered":"2023-10-09T05:07:46.785837+0000","last_clean":"2023-10-09T05:07:46.785837+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:07:46.785837+0000","last_undegraded":"2023-10-09T05:07:46.785837+0000","last_fullsized":"2023-10-09T05:07:46.785837+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.794976+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:07:47.794976+0000","last_peered":"2023-10-09T05:07:47.794976+0000","last_clean":"2023-10-09T05:07:47.794976+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:07:47.794976+0000","last_undegraded":"2023-10-09T05:07:47.794976+0000","last_fullsized":"2023-10-09T05:07:47.794976+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":161,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795684+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:07:47.795684+0000","last_peered":"2023-10-09T05:07:47.795684+0000","last_clean":"2023-10-09T05:07:47.795684+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:07:47.795684+0000","last_undegraded":"2023-10-09T05:07:47.795684+0000","last_fullsized":"2023-10-09T05:07:47.795684+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":111,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.786110+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:07:46.786110+0000","last_peered":"2023-10-09T05:07:46.786110+0000","last_clean":"2023-10-09T05:07:46.786110+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:07:46.786110+0000","last_undegraded":"2023-10-09T05:07:46.786110+0000","last_fullsized":"2023-10-09T05:07:46.786110+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":835,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.269354+0000","last_change":"2023-10-09T05:07:31.951921+0000","last_active":"2023-10-09T05:07:46.269354+0000","last_peered":"2023-10-09T05:07:46.269354+0000","last_clean":"2023-10-09T05:07:46.269354+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:07:46.269354+0000","last_undegraded":"2023-10-09T05:07:46.269354+0000","last_fullsized":"2023-10-09T05:07:46.269354+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.794564+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:07:47.794564+0000","last_peered":"2023-10-09T05:07:47.794564+0000","last_clean":"2023-10-09T05:07:47.794564+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:07:47.794564+0000","last_undegraded":"2023-10-09T05:07:47.794564+0000","last_fullsized":"2023-10-09T05:07:47.794564+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":775,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.268867+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:07:46.268867+0000","last_peered":"2023-10-09T05:07:46.268867+0000","last_clean":"2023-10-09T05:07:46.268867+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:07:46.268867+0000","last_undegraded":"2023-10-09T05:07:46.268867+0000","last_fullsized":"2023-10-09T05:07:46.268867+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":54,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.794397+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:07:47.794397+0000","last_peered":"2023-10-09T05:07:47.794397+0000","last_clean":"2023-10-09T05:07:47.794397+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:07:47.794397+0000","last_undegraded":"2023-10-09T05:07:47.794397+0000","last_fullsized":"2023-10-09T05:07:47.794397+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":316,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.792554+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:07:46.792554+0000","last_peered":"2023-10-09T05:07:46.792554+0000","last_clean":"2023-10-09T05:07:46.792554+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:07:46.792554+0000","last_undegraded":"2023-10-09T05:07:46.792554+0000","last_fullsized":"2023-10-09T05:07:46.792554+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":324,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.791838+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:07:46.791838+0000","last_peered":"2023-10-09T05:07:46.791838+0000","last_clean":"2023-10-09T05:07:46.791838+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:07:46.791838+0000","last_undegraded":"2023-10-09T05:07:46.791838+0000","last_fullsized":"2023-10-09T05:07:46.791838+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":74,"reported_epoch":693,"state":"active+clean","last_fresh":"2023-10-09T05:07:46.269010+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:07:46.269010+0000","last_peered":"2023-10-09T05:07:46.269010+0000","last_clean":"2023-10-09T05:07:46.269010+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:07:46.269010+0000","last_undegraded":"2023-10-09T05:07:46.269010+0000","last_fullsized":"2023-10-09T05:07:46.269010+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-10-09T05:07:47.795333+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:07:47.795333+0000","last_peered":"2023-10-09T05:07:47.795333+0000","last_clean":"2023-10-09T05:07:47.795333+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:07:47.795333+0000","last_undegraded":"2023-10-09T05:07:47.795333+0000","last_fullsized":"2023-10-09T05:07:47.795333+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":17710,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":6177690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":25,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234726,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2280200,"kb_used_data":766848,"kb_used_omap":0,"kb_used_meta":1513344,"kb_avail":91464952,"statfs":{"total":95995035648,"available":93660110848,"internally_reserved":0,"allocated":785252352,"data_stored":783484453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1549664256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:18 2023","interfaces":[{"interface":"back","average":{"1min":13.488,"5min":8.5220000000000002,"15min":3.4199999999999999},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23599999999999999},"max":{"1min":160.38999999999999,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":14.443,"5min":8.7780000000000005,"15min":3.6429999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":160.47499999999999,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.379}]},{"osd":1,"last update":"Mon Oct 9 05:07:18 2023","interfaces":[{"interface":"back","average":{"1min":33.710000000000001,"5min":9.9939999999999998,"15min":4.1239999999999997},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.26500000000000001},"max":{"1min":322.31999999999999,"5min":322.31999999999999,"15min":322.31999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":33.951999999999998,"5min":9.9659999999999993,"15min":4.0800000000000001},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.27800000000000002},"max":{"1min":322.14800000000002,"5min":322.14800000000002,"15min":322.14800000000002},"last":0.74199999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.7469999999999999,"15min":3.8620000000000001},"min":{"1min":0.35099999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3470000000000004,"5min":64.480999999999995,"15min":207.96600000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.7090000000000001,"15min":3.9550000000000001},"min":{"1min":0.35199999999999998,"5min":0.312,"15min":0.29699999999999999},"max":{"1min":6.3579999999999997,"5min":55.134,"15min":207.97900000000001},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:07:12 2023","interfaces":[{"interface":"back","average":{"1min":39.868000000000002,"5min":15.595000000000001,"15min":6.3470000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":349.863,"5min":349.863,"15min":349.863},"last":0.72299999999999998},{"interface":"front","average":{"1min":39.591000000000001,"5min":15.047000000000001,"15min":6.1660000000000004},"min":{"1min":0.32900000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":349.67500000000001,"5min":349.67500000000001,"15min":349.67500000000001},"last":0.49399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.38900000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.145,"5min":1.145,"15min":1.145},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.51600000000000001,"15min":0.48699999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.0609999999999999,"5min":3.0609999999999999,"15min":3.0609999999999999},"last":0.81299999999999994}]},{"osd":5,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3849999999999998,"5min":1.847,"15min":1.897},"min":{"1min":0.33200000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":74.915000000000006,"5min":74.915000000000006,"15min":131.84899999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":4.4939999999999998,"5min":2,"15min":1.99},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":74.972999999999999,"5min":74.972999999999999,"15min":135.74199999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Oct 9 05:06:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.90000000000000002,"15min":1.0680000000000001},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":2.1829999999999998,"5min":10.183999999999999,"15min":57.168999999999997},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.0860000000000001,"15min":1.218},"min":{"1min":0.32900000000000001,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":1.7,"5min":21.097000000000001,"15min":57.183999999999997},"last":0.59399999999999997}]}]},{"osd":6,"up_from":46,"seq":197568496358,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518248,"kb_used_data":7648,"kb_used_omap":0,"kb_used_meta":1510592,"kb_avail":92226904,"statfs":{"total":95995035648,"available":94440349696,"internally_reserved":0,"allocated":7831552,"data_stored":6073268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1546846208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":5.5570000000000004,"5min":4.1929999999999996,"15min":3.077},"min":{"1min":0.22600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":40.877000000000002,"5min":65.721999999999994,"15min":321.346},"last":0.41799999999999998},{"interface":"front","average":{"1min":5.8140000000000001,"5min":4.266,"15min":2.6280000000000001},"min":{"1min":0.27400000000000002,"5min":0.255,"15min":0.255},"max":{"1min":40.887,"5min":65.585999999999999,"15min":169.119},"last":0.40000000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":4.601,"5min":2.1880000000000002,"15min":2.1850000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":43.604999999999997,"5min":43.604999999999997,"15min":90.278999999999996},"last":0.33500000000000002},{"interface":"front","average":{"1min":4.4729999999999999,"5min":2.4119999999999999,"15min":2.1640000000000001},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":44.697000000000003,"5min":44.697000000000003,"15min":90.414000000000001},"last":0.46500000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":1.925,"15min":2.101},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.25600000000000001},"max":{"1min":7.1159999999999997,"5min":23.943000000000001,"15min":133.535},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.5030000000000001,"5min":1.907,"15min":2.036},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.30599999999999999},"max":{"1min":8.0039999999999996,"5min":25.978999999999999,"15min":133.51900000000001},"last":0.48599999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":4.0990000000000002,"5min":3.7749999999999999,"15min":3.137},"min":{"1min":0.45900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":42.366,"5min":64.355999999999995,"15min":147.43600000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":4.4500000000000002,"5min":3.891,"15min":3.2080000000000002},"min":{"1min":0.41699999999999998,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":45.814999999999998,"5min":56.133000000000003,"15min":147.41999999999999},"last":0.45500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.89700000000000002,"15min":0.78900000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3769999999999998,"5min":18.186,"15min":18.186},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0589999999999999,"15min":1.022},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.2469999999999999,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:07:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.82399999999999995,"15min":0.94699999999999995},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.9539999999999997,"5min":7.9539999999999997,"15min":25.934000000000001},"last":0.621},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.91100000000000003,"15min":0.91500000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":5.2990000000000004,"5min":16.789000000000001,"15min":16.789000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.86499999999999999,"15min":0.998},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":7.5039999999999996,"5min":7.5039999999999996,"15min":36.204999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.0760000000000001,"15min":0.98599999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.3399999999999999,"5min":20.041,"15min":20.041},"last":1.5009999999999999}]}]},{"osd":1,"up_from":118,"seq":506806141425,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580988,"kb_used_data":168628,"kb_used_omap":4,"kb_used_meta":1412347,"kb_avail":92164164,"statfs":{"total":95995035648,"available":94376103936,"internally_reserved":0,"allocated":172675072,"data_stored":168949673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1446244238},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9059999999999997,"5min":5.7039999999999997,"15min":2.5880000000000001},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":42.058999999999997,"5min":100.247,"15min":100.247},"last":0.36599999999999999},{"interface":"front","average":{"1min":31.283000000000001,"5min":11.256,"15min":4.4020000000000001},"min":{"1min":0.318,"5min":0.247,"15min":0.214},"max":{"1min":554.62,"5min":554.62,"15min":554.62},"last":0.46300000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:06:48 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":1.8340000000000001,"15min":2.7120000000000002},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.393000000000001,"5min":12.460000000000001,"15min":263.089},"last":0.50600000000000001},{"interface":"front","average":{"1min":4.359,"5min":2.3860000000000001,"15min":2.8929999999999998},"min":{"1min":0.36399999999999999,"5min":0.252,"15min":0.222},"max":{"1min":19.440000000000001,"5min":19.440000000000001,"15min":254.50899999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:07:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":8.1950000000000003,"15min":3.9550000000000001},"min":{"1min":0.26400000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.0279999999999996,"5min":235.43600000000001,"15min":235.43600000000001},"last":0.433},{"interface":"front","average":{"1min":26.606999999999999,"5min":12.436999999999999,"15min":5.1699999999999999},"min":{"1min":0.29999999999999999,"5min":0.249,"15min":0.249},"max":{"1min":554.84400000000005,"5min":554.84400000000005,"15min":554.84400000000005},"last":0.49399999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":25.523,"5min":6.9199999999999999,"15min":3.1560000000000001},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":554.44000000000005,"5min":554.44000000000005,"15min":554.44000000000005},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.409,"5min":1.871,"15min":1.2549999999999999},"min":{"1min":0.32600000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.5640000000000001,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.59899999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.305,"5min":7.766,"15min":3.3719999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":554.54700000000003,"5min":554.54700000000003,"15min":554.54700000000003},"last":0.56999999999999995},{"interface":"front","average":{"1min":5.1310000000000002,"5min":2.6259999999999999,"15min":1.744},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":80.390000000000001,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.69299999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.236999999999998,"5min":7.7699999999999996,"15min":3.387},"min":{"1min":0.40400000000000003,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":554.36500000000001,"5min":554.36500000000001,"15min":554.36500000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":5.0069999999999997,"5min":2.5939999999999999,"15min":1.7350000000000001},"min":{"1min":0.41999999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":78.278000000000006,"5min":78.278000000000006,"15min":78.278000000000006},"last":0.65200000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.968,"15min":1.5900000000000001},"min":{"1min":0.48299999999999998,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.4850000000000003,"5min":18.899999999999999,"15min":19.664999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.589,"5min":1.6240000000000001,"15min":1.397},"min":{"1min":0.47299999999999998,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":7.0099999999999998,"5min":11.903,"15min":11.903},"last":0.626}]}]},{"osd":0,"up_from":91,"seq":390842024509,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1348756,"kb_used_data":117068,"kb_used_omap":5,"kb_used_meta":1231674,"kb_avail":92396396,"statfs":{"total":95995035648,"available":94613909504,"internally_reserved":0,"allocated":119877632,"data_stored":116850592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1261234810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:07:13 2023","interfaces":[{"interface":"back","average":{"1min":3.395,"5min":3.6909999999999998,"15min":2.6059999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19600000000000001},"max":{"1min":17.73,"5min":53.176000000000002,"15min":155.411},"last":0.39800000000000002},{"interface":"front","average":{"1min":3.7160000000000002,"5min":4.0359999999999996,"15min":2.6520000000000001},"min":{"1min":0.39100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":17.594000000000001,"5min":53.372999999999998,"15min":154.75},"last":0.41399999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:34 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.5990000000000002,"15min":2.8519999999999999},"min":{"1min":0.26500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":20.010999999999999,"5min":20.010999999999999,"15min":151.06399999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.7349999999999999,"5min":3.2509999999999999,"15min":3.097},"min":{"1min":0.28199999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.364000000000001,"5min":20.239000000000001,"15min":151.22399999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:07:01 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":7.7830000000000004,"15min":4.6100000000000003},"min":{"1min":0.307,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":19.096,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.726,"5min":8.3650000000000002,"15min":4.8470000000000004},"min":{"1min":0.29299999999999998,"5min":0.27500000000000002,"15min":0.26400000000000001},"max":{"1min":15.554,"5min":290.73899999999998,"15min":290.73899999999998},"last":0.502}]},{"osd":4,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9219999999999997,"5min":2.621,"15min":2.1080000000000001},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":57.360999999999997,"5min":57.360999999999997,"15min":57.360999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":4.5979999999999999,"5min":2.3839999999999999,"15min":1.8180000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":57.338000000000001,"5min":57.338000000000001,"15min":57.338000000000001},"last":0.61799999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:07:23 2023","interfaces":[{"interface":"back","average":{"1min":8.4139999999999997,"5min":3.8599999999999999,"15min":1.9870000000000001},"min":{"1min":0.42099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":106.922,"5min":106.922,"15min":106.922},"last":0.52400000000000002},{"interface":"front","average":{"1min":8.3190000000000008,"5min":3.2080000000000002,"15min":1.8340000000000001},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":106.771,"5min":106.771,"15min":106.771},"last":0.58999999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:07:23 2023","interfaces":[{"interface":"back","average":{"1min":12.031000000000001,"5min":4.5830000000000002,"15min":2.3780000000000001},"min":{"1min":0.42099999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":112.208,"5min":112.208,"15min":112.208},"last":0.55400000000000005},{"interface":"front","average":{"1min":11.976000000000001,"5min":3.9820000000000002,"15min":2.391},"min":{"1min":0.372,"5min":0.35799999999999998,"15min":0.35099999999999998},"max":{"1min":110.46899999999999,"5min":110.46899999999999,"15min":110.46899999999999},"last":0.51300000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:07:23 2023","interfaces":[{"interface":"back","average":{"1min":2.1869999999999998,"5min":2.0880000000000001,"15min":1.55},"min":{"1min":0.42899999999999999,"5min":0.41899999999999998,"15min":0.39400000000000002},"max":{"1min":13.787000000000001,"5min":13.787000000000001,"15min":21.658000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.089,"5min":1.887,"15min":1.528},"min":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":6.9020000000000001,"5min":11.842000000000001,"15min":21.68},"last":0.57599999999999996}]}]},{"osd":2,"up_from":0,"seq":949187772788,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356481,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1187940,"kb_used_data":166300,"kb_used_omap":2,"kb_used_meta":1021629,"kb_avail":92557212,"statfs":{"total":95995035648,"available":94778585088,"internally_reserved":0,"allocated":170291200,"data_stored":167173861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1046148149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":7.4809999999999999,"5min":4.6970000000000001,"15min":2.0379999999999998},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.20699999999999999},"max":{"1min":62.863,"5min":108.631,"15min":108.631},"last":0.248},{"interface":"front","average":{"1min":7.7329999999999997,"5min":4.9349999999999996,"15min":2.2429999999999999},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":62.948,"5min":110.012,"15min":110.012},"last":0.313}]},{"osd":1,"last update":"Mon Oct 9 05:07:13 2023","interfaces":[{"interface":"back","average":{"1min":2.754,"5min":2.343,"15min":1.5449999999999999},"min":{"1min":0.25700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":13.103999999999999,"5min":15.218,"15min":15.983000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.1640000000000001,"15min":1.4690000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.164,"5min":17.544,"15min":26.692},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":1.5509999999999999,"15min":2.391},"min":{"1min":0.30499999999999999,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":3.0089999999999999,"5min":11.996,"15min":212.065},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.756,"15min":2.464},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":5.7549999999999999,"5min":12.987,"15min":211.98400000000001},"last":0.38300000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.8,"15min":1.746},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9359999999999999,"5min":11.193,"15min":11.193},"last":0.45900000000000002},{"interface":"front","average":{"1min":6.8650000000000002,"5min":2.4209999999999998,"15min":1.405},"min":{"1min":0.36499999999999999,"5min":0.309,"15min":0.309},"max":{"1min":62.683999999999997,"5min":62.683999999999997,"15min":62.683999999999997},"last":0.42399999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":6.4800000000000004,"5min":2.9929999999999999,"15min":1.6950000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":75.478999999999999,"5min":75.478999999999999,"15min":75.478999999999999},"last":0.505},{"interface":"front","average":{"1min":10.881,"5min":4.0910000000000002,"15min":2.2429999999999999},"min":{"1min":0.379,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":75.459000000000003,"5min":75.459000000000003,"15min":75.459000000000003},"last":0.437}]},{"osd":6,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":11.130000000000001,"5min":3.6459999999999999,"15min":1.9019999999999999},"min":{"1min":0.40799999999999997,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":79.093999999999994,"5min":79.093999999999994,"15min":79.093999999999994},"last":0.49399999999999999},{"interface":"front","average":{"1min":11.366,"5min":3.4199999999999999,"15min":1.9950000000000001},"min":{"1min":0.39500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":78.555999999999997,"5min":78.555999999999997,"15min":78.555999999999997},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:07:14 2023","interfaces":[{"interface":"back","average":{"1min":6.226,"5min":2.4889999999999999,"15min":1.6699999999999999},"min":{"1min":0.42699999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":63.017000000000003,"5min":63.017000000000003,"15min":63.017000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":6.4470000000000001,"5min":2.4740000000000002,"15min":1.4379999999999999},"min":{"1min":0.47599999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":62.808,"5min":62.808,"15min":62.808},"last":0.59099999999999997}]}]},{"osd":4,"up_from":626,"seq":2688649527350,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1614312,"kb_used_data":1213216,"kb_used_omap":22,"kb_used_meta":401065,"kb_avail":92130840,"statfs":{"total":95995035648,"available":94341980160,"internally_reserved":0,"allocated":1242333184,"data_stored":1240574167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":410691405},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":5.3360000000000003,"5min":4.3310000000000004,"15min":2.9710000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":70.512,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":5.9180000000000001,"5min":4.8550000000000004,"15min":3.7320000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":70.813999999999993,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.51300000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4710000000000001,"5min":5.2199999999999998,"15min":3.1589999999999998},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":103.997,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":7.7279999999999998,"5min":4.8929999999999998,"15min":2.8980000000000001},"min":{"1min":0.34100000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.333,"5min":182.41,"15min":182.41},"last":0.52200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.907,"15min":1.728},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2210000000000001,"5min":11.051,"15min":11.051},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.913,"15min":1.762},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.234,"5min":11.036,"15min":11.036},"last":0.56399999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":13.058,"5min":7.5650000000000004,"15min":3.3050000000000002},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.318},"max":{"1min":109.53400000000001,"5min":270.767,"15min":270.767},"last":0.44900000000000001},{"interface":"front","average":{"1min":13.085000000000001,"5min":7.7469999999999999,"15min":3.8109999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":109.84,"5min":270.274,"15min":270.274},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":8.9659999999999993,"5min":2.0960000000000001,"15min":0.95599999999999996},"min":{"1min":0.25900000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":134.82900000000001,"5min":134.82900000000001,"15min":134.82900000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":9.0030000000000001,"5min":2.149,"15min":1.0149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":134.60400000000001,"5min":134.60400000000001,"15min":134.60400000000001},"last":0.48299999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":9.3529999999999998,"5min":2.3100000000000001,"15min":1.212},"min":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":139.77799999999999,"5min":139.77799999999999,"15min":139.77799999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":9.6219999999999999,"5min":2.2959999999999998,"15min":1.0669999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":140.78200000000001,"5min":140.78200000000001,"15min":140.78200000000001},"last":0.41899999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.53200000000000003,"15min":0.49099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.036,"5min":3.036,"15min":3.036},"last":0.498}]}]},{"osd":5,"up_from":38,"seq":163208758002,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3509508,"kb_used_data":2016188,"kb_used_omap":0,"kb_used_meta":1493312,"kb_avail":90235644,"statfs":{"total":95995035648,"available":92401299456,"internally_reserved":0,"allocated":2064576512,"data_stored":2062834245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1529151488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":12.31,"5min":6.9690000000000003,"15min":3.2719999999999998},"min":{"1min":0.27800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":212.72800000000001,"5min":212.72800000000001,"15min":212.72800000000001},"last":0.375},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.3040000000000003,"15min":3.4620000000000002},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.245},"max":{"1min":212.53999999999999,"5min":212.53999999999999,"15min":212.53999999999999},"last":0.51500000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":7.2240000000000002,"5min":3.8450000000000002,"15min":2.9129999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":113.977,"5min":121.504,"15min":262.26900000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":6.9960000000000004,"5min":3.9409999999999998,"15min":2.9950000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":112.947,"5min":121.56,"15min":262.291},"last":0.41099999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.407,"15min":1.9239999999999999},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":3.0030000000000001,"5min":10.382999999999999,"15min":106.238},"last":1.907},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.843,"15min":2.1269999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":8.9239999999999995,"5min":8.9239999999999995,"15min":104.31699999999999},"last":0.44500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5759999999999996,"5min":7.8410000000000002,"15min":3.7989999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.318},"max":{"1min":75.635000000000005,"5min":125.824,"15min":125.824},"last":0.88},{"interface":"front","average":{"1min":5.8579999999999997,"5min":7.8010000000000002,"15min":3.7770000000000001},"min":{"1min":0.38700000000000001,"5min":0.36199999999999999,"15min":0.316},"max":{"1min":75.656999999999996,"5min":125.761,"15min":125.761},"last":1.885}]},{"osd":4,"last update":"Mon Oct 9 05:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.433},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.94399999999999995,"5min":1.99,"15min":1.99},"last":0.497},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.45900000000000002,"15min":0.46000000000000002},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.165,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.48199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.502,"15min":0.71999999999999997},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":4.3360000000000003,"5min":4.3360000000000003,"15min":11.093},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.57899999999999996,"15min":0.77400000000000002},"min":{"1min":0.309,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.0590000000000002,"5min":5.9329999999999998,"15min":13.083},"last":0.46200000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:07:10 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.68700000000000006,"15min":0.86699999999999999},"min":{"1min":0.309,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":1.093,"5min":8.3520000000000003,"15min":9.6809999999999992},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.90200000000000002,"15min":0.88500000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":2.0489999999999999,"5min":11.791,"15min":13.233000000000001},"last":0.53100000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:07:55.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-09T05:07:55.717 INFO:tasks.workunit.client.0.smithi029.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T05:07:55.747 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:07:55.747 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T05:07:55.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:55.821 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:55.905 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:07:55.926 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:55.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:55.936 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:07:55.962 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:07:55.985 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:07:56.004 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:07:56.024 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:07:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:55 smithi177 ceph-mon[195797]: 1.2 scrub starts 2023-10-09T05:07:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:55 smithi177 ceph-mon[195797]: 1.2 scrub ok 2023-10-09T05:07:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:55 smithi177 ceph-mon[195797]: 1.9 scrub starts 2023-10-09T05:07:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:55 smithi177 ceph-mon[195797]: 1.9 scrub ok 2023-10-09T05:07:56.031 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:56.043 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[223858]: 1.2 scrub starts 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[223858]: 1.2 scrub ok 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[223858]: 1.9 scrub starts 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[223858]: 1.9 scrub ok 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[216903]: 1.2 scrub starts 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[216903]: 1.2 scrub ok 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[216903]: 1.9 scrub starts 2023-10-09T05:07:56.057 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:55 smithi029 ceph-mon[216903]: 1.9 scrub ok 2023-10-09T05:07:56.060 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:56.063 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:07:56.065 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.065 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.065 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.066 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.066 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.066 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.066 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.066 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:07:56.866 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:07:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:56.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:07:56.971 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:07:56.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1100245006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:56.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[216903]: pgmap v2240: 83 pgs: 3 peering, 80 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:56.999 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[216903]: 1.4 deep-scrub starts 2023-10-09T05:07:57.000 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[216903]: 1.4 deep-scrub ok 2023-10-09T05:07:57.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1100245006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:57.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[223858]: pgmap v2240: 83 pgs: 3 peering, 80 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:57.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[223858]: 1.4 deep-scrub starts 2023-10-09T05:07:57.000 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:56 smithi029 ceph-mon[223858]: 1.4 deep-scrub ok 2023-10-09T05:07:57.003 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:57.065 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:07:57.065 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:07:57.253 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:07:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:57.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:07:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:56 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1100245006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:07:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:56 smithi177 ceph-mon[195797]: pgmap v2240: 83 pgs: 3 peering, 80 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:56 smithi177 ceph-mon[195797]: 1.4 deep-scrub starts 2023-10-09T05:07:57.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:56 smithi177 ceph-mon[195797]: 1.4 deep-scrub ok 2023-10-09T05:07:57.358 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:57.390 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:57.440 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:57.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:57.545 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:07:57.573 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:57.616 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:07:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:57.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:07:57.723 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:07:57.753 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:57.866 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T05:07:58.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:57 smithi029 ceph-mon[216903]: 1.7 scrub starts 2023-10-09T05:07:58.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:57 smithi029 ceph-mon[216903]: 1.7 scrub ok 2023-10-09T05:07:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:57 smithi029 ceph-mon[223858]: 1.7 scrub starts 2023-10-09T05:07:58.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:57 smithi029 ceph-mon[223858]: 1.7 scrub ok 2023-10-09T05:07:58.189 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T05:07:58.245 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:07:58.246 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:07:58.247 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T05:07:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:57 smithi177 ceph-mon[195797]: 1.7 scrub starts 2023-10-09T05:07:58.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:57 smithi177 ceph-mon[195797]: 1.7 scrub ok 2023-10-09T05:07:58.506 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:07:58.507 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:07:58.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:58.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:07:58.611 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:07:58.643 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:58.689 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T05:07:58.689 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T05:07:58.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:07:58.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:07:58.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:07:58.771 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-09T05:07:58.772 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-09T05:07:58.772 DEBUG:teuthology.orchestra.run.smithi029:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-09T05:07:58.802 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:07:58.847 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:07:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:58 smithi177 ceph-mon[195797]: pgmap v2241: 83 pgs: 83 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:07:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]: dispatch 2023-10-09T05:07:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:59.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[216903]: pgmap v2241: 83 pgs: 83 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]: dispatch 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[223858]: pgmap v2241: 83 pgs: 83 active+clean; 477 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-09T05:07:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]: dispatch 2023-10-09T05:07:59.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:07:59.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:07:59.272 INFO:tasks.workunit.client.0.smithi029.stdout:Running main() from gmock_main.cc 2023-10-09T05:07:59.272 INFO:tasks.workunit.client.0.smithi029.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-09T05:07:59.272 INFO:tasks.workunit.client.0.smithi029.stdout:[----------] Global test environment set-up. 2023-10-09T05:07:59.272 INFO:tasks.workunit.client.0.smithi029.stdout:[----------] 70 tests from TestClsRbd 2023-10-09T05:08:00.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: 1.3 scrub starts 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: 1.3 scrub ok 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]': finished 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: osdmap e702: 8 total, 8 up, 7 in 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: 1.1 scrub starts 2023-10-09T05:08:00.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[223858]: 1.1 scrub ok 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: 1.3 scrub starts 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: 1.3 scrub ok 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]': finished 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: osdmap e702: 8 total, 8 up, 7 in 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: 1.1 scrub starts 2023-10-09T05:08:00.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:07:59 smithi029 ceph-mon[216903]: 1.1 scrub ok 2023-10-09T05:08:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: 1.3 scrub starts 2023-10-09T05:08:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: 1.3 scrub ok 2023-10-09T05:08:00.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 7]}]': finished 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: osdmap e702: 8 total, 8 up, 7 in 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: failed to encode map e702 with expected crc 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: 1.1 scrub starts 2023-10-09T05:08:00.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:07:59 smithi177 ceph-mon[195797]: 1.1 scrub ok 2023-10-09T05:08:00.840 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-09T05:08:00.862 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_all_features (22 ms) 2023-10-09T05:08:00.862 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.copyup 2023-10-09T05:08:01.070 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.copyup (208 ms) 2023-10-09T05:08:01.070 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-09T05:08:01.105 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.sparse_copyup (35 ms) 2023-10-09T05:08:01.105 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-09T05:08:01.112 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2023-10-09T05:08:01.112 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-09T05:08:01.123 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-10-09T05:08:01.124 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-09T05:08:01.153 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.directory_methods (29 ms) 2023-10-09T05:08:01.153 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.create 2023-10-09T05:08:01.166 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.create (12 ms) 2023-10-09T05:08:01.166 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_features 2023-10-09T05:08:01.167 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-10-09T05:08:01.167 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-09T05:08:01.169 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2023-10-09T05:08:01.169 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-09T05:08:01.170 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-09T05:08:01.170 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-09T05:08:01.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: osdmap e703: 8 total, 8 up, 7 in 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1863743491' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi029-712857-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[223858]: pgmap v2244: 115 pgs: 32 unknown, 83 active+clean; 587 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: osdmap e703: 8 total, 8 up, 7 in 2023-10-09T05:08:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1863743491' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi029-712857-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:08:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:00 smithi029 ceph-mon[216903]: pgmap v2244: 115 pgs: 32 unknown, 83 active+clean; 587 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:08:01.172 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-10-09T05:08:01.172 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-09T05:08:01.174 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-09T05:08:01.174 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-09T05:08:01.177 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-10-09T05:08:01.177 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.get_size 2023-10-09T05:08:01.181 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-09T05:08:01.181 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.set_size 2023-10-09T05:08:01.185 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2023-10-09T05:08:01.186 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-09T05:08:01.204 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-10-09T05:08:01.204 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-09T05:08:01.211 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-10-09T05:08:01.212 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-09T05:08:01.251 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.parents_v1 (39 ms) 2023-10-09T05:08:01.251 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: osdmap e703: 8 total, 8 up, 7 in 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1863743491' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi029-712857-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: failed to encode map e703 with expected crc 2023-10-09T05:08:01.278 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:00 smithi177 ceph-mon[195797]: pgmap v2244: 115 pgs: 32 unknown, 83 active+clean; 587 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:08:01.279 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2023-10-09T05:08:01.280 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-09T05:08:01.302 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2023-10-09T05:08:01.302 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-09T05:08:01.309 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2023-10-09T05:08:01.309 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-09T05:08:01.318 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-10-09T05:08:01.318 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-09T05:08:01.320 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-10-09T05:08:01.320 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-09T05:08:01.326 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-10-09T05:08:01.326 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-09T05:08:01.330 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-10-09T05:08:01.330 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-09T05:08:01.330 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-10-09T05:08:01.330 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-09T05:08:01.336 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2023-10-09T05:08:01.336 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-09T05:08:01.341 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2023-10-09T05:08:01.342 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.flags 2023-10-09T05:08:01.346 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-10-09T05:08:01.346 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.metadata 2023-10-09T05:08:01.369 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2023-10-09T05:08:01.370 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.set_features 2023-10-09T05:08:01.376 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2023-10-09T05:08:01.376 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.mirror 2023-10-09T05:08:01.409 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.mirror (32 ms) 2023-10-09T05:08:01.409 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-09T05:08:01.423 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-10-09T05:08:01.423 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-09T05:08:02.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: 1.5 scrub starts 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: 1.5 scrub ok 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1863743491' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi029-712857-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: osdmap e704: 8 total, 8 up, 7 in 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[223858]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: 1.5 scrub starts 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: 1.5 scrub ok 2023-10-09T05:08:02.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1863743491' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi029-712857-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:08:02.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: osdmap e704: 8 total, 8 up, 7 in 2023-10-09T05:08:02.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:01 smithi029 ceph-mon[216903]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: 1.5 scrub starts 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: 1.5 scrub ok 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1863743491' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi029-712857-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: osdmap e704: 8 total, 8 up, 7 in 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: failed to encode map e704 with expected crc 2023-10-09T05:08:02.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:01 smithi177 ceph-mon[195797]: failed to encode map e704 with expected crc 2023-10-09T05:08:03.123 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:08:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:08:02] "GET /metrics HTTP/1.1" 200 34248 "" "Prometheus/2.43.0" 2023-10-09T05:08:03.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[216903]: failed to encode map e705 with expected crc 2023-10-09T05:08:03.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[216903]: osdmap e705: 8 total, 8 up, 7 in 2023-10-09T05:08:03.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[216903]: failed to encode map e705 with expected crc 2023-10-09T05:08:03.123 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[216903]: pgmap v2247: 115 pgs: 32 unknown, 83 active+clean; 587 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:08:03.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[223858]: failed to encode map e705 with expected crc 2023-10-09T05:08:03.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[223858]: osdmap e705: 8 total, 8 up, 7 in 2023-10-09T05:08:03.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[223858]: failed to encode map e705 with expected crc 2023-10-09T05:08:03.124 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:02 smithi029 ceph-mon[223858]: pgmap v2247: 115 pgs: 32 unknown, 83 active+clean; 587 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:08:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:02 smithi177 ceph-mon[195797]: failed to encode map e705 with expected crc 2023-10-09T05:08:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:02 smithi177 ceph-mon[195797]: osdmap e705: 8 total, 8 up, 7 in 2023-10-09T05:08:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:02 smithi177 ceph-mon[195797]: failed to encode map e705 with expected crc 2023-10-09T05:08:03.277 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:02 smithi177 ceph-mon[195797]: pgmap v2247: 115 pgs: 32 unknown, 83 active+clean; 587 KiB data, 12 GiB used, 614 GiB / 626 GiB avail 2023-10-09T05:08:03.885 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:03.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:03.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:03.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:04.017 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:04.066 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:04.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:04.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:04.098 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.mirror_image_status (2675 ms) 2023-10-09T05:08:04.099 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-09T05:08:04.172 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:04.197 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.mirror_image_map (98 ms) 2023-10-09T05:08:04.197 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-09T05:08:04.199 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:04.205 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2023-10-09T05:08:04.205 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-09T05:08:04.217 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2023-10-09T05:08:04.217 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-09T05:08:04.219 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-09T05:08:04.220 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-09T05:08:04.222 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-10-09T05:08:04.222 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-09T05:08:04.225 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-10-09T05:08:04.225 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-09T05:08:04.232 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-10-09T05:08:04.232 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-09T05:08:04.235 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-10-09T05:08:04.236 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-09T05:08:04.240 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-10-09T05:08:04.240 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-09T05:08:04.242 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-09T05:08:04.243 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-09T05:08:04.246 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2023-10-09T05:08:04.246 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-09T05:08:04.247 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:04.249 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2023-10-09T05:08:04.249 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-09T05:08:04.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:04.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:04.253 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-10-09T05:08:04.253 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-09T05:08:04.255 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-10-09T05:08:04.255 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-09T05:08:04.258 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-09T05:08:04.258 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-09T05:08:04.261 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-09T05:08:04.261 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-09T05:08:04.263 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-09T05:08:04.263 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-09T05:08:04.264 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-09T05:08:04.265 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-09T05:08:04.267 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-09T05:08:04.267 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-09T05:08:04.270 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-09T05:08:04.270 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-09T05:08:04.272 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-10-09T05:08:04.272 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-09T05:08:04.275 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-10-09T05:08:04.275 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-09T05:08:04.291 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2023-10-09T05:08:04.291 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-09T05:08:04.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:04.357 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-09T05:08:04.365 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:04.394 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:04.510 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:04.512 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (221 ms) 2023-10-09T05:08:04.512 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-09T05:08:04.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:04.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:04.516 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-10-09T05:08:04.516 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-09T05:08:04.518 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-10-09T05:08:04.518 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-09T05:08:04.525 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-10-09T05:08:04.525 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.op_features 2023-10-09T05:08:04.534 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.op_features (9 ms) 2023-10-09T05:08:04.535 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-09T05:08:04.562 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.clone_parent (27 ms) 2023-10-09T05:08:04.562 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-09T05:08:04.570 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-10-09T05:08:04.570 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-09T05:08:04.583 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-10-09T05:08:04.583 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-09T05:08:04.590 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-10-09T05:08:04.590 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.migration 2023-10-09T05:08:04.603 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.migration (12 ms) 2023-10-09T05:08:04.603 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-09T05:08:04.616 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:04.618 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.migration_v1 (16 ms) 2023-10-09T05:08:04.618 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-09T05:08:04.641 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:04.682 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:04.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:04.789 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:04.815 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:04.857 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:04.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:04.964 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:04.991 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:05.018 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:05.034 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:05.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:05.141 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:05.166 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:05.208 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:05.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:05.315 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:05.344 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:05.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[216903]: pgmap v2248: 115 pgs: 12 unknown, 103 active+clean; 483 KiB data, 11 GiB used, 614 GiB / 626 GiB avail; 1.9 MiB/s rd, 878 KiB/s wr, 692 op/s 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[223858]: pgmap v2248: 115 pgs: 12 unknown, 103 active+clean; 483 KiB data, 11 GiB used, 614 GiB / 626 GiB avail; 1.9 MiB/s rd, 878 KiB/s wr, 692 op/s 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:05.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:05 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:05.929 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:05.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:05 smithi177 ceph-mon[195797]: pgmap v2248: 115 pgs: 12 unknown, 103 active+clean; 483 KiB data, 11 GiB used, 614 GiB / 626 GiB avail; 1.9 MiB/s rd, 878 KiB/s wr, 692 op/s 2023-10-09T05:08:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:05 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:06.033 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:06.062 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:06.172 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:06.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:06.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:06.306 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:06.399 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:08:06.468 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:06.567 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1948 ms) 2023-10-09T05:08:06.567 INFO:tasks.workunit.client.0.smithi029.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-09T05:08:06.570 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: 1.a scrub starts 2023-10-09T05:08:06.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: 1.a scrub starts 2023-10-09T05:08:06.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: 1.a scrub ok 2023-10-09T05:08:06.614 INFO:tasks.workunit.client.0.smithi029.stdout:[ OK ] TestClsRbd.sparsify (48 ms) 2023-10-09T05:08:06.704 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:06.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:06.808 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:06.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: 1.a scrub starts 2023-10-09T05:08:06.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: 1.a scrub ok 2023-10-09T05:08:06.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: failed to encode map e706 with expected crc 2023-10-09T05:08:06.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: failed to encode map e706 with expected crc 2023-10-09T05:08:06.837 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: osdmap e706: 8 total, 8 up, 7 in 2023-10-09T05:08:06.838 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:08:06.838 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:08:06.838 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:06.840 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:06.888 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:06.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:06.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: failed to encode map e706 with expected crc 2023-10-09T05:08:06.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: failed to encode map e706 with expected crc 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: osdmap e706: 8 total, 8 up, 7 in 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: 1.a scrub ok 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: failed to encode map e706 with expected crc 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: failed to encode map e706 with expected crc 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: osdmap e706: 8 total, 8 up, 7 in 2023-10-09T05:08:06.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:08:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:08:06.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:06.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:07.024 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:07.337 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:07.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:07.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:07.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:07.469 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:07.567 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:07.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:07.585 INFO:tasks.workunit.client.0.smithi029.stdout:[----------] 70 tests from TestClsRbd (5775 ms total) 2023-10-09T05:08:07.585 INFO:tasks.workunit.client.0.smithi029.stdout: 2023-10-09T05:08:07.585 INFO:tasks.workunit.client.0.smithi029.stdout:[----------] Global test environment tear-down 2023-10-09T05:08:07.586 INFO:tasks.workunit.client.0.smithi029.stdout:[==========] 70 tests from 1 test suite ran. (8314 ms total) 2023-10-09T05:08:07.586 INFO:tasks.workunit.client.0.smithi029.stdout:[ PASSED ] 70 tests. 2023-10-09T05:08:07.588 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is set 2023-10-09T05:08:07.588 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T05:08:07.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[216903]: pgmap v2250: 115 pgs: 115 active+clean; 431 KiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1.7 MiB/s rd, 834 KiB/s wr, 694 op/s 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[216903]: failed to encode map e707 with expected crc 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[216903]: osdmap e707: 8 total, 8 up, 7 in 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[216903]: failed to encode map e707 with expected crc 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[216903]: failed to encode map e707 with expected crc 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/218861278' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:08:07.609 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[223858]: pgmap v2250: 115 pgs: 115 active+clean; 431 KiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1.7 MiB/s rd, 834 KiB/s wr, 694 op/s 2023-10-09T05:08:07.610 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[223858]: failed to encode map e707 with expected crc 2023-10-09T05:08:07.610 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[223858]: osdmap e707: 8 total, 8 up, 7 in 2023-10-09T05:08:07.610 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[223858]: failed to encode map e707 with expected crc 2023-10-09T05:08:07.610 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[223858]: failed to encode map e707 with expected crc 2023-10-09T05:08:07.610 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/218861278' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:08:07.643 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-09T05:08:07.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:08:07.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:07.721 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:07.923 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:07.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:07.931 DEBUG:teuthology.parallel:result is None 2023-10-09T05:08:07.931 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:08:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:07 smithi177 ceph-mon[195797]: pgmap v2250: 115 pgs: 115 active+clean; 431 KiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1.7 MiB/s rd, 834 KiB/s wr, 694 op/s 2023-10-09T05:08:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:07 smithi177 ceph-mon[195797]: failed to encode map e707 with expected crc 2023-10-09T05:08:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:07 smithi177 ceph-mon[195797]: osdmap e707: 8 total, 8 up, 7 in 2023-10-09T05:08:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:07 smithi177 ceph-mon[195797]: failed to encode map e707 with expected crc 2023-10-09T05:08:08.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:07 smithi177 ceph-mon[195797]: failed to encode map e707 with expected crc 2023-10-09T05:08:08.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/218861278' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-09T05:08:08.028 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:08.054 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:08.098 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:08:08.099 DEBUG:teuthology.orchestra.run.smithi029:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:08.277 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:08:08.277 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:08:08.277 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-09T05:08:08.277 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:08:08.278 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:08:08.278 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:08:08.278 DEBUG:teuthology.orchestra.run.smithi029:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:08.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:08.295 INFO:teuthology.orchestra.run.smithi029.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:08:08.296 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:08.355 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:08.355 DEBUG:teuthology.orchestra.run.smithi029:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:08:08.355 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:08.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:08.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:08.416 INFO:tasks.workunit:timeout=3h 2023-10-09T05:08:08.416 INFO:tasks.workunit:cleanup=True 2023-10-09T05:08:08.417 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:08:08.460 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:08.474 INFO:tasks.workunit.client.0.smithi029.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-09T05:08:08.490 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:08.537 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:08.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:08.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:08.671 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/218861278' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:08:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: osdmap e708: 8 total, 8 up, 7 in 2023-10-09T05:08:08.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:08 smithi177 ceph-mon[195797]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:08.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/218861278' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: osdmap e708: 8 total, 8 up, 7 in 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[223858]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/218861278' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-09T05:08:08.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: osdmap e708: 8 total, 8 up, 7 in 2023-10-09T05:08:08.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: failed to encode map e708 with expected crc 2023-10-09T05:08:08.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:08 smithi029 ceph-mon[216903]: failed to encode map e708 with expected crc 2023-10-09T05:08:09.472 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:09.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:09.576 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:09.580 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:09.608 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:09.656 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:09.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:09.763 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:09.791 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:09.826 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:09 smithi029 ceph-mon[216903]: pgmap v2253: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:09.826 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:09 smithi029 ceph-mon[223858]: pgmap v2253: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:09.836 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:09.841 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:09 smithi177 ceph-mon[195797]: pgmap v2253: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:09.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:09.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:09.943 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:09.974 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:10.272 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:10.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:10.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:10.377 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:10.406 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:10.453 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:10.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:10.511 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:08:10.511 INFO:teuthology.orchestra.run.smithi029.stdout:{"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":3027,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":708,"num_osds":8,"num_up_osds":8,"osd_up_since":1696827803,"num_in_osds":7,"osd_in_since":1696827750,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83}],"num_pgs":83,"num_pools":4,"num_objects":23,"data_bytes":386854,"bytes_used":11793637376,"bytes_avail":660171612160,"bytes_total":671965249536},"fsmap":{"epoch":8,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi029.mozfhx","status":"up:active","gid":14649}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"}},"servicemap":{"epoch":37,"modified":"2023-10-09T05:07:45.912401+0000","services":{}},"progress_events":{"160b7126-0741-4672-bda0-6510f73fc187":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"3e2188b9-b3a5-43a7-9e0a-d4eba1add258":{"message":"Upgrade to 18.2.0-598-gb4fa663f (12m)\n [============................] (remaining: 14m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-09T05:08:10.560 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:10.564 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:08:10.564 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:08:10.587 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:10.631 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:10.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:10.737 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:10.768 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:10.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:10 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1940039690' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:08:10.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:10 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1940039690' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:08:10.841 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:10 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1940039690' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-09T05:08:11.610 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-09T05:08:11.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:11 smithi029 ceph-mon[216903]: pgmap v2254: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:11 smithi029 ceph-mon[223858]: pgmap v2254: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:11 smithi177 ceph-mon[195797]: pgmap v2254: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:12.226 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:12.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:12.331 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:12.360 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:12.560 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:12.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:12.664 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:12.697 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:12.746 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:12.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:12.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:12.851 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:12.880 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:13.098 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:08:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:08:12] "GET /metrics HTTP/1.1" 200 34248 "" "Prometheus/2.43.0" 2023-10-09T05:08:13.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:13.106 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:13.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:13.127 DEBUG:teuthology.orchestra.run.smithi029:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-09T05:08:13.184 INFO:tasks.workunit.client.0.smithi029.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-09T05:08:13.186 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:08:13.186 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-09T05:08:13.207 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:13.238 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:13.259 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-09T05:08:13.286 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:13.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:13.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:13.392 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:13.420 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:13.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:13.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:13.479 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-09T05:08:13.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:13.595 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:13.637 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:13.638 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-09T05:08:13.640 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:08:13.640 INFO:tasks.workunit.client.0.smithi029.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-09T05:08:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:13.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:13.687 INFO:tasks.workunit.client.0.smithi029.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-09T05:08:13.691 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2023-10-09T05:08:13.691 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-09T05:08:13.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:13.749 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-09T05:08:13.751 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-09T05:08:13.751 DEBUG:teuthology.orchestra.run.smithi029:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-09T05:08:13.771 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:13.811 INFO:tasks.workunit.client.0.smithi029.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-09T05:08:13.811 INFO:tasks.workunit.client.0.smithi029.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-09T05:08:13.811 INFO:tasks.workunit.client.0.smithi029.stderr:+ TMPDIR=/tmp/rbd_import_export_715438 2023-10-09T05:08:13.811 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm -rf /tmp/rbd_import_export_715438 2023-10-09T05:08:13.812 INFO:tasks.workunit.client.0.smithi029.stderr:+ mkdir /tmp/rbd_import_export_715438 2023-10-09T05:08:13.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:13.813 INFO:tasks.workunit.client.0.smithi029.stderr:+ trap 'rm -rf /tmp/rbd_import_export_715438' INT TERM EXIT 2023-10-09T05:08:13.813 INFO:tasks.workunit.client.0.smithi029.stderr:+ mkdir foo.715438 2023-10-09T05:08:13.814 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import foo.715438 foo.dir 2023-10-09T05:08:13.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:13.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:13.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:13 smithi029 ceph-mon[216903]: pgmap v2255: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:13.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:13 smithi029 ceph-mon[223858]: pgmap v2255: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:13.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:13.945 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:13 smithi177 ceph-mon[195797]: pgmap v2255: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:14.625 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: cannot import a directory 2023-10-09T05:08:14.625 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 0% complete...failed. 2023-10-09T05:08:14.625 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: import failed: (21) Is a directory 2023-10-09T05:08:14.628 INFO:tasks.workunit.client.0.smithi029.stderr:+ true 2023-10-09T05:08:14.628 INFO:tasks.workunit.client.0.smithi029.stderr:+ rmdir foo.715438 2023-10-09T05:08:14.628 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_715438/img bs=1k count=1 seek=10 2023-10-09T05:08:14.629 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records in 2023-10-09T05:08:14.630 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records out 2023-10-09T05:08:14.630 INFO:tasks.workunit.client.0.smithi029.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.4265e-05 s, 18.9 MB/s 2023-10-09T05:08:14.630 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_715438/img bs=1k count=10 seek=100 2023-10-09T05:08:14.630 INFO:tasks.workunit.client.0.smithi029.stderr:10+0 records in 2023-10-09T05:08:14.631 INFO:tasks.workunit.client.0.smithi029.stderr:10+0 records out 2023-10-09T05:08:14.631 INFO:tasks.workunit.client.0.smithi029.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.7194e-05 s, 152 MB/s 2023-10-09T05:08:14.631 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_715438/img bs=1k count=100 seek=1000 2023-10-09T05:08:14.631 INFO:tasks.workunit.client.0.smithi029.stderr:70+1 records in 2023-10-09T05:08:14.631 INFO:tasks.workunit.client.0.smithi029.stderr:70+1 records out 2023-10-09T05:08:14.631 INFO:tasks.workunit.client.0.smithi029.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000210093 s, 343 MB/s 2023-10-09T05:08:14.632 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_715438/img bs=1k seek=10000 2023-10-09T05:08:14.632 INFO:tasks.workunit.client.0.smithi029.stderr:139+1 records in 2023-10-09T05:08:14.633 INFO:tasks.workunit.client.0.smithi029.stderr:139+1 records out 2023-10-09T05:08:14.633 INFO:tasks.workunit.client.0.smithi029.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000377291 s, 380 MB/s 2023-10-09T05:08:14.633 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_715438/img bs=1k seek=100000 2023-10-09T05:08:14.773 INFO:tasks.workunit.client.0.smithi029.stderr:70+1 records in 2023-10-09T05:08:14.773 INFO:tasks.workunit.client.0.smithi029.stderr:70+1 records out 2023-10-09T05:08:14.773 INFO:tasks.workunit.client.0.smithi029.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.13955 s, 517 kB/s 2023-10-09T05:08:14.773 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_715438/img bs=1k seek=1000000 2023-10-09T05:08:14.774 INFO:tasks.workunit.client.0.smithi029.stderr:189+1 records in 2023-10-09T05:08:14.775 INFO:tasks.workunit.client.0.smithi029.stderr:189+1 records out 2023-10-09T05:08:14.775 INFO:tasks.workunit.client.0.smithi029.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000517304 s, 375 MB/s 2023-10-09T05:08:14.775 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm testimg 2023-10-09T05:08:14.815 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 0% complete...failed. 2023-10-09T05:08:14.815 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:08:14.818 INFO:tasks.workunit.client.0.smithi029.stderr:+ true 2023-10-09T05:08:14.818 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format /tmp/rbd_import_export_715438/img testimg 2023-10-09T05:08:15.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:15 smithi029 ceph-mon[216903]: pgmap v2256: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:15 smithi029 ceph-mon[223858]: pgmap v2256: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:15 smithi177 ceph-mon[195797]: pgmap v2256: 83 pgs: 83 active+clean; 378 KiB data, 11 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:08:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[216903]: failed to encode map e709 with expected crc 2023-10-09T05:08:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[216903]: osdmap e709: 8 total, 8 up, 7 in 2023-10-09T05:08:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[216903]: failed to encode map e709 with expected crc 2023-10-09T05:08:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[216903]: failed to encode map e709 with expected crc 2023-10-09T05:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[223858]: failed to encode map e709 with expected crc 2023-10-09T05:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[223858]: osdmap e709: 8 total, 8 up, 7 in 2023-10-09T05:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[223858]: failed to encode map e709 with expected crc 2023-10-09T05:08:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:16 smithi029 ceph-mon[223858]: failed to encode map e709 with expected crc 2023-10-09T05:08:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:16 smithi177 ceph-mon[195797]: failed to encode map e709 with expected crc 2023-10-09T05:08:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:16 smithi177 ceph-mon[195797]: osdmap e709: 8 total, 8 up, 7 in 2023-10-09T05:08:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:16 smithi177 ceph-mon[195797]: failed to encode map e709 with expected crc 2023-10-09T05:08:17.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:16 smithi177 ceph-mon[195797]: failed to encode map e709 with expected crc 2023-10-09T05:08:17.169 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-09T05:08:17.170 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:17.172 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg /tmp/rbd_import_export_715438/img2 2023-10-09T05:08:17.454 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:17.457 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg - 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[216903]: pgmap v2258: 83 pgs: 83 active+clean; 378 KiB data, 10 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[216903]: failed to encode map e710 with expected crc 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[216903]: failed to encode map e710 with expected crc 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[216903]: osdmap e710: 8 total, 8 up, 7 in 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[216903]: failed to encode map e710 with expected crc 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[223858]: pgmap v2258: 83 pgs: 83 active+clean; 378 KiB data, 10 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[223858]: failed to encode map e710 with expected crc 2023-10-09T05:08:17.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[223858]: failed to encode map e710 with expected crc 2023-10-09T05:08:17.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[223858]: osdmap e710: 8 total, 8 up, 7 in 2023-10-09T05:08:17.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:17 smithi029 ceph-mon[223858]: failed to encode map e710 with expected crc 2023-10-09T05:08:17.939 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-09T05:08:17.951 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-09T05:08:17.951 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:08:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:17 smithi177 ceph-mon[195797]: pgmap v2258: 83 pgs: 83 active+clean; 378 KiB data, 10 GiB used, 615 GiB / 626 GiB avail 2023-10-09T05:08:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:17 smithi177 ceph-mon[195797]: failed to encode map e710 with expected crc 2023-10-09T05:08:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:17 smithi177 ceph-mon[195797]: failed to encode map e710 with expected crc 2023-10-09T05:08:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:17 smithi177 ceph-mon[195797]: osdmap e710: 8 total, 8 up, 7 in 2023-10-09T05:08:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:17 smithi177 ceph-mon[195797]: failed to encode map e710 with expected crc 2023-10-09T05:08:18.273 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:18.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:18.287 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:18.291 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm testimg 2023-10-09T05:08:18.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:18.406 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T05:08:18.412 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img2 2023-10-09T05:08:18.421 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:18.503 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:18.527 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:18.549 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:18.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:18.641 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:18.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:18.706 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:19.163 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:19.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:19.212 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:08:19.237 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:08:19.261 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img3 2023-10-09T05:08:19.265 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:08:19.268 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:19.288 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:08:19.301 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:19.309 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:08:19.329 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:08:19.350 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:08:19.352 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:19.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:19.370 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:08:19.371 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.371 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.371 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.372 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.372 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.372 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.372 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.372 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:08:19.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:19.494 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:19.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:19 smithi029 ceph-mon[216903]: pgmap v2260: 83 pgs: 83 active+clean; 2.3 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 18 KiB/s rd, 18 KiB/s wr, 0 op/s 2023-10-09T05:08:19.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3851273340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:19.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:19 smithi029 ceph-mon[223858]: pgmap v2260: 83 pgs: 83 active+clean; 2.3 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 18 KiB/s rd, 18 KiB/s wr, 0 op/s 2023-10-09T05:08:19.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3851273340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:19.808 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img2 /tmp/rbd_import_export_715438/img3 2023-10-09T05:08:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:19 smithi177 ceph-mon[195797]: pgmap v2260: 83 pgs: 83 active+clean; 2.3 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 18 KiB/s rd, 18 KiB/s wr, 0 op/s 2023-10-09T05:08:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3851273340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:20.083 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format - testimg 2023-10-09T05:08:20.606 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:20.607 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:20.610 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg /tmp/rbd_import_export_715438/img2 2023-10-09T05:08:20.913 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:20.917 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg - 2023-10-09T05:08:21.160 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:21.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:21.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:21.298 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:21.717 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:21.720 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm testimg 2023-10-09T05:08:21.808 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-09T05:08:21.811 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img2 2023-10-09T05:08:21.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:21 smithi029 ceph-mon[216903]: pgmap v2261: 83 pgs: 83 active+clean; 5.8 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-09T05:08:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:21 smithi029 ceph-mon[216903]: osdmap e711: 8 total, 8 up, 7 in 2023-10-09T05:08:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:21 smithi029 ceph-mon[223858]: pgmap v2261: 83 pgs: 83 active+clean; 5.8 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-09T05:08:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:21 smithi029 ceph-mon[223858]: osdmap e711: 8 total, 8 up, 7 in 2023-10-09T05:08:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:21 smithi177 ceph-mon[195797]: pgmap v2261: 83 pgs: 83 active+clean; 5.8 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-09T05:08:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:21 smithi177 ceph-mon[195797]: osdmap e711: 8 total, 8 up, 7 in 2023-10-09T05:08:22.510 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img3 2023-10-09T05:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[216903]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[216903]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[216903]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[216903]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.922 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:08:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:08:22] "GET /metrics HTTP/1.1" 200 34242 "" "Prometheus/2.43.0" 2023-10-09T05:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[223858]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[223858]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[223858]: failed to encode map e711 with expected crc 2023-10-09T05:08:22.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:22 smithi029 ceph-mon[223858]: failed to encode map e711 with expected crc 2023-10-09T05:08:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:22 smithi177 ceph-mon[195797]: failed to encode map e711 with expected crc 2023-10-09T05:08:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:22 smithi177 ceph-mon[195797]: failed to encode map e711 with expected crc 2023-10-09T05:08:23.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:22 smithi177 ceph-mon[195797]: failed to encode map e711 with expected crc 2023-10-09T05:08:23.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:22 smithi177 ceph-mon[195797]: failed to encode map e711 with expected crc 2023-10-09T05:08:23.043 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img2 /tmp/rbd_import_export_715438/img3 2023-10-09T05:08:23.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:23 smithi029 ceph-mon[216903]: pgmap v2263: 83 pgs: 83 active+clean; 5.8 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 118 KiB/s rd, 80 KiB/s wr, 23 op/s 2023-10-09T05:08:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:23 smithi029 ceph-mon[223858]: pgmap v2263: 83 pgs: 83 active+clean; 5.8 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 118 KiB/s rd, 80 KiB/s wr, 23 op/s 2023-10-09T05:08:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:23 smithi177 ceph-mon[195797]: pgmap v2263: 83 pgs: 83 active+clean; 5.8 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 118 KiB/s rd, 80 KiB/s wr, 23 op/s 2023-10-09T05:08:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:25 smithi029 ceph-mon[216903]: pgmap v2264: 83 pgs: 83 active+clean; 378 KiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 316 KiB/s rd, 130 KiB/s wr, 100 op/s 2023-10-09T05:08:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:25 smithi029 ceph-mon[223858]: pgmap v2264: 83 pgs: 83 active+clean; 378 KiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 316 KiB/s rd, 130 KiB/s wr, 100 op/s 2023-10-09T05:08:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:25 smithi177 ceph-mon[195797]: pgmap v2264: 83 pgs: 83 active+clean; 378 KiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 316 KiB/s rd, 130 KiB/s wr, 100 op/s 2023-10-09T05:08:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:27 smithi029 ceph-mon[216903]: pgmap v2265: 83 pgs: 83 active+clean; 378 KiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 271 KiB/s rd, 111 KiB/s wr, 86 op/s 2023-10-09T05:08:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:27 smithi029 ceph-mon[223858]: pgmap v2265: 83 pgs: 83 active+clean; 378 KiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 271 KiB/s rd, 111 KiB/s wr, 86 op/s 2023-10-09T05:08:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:27 smithi177 ceph-mon[195797]: pgmap v2265: 83 pgs: 83 active+clean; 378 KiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 271 KiB/s rd, 111 KiB/s wr, 86 op/s 2023-10-09T05:08:28.751 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd help export 2023-10-09T05:08:28.751 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep -q export-format 2023-10-09T05:08:28.772 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_715438/img bs=1k count=10 seek=100 2023-10-09T05:08:28.773 INFO:tasks.workunit.client.0.smithi029.stderr:10+0 records in 2023-10-09T05:08:28.773 INFO:tasks.workunit.client.0.smithi029.stderr:10+0 records out 2023-10-09T05:08:28.773 INFO:tasks.workunit.client.0.smithi029.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.9848e-05 s, 147 MB/s 2023-10-09T05:08:28.773 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format /tmp/rbd_import_export_715438/img testimg 2023-10-09T05:08:28.828 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:28.828 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:28.831 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap create testimg@snap 2023-10-09T05:08:28.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:28.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:29.667 INFO:tasks.workunit.client.0.smithi029.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:08:29.674 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd image-meta set testimg key1 value1 2023-10-09T05:08:29.731 INFO:tasks.workunit.client.0.smithi029.stderr:++ rbd image-meta list testimg 2023-10-09T05:08:29.773 INFO:tasks.workunit.client.0.smithi029.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-09T05:08:29.773 INFO:tasks.workunit.client.0.smithi029.stderr: 2023-10-09T05:08:29.773 INFO:tasks.workunit.client.0.smithi029.stderr:Key Value 2023-10-09T05:08:29.773 INFO:tasks.workunit.client.0.smithi029.stderr:key1 value1' 2023-10-09T05:08:29.773 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:29.814 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:29.817 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_715438/img_v2 testimg_import 2023-10-09T05:08:29.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:29 smithi029 ceph-mon[216903]: pgmap v2266: 83 pgs: 83 active+clean; 378 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 238 KiB/s rd, 90 KiB/s wr, 79 op/s 2023-10-09T05:08:29.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:29 smithi029 ceph-mon[223858]: pgmap v2266: 83 pgs: 83 active+clean; 378 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 238 KiB/s rd, 90 KiB/s wr, 79 op/s 2023-10-09T05:08:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:29 smithi177 ceph-mon[195797]: pgmap v2266: 83 pgs: 83 active+clean; 378 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 238 KiB/s rd, 90 KiB/s wr, 79 op/s 2023-10-09T05:08:30.029 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:30.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:30.133 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:30.177 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:30.377 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:30.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:30.480 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:30.514 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:30.563 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:30.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:30.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:30.645 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:08:30.648 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import 2023-10-09T05:08:30.669 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout:rbd image 'testimg_import': 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: size 110 KiB in 1 objects 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: order 22 (4 MiB objects) 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: snapshot_count: 1 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: id: baa749f0b7b6 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: block_name_prefix: rbd_data.baa749f0b7b6 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: format: 2 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: op_features: 2023-10-09T05:08:30.682 INFO:tasks.workunit.client.0.smithi029.stdout: flags: 2023-10-09T05:08:30.683 INFO:tasks.workunit.client.0.smithi029.stdout: create_timestamp: Mon Oct 9 05:08:29 2023 2023-10-09T05:08:30.683 INFO:tasks.workunit.client.0.smithi029.stdout: access_timestamp: Mon Oct 9 05:08:29 2023 2023-10-09T05:08:30.683 INFO:tasks.workunit.client.0.smithi029.stdout: modify_timestamp: Mon Oct 9 05:08:29 2023 2023-10-09T05:08:30.685 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import@snap 2023-10-09T05:08:30.696 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:30.721 INFO:tasks.workunit.client.0.smithi029.stdout:rbd image 'testimg_import': 2023-10-09T05:08:30.722 INFO:tasks.workunit.client.0.smithi029.stdout: size 110 KiB in 1 objects 2023-10-09T05:08:30.722 INFO:tasks.workunit.client.0.smithi029.stdout: order 22 (4 MiB objects) 2023-10-09T05:08:30.722 INFO:tasks.workunit.client.0.smithi029.stdout: snapshot_count: 1 2023-10-09T05:08:30.722 INFO:tasks.workunit.client.0.smithi029.stdout: id: baa749f0b7b6 2023-10-09T05:08:30.722 INFO:tasks.workunit.client.0.smithi029.stdout: block_name_prefix: rbd_data.baa749f0b7b6 2023-10-09T05:08:30.723 INFO:tasks.workunit.client.0.smithi029.stdout: format: 2 2023-10-09T05:08:30.723 INFO:tasks.workunit.client.0.smithi029.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-09T05:08:30.723 INFO:tasks.workunit.client.0.smithi029.stdout: op_features: 2023-10-09T05:08:30.723 INFO:tasks.workunit.client.0.smithi029.stdout: flags: 2023-10-09T05:08:30.724 INFO:tasks.workunit.client.0.smithi029.stdout: create_timestamp: Mon Oct 9 05:08:29 2023 2023-10-09T05:08:30.724 INFO:tasks.workunit.client.0.smithi029.stdout: access_timestamp: Mon Oct 9 05:08:29 2023 2023-10-09T05:08:30.724 INFO:tasks.workunit.client.0.smithi029.stdout: modify_timestamp: Mon Oct 9 05:08:29 2023 2023-10-09T05:08:30.724 INFO:tasks.workunit.client.0.smithi029.stdout: protected: False 2023-10-09T05:08:30.725 INFO:tasks.workunit.client.0.smithi029.stderr:++ rbd image-meta list testimg_import 2023-10-09T05:08:30.739 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:30.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr: 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:Key Value 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:key1 value1' 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr: 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:Key Value 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr: 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:Key Value 2023-10-09T05:08:30.767 INFO:tasks.workunit.client.0.smithi029.stderr:key1 value1' ']' 2023-10-09T05:08:30.768 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg_import /tmp/rbd_import_export_715438/img_import 2023-10-09T05:08:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:30 smithi177 ceph-mon[195797]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:30 smithi177 ceph-mon[195797]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:30 smithi177 ceph-mon[195797]: osdmap e712: 8 total, 8 up, 7 in 2023-10-09T05:08:30.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:30 smithi177 ceph-mon[195797]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.802 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete...done. 2023-10-09T05:08:30.805 INFO:tasks.workunit.client.0.smithi029.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img_import 2023-10-09T05:08:30.806 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp -l /tmp/rbd_import_export_715438/img /tmp/rbd_import_export_715438/img_import 2023-10-09T05:08:30.806 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/img --format %b 2023-10-09T05:08:30.807 INFO:tasks.workunit.client.0.smithi029.stderr:+ origsize=24 2023-10-09T05:08:30.808 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/img_import --format %b 2023-10-09T05:08:30.809 INFO:tasks.workunit.client.0.smithi029.stderr:+ exportsize=224 2023-10-09T05:08:30.809 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=200 2023-10-09T05:08:30.809 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=200 2023-10-09T05:08:30.809 INFO:tasks.workunit.client.0.smithi029.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-09T05:08:30.809 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_715438/img_snap 2023-10-09T05:08:30.846 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:30.849 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete...done. 2023-10-09T05:08:30.852 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_715438/img_snap_import 2023-10-09T05:08:30.872 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:30.889 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete...done. 2023-10-09T05:08:30.892 INFO:tasks.workunit.client.0.smithi029.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_715438/img_snap /tmp/rbd_import_export_715438/img_snap_import 2023-10-09T05:08:30.892 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp -l /tmp/rbd_import_export_715438/img_snap /tmp/rbd_import_export_715438/img_snap_import 2023-10-09T05:08:30.893 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/img_snap --format %b 2023-10-09T05:08:30.894 INFO:tasks.workunit.client.0.smithi029.stderr:+ origsize=224 2023-10-09T05:08:30.894 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/img_snap_import --format %b 2023-10-09T05:08:30.896 INFO:tasks.workunit.client.0.smithi029.stderr:+ exportsize=224 2023-10-09T05:08:30.896 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:30.896 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:30.896 INFO:tasks.workunit.client.0.smithi029.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:08:30.896 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:30.896 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_import 2023-10-09T05:08:30.897 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_snap 2023-10-09T05:08:30.897 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_snap_import 2023-10-09T05:08:30.898 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap rm testimg_import@snap 2023-10-09T05:08:30.913 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:30.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:30.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[216903]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[216903]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[216903]: osdmap e712: 8 total, 8 up, 7 in 2023-10-09T05:08:30.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[216903]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[223858]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[223858]: failed to encode map e712 with expected crc 2023-10-09T05:08:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[223858]: osdmap e712: 8 total, 8 up, 7 in 2023-10-09T05:08:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:30 smithi029 ceph-mon[223858]: failed to encode map e712 with expected crc 2023-10-09T05:08:31.020 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:31.048 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:31.058 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:08:31.059 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:08:31.088 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:31.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:31.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:31.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:31.219 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:31.261 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:31.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:31.278 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:08:31.278 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:08:31.278 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:08:31.278 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:08:31.278 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:92266 14.3 14:c00c27be:::benchmark_data_smithi029_550670_object14655:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e614)", 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:02:58.952190+0000", 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 332.235215638, 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9514774369999999, 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.279 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92266 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952190+0000", 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952190+0000", 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.280 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952188+0000", 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.952196+0000", 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9359999999999999e-06 2023-10-09T05:08:31.281 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.958844+0000", 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066482440000000002 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.958866+0000", 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2206000000000001e-05 2023-10-09T05:08:31.282 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.964761+0000", 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0058949500000000004 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:58.964773+0000", 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2123000000000001e-05 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.283 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.132948+0000", 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1681750040000001 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.134184+0000", 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0012359319999999999 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.872115+0000", 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73793104899999995 2023-10-09T05:08:31.284 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.873515+0000", 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013995679999999999 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.878743+0000", 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0052280269999999997 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.285 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.879886+0000", 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001143126 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.882208+0000", 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023221399999999999 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.884422+0000", 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022137929999999999 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.286 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.884806+0000", 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000384292 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.899658+0000", 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014851544 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.900363+0000", 2023-10-09T05:08:31.287 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000705027 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:00.903667+0000", 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033043550000000001 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93386 14.b 14:d69ddaa2:::benchmark_data_smithi029_550670_object15775:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:08:31.288 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.85898582999999, 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1644230159999998, 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93386 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.289 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328419+0000", 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:31.290 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328422+0000", 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3869999999999999e-06 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328429+0000", 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2019999999999998e-06 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.291 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328435+0000", 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8379999999999998e-06 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328494+0000", 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9197999999999998e-05 2023-10-09T05:08:31.292 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328505+0000", 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0804e-05 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340759+0000", 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012254243999999999 2023-10-09T05:08:31.293 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340767+0000", 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1489999999999997e-06 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.490908+0000", 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1501412469999996 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.294 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.492842+0000", 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001934479 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93396 14.1b 14:d8f43944:::benchmark_data_smithi029_550670_object15785:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.85895872499998, 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6073650639999997, 2023-10-09T05:08:31.295 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93396 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.296 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328446+0000", 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328449+0000", 2023-10-09T05:08:31.297 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5550000000000001e-06 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328455+0000", 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8199999999999999e-06 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328461+0000", 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4519999999999996e-06 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.298 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328571+0000", 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000109665 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328578+0000", 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4159999999999998e-06 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.299 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340728+0000", 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012150466 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340737+0000", 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2320000000000001e-06 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.555267+0000", 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2145304489999997 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.300 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.560223+0000", 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004956088 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.578749+0000", 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.018526206999999999 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:31.301 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.583563+0000", 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048131340000000002 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.890933+0000", 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.30737034099999999 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.302 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.893611+0000", 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026780860000000001 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.898539+0000", 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049277770000000004 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.303 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.927690+0000", 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029151626 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.927958+0000", 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000267335 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.928819+0000", 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00086092500000000001 2023-10-09T05:08:31.304 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.930225+0000", 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001406584 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.935811+0000", 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005586298 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.305 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93399 14.b 14:d3f904cb:::benchmark_data_smithi029_550670_object15788:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.858879704, 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4290976730000002, 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.306 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93399 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.307 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328525+0000", 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328528+0000", 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2449999999999998e-06 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.308 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328536+0000", 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5859999999999999e-06 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328542+0000", 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.28e-06 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328609+0000", 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6457000000000004e-05 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.309 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328617+0000", 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0630000000000006e-06 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340788+0000", 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.01217118 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.310 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340795+0000", 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9949999999999999e-06 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.511929+0000", 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1711344940000004 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515822+0000", 2023-10-09T05:08:31.311 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0038923809999999999 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.537272+0000", 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.021450818 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.540947+0000", 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036743560000000001 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.312 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.644480+0000", 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10353306700000001 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.647474+0000", 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002994343 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.313 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.723802+0000", 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076327964999999998 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.314 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.741136+0000", 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017333634000000001 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.751071+0000", 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.009935078 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.751762+0000", 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00069107999999999995 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.315 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.752883+0000", 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011206230000000001 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.757623+0000", 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047405010000000003 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.316 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93409 14.b 14:d4aef484:::benchmark_data_smithi029_550670_object15798:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328604+0000", 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.858801357, 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6414218070000004, 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93409 2023-10-09T05:08:31.317 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328604+0000", 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328604+0000", 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.318 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328603+0000", 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328606+0000", 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.136e-06 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328611+0000", 2023-10-09T05:08:31.319 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8850000000000004e-06 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328617+0000", 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0880000000000002e-06 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328645+0000", 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8212e-05 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.320 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328652+0000", 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6320000000000008e-06 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340814+0000", 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012161929 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.321 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340821+0000", 2023-10-09T05:08:31.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9299999999999997e-06 2023-10-09T05:08:31.322 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.322 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.322 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.322 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.516308+0000", 2023-10-09T05:08:31.322 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1754863630000001 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.519571+0000", 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032632640000000001 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.585755+0000", 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.066184280999999998 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.323 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.587974+0000", 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022187729999999998 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.937859+0000", 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.34988536999999997 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.324 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.941380+0000", 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003520787 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.948853+0000", 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074730439999999999 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.954870+0000", 2023-10-09T05:08:31.325 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006017166 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.955270+0000", 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000399792 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.961354+0000", 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060840929999999996 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.326 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.963250+0000", 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001896107 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.970026+0000", 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0067752860000000002 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.327 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93412 14.b 14:d2c07cf8:::benchmark_data_smithi029_550670_object15801:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.85878284699999, 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4372284039999998, 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93412 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.328 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:08:31.329 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328622+0000", 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328624+0000", 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1090000000000001e-06 2023-10-09T05:08:31.330 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328630+0000", 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8420000000000001e-06 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328635+0000", 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.13e-06 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.331 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328677+0000", 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2333999999999998e-05 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting_for_map not empty", 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328685+0000", 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6750000000000002e-06 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340840+0000", 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012154531999999999 2023-10-09T05:08:31.332 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.340846+0000", 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.2430000000000002e-06 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.519895+0000", 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.179048892 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.333 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.523331+0000", 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034361249999999999 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.552508+0000", 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.029177214 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.554502+0000", 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019938569999999999 2023-10-09T05:08:31.334 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.741382+0000", 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.186879822 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.743460+0000", 2023-10-09T05:08:31.335 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020784990000000001 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.747621+0000", 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041610090000000002 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.758320+0000", 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010698322 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.336 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.758544+0000", 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000224291 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.759477+0000", 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00093306699999999999 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.760812+0000", 2023-10-09T05:08:31.337 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001334855 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.765851+0000", 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050388869999999997 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.338 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93416 14.16 14:684cfe34:::benchmark_data_smithi029_550670_object15805:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e628)", 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 305.85876593400002, 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2858180489999995, 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93416 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.339 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.340 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328639+0000", 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328641+0000", 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.063e-06 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.341 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328647+0000", 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8769999999999998e-06 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.328652+0000", 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2009999999999998e-06 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.329007+0000", 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035455600000000001 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.342 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.343 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:08:31.343 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.329031+0000", 2023-10-09T05:08:31.343 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4559999999999999e-05 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.337619+0000", 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085875819999999999 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.345 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.337630+0000", 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0822000000000001e-05 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.449356+0000", 2023-10-09T05:08:31.346 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.1117265169999992 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452444+0000", 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030876950000000001 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.472192+0000", 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019747752 2023-10-09T05:08:31.347 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.473616+0000", 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014240419999999999 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.586181+0000", 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.11256505999999999 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.348 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.589201+0000", 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0030202639999999999 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.595371+0000", 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061696779999999996 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.603917+0000", 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0085462880000000008 2023-10-09T05:08:31.349 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.604966+0000", 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00104896 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.350 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.605313+0000", 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034676100000000001 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.607032+0000", 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017197169999999999 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.614457+0000", 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074249579999999997 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.351 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 629/597 [PushOp(14:cc36aedf:::benchmark_data_smithi029_550670_object46560:head, version: 445'2960, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:cc36aedf:::benchmark_data_smithi029_550670_object46560:head@445'2960, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:26.410300+0000", 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 304.77710535699998, 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1977970630000003, 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.352 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410300+0000", 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410300+0000", 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.353 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410299+0000", 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410337+0000", 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8713e-05 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410353+0000", 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5715000000000001e-05 2023-10-09T05:08:31.354 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.410370+0000", 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7251000000000001e-05 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.606870+0000", 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1965000789999998 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.355 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.606878+0000", 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9400000000000002e-06 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.608097+0000", 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001218528 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.356 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(14.13 629/597 [PushOp(14:c9d12152:::benchmark_data_smithi029_550670_object46566:head, version: 445'2961, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(14:c9d12152:::benchmark_data_smithi029_550670_object46566:head@445'2961, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:26.423230+0000", 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 304.76417500899998, 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1900266359999998, 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.357 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423230+0000", 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423230+0000", 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423229+0000", 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:31.358 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423269+0000", 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0085e-05 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423290+0000", 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1591e-05 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.359 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:26.423309+0000", 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8742000000000001e-05 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.611347+0000", 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.188037445 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.360 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.611359+0000", 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2364000000000001e-05 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.613257+0000", 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001897805 2023-10-09T05:08:31.361 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.362 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.362 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.362 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.362 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.362 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93422 14.13 e629/597 14:c932bd54:::benchmark_data_smithi029_550670_object15811:head v 629'5452, mlcod=445'2858)", 2023-10-09T05:08:31.362 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:28.668401+0000", 2023-10-09T05:08:31.363 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 302.519004043, 2023-10-09T05:08:31.363 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8474744859999999, 2023-10-09T05:08:31.363 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.363 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.363 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.363 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.364 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668401+0000", 2023-10-09T05:08:31.364 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.364 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.364 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.364 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668401+0000", 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668400+0000", 2023-10-09T05:08:31.365 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668411+0000", 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1674e-05 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.366 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668430+0000", 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9023999999999999e-05 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.668442+0000", 2023-10-09T05:08:31.367 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1928e-05 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.506278+0000", 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8378357049999998 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.368 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.506305+0000", 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7076e-05 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515837+0000", 2023-10-09T05:08:31.369 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0095318950000000003 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515876+0000", 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8557999999999999e-05 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.370 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93428 14.13 e629/597 14:c93fb668:::benchmark_data_smithi029_550670_object15817:head v 629'5453, mlcod=445'2858)", 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:28.714692+0000", 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 302.47271350599999, 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8012098910000001, 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.371 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714692+0000", 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714692+0000", 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.372 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714690+0000", 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714701+0000", 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0254999999999999e-05 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.373 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714714+0000", 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3319000000000001e-05 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.714724+0000", 2023-10-09T05:08:31.374 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7990000000000005e-06 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.508239+0000", 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7935153970000002 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.508266+0000", 2023-10-09T05:08:31.375 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6548999999999999e-05 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515879+0000", 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007613519 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.515902+0000", 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2342999999999999e-05 2023-10-09T05:08:31.376 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.377 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.377 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.377 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.377 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.377 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93442 14.1b 14:dd3cf304:::benchmark_data_smithi029_550670_object15831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:08:31.377 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:28.905854+0000", 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 302.28155158099997, 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0445410229999998, 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.378 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93442 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905854+0000", 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905854+0000", 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.379 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905851+0000", 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.380 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905861+0000", 2023-10-09T05:08:31.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3379999999999994e-06 2023-10-09T05:08:31.381 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.381 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.381 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.381 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905876+0000", 2023-10-09T05:08:31.381 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5471000000000001e-05 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.905887+0000", 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0397000000000001e-05 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.382 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.560607+0000", 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.654720014 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.563852+0000", 2023-10-09T05:08:31.383 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032456310000000001 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.599630+0000", 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.035777825999999999 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.384 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.601885+0000", 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022552140000000002 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.385 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.915030+0000", 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.313144804 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.917683+0000", 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002652387 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:08:31.386 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.922705+0000", 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050228260000000002 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.936744+0000", 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014038185999999999 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.937094+0000", 2023-10-09T05:08:31.387 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035057099999999999 2023-10-09T05:08:31.388 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.388 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.940419+0000", 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033246009999999999 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.942291+0000", 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018717810000000001 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.389 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.390 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.950395+0000", 2023-10-09T05:08:31.390 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0081040590000000003 2023-10-09T05:08:31.390 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.390 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.390 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.390 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93447 14.16 14:6acb3bd1:::benchmark_data_smithi029_550670_object15836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:30.831436+0000", 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 300.35596953700002, 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.975447398, 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.391 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.392 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.392 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.392 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93447 2023-10-09T05:08:31.392 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.392 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.392 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.393 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831436+0000", 2023-10-09T05:08:31.393 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.393 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.393 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.393 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831436+0000", 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831433+0000", 2023-10-09T05:08:31.394 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999976 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831446+0000", 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2469999999999999e-05 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.395 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831464+0000", 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7943000000000001e-05 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:30.831478+0000", 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4232e-05 2023-10-09T05:08:31.396 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452902+0000", 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6214245329999999 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.456340+0000", 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034377259999999999 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.397 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.498891+0000", 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.042551154000000001 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.500720+0000", 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018290450000000001 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.398 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.747804+0000", 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.247083261 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.750643+0000", 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028398680000000002 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.399 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.757126+0000", 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0064828990000000003 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.764610+0000", 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074833529999999999 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.767128+0000", 2023-10-09T05:08:31.400 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025186230000000002 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.797584+0000", 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030455612 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.799461+0000", 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0018771689999999999 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.401 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.806883+0000", 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007421871 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93455 14.16 14:6990cb23:::benchmark_data_smithi029_550670_object15844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:31.524915+0000", 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 299.66248970700002, 2023-10-09T05:08:31.402 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1630371140000002, 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93455 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524915+0000", 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.403 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524915+0000", 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524914+0000", 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.404 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524922+0000", 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0220000000000004e-06 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524937+0000", 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.467e-05 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.524947+0000", 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0023999999999999e-05 2023-10-09T05:08:31.405 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.456674+0000", 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.931727075 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.406 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.459565+0000", 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002890798 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.484873+0000", 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025308074 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.486610+0000", 2023-10-09T05:08:31.407 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017378389999999999 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.655312+0000", 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16870205799999999 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.658075+0000", 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027629289999999999 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.408 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.662487+0000", 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044119500000000004 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.672469+0000", 2023-10-09T05:08:31.409 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0099814220000000002 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.674550+0000", 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0020812579999999999 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.679292+0000", 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047416660000000003 2023-10-09T05:08:31.410 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.680828+0000", 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015360129999999999 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.687953+0000", 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.007124816 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.411 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93459 14.6 14:6034b7a4:::benchmark_data_smithi029_550670_object15848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:31.539802+0000", 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 299.64760340200002, 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2219217410000001, 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.412 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93459 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539802+0000", 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.413 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539802+0000", 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539800+0000", 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.414 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539809+0000", 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2410000000000001e-06 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539826+0000", 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6875e-05 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.415 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.539839+0000", 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2816000000000001e-05 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.487287+0000", 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9474478159999999 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.490853+0000", 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035662910000000001 2023-10-09T05:08:31.416 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.515785+0000", 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.024931496000000001 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.518582+0000", 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027977129999999998 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.417 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.720360+0000", 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20177731900000001 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.723968+0000", 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036078809999999998 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.730738+0000", 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.006770463 2023-10-09T05:08:31.418 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.746843+0000", 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016105209999999998 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.749571+0000", 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002728252 2023-10-09T05:08:31.419 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.751066+0000", 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014948209999999999 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.752990+0000", 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019237620000000001 2023-10-09T05:08:31.420 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.761723+0000", 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0087334360000000007 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93460 14.16 14:686632a8:::benchmark_data_smithi029_550670_object15849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:31.638258+0000", 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 299.54914676700002, 2023-10-09T05:08:31.421 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1443809819999999, 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93460 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.422 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638258+0000", 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638258+0000", 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.423 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638256+0000", 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638268+0000", 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1698e-05 2023-10-09T05:08:31.424 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638286+0000", 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7572999999999999e-05 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:31.638298+0000", 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.202e-05 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.425 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.501793+0000", 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.863495178 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.504460+0000", 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026674910000000001 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.523685+0000", 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.019225123 2023-10-09T05:08:31.426 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.525268+0000", 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015823790000000001 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.731965+0000", 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.20669738900000001 2023-10-09T05:08:31.427 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.734608+0000", 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002642564 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.740445+0000", 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005837676 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.428 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.762584+0000", 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022138415000000002 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.763646+0000", 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010624269999999999 2023-10-09T05:08:31.429 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.430 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.430 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.430 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.774272+0000", 2023-10-09T05:08:31.430 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010625874 2023-10-09T05:08:31.430 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.775711+0000", 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014388789999999999 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.431 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.782639+0000", 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069283929999999997 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.432 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93503 14.6 14:6687bdb8:::benchmark_data_smithi029_550670_object15892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:32.188842+0000", 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 298.99856358099998, 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6964581760000002, 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.433 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93503 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188842+0000", 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.434 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188842+0000", 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188840+0000", 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.435 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188849+0000", 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2369999999999998e-06 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188867+0000", 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8096000000000001e-05 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.188877+0000", 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0236e-05 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.436 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.519062+0000", 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3301848039999999 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.523363+0000", 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043013169999999998 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.437 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.538989+0000", 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015626225000000001 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.540399+0000", 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014099169999999999 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.820654+0000", 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.28025441899999998 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.438 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.823802+0000", 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031477179999999999 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.830337+0000", 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0065356770000000002 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.439 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.874290+0000", 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043952721 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.875405+0000", 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001115322 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.875761+0000", 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035534100000000001 2023-10-09T05:08:31.440 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.877530+0000", 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017697640000000001 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.885300+0000", 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0077693809999999997 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.441 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93510 14.6 14:6787e907:::benchmark_data_smithi029_550670_object15899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:32.323702+0000", 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 298.86370267500001, 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5738554680000001, 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93510 2023-10-09T05:08:31.442 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323702+0000", 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323702+0000", 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.443 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323700+0000", 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323712+0000", 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1868000000000001e-05 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323729+0000", 2023-10-09T05:08:31.444 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6815000000000001e-05 2023-10-09T05:08:31.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.445 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.445 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.445 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:32.323740+0000", 2023-10-09T05:08:31.445 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1317e-05 2023-10-09T05:08:31.445 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.540685+0000", 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2169450450000001 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:31.446 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.544160+0000", 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034742140000000002 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.588340+0000", 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044180187000000003 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.447 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.589693+0000", 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001353314 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.859742+0000", 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.27004854900000003 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.863107+0000", 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033654710000000001 2023-10-09T05:08:31.448 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.869068+0000", 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.005960941 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.886409+0000", 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.017341354 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:31.449 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.886728+0000", 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031839100000000001 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.887739+0000", 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0010109030000000001 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.889526+0000", 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001787723 2023-10-09T05:08:31.450 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.897558+0000", 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0080314870000000003 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93476 14.1e e630/628 14:7b4313ab:::benchmark_data_smithi029_550670_object15865:head v 630'5763, mlcod=0'0)", 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:33.290870+0000", 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 297.896535492, 2023-10-09T05:08:31.451 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1616698269999999, 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290870+0000", 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290870+0000", 2023-10-09T05:08:31.452 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290869+0000", 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290876+0000", 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4189999999999996e-06 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.453 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290891+0000", 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4423000000000001e-05 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.290901+0000", 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.91e-06 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.454 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.447763+0000", 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.156862324 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.447787+0000", 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3946000000000001e-05 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452499+0000", 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0047118480000000003 2023-10-09T05:08:31.455 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452539+0000", 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0788999999999999e-05 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93508 14.1e e630/628 14:7b545f8b:::benchmark_data_smithi029_550670_object15897:head v 630'5764, mlcod=0'0)", 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:33.293751+0000", 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 297.89365382199998, 2023-10-09T05:08:31.456 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1588168030000001, 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293751+0000", 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293751+0000", 2023-10-09T05:08:31.457 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293750+0000", 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293760+0000", 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8030000000000008e-06 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.458 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293772+0000", 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2007e-05 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:33.293781+0000", 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.0059999999999998e-06 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.448529+0000", 2023-10-09T05:08:31.459 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.154747612 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.448543+0000", 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4457999999999999e-05 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452543+0000", 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039998309999999997 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:31.460 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:34.452568+0000", 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5148e-05 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:31.461 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:08:31.461 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-09T05:08:31.462 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:31.495 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:31.638 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is set 2023-10-09T05:08:31.731 INFO:tasks.workunit.client.0.smithi029.stderr: Removing snap: 100% complete...done. 2023-10-09T05:08:31.738 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg_import 2023-10-09T05:08:31.821 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:31.823 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap rm testimg@snap 2023-10-09T05:08:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: pgmap v2268: 83 pgs: 83 active+clean; 488 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 203 KiB/s rd, 62 KiB/s wr, 75 op/s 2023-10-09T05:08:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: osdmap e713: 8 total, 8 up, 7 in 2023-10-09T05:08:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/822628228' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: pgmap v2268: 83 pgs: 83 active+clean; 488 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 203 KiB/s rd, 62 KiB/s wr, 75 op/s 2023-10-09T05:08:31.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: osdmap e713: 8 total, 8 up, 7 in 2023-10-09T05:08:31.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: failed to encode map e713 with expected crc 2023-10-09T05:08:31.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:31 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/822628228' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: pgmap v2268: 83 pgs: 83 active+clean; 488 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 203 KiB/s rd, 62 KiB/s wr, 75 op/s 2023-10-09T05:08:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: failed to encode map e713 with expected crc 2023-10-09T05:08:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: osdmap e713: 8 total, 8 up, 7 in 2023-10-09T05:08:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: failed to encode map e713 with expected crc 2023-10-09T05:08:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: failed to encode map e713 with expected crc 2023-10-09T05:08:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: failed to encode map e713 with expected crc 2023-10-09T05:08:32.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:31 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/822628228' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:32.646 INFO:tasks.workunit.client.0.smithi029.stderr: Removing snap: 100% complete...done. 2023-10-09T05:08:32.655 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm testimg 2023-10-09T05:08:32.748 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:32.751 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --order 20 /tmp/rbd_import_export_715438/img testimg 2023-10-09T05:08:32.805 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:32.806 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:08:32.808 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:32.847 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:32.850 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_715438/img_v2 testimg_import 2023-10-09T05:08:32.912 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:08:32.916 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import 2023-10-09T05:08:32.916 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep order 2023-10-09T05:08:32.916 INFO:tasks.workunit.client.0.smithi029.stderr:+ awk '{print $2}' 2023-10-09T05:08:32.916 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep 20 2023-10-09T05:08:32.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:08:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:08:32] "GET /metrics HTTP/1.1" 200 34285 "" "Prometheus/2.43.0" 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: from='client.37935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/822628228' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: osdmap e714: 8 total, 8 up, 7 in 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[216903]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: from='client.37935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/822628228' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: osdmap e714: 8 total, 8 up, 7 in 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:32 smithi029 ceph-mon[223858]: failed to encode map e714 with expected crc 2023-10-09T05:08:32.953 INFO:tasks.workunit.client.0.smithi029.stdout:20 2023-10-09T05:08:32.953 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:33.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: from='client.37935 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: failed to encode map e714 with expected crc 2023-10-09T05:08:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/822628228' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-09T05:08:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: osdmap e714: 8 total, 8 up, 7 in 2023-10-09T05:08:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: failed to encode map e714 with expected crc 2023-10-09T05:08:33.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: failed to encode map e714 with expected crc 2023-10-09T05:08:33.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:32 smithi177 ceph-mon[195797]: failed to encode map e714 with expected crc 2023-10-09T05:08:33.066 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg_import 2023-10-09T05:08:33.153 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:33.156 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg 2023-10-09T05:08:33.247 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:33.250 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_715438/img testimg 2023-10-09T05:08:33.297 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:33.301 INFO:tasks.workunit.client.0.smithi029.stderr:++ rbd info testimg 2023-10-09T05:08:33.301 INFO:tasks.workunit.client.0.smithi029.stderr:++ grep features 2023-10-09T05:08:33.338 INFO:tasks.workunit.client.0.smithi029.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-09T05:08:33.338 INFO:tasks.workunit.client.0.smithi029.stderr: op_features: ' 2023-10-09T05:08:33.339 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:33.375 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:33.378 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_715438/img_v2 testimg_import 2023-10-09T05:08:33.429 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:08:33.432 INFO:tasks.workunit.client.0.smithi029.stderr:++ rbd info testimg_import 2023-10-09T05:08:33.432 INFO:tasks.workunit.client.0.smithi029.stderr:++ grep features 2023-10-09T05:08:33.468 INFO:tasks.workunit.client.0.smithi029.stderr:+ FEATURES_AFTER=' features: layering 2023-10-09T05:08:33.469 INFO:tasks.workunit.client.0.smithi029.stderr: op_features: ' 2023-10-09T05:08:33.469 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' ' features: layering 2023-10-09T05:08:33.469 INFO:tasks.workunit.client.0.smithi029.stderr: op_features: ' '!=' ' features: layering 2023-10-09T05:08:33.469 INFO:tasks.workunit.client.0.smithi029.stderr: op_features: ' ']' 2023-10-09T05:08:33.469 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:33.469 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg_import 2023-10-09T05:08:33.540 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:33.543 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg 2023-10-09T05:08:33.587 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2269,"stamp":"2023-10-09T05:08:30.627253+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":499529,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":734,"num_read_kb":2612,"num_write":208,"num_write_kb":1297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":972444,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1964,"ondisk_log_size":1964,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9988604,"kb_used_data":1404548,"kb_used_omap":34,"kb_used_meta":8583965,"kb_avail":646227460,"statfs":{"total":671965249536,"available":661736919040,"internally_reserved":0,"allocated":1438257152,"data_stored":1421114858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8789980554},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-5573944,"num_objects":-3,"num_object_clones":0,"num_object_copies":-9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3,"num_whiteouts":0,"num_read":563,"num_read_kb":1782,"num_write":100,"num_write_kb":545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.705835"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":213,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599417+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:08:16.599417+0000","last_peered":"2023-10-09T05:08:16.599417+0000","last_clean":"2023-10-09T05:08:16.599417+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:08:16.599417+0000","last_undegraded":"2023-10-09T05:08:16.599417+0000","last_fullsized":"2023-10-09T05:08:16.599417+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":359,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607934+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:08:16.607934+0000","last_peered":"2023-10-09T05:08:16.607934+0000","last_clean":"2023-10-09T05:08:16.607934+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:08:16.607934+0000","last_undegraded":"2023-10-09T05:08:16.607934+0000","last_fullsized":"2023-10-09T05:08:16.607934+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":136,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596492+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:08:16.596492+0000","last_peered":"2023-10-09T05:08:16.596492+0000","last_clean":"2023-10-09T05:08:16.596492+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:08:16.596492+0000","last_undegraded":"2023-10-09T05:08:16.596492+0000","last_fullsized":"2023-10-09T05:08:16.596492+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":225,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607782+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:08:16.607782+0000","last_peered":"2023-10-09T05:08:16.607782+0000","last_clean":"2023-10-09T05:08:16.607782+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:08:16.607782+0000","last_undegraded":"2023-10-09T05:08:16.607782+0000","last_fullsized":"2023-10-09T05:08:16.607782+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":341,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607345+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:08:16.607345+0000","last_peered":"2023-10-09T05:08:16.607345+0000","last_clean":"2023-10-09T05:08:16.607345+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:08:16.607345+0000","last_undegraded":"2023-10-09T05:08:16.607345+0000","last_fullsized":"2023-10-09T05:08:16.607345+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":237,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603595+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:08:16.603595+0000","last_peered":"2023-10-09T05:08:16.603595+0000","last_clean":"2023-10-09T05:08:16.603595+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:08:16.603595+0000","last_undegraded":"2023-10-09T05:08:16.603595+0000","last_fullsized":"2023-10-09T05:08:16.603595+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":84,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601767+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:08:16.601767+0000","last_peered":"2023-10-09T05:08:16.601767+0000","last_clean":"2023-10-09T05:08:16.601767+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:08:16.601767+0000","last_undegraded":"2023-10-09T05:08:16.601767+0000","last_fullsized":"2023-10-09T05:08:16.601767+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":333,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:18.897596+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:08:18.897596+0000","last_peered":"2023-10-09T05:08:18.897596+0000","last_clean":"2023-10-09T05:08:18.897596+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:08:18.897596+0000","last_undegraded":"2023-10-09T05:08:18.897596+0000","last_fullsized":"2023-10-09T05:08:18.897596+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601357+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:08:16.601357+0000","last_peered":"2023-10-09T05:08:16.601357+0000","last_clean":"2023-10-09T05:08:16.601357+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:08:16.601357+0000","last_undegraded":"2023-10-09T05:08:16.601357+0000","last_fullsized":"2023-10-09T05:08:16.601357+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":323,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.181850+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:08:21.181850+0000","last_peered":"2023-10-09T05:08:21.181850+0000","last_clean":"2023-10-09T05:08:21.181850+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:08:21.181850+0000","last_undegraded":"2023-10-09T05:08:21.181850+0000","last_fullsized":"2023-10-09T05:08:21.181850+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":393,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607859+0000","last_change":"2023-10-09T05:07:23.305378+0000","last_active":"2023-10-09T05:08:16.607859+0000","last_peered":"2023-10-09T05:08:16.607859+0000","last_clean":"2023-10-09T05:08:16.607859+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:08:16.607859+0000","last_undegraded":"2023-10-09T05:08:16.607859+0000","last_fullsized":"2023-10-09T05:08:16.607859+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":990,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":136,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596598+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:08:16.596598+0000","last_peered":"2023-10-09T05:08:16.596598+0000","last_clean":"2023-10-09T05:08:16.596598+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:08:16.596598+0000","last_undegraded":"2023-10-09T05:08:16.596598+0000","last_fullsized":"2023-10-09T05:08:16.596598+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":166,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599577+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:08:16.599577+0000","last_peered":"2023-10-09T05:08:16.599577+0000","last_clean":"2023-10-09T05:08:16.599577+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:08:16.599577+0000","last_undegraded":"2023-10-09T05:08:16.599577+0000","last_fullsized":"2023-10-09T05:08:16.599577+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":181,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600039+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:08:16.600039+0000","last_peered":"2023-10-09T05:08:16.600039+0000","last_clean":"2023-10-09T05:08:16.600039+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:08:16.600039+0000","last_undegraded":"2023-10-09T05:08:16.600039+0000","last_fullsized":"2023-10-09T05:08:16.600039+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":187,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599926+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:08:16.599926+0000","last_peered":"2023-10-09T05:08:16.599926+0000","last_clean":"2023-10-09T05:08:16.599926+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:08:16.599926+0000","last_undegraded":"2023-10-09T05:08:16.599926+0000","last_fullsized":"2023-10-09T05:08:16.599926+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":295,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607817+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:08:16.607817+0000","last_peered":"2023-10-09T05:08:16.607817+0000","last_clean":"2023-10-09T05:08:16.607817+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:08:16.607817+0000","last_undegraded":"2023-10-09T05:08:16.607817+0000","last_fullsized":"2023-10-09T05:08:16.607817+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":215,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607288+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:08:16.607288+0000","last_peered":"2023-10-09T05:08:16.607288+0000","last_clean":"2023-10-09T05:08:16.607288+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:08:16.607288+0000","last_undegraded":"2023-10-09T05:08:16.607288+0000","last_fullsized":"2023-10-09T05:08:16.607288+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":88,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601216+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:08:16.601216+0000","last_peered":"2023-10-09T05:08:16.601216+0000","last_clean":"2023-10-09T05:08:16.601216+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:08:16.601216+0000","last_undegraded":"2023-10-09T05:08:16.601216+0000","last_fullsized":"2023-10-09T05:08:16.601216+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":150,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599415+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:08:16.599415+0000","last_peered":"2023-10-09T05:08:16.599415+0000","last_clean":"2023-10-09T05:08:16.599415+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:08:16.599415+0000","last_undegraded":"2023-10-09T05:08:16.599415+0000","last_fullsized":"2023-10-09T05:08:16.599415+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":349,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607906+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:08:16.607906+0000","last_peered":"2023-10-09T05:08:16.607906+0000","last_clean":"2023-10-09T05:08:16.607906+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:08:16.607906+0000","last_undegraded":"2023-10-09T05:08:16.607906+0000","last_fullsized":"2023-10-09T05:08:16.607906+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":345,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603176+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:08:16.603176+0000","last_peered":"2023-10-09T05:08:16.603176+0000","last_clean":"2023-10-09T05:08:16.603176+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:08:16.603176+0000","last_undegraded":"2023-10-09T05:08:16.603176+0000","last_fullsized":"2023-10-09T05:08:16.603176+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":207,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.608037+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:08:16.608037+0000","last_peered":"2023-10-09T05:08:16.608037+0000","last_clean":"2023-10-09T05:08:16.608037+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:08:16.608037+0000","last_undegraded":"2023-10-09T05:08:16.608037+0000","last_fullsized":"2023-10-09T05:08:16.608037+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":240,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600089+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:08:16.600089+0000","last_peered":"2023-10-09T05:08:16.600089+0000","last_clean":"2023-10-09T05:08:16.600089+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:08:16.600089+0000","last_undegraded":"2023-10-09T05:08:16.600089+0000","last_fullsized":"2023-10-09T05:08:16.600089+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":98,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600694+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:08:16.600694+0000","last_peered":"2023-10-09T05:08:16.600694+0000","last_clean":"2023-10-09T05:08:16.600694+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:08:16.600694+0000","last_undegraded":"2023-10-09T05:08:16.600694+0000","last_fullsized":"2023-10-09T05:08:16.600694+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":74,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599649+0000","last_change":"2023-10-09T05:07:31.961439+0000","last_active":"2023-10-09T05:08:16.599649+0000","last_peered":"2023-10-09T05:08:16.599649+0000","last_clean":"2023-10-09T05:08:16.599649+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:08:16.599649+0000","last_undegraded":"2023-10-09T05:08:16.599649+0000","last_fullsized":"2023-10-09T05:08:16.599649+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":197,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599278+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:08:16.599278+0000","last_peered":"2023-10-09T05:08:16.599278+0000","last_clean":"2023-10-09T05:08:16.599278+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:08:16.599278+0000","last_undegraded":"2023-10-09T05:08:16.599278+0000","last_fullsized":"2023-10-09T05:08:16.599278+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":92,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601201+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:08:16.601201+0000","last_peered":"2023-10-09T05:08:16.601201+0000","last_clean":"2023-10-09T05:08:16.601201+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:08:16.601201+0000","last_undegraded":"2023-10-09T05:08:16.601201+0000","last_fullsized":"2023-10-09T05:08:16.601201+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":257,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603308+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:08:16.603308+0000","last_peered":"2023-10-09T05:08:16.603308+0000","last_clean":"2023-10-09T05:08:16.603308+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:08:16.603308+0000","last_undegraded":"2023-10-09T05:08:16.603308+0000","last_fullsized":"2023-10-09T05:08:16.603308+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":770,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601348+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:08:16.601348+0000","last_peered":"2023-10-09T05:08:16.601348+0000","last_clean":"2023-10-09T05:08:16.601348+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:08:16.601348+0000","last_undegraded":"2023-10-09T05:08:16.601348+0000","last_fullsized":"2023-10-09T05:08:16.601348+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":147,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596297+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:08:16.596297+0000","last_peered":"2023-10-09T05:08:16.596297+0000","last_clean":"2023-10-09T05:08:16.596297+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:08:16.596297+0000","last_undegraded":"2023-10-09T05:08:16.596297+0000","last_fullsized":"2023-10-09T05:08:16.596297+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1156,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607191+0000","last_change":"2023-10-09T05:07:54.997231+0000","last_active":"2023-10-09T05:08:16.607191+0000","last_peered":"2023-10-09T05:08:16.607191+0000","last_clean":"2023-10-09T05:08:16.607191+0000","last_became_active":"2023-10-09T05:07:53.144659+0000","last_became_peered":"2023-10-09T05:07:53.144659+0000","last_unstale":"2023-10-09T05:08:16.607191+0000","last_undegraded":"2023-10-09T05:08:16.607191+0000","last_fullsized":"2023-10-09T05:08:16.607191+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:54.997107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:54.997107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:13.843255+0000","scrub_duration":0.0021928569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":316,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607390+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:08:16.607390+0000","last_peered":"2023-10-09T05:08:16.607390+0000","last_clean":"2023-10-09T05:08:16.607390+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:08:16.607390+0000","last_undegraded":"2023-10-09T05:08:16.607390+0000","last_fullsized":"2023-10-09T05:08:16.607390+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":149,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599565+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:08:16.599565+0000","last_peered":"2023-10-09T05:08:16.599565+0000","last_clean":"2023-10-09T05:08:16.599565+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:08:16.599565+0000","last_undegraded":"2023-10-09T05:08:16.599565+0000","last_fullsized":"2023-10-09T05:08:16.599565+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1139,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600558+0000","last_change":"2023-10-09T05:07:53.145382+0000","last_active":"2023-10-09T05:08:16.600558+0000","last_peered":"2023-10-09T05:08:16.600558+0000","last_clean":"2023-10-09T05:08:16.600558+0000","last_became_active":"2023-10-09T05:07:53.144898+0000","last_became_peered":"2023-10-09T05:07:53.144898+0000","last_unstale":"2023-10-09T05:08:16.600558+0000","last_undegraded":"2023-10-09T05:08:16.600558+0000","last_fullsized":"2023-10-09T05:08:16.600558+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:48.624604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:48.624604+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":254,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603221+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:08:16.603221+0000","last_peered":"2023-10-09T05:08:16.603221+0000","last_clean":"2023-10-09T05:08:16.603221+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:08:16.603221+0000","last_undegraded":"2023-10-09T05:08:16.603221+0000","last_fullsized":"2023-10-09T05:08:16.603221+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":69,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599607+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:08:16.599607+0000","last_peered":"2023-10-09T05:08:16.599607+0000","last_clean":"2023-10-09T05:08:16.599607+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:08:16.599607+0000","last_undegraded":"2023-10-09T05:08:16.599607+0000","last_fullsized":"2023-10-09T05:08:16.599607+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599286+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:08:16.599286+0000","last_peered":"2023-10-09T05:08:16.599286+0000","last_clean":"2023-10-09T05:08:16.599286+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:08:16.599286+0000","last_undegraded":"2023-10-09T05:08:16.599286+0000","last_fullsized":"2023-10-09T05:08:16.599286+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":184,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600019+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:08:16.600019+0000","last_peered":"2023-10-09T05:08:16.600019+0000","last_clean":"2023-10-09T05:08:16.600019+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:08:16.600019+0000","last_undegraded":"2023-10-09T05:08:16.600019+0000","last_fullsized":"2023-10-09T05:08:16.600019+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1164,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600919+0000","last_change":"2023-10-09T05:08:05.173567+0000","last_active":"2023-10-09T05:08:16.600919+0000","last_peered":"2023-10-09T05:08:16.600919+0000","last_clean":"2023-10-09T05:08:16.600919+0000","last_became_active":"2023-10-09T05:07:53.145000+0000","last_became_peered":"2023-10-09T05:07:53.145000+0000","last_unstale":"2023-10-09T05:08:16.600919+0000","last_undegraded":"2023-10-09T05:08:16.600919+0000","last_fullsized":"2023-10-09T05:08:16.600919+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:05.173448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:05.173448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:52.691557+0000","scrub_duration":0.001419465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.181196+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:08:21.181196+0000","last_peered":"2023-10-09T05:08:21.181196+0000","last_clean":"2023-10-09T05:08:21.181196+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:08:21.181196+0000","last_undegraded":"2023-10-09T05:08:21.181196+0000","last_fullsized":"2023-10-09T05:08:21.181196+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":139,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596425+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:08:16.596425+0000","last_peered":"2023-10-09T05:08:16.596425+0000","last_clean":"2023-10-09T05:08:16.596425+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:08:16.596425+0000","last_undegraded":"2023-10-09T05:08:16.596425+0000","last_fullsized":"2023-10-09T05:08:16.596425+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1344,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603604+0000","last_change":"2023-10-09T05:07:56.681356+0000","last_active":"2023-10-09T05:08:16.603604+0000","last_peered":"2023-10-09T05:08:16.603604+0000","last_clean":"2023-10-09T05:08:16.603604+0000","last_became_active":"2023-10-09T05:07:52.147126+0000","last_became_peered":"2023-10-09T05:07:52.147126+0000","last_unstale":"2023-10-09T05:08:16.603604+0000","last_undegraded":"2023-10-09T05:08:16.603604+0000","last_fullsized":"2023-10-09T05:08:16.603604+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:56.681273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.681273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:27.660502+0000","scrub_duration":0.001633844,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"711'33","reported_seq":179,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.801148+0000","last_change":"2023-10-09T05:08:16.597175+0000","last_active":"2023-10-09T05:08:21.801148+0000","last_peered":"2023-10-09T05:08:21.801148+0000","last_clean":"2023-10-09T05:08:21.801148+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:08:21.801148+0000","last_undegraded":"2023-10-09T05:08:21.801148+0000","last_fullsized":"2023-10-09T05:08:21.801148+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":33,"log_dups_size":0,"ondisk_log_size":33,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037990499999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":96,"num_read_kb":453,"num_write":39,"num_write_kb":214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":88,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601336+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:08:16.601336+0000","last_peered":"2023-10-09T05:08:16.601336+0000","last_clean":"2023-10-09T05:08:16.601336+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:08:16.601336+0000","last_undegraded":"2023-10-09T05:08:16.601336+0000","last_fullsized":"2023-10-09T05:08:16.601336+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":145,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596994+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:08:16.596994+0000","last_peered":"2023-10-09T05:08:16.596994+0000","last_clean":"2023-10-09T05:08:16.596994+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:08:16.596994+0000","last_undegraded":"2023-10-09T05:08:16.596994+0000","last_fullsized":"2023-10-09T05:08:16.596994+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1153,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.600697+0000","last_change":"2023-10-09T05:07:59.084272+0000","last_active":"2023-10-09T05:08:16.600697+0000","last_peered":"2023-10-09T05:08:16.600697+0000","last_clean":"2023-10-09T05:08:16.600697+0000","last_became_active":"2023-10-09T05:07:53.144574+0000","last_became_peered":"2023-10-09T05:07:53.144574+0000","last_unstale":"2023-10-09T05:08:16.600697+0000","last_undegraded":"2023-10-09T05:08:16.600697+0000","last_fullsized":"2023-10-09T05:08:16.600697+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:59.084186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:59.084186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:02.285887+0000","scrub_duration":0.001940289,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"711'11","reported_seq":840,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.798107+0000","last_change":"2023-10-09T05:08:16.602122+0000","last_active":"2023-10-09T05:08:21.798107+0000","last_peered":"2023-10-09T05:08:21.798107+0000","last_clean":"2023-10-09T05:08:21.798107+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:08:21.798107+0000","last_undegraded":"2023-10-09T05:08:21.798107+0000","last_fullsized":"2023-10-09T05:08:21.798107+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001099332,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":30,"num_read_kb":860,"num_write":11,"num_write_kb":420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":176,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599732+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:08:16.599732+0000","last_peered":"2023-10-09T05:08:16.599732+0000","last_clean":"2023-10-09T05:08:16.599732+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:08:16.599732+0000","last_undegraded":"2023-10-09T05:08:16.599732+0000","last_fullsized":"2023-10-09T05:08:16.599732+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":18,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601006+0000","last_change":"2023-10-09T05:07:59.843178+0000","last_active":"2023-10-09T05:08:16.601006+0000","last_peered":"2023-10-09T05:08:16.601006+0000","last_clean":"2023-10-09T05:08:16.601006+0000","last_became_active":"2023-10-09T05:07:59.842875+0000","last_became_peered":"2023-10-09T05:07:59.842875+0000","last_unstale":"2023-10-09T05:08:16.601006+0000","last_undegraded":"2023-10-09T05:08:16.601006+0000","last_fullsized":"2023-10-09T05:08:16.601006+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:54.404519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1165,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599148+0000","last_change":"2023-10-09T05:07:54.675526+0000","last_active":"2023-10-09T05:08:16.599148+0000","last_peered":"2023-10-09T05:08:16.599148+0000","last_clean":"2023-10-09T05:08:16.599148+0000","last_became_active":"2023-10-09T05:07:52.148044+0000","last_became_peered":"2023-10-09T05:07:52.148044+0000","last_unstale":"2023-10-09T05:08:16.599148+0000","last_undegraded":"2023-10-09T05:08:16.599148+0000","last_fullsized":"2023-10-09T05:08:16.599148+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:54.675443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:54.675443+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:41.957176+0000","scrub_duration":0.001713099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"711'8","reported_seq":238,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:28.861398+0000","last_change":"2023-10-09T05:08:16.608680+0000","last_active":"2023-10-09T05:08:28.861398+0000","last_peered":"2023-10-09T05:08:28.861398+0000","last_clean":"2023-10-09T05:08:28.861398+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:08:28.861398+0000","last_undegraded":"2023-10-09T05:08:28.861398+0000","last_fullsized":"2023-10-09T05:08:28.861398+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001263225,"stat_sum":{"num_bytes":16,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":32,"num_read_kb":288,"num_write":7,"num_write_kb":143,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":126,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.136206+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:08:21.136206+0000","last_peered":"2023-10-09T05:08:21.136206+0000","last_clean":"2023-10-09T05:08:21.136206+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:08:21.136206+0000","last_undegraded":"2023-10-09T05:08:21.136206+0000","last_fullsized":"2023-10-09T05:08:21.136206+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":61,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601406+0000","last_change":"2023-10-09T05:07:59.843239+0000","last_active":"2023-10-09T05:08:16.601406+0000","last_peered":"2023-10-09T05:08:16.601406+0000","last_clean":"2023-10-09T05:08:16.601406+0000","last_became_active":"2023-10-09T05:07:59.842872+0000","last_became_peered":"2023-10-09T05:07:59.842872+0000","last_unstale":"2023-10-09T05:08:16.601406+0000","last_undegraded":"2023-10-09T05:08:16.601406+0000","last_fullsized":"2023-10-09T05:08:16.601406+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:02.967485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"711'9","reported_seq":120,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.808962+0000","last_change":"2023-10-09T05:08:16.601502+0000","last_active":"2023-10-09T05:08:21.808962+0000","last_peered":"2023-10-09T05:08:21.808962+0000","last_clean":"2023-10-09T05:08:21.808962+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:08:21.808962+0000","last_undegraded":"2023-10-09T05:08:21.808962+0000","last_fullsized":"2023-10-09T05:08:21.808962+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000569795,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":8,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1324,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603056+0000","last_change":"2023-10-09T05:07:50.678818+0000","last_active":"2023-10-09T05:08:16.603056+0000","last_peered":"2023-10-09T05:08:16.603056+0000","last_clean":"2023-10-09T05:08:16.603056+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:08:16.603056+0000","last_undegraded":"2023-10-09T05:08:16.603056+0000","last_fullsized":"2023-10-09T05:08:16.603056+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:50.678749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:50.678749+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:08.472240+0000","scrub_duration":0.0015712409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":205,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607408+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:08:16.607408+0000","last_peered":"2023-10-09T05:08:16.607408+0000","last_clean":"2023-10-09T05:08:16.607408+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:08:16.607408+0000","last_undegraded":"2023-10-09T05:08:16.607408+0000","last_fullsized":"2023-10-09T05:08:16.607408+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":231,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603592+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:08:16.603592+0000","last_peered":"2023-10-09T05:08:16.603592+0000","last_clean":"2023-10-09T05:08:16.603592+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:08:16.603592+0000","last_undegraded":"2023-10-09T05:08:16.603592+0000","last_fullsized":"2023-10-09T05:08:16.603592+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1171,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.595726+0000","last_change":"2023-10-09T05:07:58.306714+0000","last_active":"2023-10-09T05:08:16.595726+0000","last_peered":"2023-10-09T05:08:16.595726+0000","last_clean":"2023-10-09T05:08:16.595726+0000","last_became_active":"2023-10-09T05:07:52.143205+0000","last_became_peered":"2023-10-09T05:07:52.143205+0000","last_unstale":"2023-10-09T05:08:16.595726+0000","last_undegraded":"2023-10-09T05:08:16.595726+0000","last_fullsized":"2023-10-09T05:08:16.595726+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:58.306600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:58.306600+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.900624+0000","scrub_duration":0.001694367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"711'35","reported_seq":292,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.805438+0000","last_change":"2023-10-09T05:08:16.596429+0000","last_active":"2023-10-09T05:08:21.805438+0000","last_peered":"2023-10-09T05:08:21.805438+0000","last_clean":"2023-10-09T05:08:21.805438+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:08:21.805438+0000","last_undegraded":"2023-10-09T05:08:21.805438+0000","last_fullsized":"2023-10-09T05:08:21.805438+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":35,"log_dups_size":0,"ondisk_log_size":35,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022399999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":303,"num_read_kb":586,"num_write":25,"num_write_kb":170,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599124+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:08:16.599124+0000","last_peered":"2023-10-09T05:08:16.599124+0000","last_clean":"2023-10-09T05:08:16.599124+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:08:16.599124+0000","last_undegraded":"2023-10-09T05:08:16.599124+0000","last_fullsized":"2023-10-09T05:08:16.599124+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":165,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:25.315599+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:08:25.315599+0000","last_peered":"2023-10-09T05:08:25.315599+0000","last_clean":"2023-10-09T05:08:25.315599+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:08:25.315599+0000","last_undegraded":"2023-10-09T05:08:25.315599+0000","last_fullsized":"2023-10-09T05:08:25.315599+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1338,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.603607+0000","last_change":"2023-10-09T05:07:53.688815+0000","last_active":"2023-10-09T05:08:16.603607+0000","last_peered":"2023-10-09T05:08:16.603607+0000","last_clean":"2023-10-09T05:08:16.603607+0000","last_became_active":"2023-10-09T05:07:52.149137+0000","last_became_peered":"2023-10-09T05:07:52.149137+0000","last_unstale":"2023-10-09T05:08:16.603607+0000","last_undegraded":"2023-10-09T05:08:16.603607+0000","last_fullsized":"2023-10-09T05:08:16.603607+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:53.688730+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:53.688730+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:38.094584+0000","scrub_duration":0.0019408190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"711'1","reported_seq":182,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.143313+0000","last_change":"2023-10-09T05:08:16.600328+0000","last_active":"2023-10-09T05:08:21.143313+0000","last_peered":"2023-10-09T05:08:21.143313+0000","last_clean":"2023-10-09T05:08:21.143313+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:08:21.143313+0000","last_undegraded":"2023-10-09T05:08:21.143313+0000","last_fullsized":"2023-10-09T05:08:21.143313+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042681000000000001,"stat_sum":{"num_bytes":112640,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596569+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:08:16.596569+0000","last_peered":"2023-10-09T05:08:16.596569+0000","last_clean":"2023-10-09T05:08:16.596569+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:08:16.596569+0000","last_undegraded":"2023-10-09T05:08:16.596569+0000","last_fullsized":"2023-10-09T05:08:16.596569+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":88,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601120+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:08:16.601120+0000","last_peered":"2023-10-09T05:08:16.601120+0000","last_clean":"2023-10-09T05:08:16.601120+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:08:16.601120+0000","last_undegraded":"2023-10-09T05:08:16.601120+0000","last_fullsized":"2023-10-09T05:08:16.601120+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1168,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599415+0000","last_change":"2023-10-09T05:08:00.730177+0000","last_active":"2023-10-09T05:08:16.599415+0000","last_peered":"2023-10-09T05:08:16.599415+0000","last_clean":"2023-10-09T05:08:16.599415+0000","last_became_active":"2023-10-09T05:07:53.144728+0000","last_became_peered":"2023-10-09T05:07:53.144728+0000","last_unstale":"2023-10-09T05:08:16.599415+0000","last_undegraded":"2023-10-09T05:08:16.599415+0000","last_fullsized":"2023-10-09T05:08:16.599415+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:00.730094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:00.730094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:30.129973+0000","scrub_duration":0.0018014909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":342,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.608159+0000","last_change":"2023-10-09T05:07:23.309039+0000","last_active":"2023-10-09T05:08:16.608159+0000","last_peered":"2023-10-09T05:08:16.608159+0000","last_clean":"2023-10-09T05:08:16.608159+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:08:16.608159+0000","last_undegraded":"2023-10-09T05:08:16.608159+0000","last_fullsized":"2023-10-09T05:08:16.608159+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"711'38","reported_seq":261,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.137343+0000","last_change":"2023-10-09T05:08:16.597100+0000","last_active":"2023-10-09T05:08:21.137343+0000","last_peered":"2023-10-09T05:08:21.137343+0000","last_clean":"2023-10-09T05:08:21.137343+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:08:21.137343+0000","last_undegraded":"2023-10-09T05:08:21.137343+0000","last_fullsized":"2023-10-09T05:08:21.137343+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":38,"log_dups_size":0,"ondisk_log_size":38,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043862099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":214,"num_read_kb":185,"num_write":32,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607123+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:08:16.607123+0000","last_peered":"2023-10-09T05:08:16.607123+0000","last_clean":"2023-10-09T05:08:16.607123+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:08:16.607123+0000","last_undegraded":"2023-10-09T05:08:16.607123+0000","last_fullsized":"2023-10-09T05:08:16.607123+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"701'162","reported_seq":1134,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607583+0000","last_change":"2023-10-09T05:07:56.029382+0000","last_active":"2023-10-09T05:08:16.607583+0000","last_peered":"2023-10-09T05:08:16.607583+0000","last_clean":"2023-10-09T05:08:16.607583+0000","last_became_active":"2023-10-09T05:07:53.144623+0000","last_became_peered":"2023-10-09T05:07:53.144623+0000","last_unstale":"2023-10-09T05:08:16.607583+0000","last_undegraded":"2023-10-09T05:08:16.607583+0000","last_fullsized":"2023-10-09T05:08:16.607583+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":3,"last_scrub":"701'162","last_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_deep_scrub":"701'162","last_deep_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.029274+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:37.229597+0000","scrub_duration":0.005003141,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"711'2","reported_seq":236,"reported_epoch":711,"state":"active+clean","last_fresh":"2023-10-09T05:08:21.144492+0000","last_change":"2023-10-09T05:08:16.603975+0000","last_active":"2023-10-09T05:08:21.144492+0000","last_peered":"2023-10-09T05:08:21.144492+0000","last_clean":"2023-10-09T05:08:21.144492+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:08:21.144492+0000","last_undegraded":"2023-10-09T05:08:21.144492+0000","last_fullsized":"2023-10-09T05:08:21.144492+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000276552,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2,"num_read_kb":144,"num_write":2,"num_write_kb":72,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":146,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596487+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:08:16.596487+0000","last_peered":"2023-10-09T05:08:16.596487+0000","last_clean":"2023-10-09T05:08:16.596487+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:08:16.596487+0000","last_undegraded":"2023-10-09T05:08:16.596487+0000","last_fullsized":"2023-10-09T05:08:16.596487+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":186,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599162+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:08:16.599162+0000","last_peered":"2023-10-09T05:08:16.599162+0000","last_clean":"2023-10-09T05:08:16.599162+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:08:16.599162+0000","last_undegraded":"2023-10-09T05:08:16.599162+0000","last_fullsized":"2023-10-09T05:08:16.599162+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":176,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599969+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:08:16.599969+0000","last_peered":"2023-10-09T05:08:16.599969+0000","last_clean":"2023-10-09T05:08:16.599969+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:08:16.599969+0000","last_undegraded":"2023-10-09T05:08:16.599969+0000","last_fullsized":"2023-10-09T05:08:16.599969+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":127,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.596662+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:08:16.596662+0000","last_peered":"2023-10-09T05:08:16.596662+0000","last_clean":"2023-10-09T05:08:16.596662+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:08:16.596662+0000","last_undegraded":"2023-10-09T05:08:16.596662+0000","last_fullsized":"2023-10-09T05:08:16.596662+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":851,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601505+0000","last_change":"2023-10-09T05:07:31.951921+0000","last_active":"2023-10-09T05:08:16.601505+0000","last_peered":"2023-10-09T05:08:16.601505+0000","last_clean":"2023-10-09T05:08:16.601505+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:08:16.601505+0000","last_undegraded":"2023-10-09T05:08:16.601505+0000","last_fullsized":"2023-10-09T05:08:16.601505+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":69,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599459+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:08:16.599459+0000","last_peered":"2023-10-09T05:08:16.599459+0000","last_clean":"2023-10-09T05:08:16.599459+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:08:16.599459+0000","last_undegraded":"2023-10-09T05:08:16.599459+0000","last_fullsized":"2023-10-09T05:08:16.599459+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":791,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601529+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:08:16.601529+0000","last_peered":"2023-10-09T05:08:16.601529+0000","last_clean":"2023-10-09T05:08:16.601529+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:08:16.601529+0000","last_undegraded":"2023-10-09T05:08:16.601529+0000","last_fullsized":"2023-10-09T05:08:16.601529+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":69,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.599211+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:08:16.599211+0000","last_peered":"2023-10-09T05:08:16.599211+0000","last_clean":"2023-10-09T05:08:16.599211+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:08:16.599211+0000","last_undegraded":"2023-10-09T05:08:16.599211+0000","last_fullsized":"2023-10-09T05:08:16.599211+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":332,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607868+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:08:16.607868+0000","last_peered":"2023-10-09T05:08:16.607868+0000","last_clean":"2023-10-09T05:08:16.607868+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:08:16.607868+0000","last_undegraded":"2023-10-09T05:08:16.607868+0000","last_fullsized":"2023-10-09T05:08:16.607868+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":340,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.607937+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:08:16.607937+0000","last_peered":"2023-10-09T05:08:16.607937+0000","last_clean":"2023-10-09T05:08:16.607937+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:08:16.607937+0000","last_undegraded":"2023-10-09T05:08:16.607937+0000","last_fullsized":"2023-10-09T05:08:16.607937+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":90,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.601575+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:08:16.601575+0000","last_peered":"2023-10-09T05:08:16.601575+0000","last_clean":"2023-10-09T05:08:16.601575+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:08:16.601575+0000","last_undegraded":"2023-10-09T05:08:16.601575+0000","last_fullsized":"2023-10-09T05:08:16.601575+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":187,"reported_epoch":710,"state":"active+clean","last_fresh":"2023-10-09T05:08:16.598991+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:08:16.598991+0000","last_peered":"2023-10-09T05:08:16.598991+0000","last_clean":"2023-10-09T05:08:16.598991+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:08:16.598991+0000","last_undegraded":"2023-10-09T05:08:16.598991+0000","last_fullsized":"2023-10-09T05:08:16.598991+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":17710,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":381350,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":112675,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":689,"num_read_kb":2528,"num_write":125,"num_write_kb":1049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":30844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234733,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1751216,"kb_used_data":233768,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":91993936,"statfs":{"total":95995035648,"available":94201790464,"internally_reserved":0,"allocated":239378432,"data_stored":237605148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":8.4380000000000006,"15min":3.371},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23599999999999999},"max":{"1min":0.74299999999999999,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.90300000000000002,"5min":8.8070000000000004,"15min":3.6389999999999998},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.7190000000000003,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.59599999999999997}]},{"osd":1,"last update":"Mon Oct 9 05:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":9.9890000000000008,"15min":4.1050000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26500000000000001},"max":{"1min":4.8730000000000002,"5min":322.31999999999999,"15min":322.31999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.68200000000000005,"5min":9.907,"15min":4.0529999999999999},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":2.4409999999999998,"5min":322.14800000000002,"15min":322.14800000000002},"last":0.70599999999999996}]},{"osd":2,"last update":"Mon Oct 9 05:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.8100000000000001,"15min":3.0329999999999999},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.6930000000000001,"5min":11.948,"15min":154.76400000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.8109999999999999,"15min":3.077},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.312},"max":{"1min":3.0310000000000001,"5min":11.968999999999999,"15min":154.80000000000001},"last":0.56299999999999994}]},{"osd":3,"last update":"Mon Oct 9 05:08:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":15.032,"15min":6.2939999999999996},"min":{"1min":0.42499999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.181,"5min":349.863,"15min":349.863},"last":0.623},{"interface":"front","average":{"1min":0.98899999999999999,"5min":14.420999999999999,"15min":6.1269999999999998},"min":{"1min":0.39900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.3150000000000004,"5min":349.67500000000001,"15min":349.67500000000001},"last":0.66800000000000004}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.38900000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.145,"5min":1.145,"15min":1.145},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.51600000000000001,"15min":0.48699999999999999},"min":{"1min":0.32700000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.0609999999999999,"5min":3.0609999999999999,"15min":3.0609999999999999},"last":0.44500000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.3180000000000001,"15min":1.6319999999999999},"min":{"1min":0.34999999999999998,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":2.7189999999999999,"5min":74.915000000000006,"15min":131.84899999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.67000000000000004,"5min":1.3240000000000001,"15min":1.8180000000000001},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":3.274,"5min":74.972999999999999,"15min":135.74199999999999},"last":0.51200000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.5,"15min":0.94399999999999995},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.88600000000000001,"5min":2.1829999999999998,"15min":57.168999999999997},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.54200000000000004,"15min":1.073},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":57.183999999999997},"last":0.63900000000000001}]}]},{"osd":6,"up_from":46,"seq":197568496366,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1522160,"kb_used_data":7464,"kb_used_omap":0,"kb_used_meta":1514688,"kb_avail":92222992,"statfs":{"total":95995035648,"available":94436343808,"internally_reserved":0,"allocated":7643136,"data_stored":5870534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1551040512},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":5.0940000000000003,"15min":3.383},"min":{"1min":0.252,"5min":0.218,"15min":0.218},"max":{"1min":122.07899999999999,"5min":122.07899999999999,"15min":321.346},"last":0.74299999999999999},{"interface":"front","average":{"1min":6.1070000000000002,"5min":5.2140000000000004,"15min":2.9169999999999998},"min":{"1min":0.26500000000000001,"5min":0.255,"15min":0.255},"max":{"1min":122.107,"5min":122.107,"15min":169.119},"last":0.754}]},{"osd":1,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":2.1429999999999998,"15min":2.153},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.2879999999999998,"5min":43.604999999999997,"15min":90.278999999999996},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.274,"15min":2.1440000000000001},"min":{"1min":0.33500000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":3.5310000000000001,"5min":44.697000000000003,"15min":90.414000000000001},"last":0.86499999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.8260000000000001,"15min":2.04},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25600000000000001},"max":{"1min":8.4800000000000004,"5min":11.032999999999999,"15min":133.535},"last":0.81299999999999994},{"interface":"front","average":{"1min":1.073,"5min":1.6599999999999999,"15min":2.0230000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30599999999999999},"max":{"1min":3.6579999999999999,"5min":8.5920000000000005,"15min":133.51900000000001},"last":0.78000000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":6.1749999999999998,"5min":4.6959999999999997,"15min":3.472},"min":{"1min":0.38900000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":129.499,"5min":129.499,"15min":147.43600000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":6.976,"5min":4.9690000000000003,"15min":3.5680000000000001},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":130.994,"5min":130.994,"15min":147.41999999999999},"last":0.85099999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.89700000000000002,"15min":0.78900000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.3769999999999998,"5min":18.186,"15min":18.186},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.0589999999999999,"15min":1.022},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.2469999999999999,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.65400000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.67800000000000005,"15min":0.85399999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.4350000000000001,"5min":7.9539999999999997,"15min":25.934000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.70399999999999996,"15min":0.84199999999999997},"min":{"1min":0.29799999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":4.1909999999999998,"5min":7.2220000000000004,"15min":16.789000000000001},"last":0.82899999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.74299999999999999,"15min":0.80500000000000005},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":2.3879999999999999,"5min":7.5039999999999996,"15min":7.5039999999999996},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.79800000000000004,"15min":0.90600000000000003},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.315},"max":{"1min":3.4009999999999998,"5min":8.3399999999999999,"15min":20.041},"last":0.69299999999999995}]}]},{"osd":1,"up_from":118,"seq":506806141433,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1425876,"kb_used_data":9420,"kb_used_omap":4,"kb_used_meta":1416443,"kb_avail":92319276,"statfs":{"total":95995035648,"available":94534938624,"internally_reserved":0,"allocated":9646080,"data_stored":5840654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1450438542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9059999999999997,"5min":5.7039999999999997,"15min":2.5880000000000001},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":42.058999999999997,"5min":100.247,"15min":100.247},"last":0.42699999999999999},{"interface":"front","average":{"1min":31.283000000000001,"5min":11.256,"15min":4.4020000000000001},"min":{"1min":0.318,"5min":0.247,"15min":0.214},"max":{"1min":554.62,"5min":554.62,"15min":554.62},"last":0.371}]},{"osd":2,"last update":"Mon Oct 9 05:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":2.0569999999999999,"15min":2.7589999999999999},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.186,"5min":12.460000000000001,"15min":263.089},"last":0.34100000000000003},{"interface":"front","average":{"1min":28.771999999999998,"5min":8.0389999999999997,"15min":4.7350000000000003},"min":{"1min":0.26000000000000001,"5min":0.252,"15min":0.222},"max":{"1min":554.31200000000001,"5min":554.31200000000001,"15min":554.31200000000001},"last":0.51500000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:08:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":8.1850000000000005,"15min":3.915},"min":{"1min":0.27700000000000002,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":2.3010000000000002,"5min":235.43600000000001,"15min":235.43600000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.92100000000000004,"5min":12.523,"15min":5.1559999999999997},"min":{"1min":0.32300000000000001,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":4.2119999999999997,"5min":554.84400000000005,"15min":554.84400000000005},"last":0.44600000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":25.523,"5min":6.9199999999999999,"15min":3.1560000000000001},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":554.44000000000005,"5min":554.44000000000005,"15min":554.44000000000005},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.409,"5min":1.871,"15min":1.2549999999999999},"min":{"1min":0.32600000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.5640000000000001,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.59599999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.305,"5min":7.766,"15min":3.3719999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":554.54700000000003,"5min":554.54700000000003,"15min":554.54700000000003},"last":0.67800000000000005},{"interface":"front","average":{"1min":5.1310000000000002,"5min":2.6259999999999999,"15min":1.744},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":80.390000000000001,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.56200000000000006}]},{"osd":6,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.236999999999998,"5min":7.7699999999999996,"15min":3.387},"min":{"1min":0.40400000000000003,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":554.36500000000001,"5min":554.36500000000001,"15min":554.36500000000001},"last":0.72899999999999998},{"interface":"front","average":{"1min":5.0069999999999997,"5min":2.5939999999999999,"15min":1.7350000000000001},"min":{"1min":0.41999999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":78.278000000000006,"5min":78.278000000000006,"15min":78.278000000000006},"last":0.69699999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.968,"15min":1.5900000000000001},"min":{"1min":0.48299999999999998,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.4850000000000003,"5min":18.899999999999999,"15min":19.664999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.589,"5min":1.6240000000000001,"15min":1.397},"min":{"1min":0.47299999999999998,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":7.0099999999999998,"5min":11.903,"15min":11.903},"last":0.70499999999999996}]}]},{"osd":0,"up_from":91,"seq":390842024517,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1269376,"kb_used_data":41784,"kb_used_omap":5,"kb_used_meta":1227578,"kb_avail":92475776,"statfs":{"total":95995035648,"available":94695194624,"internally_reserved":0,"allocated":42786816,"data_stored":39714286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1257040506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:08:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":3.9220000000000002,"15min":2.6549999999999998},"min":{"1min":0.22600000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":17.704999999999998,"5min":53.176000000000002,"15min":155.411},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.3090000000000002,"5min":4.3460000000000001,"15min":2.7200000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":17.581,"5min":53.372999999999998,"15min":154.75},"last":0.36099999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:34 2023","interfaces":[{"interface":"back","average":{"1min":2.528,"5min":2.5990000000000002,"15min":2.8519999999999999},"min":{"1min":0.26500000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":20.010999999999999,"5min":20.010999999999999,"15min":151.06399999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.7349999999999999,"5min":3.2509999999999999,"15min":3.097},"min":{"1min":0.28199999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.364000000000001,"5min":20.239000000000001,"15min":151.22399999999999},"last":0.28699999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:08:04 2023","interfaces":[{"interface":"back","average":{"1min":14.112,"5min":10.515000000000001,"15min":5.5140000000000002},"min":{"1min":0.32100000000000001,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":267.63799999999998,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":13.535,"5min":10.983000000000001,"15min":5.7149999999999999},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":267.66199999999998,"5min":290.73899999999998,"15min":290.73899999999998},"last":0.39000000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9219999999999997,"5min":2.621,"15min":2.1080000000000001},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":57.360999999999997,"5min":57.360999999999997,"15min":57.360999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":4.5979999999999999,"5min":2.3839999999999999,"15min":1.8180000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":57.338000000000001,"5min":57.338000000000001,"15min":57.338000000000001},"last":0.46899999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":3.9119999999999999,"15min":1.9930000000000001},"min":{"1min":0.35799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.9450000000000003,"5min":106.922,"15min":106.922},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.2400000000000002,"15min":1.8340000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.34899999999999998},"max":{"1min":8.0749999999999993,"5min":106.771,"15min":106.771},"last":0.48399999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":4.6280000000000001,"15min":2.3929999999999998},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.9889999999999999,"5min":112.208,"15min":112.208},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.1120000000000001,"5min":3.9870000000000001,"15min":2.4140000000000001},"min":{"1min":0.375,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":8.0440000000000005,"5min":110.46899999999999,"15min":110.46899999999999},"last":0.45500000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":2.044,"15min":1.538},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.39400000000000002},"max":{"1min":8.0079999999999991,"5min":13.787000000000001,"15min":21.658000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.8500000000000001,"15min":1.5109999999999999},"min":{"1min":0.45400000000000001,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":7.5419999999999998,"5min":11.842000000000001,"15min":21.68},"last":0.55300000000000005}]}]},{"osd":2,"up_from":0,"seq":949187772796,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356489,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049516,"kb_used_data":36068,"kb_used_omap":2,"kb_used_meta":1013437,"kb_avail":92695636,"statfs":{"total":95995035648,"available":94920331264,"internally_reserved":0,"allocated":36933632,"data_stored":33751245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1037759541},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":4.6500000000000004,"15min":2.0499999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":1.595,"5min":108.631,"15min":108.631},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.81999999999999995,"5min":4.9509999999999996,"15min":2.2709999999999999},"min":{"1min":0.31,"5min":0.247,"15min":0.247},"max":{"1min":6.4020000000000001,"5min":110.012,"15min":110.012},"last":0.41599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":2.472,"15min":1.516},"min":{"1min":0.27500000000000002,"5min":0.223,"15min":0.221},"max":{"1min":6.1219999999999999,"5min":15.218,"15min":15.218},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.21,"15min":1.419},"min":{"1min":0.29599999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.9509999999999996,"5min":17.544,"15min":26.692},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Oct 9 05:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":1.5509999999999999,"15min":2.391},"min":{"1min":0.30499999999999999,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":3.0089999999999999,"5min":11.996,"15min":212.065},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.756,"15min":2.464},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":5.7549999999999999,"5min":12.987,"15min":211.98400000000001},"last":0.46000000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:07:31 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.8,"15min":1.746},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.9359999999999999,"5min":11.193,"15min":11.193},"last":0.50800000000000001},{"interface":"front","average":{"1min":6.8650000000000002,"5min":2.4209999999999998,"15min":1.405},"min":{"1min":0.36499999999999999,"5min":0.309,"15min":0.309},"max":{"1min":62.683999999999997,"5min":62.683999999999997,"15min":62.683999999999997},"last":0.46999999999999997}]},{"osd":5,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":3.0110000000000001,"15min":1.619},"min":{"1min":0.36599999999999999,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":6.9939999999999998,"5min":75.478999999999999,"15min":75.478999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":4.0499999999999998,"15min":2.1560000000000001},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":3.9689999999999999,"5min":75.459000000000003,"15min":75.459000000000003},"last":0.58099999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":3.5070000000000001,"15min":1.889},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34799999999999998},"max":{"1min":3.6560000000000001,"5min":79.093999999999994,"15min":79.093999999999994},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.512,"5min":3.5379999999999998,"15min":2.0289999999999999},"min":{"1min":0.44700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.0060000000000002,"5min":78.555999999999997,"15min":78.555999999999997},"last":0.53600000000000003}]},{"osd":7,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":2.4830000000000001,"15min":1.669},"min":{"1min":0.42999999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.5640000000000001,"5min":63.017000000000003,"15min":63.017000000000003},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.871,"5min":2.351,"15min":1.4079999999999999},"min":{"1min":0.38900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.8119999999999998,"5min":62.808,"15min":62.808},"last":0.65700000000000003}]}]},{"osd":4,"up_from":626,"seq":2688649527358,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":790712,"kb_used_data":385520,"kb_used_omap":22,"kb_used_meta":405161,"kb_avail":92954440,"statfs":{"total":95995035648,"available":95185346560,"internally_reserved":0,"allocated":394772480,"data_stored":393001174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":414885709},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":5.3360000000000003,"5min":4.3310000000000004,"15min":2.9710000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":70.512,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":5.9180000000000001,"5min":4.8550000000000004,"15min":3.7320000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":70.813999999999993,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.61499999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4710000000000001,"5min":5.2199999999999998,"15min":3.1589999999999998},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":103.997,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":7.7279999999999998,"5min":4.8929999999999998,"15min":2.8980000000000001},"min":{"1min":0.34100000000000003,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.333,"5min":182.41,"15min":182.41},"last":0.55000000000000004}]},{"osd":2,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.907,"15min":1.728},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2210000000000001,"5min":11.051,"15min":11.051},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.913,"15min":1.762},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.234,"5min":11.036,"15min":11.036},"last":0.73699999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":13.058,"5min":7.5650000000000004,"15min":3.3050000000000002},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.318},"max":{"1min":109.53400000000001,"5min":270.767,"15min":270.767},"last":0.60099999999999998},{"interface":"front","average":{"1min":13.085000000000001,"5min":7.7469999999999999,"15min":3.8109999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":109.84,"5min":270.274,"15min":270.274},"last":0.70299999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":8.9659999999999993,"5min":2.0960000000000001,"15min":0.95599999999999996},"min":{"1min":0.25900000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":134.82900000000001,"5min":134.82900000000001,"15min":134.82900000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":9.0030000000000001,"5min":2.149,"15min":1.0149999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":134.60400000000001,"5min":134.60400000000001,"15min":134.60400000000001},"last":0.57299999999999995}]},{"osd":6,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":9.3529999999999998,"5min":2.3100000000000001,"15min":1.212},"min":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":139.77799999999999,"5min":139.77799999999999,"15min":139.77799999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":9.6219999999999999,"5min":2.2959999999999998,"15min":1.0669999999999999},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":140.78200000000001,"5min":140.78200000000001,"15min":140.78200000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.53200000000000003,"15min":0.49099999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.036,"5min":3.036,"15min":3.036},"last":0.48299999999999998}]}]},{"osd":5,"up_from":38,"seq":163208758010,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2179748,"kb_used_data":690524,"kb_used_omap":0,"kb_used_meta":1489216,"kb_avail":91565404,"statfs":{"total":95995035648,"available":93762973696,"internally_reserved":0,"allocated":707096576,"data_stored":705331817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1524957184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":12.31,"5min":6.9690000000000003,"15min":3.2719999999999998},"min":{"1min":0.27800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":212.72800000000001,"5min":212.72800000000001,"15min":212.72800000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":12.295999999999999,"5min":7.3040000000000003,"15min":3.4620000000000002},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.245},"max":{"1min":212.53999999999999,"5min":212.53999999999999,"15min":212.53999999999999},"last":0.74099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:07:29 2023","interfaces":[{"interface":"back","average":{"1min":7.2240000000000002,"5min":3.8450000000000002,"15min":2.9129999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":113.977,"5min":121.504,"15min":262.26900000000001},"last":0.72199999999999998},{"interface":"front","average":{"1min":6.9960000000000004,"5min":3.9409999999999998,"15min":2.9950000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":112.947,"5min":121.56,"15min":262.291},"last":0.626}]},{"osd":2,"last update":"Mon Oct 9 05:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.407,"15min":1.9239999999999999},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":3.0030000000000001,"5min":10.382999999999999,"15min":106.238},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.843,"15min":2.1269999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":8.9239999999999995,"5min":8.9239999999999995,"15min":104.31699999999999},"last":0.67300000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:08:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":6.9809999999999999,"15min":3.7810000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9409999999999998,"5min":125.824,"15min":125.824},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.48,"5min":6.7469999999999999,"15min":3.798},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.4299999999999997,"5min":125.761,"15min":125.761},"last":0.64400000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:07:32 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.433},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.94399999999999995,"5min":1.99,"15min":1.99},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.45900000000000002,"15min":0.46000000000000002},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.165,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:07:39 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.502,"15min":0.71999999999999997},"min":{"1min":0.28899999999999998,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":4.3360000000000003,"5min":4.3360000000000003,"15min":11.093},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.57899999999999996,"15min":0.77400000000000002},"min":{"1min":0.309,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.0590000000000002,"5min":5.9329999999999998,"15min":13.083},"last":0.54900000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.52300000000000002,"15min":0.75600000000000001},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":2.1019999999999999,"5min":2.1019999999999999,"15min":8.7720000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.54600000000000004,"15min":0.80300000000000005},"min":{"1min":0.35699999999999998,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":1.845,"5min":2.0489999999999999,"15min":11.791},"last":0.50800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":20531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":16,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":10240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:08:33.616 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:33.619 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_715438/img testimg 2023-10-09T05:08:33.681 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:33.684 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:33.724 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:33.727 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_715438/img_v2 testimg_import 2023-10-09T05:08:33.788 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:08:33.791 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import 2023-10-09T05:08:33.791 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep 'stripe unit' 2023-10-09T05:08:33.791 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-09T05:08:33.828 INFO:tasks.workunit.client.0.smithi029.stdout: stripe unit: 4 KiB 2023-10-09T05:08:33.828 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import 2023-10-09T05:08:33.828 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep 'stripe count' 2023-10-09T05:08:33.829 INFO:tasks.workunit.client.0.smithi029.stderr:+ awk '{print $3}' 2023-10-09T05:08:33.829 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep 1000 2023-10-09T05:08:33.865 INFO:tasks.workunit.client.0.smithi029.stdout:1000 2023-10-09T05:08:33.866 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/img_v2 2023-10-09T05:08:33.866 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg_import 2023-10-09T05:08:33.869 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: pgmap v2271: 83 pgs: 83 active+clean; 488 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 341 B/s rd, 1.8 KiB/s wr, 1 op/s 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: osdmap e715: 8 total, 8 up, 7 in 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.872 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[223858]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: pgmap v2271: 83 pgs: 83 active+clean; 488 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 341 B/s rd, 1.8 KiB/s wr, 1 op/s 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: osdmap e715: 8 total, 8 up, 7 in 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:33 smithi029 ceph-mon[216903]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:33.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:33.954 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T05:08:33.958 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg 2023-10-09T05:08:33.975 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:33.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: pgmap v2271: 83 pgs: 83 active+clean; 488 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 341 B/s rd, 1.8 KiB/s wr, 1 op/s 2023-10-09T05:08:33.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: osdmap e715: 8 total, 8 up, 7 in 2023-10-09T05:08:33.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: failed to encode map e715 with expected crc 2023-10-09T05:08:33.999 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: failed to encode map e715 with expected crc 2023-10-09T05:08:34.000 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: failed to encode map e715 with expected crc 2023-10-09T05:08:34.000 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:33 smithi177 ceph-mon[195797]: failed to encode map e715 with expected crc 2023-10-09T05:08:34.002 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:34.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-09T05:08:34.048 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:34.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:34.056 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-09T05:08:34.060 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_715438/img testimg 2023-10-09T05:08:34.091 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:34.125 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:34.129 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap create testimg@snap1 2023-10-09T05:08:34.153 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:34.185 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:34.289 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:34.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:34.394 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:34.421 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:34.462 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:34.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:34.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:34.594 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:34.636 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:34.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:34.654 INFO:tasks.workunit.client.0.smithi029.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:08:34.660 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap create testimg@snap2 2023-10-09T05:08:34.743 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:34.770 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:34.812 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:34.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:34.919 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:34.945 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:34.987 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:34.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:34.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:35.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:35.123 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:35.169 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:35.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:35.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:35.299 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:35.342 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:35.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:35.448 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:35.477 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:35.519 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:35.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:35.624 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:35.650 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:35.663 INFO:tasks.workunit.client.0.smithi029.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-09T05:08:35.670 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap protect testimg@snap2 2023-10-09T05:08:35.694 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:35.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:35.715 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_715438/snap_protect 2023-10-09T05:08:35.727 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:35.764 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:35.769 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_715438/snap_protect testimg_import 2023-10-09T05:08:35.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:35.830 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:35.885 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:35.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:35.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:35.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[216903]: pgmap v2273: 83 pgs: 83 active+clean; 608 KiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 240 KiB/s rd, 22 KiB/s wr, 311 op/s 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[216903]: failed to encode map e716 with expected crc 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[216903]: failed to encode map e716 with expected crc 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[216903]: osdmap e716: 8 total, 8 up, 7 in 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[216903]: failed to encode map e716 with expected crc 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[223858]: pgmap v2273: 83 pgs: 83 active+clean; 608 KiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 240 KiB/s rd, 22 KiB/s wr, 311 op/s 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[223858]: failed to encode map e716 with expected crc 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[223858]: failed to encode map e716 with expected crc 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[223858]: osdmap e716: 8 total, 8 up, 7 in 2023-10-09T05:08:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:35 smithi029 ceph-mon[223858]: failed to encode map e716 with expected crc 2023-10-09T05:08:35.980 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:08:35.989 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:36.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:35 smithi177 ceph-mon[195797]: pgmap v2273: 83 pgs: 83 active+clean; 608 KiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 240 KiB/s rd, 22 KiB/s wr, 311 op/s 2023-10-09T05:08:36.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:35 smithi177 ceph-mon[195797]: failed to encode map e716 with expected crc 2023-10-09T05:08:36.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:35 smithi177 ceph-mon[195797]: failed to encode map e716 with expected crc 2023-10-09T05:08:36.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:35 smithi177 ceph-mon[195797]: osdmap e716: 8 total, 8 up, 7 in 2023-10-09T05:08:36.014 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:35 smithi177 ceph-mon[195797]: failed to encode map e716 with expected crc 2023-10-09T05:08:36.016 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:36.057 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:36.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:36.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:36.164 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:36.189 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:36.232 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:36.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:36.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:36.370 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:36.385 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:08:36.407 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:08:36.426 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:08:36.446 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:08:36.465 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:08:36.484 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:08:36.509 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:08:36.538 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:08:36.540 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.540 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.540 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.540 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.540 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.540 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.541 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.541 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-09T05:08:36.594 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:08:36.595 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:36.621 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:36.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:36.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 2023-10-09T05:08:36.754 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:36.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[216903]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[216903]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[216903]: osdmap e717: 8 total, 8 up, 7 in 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[216903]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[216903]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3013879794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[223858]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[223858]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[223858]: osdmap e717: 8 total, 8 up, 7 in 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[223858]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[223858]: failed to encode map e717 with expected crc 2023-10-09T05:08:36.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3013879794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:36 smithi177 ceph-mon[195797]: failed to encode map e717 with expected crc 2023-10-09T05:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:36 smithi177 ceph-mon[195797]: failed to encode map e717 with expected crc 2023-10-09T05:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:36 smithi177 ceph-mon[195797]: osdmap e717: 8 total, 8 up, 7 in 2023-10-09T05:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:36 smithi177 ceph-mon[195797]: failed to encode map e717 with expected crc 2023-10-09T05:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:36 smithi177 ceph-mon[195797]: failed to encode map e717 with expected crc 2023-10-09T05:08:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3013879794' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:37.688 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-09T05:08:37.692 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import@snap1 2023-10-09T05:08:37.692 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep 'protected: False' 2023-10-09T05:08:37.736 INFO:tasks.workunit.client.0.smithi029.stdout: protected: False 2023-10-09T05:08:37.737 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd info testimg_import@snap2 2023-10-09T05:08:37.737 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep 'protected: True' 2023-10-09T05:08:37.779 INFO:tasks.workunit.client.0.smithi029.stdout: protected: True 2023-10-09T05:08:37.780 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/snap_protect 2023-10-09T05:08:37.780 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap unprotect testimg@snap2 2023-10-09T05:08:37.824 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-09T05:08:37.867 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap purge testimg 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[216903]: pgmap v2276: 83 pgs: 83 active+clean; 498 KiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 255 KiB/s rd, 25 KiB/s wr, 330 op/s 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[216903]: failed to encode map e718 with expected crc 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[216903]: osdmap e718: 8 total, 8 up, 7 in 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[216903]: failed to encode map e718 with expected crc 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[223858]: pgmap v2276: 83 pgs: 83 active+clean; 498 KiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 255 KiB/s rd, 25 KiB/s wr, 330 op/s 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[223858]: failed to encode map e718 with expected crc 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[223858]: osdmap e718: 8 total, 8 up, 7 in 2023-10-09T05:08:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:37 smithi029 ceph-mon[223858]: failed to encode map e718 with expected crc 2023-10-09T05:08:37.958 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-09T05:08:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:37.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.1' 2023-10-09T05:08:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:37 smithi177 ceph-mon[195797]: pgmap v2276: 83 pgs: 83 active+clean; 498 KiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 255 KiB/s rd, 25 KiB/s wr, 330 op/s 2023-10-09T05:08:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:37 smithi177 ceph-mon[195797]: failed to encode map e718 with expected crc 2023-10-09T05:08:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:37 smithi177 ceph-mon[195797]: osdmap e718: 8 total, 8 up, 7 in 2023-10-09T05:08:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:37 smithi177 ceph-mon[195797]: failed to encode map e718 with expected crc 2023-10-09T05:08:38.062 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:38.088 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:38.130 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:38.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:38.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:38.260 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:38.302 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:38.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:38.408 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:38.431 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:38.434 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:38.477 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:38.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:38.584 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 2023-10-09T05:08:38.614 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:38 smithi177 ceph-mon[195797]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:38 smithi177 ceph-mon[195797]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:38 smithi177 ceph-mon[195797]: osdmap e719: 8 total, 8 up, 7 in 2023-10-09T05:08:38.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:38 smithi177 ceph-mon[195797]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[223858]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[223858]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[223858]: osdmap e719: 8 total, 8 up, 7 in 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[223858]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[216903]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[216903]: failed to encode map e719 with expected crc 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[216903]: osdmap e719: 8 total, 8 up, 7 in 2023-10-09T05:08:38.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:38 smithi029 ceph-mon[216903]: failed to encode map e719 with expected crc 2023-10-09T05:08:39.473 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-09T05:08:39.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:39.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.3' 2023-10-09T05:08:39.578 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:39.610 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:39.618 INFO:teuthology.orchestra.run.smithi029.stdout:ERROR: (22) Invalid argument 2023-10-09T05:08:39.618 INFO:teuthology.orchestra.run.smithi029.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-09T05:08:39.679 INFO:tasks.workunit.client.0.smithi029.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T05:08:39.687 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd snap purge testimg_import 2023-10-09T05:08:39.733 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: pgmap v2279: 83 pgs: 83 active+clean; 608 KiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 187 KiB/s rd, 23 KiB/s wr, 238 op/s 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: osdmap e720: 8 total, 8 up, 7 in 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[223858]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: pgmap v2279: 83 pgs: 83 active+clean; 608 KiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 187 KiB/s rd, 23 KiB/s wr, 238 op/s 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: osdmap e720: 8 total, 8 up, 7 in 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.735 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:39 smithi029 ceph-mon[216903]: failed to encode map e720 with expected crc 2023-10-09T05:08:39.869 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:39.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:39.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:39.973 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: pgmap v2279: 83 pgs: 83 active+clean; 608 KiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 187 KiB/s rd, 23 KiB/s wr, 238 op/s 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: osdmap e720: 8 total, 8 up, 7 in 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.003 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:39 smithi177 ceph-mon[195797]: failed to encode map e720 with expected crc 2023-10-09T05:08:40.005 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:40.055 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:40.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:40.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:40.148 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-09T05:08:40.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:40.186 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:40.229 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:40.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:40.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:40.367 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:40.822 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:40.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:40.926 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:40.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:40 smithi177 ceph-mon[195797]: failed to encode map e721 with expected crc 2023-10-09T05:08:40.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:40 smithi177 ceph-mon[195797]: osdmap e721: 8 total, 8 up, 7 in 2023-10-09T05:08:40.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:40 smithi177 ceph-mon[195797]: failed to encode map e721 with expected crc 2023-10-09T05:08:40.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:40 smithi177 ceph-mon[195797]: failed to encode map e721 with expected crc 2023-10-09T05:08:40.951 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:40 smithi177 ceph-mon[195797]: failed to encode map e721 with expected crc 2023-10-09T05:08:40.953 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:40.995 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:40.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:41.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:41.101 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:41.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[223858]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[223858]: osdmap e721: 8 total, 8 up, 7 in 2023-10-09T05:08:41.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[223858]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[223858]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[223858]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[216903]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[216903]: osdmap e721: 8 total, 8 up, 7 in 2023-10-09T05:08:41.118 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[216903]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[216903]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:40 smithi029 ceph-mon[216903]: failed to encode map e721 with expected crc 2023-10-09T05:08:41.127 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:41.137 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-09T05:08:41.152 INFO:tasks.workunit.client.0.smithi029.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-09T05:08:41.160 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg 2023-10-09T05:08:41.176 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:41.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:41.253 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:41.256 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd remove testimg_import 2023-10-09T05:08:41.282 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:41.310 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:41.356 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:41.361 INFO:tasks.workunit.client.0.smithi029.stderr:+ tiered=0 2023-10-09T05:08:41.361 INFO:tasks.workunit.client.0.smithi029.stderr:+ ceph osd dump 2023-10-09T05:08:41.361 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep '^pool' 2023-10-09T05:08:41.361 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep ''\''rbd'\''' 2023-10-09T05:08:41.362 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep tier 2023-10-09T05:08:41.566 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:41.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:41.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:41.703 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:41.760 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:41.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:41.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:41.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[216903]: pgmap v2282: 83 pgs: 83 active+clean; 598 KiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 306 KiB/s rd, 36 KiB/s wr, 411 op/s 2023-10-09T05:08:41.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[216903]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[216903]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[216903]: osdmap e722: 8 total, 8 up, 7 in 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[216903]: osdmap e723: 8 total, 8 up, 7 in 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[216903]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[223858]: pgmap v2282: 83 pgs: 83 active+clean; 598 KiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 306 KiB/s rd, 36 KiB/s wr, 411 op/s 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[223858]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[223858]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[223858]: osdmap e722: 8 total, 8 up, 7 in 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[223858]: osdmap e723: 8 total, 8 up, 7 in 2023-10-09T05:08:41.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:41 smithi029 ceph-mon[223858]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.865 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:41.892 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:41.935 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:41.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:41 smithi177 ceph-mon[195797]: pgmap v2282: 83 pgs: 83 active+clean; 598 KiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 306 KiB/s rd, 36 KiB/s wr, 411 op/s 2023-10-09T05:08:41.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:41 smithi177 ceph-mon[195797]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:41 smithi177 ceph-mon[195797]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:41 smithi177 ceph-mon[195797]: osdmap e722: 8 total, 8 up, 7 in 2023-10-09T05:08:41.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:41 smithi177 ceph-mon[195797]: osdmap e723: 8 total, 8 up, 7 in 2023-10-09T05:08:41.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:41 smithi177 ceph-mon[195797]: failed to encode map e722 with expected crc 2023-10-09T05:08:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:41.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:42.035 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:42.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:42.066 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:42.109 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:42.239 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:42.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:42.278 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_715438/sparse1 2023-10-09T05:08:42.284 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records in 2023-10-09T05:08:42.284 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records out 2023-10-09T05:08:42.284 INFO:tasks.workunit.client.0.smithi029.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00520404 s, 201 MB/s 2023-10-09T05:08:42.284 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_715438/sparse2 2023-10-09T05:08:42.290 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records in 2023-10-09T05:08:42.290 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records out 2023-10-09T05:08:42.290 INFO:tasks.workunit.client.0.smithi029.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00515555 s, 203 MB/s 2023-10-09T05:08:42.290 INFO:tasks.workunit.client.0.smithi029.stderr:+ truncate /tmp/rbd_import_export_715438/sparse2 -s 2M 2023-10-09T05:08:42.308 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse1 2023-10-09T05:08:42.345 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:42.349 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 0% complete...failed. 2023-10-09T05:08:42.349 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:08:42.353 INFO:tasks.workunit.client.0.smithi029.stderr:+ true 2023-10-09T05:08:42.353 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_715438/sparse1 2023-10-09T05:08:42.372 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:42.416 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:42.421 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T05:08:42.422 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:42.422 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:08:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:42.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:42.424 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd ls -l 2023-10-09T05:08:42.425 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep sparse1 2023-10-09T05:08:42.425 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T05:08:42.463 INFO:tasks.workunit.client.0.smithi029.stdout:sparse1 2 MiB 2 2023-10-09T05:08:42.464 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse1 2023-10-09T05:08:42.464 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse1 2023-10-09T05:08:42.464 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse1 2023-10-09T05:08:42.464 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:42.464 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.95434bcf0c41 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.95434bcf0c41 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse1 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.95434bcf0c41\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse1 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:42.503 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse1 2023-10-09T05:08:42.504 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:42.504 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:42.523 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:42.543 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:42.543 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:42.543 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:42.543 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:42.543 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:42.554 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:42.643 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects=1 2023-10-09T05:08:42.643 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 1 2023-10-09T05:08:42.643 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T05:08:42.643 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export sparse1 /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:42.654 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:42.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:42.685 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:42.688 INFO:tasks.workunit.client.0.smithi029.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_715438/sparse1 /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:42.689 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp -l /tmp/rbd_import_export_715438/sparse1 /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:42.689 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:08:42.691 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse1 --format %b 2023-10-09T05:08:42.692 INFO:tasks.workunit.client.0.smithi029.stderr:+ origsize=2048 2023-10-09T05:08:42.692 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse1.out --format %b 2023-10-09T05:08:42.693 INFO:tasks.workunit.client.0.smithi029.stderr:+ exportsize=2048 2023-10-09T05:08:42.693 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:42.693 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:42.694 INFO:tasks.workunit.client.0.smithi029.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:08:42.694 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:42.694 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse1 2023-10-09T05:08:42.760 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:42.786 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T05:08:42.787 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:42.789 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse2 2023-10-09T05:08:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[216903]: failed to encode map e723 with expected crc 2023-10-09T05:08:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[216903]: failed to encode map e723 with expected crc 2023-10-09T05:08:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2543346394' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2543346394' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:42.818 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2005172563' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:08:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[223858]: failed to encode map e723 with expected crc 2023-10-09T05:08:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[223858]: failed to encode map e723 with expected crc 2023-10-09T05:08:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2543346394' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2543346394' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:42.819 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:42 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2005172563' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:08:42.829 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:42.831 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 0% complete...failed. 2023-10-09T05:08:42.831 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: delete error: (2) No such file or directory 2023-10-09T05:08:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:42.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:42.834 INFO:tasks.workunit.client.0.smithi029.stderr:+ true 2023-10-09T05:08:42.835 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_715438/sparse2 2023-10-09T05:08:42.894 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-09T05:08:42.895 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:42.895 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:08:42.898 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd ls -l 2023-10-09T05:08:42.898 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep sparse2 2023-10-09T05:08:42.899 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-09T05:08:42.935 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:42.936 INFO:tasks.workunit.client.0.smithi029.stdout:sparse2 2 MiB 2 2023-10-09T05:08:42.937 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse2 2023-10-09T05:08:42.937 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse2 2023-10-09T05:08:42.938 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse2 2023-10-09T05:08:42.938 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:42.938 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:42.958 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:42 smithi177 ceph-mon[195797]: failed to encode map e723 with expected crc 2023-10-09T05:08:42.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:42 smithi177 ceph-mon[195797]: failed to encode map e723 with expected crc 2023-10-09T05:08:42.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2543346394' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:42.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2543346394' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-09T05:08:42.959 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:42 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2005172563' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:08:42.961 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:42.974 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.bc09a994178d 2023-10-09T05:08:42.975 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.bc09a994178d 2023-10-09T05:08:42.975 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse2 2023-10-09T05:08:42.975 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse2 2023-10-09T05:08:42.976 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.bc09a994178d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:42.976 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:42.976 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse2 2023-10-09T05:08:42.976 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:42.976 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:43.003 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:43.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:43.014 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:43.014 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:43.014 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:43.014 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:43.014 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:43.046 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects=0 2023-10-09T05:08:43.046 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 0 2023-10-09T05:08:43.046 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T05:08:43.047 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export sparse2 /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:43.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:43.135 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:08:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:08:42] "GET /metrics HTTP/1.1" 200 34285 "" "Prometheus/2.43.0" 2023-10-09T05:08:43.138 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:43.162 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:43.166 INFO:tasks.workunit.client.0.smithi029.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_715438/sparse2 /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:43.166 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp -l /tmp/rbd_import_export_715438/sparse2 /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:43.168 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse2 --format %b 2023-10-09T05:08:43.170 INFO:tasks.workunit.client.0.smithi029.stderr:+ origsize=2048 2023-10-09T05:08:43.170 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse2.out --format %b 2023-10-09T05:08:43.171 INFO:tasks.workunit.client.0.smithi029.stderr:+ exportsize=2048 2023-10-09T05:08:43.171 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:43.171 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:43.171 INFO:tasks.workunit.client.0.smithi029.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:08:43.171 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:43.172 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse2 2023-10-09T05:08:43.319 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-09T05:08:43.322 INFO:tasks.workunit.client.0.smithi029.stderr:+ truncate /tmp/rbd_import_export_715438/sparse1 -s 10M 2023-10-09T05:08:43.323 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-09T05:08:43.409 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:43.410 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:43.410 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:08:43.413 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd ls -l 2023-10-09T05:08:43.413 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep sparse1 2023-10-09T05:08:43.413 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-09T05:08:43.451 INFO:tasks.workunit.client.0.smithi029.stdout:sparse1 10 MiB 2 2023-10-09T05:08:43.452 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse1 2023-10-09T05:08:43.453 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse1 2023-10-09T05:08:43.453 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse1 2023-10-09T05:08:43.453 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:43.453 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:43.492 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.95a3b7be4950 2023-10-09T05:08:43.493 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.95a3b7be4950 2023-10-09T05:08:43.493 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.95a3b7be4950\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:43.493 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse1 2023-10-09T05:08:43.494 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse1 2023-10-09T05:08:43.494 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:43.494 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse1 2023-10-09T05:08:43.494 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:43.494 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:43.530 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:43.531 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:43.531 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:43.531 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:43.531 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:43.563 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects=1 2023-10-09T05:08:43.564 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 1 2023-10-09T05:08:43.564 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T05:08:43.564 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export sparse1 /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:43.604 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:43.609 INFO:tasks.workunit.client.0.smithi029.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_715438/sparse1 /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:43.609 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp -l /tmp/rbd_import_export_715438/sparse1 /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:43.617 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse1 --format %b 2023-10-09T05:08:43.618 INFO:tasks.workunit.client.0.smithi029.stderr:+ origsize=2048 2023-10-09T05:08:43.618 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse1.out --format %b 2023-10-09T05:08:43.620 INFO:tasks.workunit.client.0.smithi029.stderr:+ exportsize=2048 2023-10-09T05:08:43.620 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:43.620 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:43.620 INFO:tasks.workunit.client.0.smithi029.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:08:43.621 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/sparse1.out 2023-10-09T05:08:43.621 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse1 2023-10-09T05:08:43.710 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T05:08:43.713 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_715438/sparse2 oflag=append conv=notrunc 2023-10-09T05:08:43.724 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records in 2023-10-09T05:08:43.724 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records out 2023-10-09T05:08:43.725 INFO:tasks.workunit.client.0.smithi029.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.010218 s, 205 MB/s 2023-10-09T05:08:43.726 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-09T05:08:43.833 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:43.833 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:43.834 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:08:43.836 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd ls -l 2023-10-09T05:08:43.836 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep sparse2 2023-10-09T05:08:43.837 INFO:tasks.workunit.client.0.smithi029.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-09T05:08:43.873 INFO:tasks.workunit.client.0.smithi029.stdout:sparse2 4 MiB 2 2023-10-09T05:08:43.873 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse2 2023-10-09T05:08:43.874 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse2 2023-10-09T05:08:43.874 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse2 2023-10-09T05:08:43.874 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:43.874 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:43.892 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: pgmap v2285: 83 pgs: 83 active+clean; 598 KiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 134 KiB/s rd, 16 KiB/s wr, 191 op/s 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: failed to encode map e724 with expected crc 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: failed to encode map e724 with expected crc 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2005172563' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: osdmap e724: 8 total, 8 up, 7 in 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: failed to encode map e724 with expected crc 2023-10-09T05:08:43.894 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: pgmap v2285: 83 pgs: 83 active+clean; 598 KiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 134 KiB/s rd, 16 KiB/s wr, 191 op/s 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: failed to encode map e724 with expected crc 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: failed to encode map e724 with expected crc 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2005172563' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: osdmap e724: 8 total, 8 up, 7 in 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: failed to encode map e724 with expected crc 2023-10-09T05:08:43.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:43.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:43.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:43.918 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.95d09e954843 2023-10-09T05:08:43.919 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.95d09e954843 2023-10-09T05:08:43.919 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse2 2023-10-09T05:08:43.919 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse2 2023-10-09T05:08:43.919 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.95d09e954843\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:43.919 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:43.921 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse2 2023-10-09T05:08:43.921 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:43.921 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:43.962 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:43.962 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:43.963 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:43.963 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:43.963 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:43.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 2023-10-09T05:08:43.998 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects='0 2023-10-09T05:08:43.999 INFO:tasks.workunit.client.0.smithi029.stderr:2 2023-10-09T05:08:43.999 INFO:tasks.workunit.client.0.smithi029.stderr:3' 2023-10-09T05:08:43.999 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 0 2 3 2023-10-09T05:08:43.999 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-09T05:08:43.999 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export sparse2 /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: pgmap v2285: 83 pgs: 83 active+clean; 598 KiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 134 KiB/s rd, 16 KiB/s wr, 191 op/s 2023-10-09T05:08:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-09T05:08:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: failed to encode map e724 with expected crc 2023-10-09T05:08:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: failed to encode map e724 with expected crc 2023-10-09T05:08:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2005172563' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:08:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: osdmap e724: 8 total, 8 up, 7 in 2023-10-09T05:08:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: failed to encode map e724 with expected crc 2023-10-09T05:08:44.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:44.036 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:44.053 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-09T05:08:44.057 INFO:tasks.workunit.client.0.smithi029.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_715438/sparse2 /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:44.057 INFO:tasks.workunit.client.0.smithi029.stderr:+ cmp -l /tmp/rbd_import_export_715438/sparse2 /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:44.062 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse2 --format %b 2023-10-09T05:08:44.063 INFO:tasks.workunit.client.0.smithi029.stderr:+ origsize=6144 2023-10-09T05:08:44.063 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse2.out --format %b 2023-10-09T05:08:44.064 INFO:tasks.workunit.client.0.smithi029.stderr:+ exportsize=6144 2023-10-09T05:08:44.064 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:44.065 INFO:tasks.workunit.client.0.smithi029.stderr:+ difference=0 2023-10-09T05:08:44.065 INFO:tasks.workunit.client.0.smithi029.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-09T05:08:44.065 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/sparse2.out 2023-10-09T05:08:44.065 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse2 2023-10-09T05:08:44.155 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-09T05:08:44.158 INFO:tasks.workunit.client.0.smithi029.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-09T05:08:44.159 INFO:tasks.workunit.client.0.smithi029.stdout:partially-sparse file imports to partially-sparse image 2023-10-09T05:08:44.159 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_715438/sparse1 sparse 2023-10-09T05:08:44.191 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-09T05:08:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:44.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.0' 2023-10-09T05:08:44.226 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-09T05:08:44.226 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:44.226 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --order is deprecated, use --object-size 2023-10-09T05:08:44.229 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse 2023-10-09T05:08:44.230 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse 2023-10-09T05:08:44.230 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse 2023-10-09T05:08:44.230 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:44.230 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:44.265 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.96038832e614 2023-10-09T05:08:44.266 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.96038832e614 2023-10-09T05:08:44.266 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse 2023-10-09T05:08:44.267 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.96038832e614\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:44.267 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse 2023-10-09T05:08:44.267 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:44.267 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse 2023-10-09T05:08:44.267 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:44.267 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:44.298 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 2023-10-09T05:08:44.323 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:44.367 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-09T05:08:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:44.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.6' 2023-10-09T05:08:44.389 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:08:44.395 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:44.395 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:44.395 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:44.396 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:44.396 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:44.429 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects=1 2023-10-09T05:08:44.429 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 1 2023-10-09T05:08:44.429 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-09T05:08:44.429 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse 2023-10-09T05:08:44.475 DEBUG:teuthology.orchestra.run.smithi029:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 2023-10-09T05:08:44.501 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:08:44.504 INFO:teuthology.orchestra.run.smithi029.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:44.550 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-09T05:08:44.553 INFO:tasks.workunit.client.0.smithi029.stdout:zeros import through stdin to sparse image 2023-10-09T05:08:44.553 INFO:tasks.workunit.client.0.smithi029.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-09T05:08:44.553 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-09T05:08:44.554 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd import --new-format - sparse 2023-10-09T05:08:44.600 INFO:tasks.workunit.client.0.smithi029.stderr:4+0 records in 2023-10-09T05:08:44.601 INFO:tasks.workunit.client.0.smithi029.stderr:4+0 records out 2023-10-09T05:08:44.601 INFO:tasks.workunit.client.0.smithi029.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0462853 s, 90.6 MB/s 2023-10-09T05:08:44.617 INFO:tasks.workunit.client.0.smithi029.stderr: Importing image: 100% complete...done. 2023-10-09T05:08:44.617 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:44.621 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse 2023-10-09T05:08:44.621 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse 2023-10-09T05:08:44.621 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse 2023-10-09T05:08:44.621 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:44.625 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:44.659 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.bcb4a30beaee 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.bcb4a30beaee 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.bcb4a30beaee\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse 2023-10-09T05:08:44.660 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:44.661 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:44.697 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:44.697 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:44.697 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:44.697 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:44.697 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:44.736 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects= 2023-10-09T05:08:44.736 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 2023-10-09T05:08:44.737 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-09T05:08:44.738 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse 2023-10-09T05:08:44.826 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:44.828 INFO:tasks.workunit.client.0.smithi029.stderr:+ echo 'zeros export to sparse file' 2023-10-09T05:08:44.829 INFO:tasks.workunit.client.0.smithi029.stdout:zeros export to sparse file 2023-10-09T05:08:44.829 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd create --new-format sparse --size 4 2023-10-09T05:08:44.868 INFO:tasks.workunit.client.0.smithi029.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-09T05:08:44.871 INFO:tasks.workunit.client.0.smithi029.stderr:++ rbd info sparse 2023-10-09T05:08:44.871 INFO:tasks.workunit.client.0.smithi029.stderr:++ grep block_name_prefix 2023-10-09T05:08:44.871 INFO:tasks.workunit.client.0.smithi029.stderr:++ awk '{print $NF;}' 2023-10-09T05:08:44.907 INFO:tasks.workunit.client.0.smithi029.stderr:+ prefix=rbd_data.bcd5b41e9b33 2023-10-09T05:08:44.908 INFO:tasks.workunit.client.0.smithi029.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-09T05:08:44.908 INFO:tasks.workunit.client.0.smithi029.stderr:++ get_image_data_pool sparse 2023-10-09T05:08:44.909 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse 2023-10-09T05:08:44.909 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse 2023-10-09T05:08:44.909 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep 'data_pool: ' 2023-10-09T05:08:44.909 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk -F: '{ print $NF }' 2023-10-09T05:08:44.945 INFO:tasks.workunit.client.0.smithi029.stderr:++ data_pool= 2023-10-09T05:08:44.945 INFO:tasks.workunit.client.0.smithi029.stderr:++ '[' -z ']' 2023-10-09T05:08:44.945 INFO:tasks.workunit.client.0.smithi029.stderr:++ data_pool=rbd 2023-10-09T05:08:44.945 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo rbd 2023-10-09T05:08:44.945 INFO:tasks.workunit.client.0.smithi029.stderr:+ rados -p rbd put rbd_data.bcd5b41e9b33.000000000000 - 2023-10-09T05:08:44.975 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records in 2023-10-09T05:08:44.976 INFO:tasks.workunit.client.0.smithi029.stderr:1+0 records out 2023-10-09T05:08:44.976 INFO:tasks.workunit.client.0.smithi029.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0671251 s, 62.5 MB/s 2023-10-09T05:08:44.992 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects sparse 2023-10-09T05:08:44.992 INFO:tasks.workunit.client.0.smithi029.stderr:++ image=sparse 2023-10-09T05:08:44.992 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rbd info sparse 2023-10-09T05:08:44.992 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep block_name_prefix 2023-10-09T05:08:44.993 INFO:tasks.workunit.client.0.smithi029.stderr:+++ awk '{print $NF;}' 2023-10-09T05:08:45.029 INFO:tasks.workunit.client.0.smithi029.stderr:++ prefix=rbd_data.bcd5b41e9b33 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:+++ grep rbd_data.bcd5b41e9b33 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:++++ get_image_data_pool sparse 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:++++ image=sparse 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sed -e 's/rbd_data.bcd5b41e9b33\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:+++ sort -u 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ rbd info sparse 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ grep 'data_pool: ' 2023-10-09T05:08:45.030 INFO:tasks.workunit.client.0.smithi029.stderr:+++++ awk -F: '{ print $NF }' 2023-10-09T05:08:45.066 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool= 2023-10-09T05:08:45.066 INFO:tasks.workunit.client.0.smithi029.stderr:++++ '[' -z ']' 2023-10-09T05:08:45.067 INFO:tasks.workunit.client.0.smithi029.stderr:++++ data_pool=rbd 2023-10-09T05:08:45.067 INFO:tasks.workunit.client.0.smithi029.stderr:++++ echo rbd 2023-10-09T05:08:45.067 INFO:tasks.workunit.client.0.smithi029.stderr:+++ rados ls -p rbd 2023-10-09T05:08:45.100 INFO:tasks.workunit.client.0.smithi029.stderr:++ objects=0 2023-10-09T05:08:45.101 INFO:tasks.workunit.client.0.smithi029.stderr:++ echo 0 2023-10-09T05:08:45.101 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-09T05:08:45.101 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/sparse 2023-10-09T05:08:45.101 INFO:tasks.workunit.client.0.smithi029.stderr:rm: cannot remove '/tmp/rbd_import_export_715438/sparse': No such file or directory 2023-10-09T05:08:45.101 INFO:tasks.workunit.client.0.smithi029.stderr:+ true 2023-10-09T05:08:45.102 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd export sparse /tmp/rbd_import_export_715438/sparse 2023-10-09T05:08:45.140 INFO:tasks.workunit.client.0.smithi029.stderr: Exporting image: 100% complete...done. 2023-10-09T05:08:45.144 INFO:tasks.workunit.client.0.smithi029.stderr:++ stat /tmp/rbd_import_export_715438/sparse --format=%b 2023-10-09T05:08:45.145 INFO:tasks.workunit.client.0.smithi029.stderr:+ '[' 0 = 0 ']' 2023-10-09T05:08:45.145 INFO:tasks.workunit.client.0.smithi029.stderr:+ rbd rm sparse 2023-10-09T05:08:45.238 INFO:tasks.workunit.client.0.smithi029.stderr: Removing image: 100% complete...done. 2023-10-09T05:08:45.241 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm /tmp/rbd_import_export_715438/sparse /tmp/rbd_import_export_715438/sparse1 /tmp/rbd_import_export_715438/sparse2 /tmp/rbd_import_export_715438/sparse3 2023-10-09T05:08:45.244 INFO:tasks.workunit.client.0.smithi029.stderr:rm: cannot remove '/tmp/rbd_import_export_715438/sparse3': No such file or directory 2023-10-09T05:08:45.244 INFO:tasks.workunit.client.0.smithi029.stderr:+ true 2023-10-09T05:08:45.244 INFO:tasks.workunit.client.0.smithi029.stderr:+ echo OK 2023-10-09T05:08:45.245 INFO:tasks.workunit.client.0.smithi029.stdout:OK 2023-10-09T05:08:45.246 INFO:tasks.workunit.client.0.smithi029.stderr:+ rm -rf /tmp/rbd_import_export_715438 2023-10-09T05:08:45.246 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-09T05:08:45.246 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-09T05:08:45.253 INFO:teuthology.orchestra.run.smithi029.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-09T05:08:45.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:45.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi029 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.2' 2023-10-09T05:08:45.293 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-09T05:08:45.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-09T05:08:45.359 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 2023-10-09T05:08:45.386 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:45.427 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-09T05:08:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:45.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.5' 2023-10-09T05:08:45.535 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 2023-10-09T05:08:45.561 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:45.564 DEBUG:teuthology.parallel:result is None 2023-10-09T05:08:45.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:08:45.592 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:08:45.593 DEBUG:teuthology.orchestra.run.smithi029:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:45.603 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-09T05:08:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:45.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.7' 2023-10-09T05:08:45.650 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:08:45.650 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-09T05:08:45.650 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-09T05:08:45.650 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-09T05:08:45.651 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-10-09T05:08:45.651 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-09T05:08:45.651 DEBUG:teuthology.orchestra.run.smithi029:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:45.708 DEBUG:teuthology.orchestra.run.smithi177:> sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 2023-10-09T05:08:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:08:45.710 INFO:teuthology.orchestra.run.smithi029.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-09T05:08:45.710 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:45.734 INFO:teuthology.orchestra.run.smithi177.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-09T05:08:45.767 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-09T05:08:45.767 DEBUG:teuthology.orchestra.run.smithi029:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-09T05:08:45.776 INFO:teuthology.orchestra.run.smithi177.stderr:Error: no container with name or ID ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-09T05:08:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-09T05:08:45.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi177 with status 125: 'sudo docker kill -s 1 ceph-486585fa-6658-11ee-8db5-212e2dc638e7-osd.4' 2023-10-09T05:08:45.824 INFO:tasks.thrashosds:joining thrashosds 2023-10-09T05:08:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:45 smithi029 ceph-mon[216903]: pgmap v2287: 83 pgs: 83 active+clean; 1.4 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 470 KiB/s rd, 496 KiB/s wr, 307 op/s 2023-10-09T05:08:45.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:45 smithi029 ceph-mon[216903]: from='client.48257 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:45 smithi029 ceph-mon[223858]: pgmap v2287: 83 pgs: 83 active+clean; 1.4 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 470 KiB/s rd, 496 KiB/s wr, 307 op/s 2023-10-09T05:08:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:45 smithi029 ceph-mon[223858]: from='client.48257 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:45 smithi177 ceph-mon[195797]: pgmap v2287: 83 pgs: 83 active+clean; 1.4 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 470 KiB/s rd, 496 KiB/s wr, 307 op/s 2023-10-09T05:08:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:45 smithi177 ceph-mon[195797]: from='client.48257 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:08:47.286 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-09T05:08:47.292 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0: {} 2023-10-09T05:08:47.312 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1: {} 2023-10-09T05:08:47.332 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2: {} 2023-10-09T05:08:47.352 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3: {} 2023-10-09T05:08:47.372 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4: {} 2023-10-09T05:08:47.391 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5: {} 2023-10-09T05:08:47.411 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6: {} 2023-10-09T05:08:47.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[216903]: pgmap v2288: 83 pgs: 83 active+clean; 2.4 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 510 KiB/s rd, 524 KiB/s wr, 233 op/s 2023-10-09T05:08:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[216903]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[216903]: osdmap e725: 8 total, 8 up, 7 in 2023-10-09T05:08:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[216903]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[216903]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[223858]: pgmap v2288: 83 pgs: 83 active+clean; 2.4 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 510 KiB/s rd, 524 KiB/s wr, 233 op/s 2023-10-09T05:08:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[223858]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[223858]: osdmap e725: 8 total, 8 up, 7 in 2023-10-09T05:08:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[223858]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:47 smithi029 ceph-mon[223858]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.431 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7: {} 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.433 INFO:teuthology.orchestra.run.smithi029.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-09T05:08:47.478 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2287,"stamp":"2023-10-09T05:08:43.926504+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1435535,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":4577,"num_read_kb":6931,"num_write":738,"num_write_kb":3659,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":972444,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2577,"ondisk_log_size":2577,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9257804,"kb_used_data":669636,"kb_used_omap":34,"kb_used_meta":8588061,"kb_avail":646958260,"statfs":{"total":671965249536,"available":662485258240,"internally_reserved":0,"allocated":685707264,"data_stored":668397586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35446,"internal_metadata":8794174858},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":823210,"num_objects":-6,"num_object_clones":0,"num_object_copies":-18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6,"num_whiteouts":0,"num_read":1146,"num_read_kb":2000,"num_write":161,"num_write_kb":2111,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.254696"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":222,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665231+0000","last_change":"2023-10-09T05:03:24.374942+0000","last_active":"2023-10-09T05:08:37.665231+0000","last_peered":"2023-10-09T05:08:37.665231+0000","last_clean":"2023-10-09T05:08:37.665231+0000","last_became_active":"2023-10-09T05:03:24.374645+0000","last_became_peered":"2023-10-09T05:03:24.374645+0000","last_unstale":"2023-10-09T05:08:37.665231+0000","last_undegraded":"2023-10-09T05:08:37.665231+0000","last_fullsized":"2023-10-09T05:08:37.665231+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:26:03.475572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":372,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203417+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:08:41.203417+0000","last_peered":"2023-10-09T05:08:41.203417+0000","last_clean":"2023-10-09T05:08:41.203417+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:08:41.203417+0000","last_undegraded":"2023-10-09T05:08:41.203417+0000","last_fullsized":"2023-10-09T05:08:41.203417+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":149,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141269+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:08:41.141269+0000","last_peered":"2023-10-09T05:08:41.141269+0000","last_clean":"2023-10-09T05:08:41.141269+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:08:41.141269+0000","last_undegraded":"2023-10-09T05:08:41.141269+0000","last_fullsized":"2023-10-09T05:08:41.141269+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":238,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203263+0000","last_change":"2023-10-09T05:03:47.984999+0000","last_active":"2023-10-09T05:08:41.203263+0000","last_peered":"2023-10-09T05:08:41.203263+0000","last_clean":"2023-10-09T05:08:41.203263+0000","last_became_active":"2023-10-09T05:03:47.984692+0000","last_became_peered":"2023-10-09T05:03:47.984692+0000","last_unstale":"2023-10-09T05:08:41.203263+0000","last_undegraded":"2023-10-09T05:08:41.203263+0000","last_fullsized":"2023-10-09T05:08:41.203263+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:19:32.164434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":354,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202998+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:08:41.202998+0000","last_peered":"2023-10-09T05:08:41.202998+0000","last_clean":"2023-10-09T05:08:41.202998+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:08:41.202998+0000","last_undegraded":"2023-10-09T05:08:41.202998+0000","last_fullsized":"2023-10-09T05:08:41.202998+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":250,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.153117+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:08:41.153117+0000","last_peered":"2023-10-09T05:08:41.153117+0000","last_clean":"2023-10-09T05:08:41.153117+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:08:41.153117+0000","last_undegraded":"2023-10-09T05:08:41.153117+0000","last_fullsized":"2023-10-09T05:08:41.153117+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681118+0000","last_change":"2023-10-09T05:03:37.301251+0000","last_active":"2023-10-09T05:08:40.681118+0000","last_peered":"2023-10-09T05:08:40.681118+0000","last_clean":"2023-10-09T05:08:40.681118+0000","last_became_active":"2023-10-09T05:03:37.300770+0000","last_became_peered":"2023-10-09T05:03:37.300770+0000","last_unstale":"2023-10-09T05:08:40.681118+0000","last_undegraded":"2023-10-09T05:08:40.681118+0000","last_fullsized":"2023-10-09T05:08:40.681118+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":633,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:32:59.186161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":346,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202933+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:08:41.202933+0000","last_peered":"2023-10-09T05:08:41.202933+0000","last_clean":"2023-10-09T05:08:41.202933+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:08:41.202933+0000","last_undegraded":"2023-10-09T05:08:41.202933+0000","last_fullsized":"2023-10-09T05:08:41.202933+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.680970+0000","last_change":"2023-10-09T05:03:47.979465+0000","last_active":"2023-10-09T05:08:40.680970+0000","last_peered":"2023-10-09T05:08:40.680970+0000","last_clean":"2023-10-09T05:08:40.680970+0000","last_became_active":"2023-10-09T05:03:47.979140+0000","last_became_peered":"2023-10-09T05:03:47.979140+0000","last_unstale":"2023-10-09T05:08:40.680970+0000","last_undegraded":"2023-10-09T05:08:40.680970+0000","last_fullsized":"2023-10-09T05:08:40.680970+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:35:17.165946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":336,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203986+0000","last_change":"2023-10-09T05:03:47.995745+0000","last_active":"2023-10-09T05:08:41.203986+0000","last_peered":"2023-10-09T05:08:41.203986+0000","last_clean":"2023-10-09T05:08:41.203986+0000","last_became_active":"2023-10-09T05:03:47.995436+0000","last_became_peered":"2023-10-09T05:03:47.995436+0000","last_unstale":"2023-10-09T05:08:41.203986+0000","last_undegraded":"2023-10-09T05:08:41.203986+0000","last_fullsized":"2023-10-09T05:08:41.203986+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:00:27.397757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":406,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203336+0000","last_change":"2023-10-09T05:07:23.305378+0000","last_active":"2023-10-09T05:08:41.203336+0000","last_peered":"2023-10-09T05:08:41.203336+0000","last_clean":"2023-10-09T05:08:41.203336+0000","last_became_active":"2023-10-09T05:03:49.707234+0000","last_became_peered":"2023-10-09T05:03:49.707234+0000","last_unstale":"2023-10-09T05:08:41.203336+0000","last_undegraded":"2023-10-09T05:08:41.203336+0000","last_fullsized":"2023-10-09T05:08:41.203336+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:13.650000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":990,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":149,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141588+0000","last_change":"2023-10-09T05:02:31.009096+0000","last_active":"2023-10-09T05:08:41.141588+0000","last_peered":"2023-10-09T05:08:41.141588+0000","last_clean":"2023-10-09T05:08:41.141588+0000","last_became_active":"2023-10-09T05:02:31.008772+0000","last_became_peered":"2023-10-09T05:02:31.008772+0000","last_unstale":"2023-10-09T05:08:41.141588+0000","last_undegraded":"2023-10-09T05:08:41.141588+0000","last_fullsized":"2023-10-09T05:08:41.141588+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:45:42.408385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":175,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665755+0000","last_change":"2023-10-09T05:03:24.353621+0000","last_active":"2023-10-09T05:08:37.665755+0000","last_peered":"2023-10-09T05:08:37.665755+0000","last_clean":"2023-10-09T05:08:37.665755+0000","last_became_active":"2023-10-09T05:03:24.353249+0000","last_became_peered":"2023-10-09T05:03:24.353249+0000","last_unstale":"2023-10-09T05:08:37.665755+0000","last_undegraded":"2023-10-09T05:08:37.665755+0000","last_fullsized":"2023-10-09T05:08:37.665755+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:34.708426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":190,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.666478+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:08:37.666478+0000","last_peered":"2023-10-09T05:08:37.666478+0000","last_clean":"2023-10-09T05:08:37.666478+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:08:37.666478+0000","last_undegraded":"2023-10-09T05:08:37.666478+0000","last_fullsized":"2023-10-09T05:08:37.666478+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":196,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.666431+0000","last_change":"2023-10-09T05:03:24.351292+0000","last_active":"2023-10-09T05:08:37.666431+0000","last_peered":"2023-10-09T05:08:37.666431+0000","last_clean":"2023-10-09T05:08:37.666431+0000","last_became_active":"2023-10-09T05:03:24.350958+0000","last_became_peered":"2023-10-09T05:03:24.350958+0000","last_unstale":"2023-10-09T05:08:37.666431+0000","last_undegraded":"2023-10-09T05:08:37.666431+0000","last_fullsized":"2023-10-09T05:08:37.666431+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:19:57.262945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":308,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203327+0000","last_change":"2023-10-09T05:02:19.765642+0000","last_active":"2023-10-09T05:08:41.203327+0000","last_peered":"2023-10-09T05:08:41.203327+0000","last_clean":"2023-10-09T05:08:41.203327+0000","last_became_active":"2023-10-09T05:02:19.765318+0000","last_became_peered":"2023-10-09T05:02:19.765318+0000","last_unstale":"2023-10-09T05:08:41.203327+0000","last_undegraded":"2023-10-09T05:08:41.203327+0000","last_fullsized":"2023-10-09T05:08:41.203327+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:08:17.518402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":228,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202844+0000","last_change":"2023-10-09T05:03:47.989721+0000","last_active":"2023-10-09T05:08:41.202844+0000","last_peered":"2023-10-09T05:08:41.202844+0000","last_clean":"2023-10-09T05:08:41.202844+0000","last_became_active":"2023-10-09T05:03:47.989384+0000","last_became_peered":"2023-10-09T05:03:47.989384+0000","last_unstale":"2023-10-09T05:08:41.202844+0000","last_undegraded":"2023-10-09T05:08:41.202844+0000","last_fullsized":"2023-10-09T05:08:41.202844+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:53:57.716896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":100,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681061+0000","last_change":"2023-10-09T05:03:36.135266+0000","last_active":"2023-10-09T05:08:40.681061+0000","last_peered":"2023-10-09T05:08:40.681061+0000","last_clean":"2023-10-09T05:08:40.681061+0000","last_became_active":"2023-10-09T05:03:36.134797+0000","last_became_peered":"2023-10-09T05:03:36.134797+0000","last_unstale":"2023-10-09T05:08:40.681061+0000","last_undegraded":"2023-10-09T05:08:40.681061+0000","last_fullsized":"2023-10-09T05:08:40.681061+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:37.862094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":159,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664867+0000","last_change":"2023-10-09T05:03:24.348911+0000","last_active":"2023-10-09T05:08:37.664867+0000","last_peered":"2023-10-09T05:08:37.664867+0000","last_clean":"2023-10-09T05:08:37.664867+0000","last_became_active":"2023-10-09T05:03:24.348522+0000","last_became_peered":"2023-10-09T05:03:24.348522+0000","last_unstale":"2023-10-09T05:08:37.664867+0000","last_undegraded":"2023-10-09T05:08:37.664867+0000","last_fullsized":"2023-10-09T05:08:37.664867+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:31.264769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":362,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203503+0000","last_change":"2023-10-09T05:03:47.989854+0000","last_active":"2023-10-09T05:08:41.203503+0000","last_peered":"2023-10-09T05:08:41.203503+0000","last_clean":"2023-10-09T05:08:41.203503+0000","last_became_active":"2023-10-09T05:03:47.989505+0000","last_became_peered":"2023-10-09T05:03:47.989505+0000","last_unstale":"2023-10-09T05:08:41.203503+0000","last_undegraded":"2023-10-09T05:08:41.203503+0000","last_fullsized":"2023-10-09T05:08:41.203503+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:24:32.350376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":358,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.152490+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:08:41.152490+0000","last_peered":"2023-10-09T05:08:41.152490+0000","last_clean":"2023-10-09T05:08:41.152490+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:08:41.152490+0000","last_undegraded":"2023-10-09T05:08:41.152490+0000","last_fullsized":"2023-10-09T05:08:41.152490+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":220,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203564+0000","last_change":"2023-10-09T05:03:47.976417+0000","last_active":"2023-10-09T05:08:41.203564+0000","last_peered":"2023-10-09T05:08:41.203564+0000","last_clean":"2023-10-09T05:08:41.203564+0000","last_became_active":"2023-10-09T05:03:47.976082+0000","last_became_peered":"2023-10-09T05:03:47.976082+0000","last_unstale":"2023-10-09T05:08:41.203564+0000","last_undegraded":"2023-10-09T05:08:41.203564+0000","last_fullsized":"2023-10-09T05:08:41.203564+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:37:38.055988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":249,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665397+0000","last_change":"2023-10-09T05:04:15.720621+0000","last_active":"2023-10-09T05:08:37.665397+0000","last_peered":"2023-10-09T05:08:37.665397+0000","last_clean":"2023-10-09T05:08:37.665397+0000","last_became_active":"2023-10-09T05:03:48.974499+0000","last_became_peered":"2023-10-09T05:03:48.974499+0000","last_unstale":"2023-10-09T05:08:37.665397+0000","last_undegraded":"2023-10-09T05:08:37.665397+0000","last_fullsized":"2023-10-09T05:08:37.665397+0000","mapping_epoch":638,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":648,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:40:31.329657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.680763+0000","last_change":"2023-10-09T05:03:47.989085+0000","last_active":"2023-10-09T05:08:40.680763+0000","last_peered":"2023-10-09T05:08:40.680763+0000","last_clean":"2023-10-09T05:08:40.680763+0000","last_became_active":"2023-10-09T05:03:47.988744+0000","last_became_peered":"2023-10-09T05:03:47.988744+0000","last_unstale":"2023-10-09T05:08:40.680763+0000","last_undegraded":"2023-10-09T05:08:40.680763+0000","last_fullsized":"2023-10-09T05:08:40.680763+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:34:26.968332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":83,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665518+0000","last_change":"2023-10-09T05:07:31.961439+0000","last_active":"2023-10-09T05:08:37.665518+0000","last_peered":"2023-10-09T05:08:37.665518+0000","last_clean":"2023-10-09T05:08:37.665518+0000","last_became_active":"2023-10-09T05:04:20.893620+0000","last_became_peered":"2023-10-09T05:04:20.893620+0000","last_unstale":"2023-10-09T05:08:37.665518+0000","last_undegraded":"2023-10-09T05:08:37.665518+0000","last_fullsized":"2023-10-09T05:08:37.665518+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:50:32.295080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":206,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664781+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:08:37.664781+0000","last_peered":"2023-10-09T05:08:37.664781+0000","last_clean":"2023-10-09T05:08:37.664781+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:08:37.664781+0000","last_undegraded":"2023-10-09T05:08:37.664781+0000","last_fullsized":"2023-10-09T05:08:37.664781+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":104,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.680978+0000","last_change":"2023-10-09T05:03:55.617442+0000","last_active":"2023-10-09T05:08:40.680978+0000","last_peered":"2023-10-09T05:08:40.680978+0000","last_clean":"2023-10-09T05:08:40.680978+0000","last_became_active":"2023-10-09T05:03:36.135198+0000","last_became_peered":"2023-10-09T05:03:36.135198+0000","last_unstale":"2023-10-09T05:08:40.680978+0000","last_undegraded":"2023-10-09T05:08:40.680978+0000","last_fullsized":"2023-10-09T05:08:40.680978+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:57:13.151821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":270,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.152740+0000","last_change":"2023-10-09T05:03:47.992282+0000","last_active":"2023-10-09T05:08:41.152740+0000","last_peered":"2023-10-09T05:08:41.152740+0000","last_clean":"2023-10-09T05:08:41.152740+0000","last_became_active":"2023-10-09T05:03:47.991972+0000","last_became_peered":"2023-10-09T05:03:47.991972+0000","last_unstale":"2023-10-09T05:08:41.152740+0000","last_undegraded":"2023-10-09T05:08:41.152740+0000","last_fullsized":"2023-10-09T05:08:41.152740+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:20:21.127119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":782,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681201+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:08:40.681201+0000","last_peered":"2023-10-09T05:08:40.681201+0000","last_clean":"2023-10-09T05:08:40.681201+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:08:40.681201+0000","last_undegraded":"2023-10-09T05:08:40.681201+0000","last_fullsized":"2023-10-09T05:08:40.681201+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":160,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141798+0000","last_change":"2023-10-09T05:03:25.350381+0000","last_active":"2023-10-09T05:08:41.141798+0000","last_peered":"2023-10-09T05:08:41.141798+0000","last_clean":"2023-10-09T05:08:41.141798+0000","last_became_active":"2023-10-09T05:03:25.350035+0000","last_became_peered":"2023-10-09T05:03:25.350035+0000","last_unstale":"2023-10-09T05:08:41.141798+0000","last_undegraded":"2023-10-09T05:08:41.141798+0000","last_fullsized":"2023-10-09T05:08:41.141798+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:43.299745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1169,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203449+0000","last_change":"2023-10-09T05:07:54.997231+0000","last_active":"2023-10-09T05:08:41.203449+0000","last_peered":"2023-10-09T05:08:41.203449+0000","last_clean":"2023-10-09T05:08:41.203449+0000","last_became_active":"2023-10-09T05:07:53.144659+0000","last_became_peered":"2023-10-09T05:07:53.144659+0000","last_unstale":"2023-10-09T05:08:41.203449+0000","last_undegraded":"2023-10-09T05:08:41.203449+0000","last_fullsized":"2023-10-09T05:08:41.203449+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:54.997107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:54.997107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:08:13.843255+0000","scrub_duration":0.0021928569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":329,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202835+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:08:41.202835+0000","last_peered":"2023-10-09T05:08:41.202835+0000","last_clean":"2023-10-09T05:08:41.202835+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:08:41.202835+0000","last_undegraded":"2023-10-09T05:08:41.202835+0000","last_fullsized":"2023-10-09T05:08:41.202835+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665676+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:08:37.665676+0000","last_peered":"2023-10-09T05:08:37.665676+0000","last_clean":"2023-10-09T05:08:37.665676+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:08:37.665676+0000","last_undegraded":"2023-10-09T05:08:37.665676+0000","last_fullsized":"2023-10-09T05:08:37.665676+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1151,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.680654+0000","last_change":"2023-10-09T05:07:53.145382+0000","last_active":"2023-10-09T05:08:40.680654+0000","last_peered":"2023-10-09T05:08:40.680654+0000","last_clean":"2023-10-09T05:08:40.680654+0000","last_became_active":"2023-10-09T05:07:53.144898+0000","last_became_peered":"2023-10-09T05:07:53.144898+0000","last_unstale":"2023-10-09T05:08:40.680654+0000","last_undegraded":"2023-10-09T05:08:40.680654+0000","last_fullsized":"2023-10-09T05:08:40.680654+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:48.624604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:48.624604+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":267,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.152697+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:08:41.152697+0000","last_peered":"2023-10-09T05:08:41.152697+0000","last_clean":"2023-10-09T05:08:41.152697+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:08:41.152697+0000","last_undegraded":"2023-10-09T05:08:41.152697+0000","last_fullsized":"2023-10-09T05:08:41.152697+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665899+0000","last_change":"2023-10-09T05:04:20.884264+0000","last_active":"2023-10-09T05:08:37.665899+0000","last_peered":"2023-10-09T05:08:37.665899+0000","last_clean":"2023-10-09T05:08:37.665899+0000","last_became_active":"2023-10-09T05:04:20.883922+0000","last_became_peered":"2023-10-09T05:04:20.883922+0000","last_unstale":"2023-10-09T05:08:37.665899+0000","last_undegraded":"2023-10-09T05:08:37.665899+0000","last_fullsized":"2023-10-09T05:08:37.665899+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:57:24.189465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664825+0000","last_change":"2023-10-09T05:03:24.338996+0000","last_active":"2023-10-09T05:08:37.664825+0000","last_peered":"2023-10-09T05:08:37.664825+0000","last_clean":"2023-10-09T05:08:37.664825+0000","last_became_active":"2023-10-09T05:03:24.338551+0000","last_became_peered":"2023-10-09T05:03:24.338551+0000","last_unstale":"2023-10-09T05:08:37.664825+0000","last_undegraded":"2023-10-09T05:08:37.664825+0000","last_fullsized":"2023-10-09T05:08:37.664825+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:51:54.127202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":193,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665693+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:08:37.665693+0000","last_peered":"2023-10-09T05:08:37.665693+0000","last_clean":"2023-10-09T05:08:37.665693+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:08:37.665693+0000","last_undegraded":"2023-10-09T05:08:37.665693+0000","last_fullsized":"2023-10-09T05:08:37.665693+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1176,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681875+0000","last_change":"2023-10-09T05:08:05.173567+0000","last_active":"2023-10-09T05:08:40.681875+0000","last_peered":"2023-10-09T05:08:40.681875+0000","last_clean":"2023-10-09T05:08:40.681875+0000","last_became_active":"2023-10-09T05:07:53.145000+0000","last_became_peered":"2023-10-09T05:07:53.145000+0000","last_unstale":"2023-10-09T05:08:40.681875+0000","last_undegraded":"2023-10-09T05:08:40.681875+0000","last_fullsized":"2023-10-09T05:08:40.681875+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:05.173448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:05.173448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:00:52.691557+0000","scrub_duration":0.001419465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202863+0000","last_change":"2023-10-09T05:03:47.972812+0000","last_active":"2023-10-09T05:08:41.202863+0000","last_peered":"2023-10-09T05:08:41.202863+0000","last_clean":"2023-10-09T05:08:41.202863+0000","last_became_active":"2023-10-09T05:03:47.972475+0000","last_became_peered":"2023-10-09T05:03:47.972475+0000","last_unstale":"2023-10-09T05:08:41.202863+0000","last_undegraded":"2023-10-09T05:08:41.202863+0000","last_fullsized":"2023-10-09T05:08:41.202863+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:39:59.046745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":152,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141469+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:08:41.141469+0000","last_peered":"2023-10-09T05:08:41.141469+0000","last_clean":"2023-10-09T05:08:41.141469+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:08:41.141469+0000","last_undegraded":"2023-10-09T05:08:41.141469+0000","last_fullsized":"2023-10-09T05:08:41.141469+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1357,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.153097+0000","last_change":"2023-10-09T05:07:56.681356+0000","last_active":"2023-10-09T05:08:41.153097+0000","last_peered":"2023-10-09T05:08:41.153097+0000","last_clean":"2023-10-09T05:08:41.153097+0000","last_became_active":"2023-10-09T05:07:52.147126+0000","last_became_peered":"2023-10-09T05:07:52.147126+0000","last_unstale":"2023-10-09T05:08:41.153097+0000","last_undegraded":"2023-10-09T05:08:41.153097+0000","last_fullsized":"2023-10-09T05:08:41.153097+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:56.681273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.681273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:25:27.660502+0000","scrub_duration":0.001633844,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"724'103","reported_seq":346,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:42.925214+0000","last_change":"2023-10-09T05:08:41.142707+0000","last_active":"2023-10-09T05:08:42.925214+0000","last_peered":"2023-10-09T05:08:42.925214+0000","last_clean":"2023-10-09T05:08:42.925214+0000","last_became_active":"2023-10-09T05:02:19.777832+0000","last_became_peered":"2023-10-09T05:02:19.777832+0000","last_unstale":"2023-10-09T05:08:42.925214+0000","last_undegraded":"2023-10-09T05:08:42.925214+0000","last_fullsized":"2023-10-09T05:08:42.925214+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:08:02.085770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047247800000000003,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":293,"num_read_kb":637,"num_write":124,"num_write_kb":259,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"3.5","version":"0'0","reported_seq":100,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681126+0000","last_change":"2023-10-09T05:03:35.985514+0000","last_active":"2023-10-09T05:08:40.681126+0000","last_peered":"2023-10-09T05:08:40.681126+0000","last_clean":"2023-10-09T05:08:40.681126+0000","last_became_active":"2023-10-09T05:03:35.985181+0000","last_became_peered":"2023-10-09T05:03:35.985181+0000","last_unstale":"2023-10-09T05:08:40.681126+0000","last_undegraded":"2023-10-09T05:08:40.681126+0000","last_fullsized":"2023-10-09T05:08:40.681126+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:14:05.074736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":159,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141506+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:08:41.141506+0000","last_peered":"2023-10-09T05:08:41.141506+0000","last_clean":"2023-10-09T05:08:41.141506+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:08:41.141506+0000","last_undegraded":"2023-10-09T05:08:41.141506+0000","last_fullsized":"2023-10-09T05:08:41.141506+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1165,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.680673+0000","last_change":"2023-10-09T05:07:59.084272+0000","last_active":"2023-10-09T05:08:40.680673+0000","last_peered":"2023-10-09T05:08:40.680673+0000","last_clean":"2023-10-09T05:08:40.680673+0000","last_became_active":"2023-10-09T05:07:53.144574+0000","last_became_peered":"2023-10-09T05:07:53.144574+0000","last_unstale":"2023-10-09T05:08:40.680673+0000","last_undegraded":"2023-10-09T05:08:40.680673+0000","last_fullsized":"2023-10-09T05:08:40.680673+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:59.084186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:59.084186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:41:02.285887+0000","scrub_duration":0.001940289,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"723'66","reported_seq":1198,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.351208+0000","last_change":"2023-10-09T05:08:41.151543+0000","last_active":"2023-10-09T05:08:41.351208+0000","last_peered":"2023-10-09T05:08:41.351208+0000","last_clean":"2023-10-09T05:08:41.351208+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:08:41.351208+0000","last_undegraded":"2023-10-09T05:08:41.351208+0000","last_fullsized":"2023-10-09T05:08:41.351208+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":66,"log_dups_size":0,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095775700000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":616,"num_read_kb":1372,"num_write":63,"num_write_kb":455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"3.3","version":"0'0","reported_seq":185,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.666317+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:08:37.666317+0000","last_peered":"2023-10-09T05:08:37.666317+0000","last_clean":"2023-10-09T05:08:37.666317+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:08:37.666317+0000","last_undegraded":"2023-10-09T05:08:37.666317+0000","last_fullsized":"2023-10-09T05:08:37.666317+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":30,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.680887+0000","last_change":"2023-10-09T05:07:59.843178+0000","last_active":"2023-10-09T05:08:40.680887+0000","last_peered":"2023-10-09T05:08:40.680887+0000","last_clean":"2023-10-09T05:08:40.680887+0000","last_became_active":"2023-10-09T05:07:59.842875+0000","last_became_peered":"2023-10-09T05:07:59.842875+0000","last_unstale":"2023-10-09T05:08:40.680887+0000","last_undegraded":"2023-10-09T05:08:40.680887+0000","last_fullsized":"2023-10-09T05:08:40.680887+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:54.404519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1174,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664529+0000","last_change":"2023-10-09T05:07:54.675526+0000","last_active":"2023-10-09T05:08:37.664529+0000","last_peered":"2023-10-09T05:08:37.664529+0000","last_clean":"2023-10-09T05:08:37.664529+0000","last_became_active":"2023-10-09T05:07:52.148044+0000","last_became_peered":"2023-10-09T05:07:52.148044+0000","last_unstale":"2023-10-09T05:08:37.664529+0000","last_undegraded":"2023-10-09T05:08:37.664529+0000","last_fullsized":"2023-10-09T05:08:37.664529+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:54.675443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:54.675443+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:48:41.957176+0000","scrub_duration":0.001713099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"724'37","reported_seq":366,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:43.866678+0000","last_change":"2023-10-09T05:08:41.204313+0000","last_active":"2023-10-09T05:08:43.866678+0000","last_peered":"2023-10-09T05:08:43.866678+0000","last_clean":"2023-10-09T05:08:43.866678+0000","last_became_active":"2023-10-09T05:02:19.788840+0000","last_became_peered":"2023-10-09T05:02:19.788840+0000","last_unstale":"2023-10-09T05:08:43.866678+0000","last_undegraded":"2023-10-09T05:08:43.866678+0000","last_fullsized":"2023-10-09T05:08:43.866678+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":37,"log_dups_size":0,"ondisk_log_size":37,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:57:35.488345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061087000000000001,"stat_sum":{"num_bytes":16,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":139,"num_read_kb":1350,"num_write":35,"num_write_kb":1197,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141121+0000","last_change":"2023-10-09T05:02:19.820082+0000","last_active":"2023-10-09T05:08:41.141121+0000","last_peered":"2023-10-09T05:08:41.141121+0000","last_clean":"2023-10-09T05:08:41.141121+0000","last_became_active":"2023-10-09T05:02:19.819738+0000","last_became_peered":"2023-10-09T05:02:19.819738+0000","last_unstale":"2023-10-09T05:08:41.141121+0000","last_undegraded":"2023-10-09T05:08:41.141121+0000","last_fullsized":"2023-10-09T05:08:41.141121+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:23:20.000845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681007+0000","last_change":"2023-10-09T05:07:59.843239+0000","last_active":"2023-10-09T05:08:40.681007+0000","last_peered":"2023-10-09T05:08:40.681007+0000","last_clean":"2023-10-09T05:08:40.681007+0000","last_became_active":"2023-10-09T05:07:59.842872+0000","last_became_peered":"2023-10-09T05:07:59.842872+0000","last_unstale":"2023-10-09T05:08:40.681007+0000","last_undegraded":"2023-10-09T05:08:40.681007+0000","last_fullsized":"2023-10-09T05:08:40.681007+0000","mapping_epoch":702,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":703,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:25:02.967485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"723'122","reported_seq":527,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:42.335687+0000","last_change":"2023-10-09T05:08:41.150760+0000","last_active":"2023-10-09T05:08:42.335687+0000","last_peered":"2023-10-09T05:08:42.335687+0000","last_clean":"2023-10-09T05:08:42.335687+0000","last_became_active":"2023-10-09T05:03:47.985043+0000","last_became_peered":"2023-10-09T05:03:47.985043+0000","last_unstale":"2023-10-09T05:08:42.335687+0000","last_undegraded":"2023-10-09T05:08:42.335687+0000","last_fullsized":"2023-10-09T05:08:42.335687+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:31:49.993926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040793699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":632,"num_read_kb":563,"num_write":98,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"1.0","version":"82'161","reported_seq":1337,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.152406+0000","last_change":"2023-10-09T05:07:50.678818+0000","last_active":"2023-10-09T05:08:41.152406+0000","last_peered":"2023-10-09T05:08:41.152406+0000","last_clean":"2023-10-09T05:08:41.152406+0000","last_became_active":"2023-10-09T05:03:51.188988+0000","last_became_peered":"2023-10-09T05:03:51.188988+0000","last_unstale":"2023-10-09T05:08:41.152406+0000","last_undegraded":"2023-10-09T05:08:41.152406+0000","last_fullsized":"2023-10-09T05:08:41.152406+0000","mapping_epoch":640,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":641,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:50.678749+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:50.678749+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:27:08.472240+0000","scrub_duration":0.0015712409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202921+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:08:41.202921+0000","last_peered":"2023-10-09T05:08:41.202921+0000","last_clean":"2023-10-09T05:08:41.202921+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:08:41.202921+0000","last_undegraded":"2023-10-09T05:08:41.202921+0000","last_fullsized":"2023-10-09T05:08:41.202921+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":244,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.153123+0000","last_change":"2023-10-09T05:03:24.351734+0000","last_active":"2023-10-09T05:08:41.153123+0000","last_peered":"2023-10-09T05:08:41.153123+0000","last_clean":"2023-10-09T05:08:41.153123+0000","last_became_active":"2023-10-09T05:03:24.351444+0000","last_became_peered":"2023-10-09T05:03:24.351444+0000","last_unstale":"2023-10-09T05:08:41.153123+0000","last_undegraded":"2023-10-09T05:08:41.153123+0000","last_fullsized":"2023-10-09T05:08:41.153123+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:49:43.142494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1184,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.141587+0000","last_change":"2023-10-09T05:07:58.306714+0000","last_active":"2023-10-09T05:08:41.141587+0000","last_peered":"2023-10-09T05:08:41.141587+0000","last_clean":"2023-10-09T05:08:41.141587+0000","last_became_active":"2023-10-09T05:07:52.143205+0000","last_became_peered":"2023-10-09T05:07:52.143205+0000","last_unstale":"2023-10-09T05:08:41.141587+0000","last_undegraded":"2023-10-09T05:08:41.141587+0000","last_fullsized":"2023-10-09T05:08:41.141587+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:58.306600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:58.306600+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.900624+0000","scrub_duration":0.001694367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"724'60","reported_seq":428,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:42.828477+0000","last_change":"2023-10-09T05:08:41.141763+0000","last_active":"2023-10-09T05:08:42.828477+0000","last_peered":"2023-10-09T05:08:42.828477+0000","last_clean":"2023-10-09T05:08:42.828477+0000","last_became_active":"2023-10-09T05:02:19.755767+0000","last_became_peered":"2023-10-09T05:02:19.755767+0000","last_unstale":"2023-10-09T05:08:42.828477+0000","last_undegraded":"2023-10-09T05:08:42.828477+0000","last_fullsized":"2023-10-09T05:08:42.828477+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":60,"log_dups_size":0,"ondisk_log_size":60,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:49:28.236939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042191800000000002,"stat_sum":{"num_bytes":1048576,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":461,"num_read_kb":729,"num_write":39,"num_write_kb":1200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"3.1","version":"0'0","reported_seq":78,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664662+0000","last_change":"2023-10-09T05:04:20.901801+0000","last_active":"2023-10-09T05:08:37.664662+0000","last_peered":"2023-10-09T05:08:37.664662+0000","last_clean":"2023-10-09T05:08:37.664662+0000","last_became_active":"2023-10-09T05:04:20.901492+0000","last_became_peered":"2023-10-09T05:04:20.901492+0000","last_unstale":"2023-10-09T05:08:37.664662+0000","last_undegraded":"2023-10-09T05:08:37.664662+0000","last_fullsized":"2023-10-09T05:08:37.664662+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:20:11.900690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":174,"reported_epoch":720,"state":"active+clean","last_fresh":"2023-10-09T05:08:38.682725+0000","last_change":"2023-10-09T05:03:47.982641+0000","last_active":"2023-10-09T05:08:38.682725+0000","last_peered":"2023-10-09T05:08:38.682725+0000","last_clean":"2023-10-09T05:08:38.682725+0000","last_became_active":"2023-10-09T05:03:47.982413+0000","last_became_peered":"2023-10-09T05:03:47.982413+0000","last_unstale":"2023-10-09T05:08:38.682725+0000","last_undegraded":"2023-10-09T05:08:38.682725+0000","last_fullsized":"2023-10-09T05:08:38.682725+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:50:23.149220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1351,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.153130+0000","last_change":"2023-10-09T05:07:53.688815+0000","last_active":"2023-10-09T05:08:41.153130+0000","last_peered":"2023-10-09T05:08:41.153130+0000","last_clean":"2023-10-09T05:08:41.153130+0000","last_became_active":"2023-10-09T05:07:52.149137+0000","last_became_peered":"2023-10-09T05:07:52.149137+0000","last_unstale":"2023-10-09T05:08:41.153130+0000","last_undegraded":"2023-10-09T05:08:41.153130+0000","last_fullsized":"2023-10-09T05:08:41.153130+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:53.688730+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:53.688730+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:38.094584+0000","scrub_duration":0.0019408190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"718'19","reported_seq":230,"reported_epoch":720,"state":"active+clean","last_fresh":"2023-10-09T05:08:38.674674+0000","last_change":"2023-10-09T05:08:38.674849+0000","last_active":"2023-10-09T05:08:38.674674+0000","last_peered":"2023-10-09T05:08:38.674674+0000","last_clean":"2023-10-09T05:08:38.674674+0000","last_became_active":"2023-10-09T05:03:24.374443+0000","last_became_peered":"2023-10-09T05:03:24.374443+0000","last_unstale":"2023-10-09T05:08:38.674674+0000","last_undegraded":"2023-10-09T05:08:38.674674+0000","last_fullsized":"2023-10-09T05:08:38.674674+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":19,"log_dups_size":0,"ondisk_log_size":19,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:17:26.116430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001662038,"stat_sum":{"num_bytes":35,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":56,"num_read_kb":59,"num_write":24,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"4","length":"2"},{"start":"8","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:42.685618+0000","last_change":"2023-10-09T05:03:36.637659+0000","last_active":"2023-10-09T05:08:42.685618+0000","last_peered":"2023-10-09T05:08:42.685618+0000","last_clean":"2023-10-09T05:08:42.685618+0000","last_became_active":"2023-10-09T05:03:36.637368+0000","last_became_peered":"2023-10-09T05:03:36.637368+0000","last_unstale":"2023-10-09T05:08:42.685618+0000","last_undegraded":"2023-10-09T05:08:42.685618+0000","last_fullsized":"2023-10-09T05:08:42.685618+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T16:03:45.475473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":100,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681503+0000","last_change":"2023-10-09T05:03:36.665195+0000","last_active":"2023-10-09T05:08:40.681503+0000","last_peered":"2023-10-09T05:08:40.681503+0000","last_clean":"2023-10-09T05:08:40.681503+0000","last_became_active":"2023-10-09T05:03:36.664819+0000","last_became_peered":"2023-10-09T05:03:36.664819+0000","last_unstale":"2023-10-09T05:08:40.681503+0000","last_undegraded":"2023-10-09T05:08:40.681503+0000","last_fullsized":"2023-10-09T05:08:40.681503+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:55:27.009119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1177,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.665855+0000","last_change":"2023-10-09T05:08:00.730177+0000","last_active":"2023-10-09T05:08:37.665855+0000","last_peered":"2023-10-09T05:08:37.665855+0000","last_clean":"2023-10-09T05:08:37.665855+0000","last_became_active":"2023-10-09T05:07:53.144728+0000","last_became_peered":"2023-10-09T05:07:53.144728+0000","last_unstale":"2023-10-09T05:08:37.665855+0000","last_undegraded":"2023-10-09T05:08:37.665855+0000","last_fullsized":"2023-10-09T05:08:37.665855+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:00.730094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:00.730094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:00:30.129973+0000","scrub_duration":0.0018014909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":355,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.203678+0000","last_change":"2023-10-09T05:07:23.309039+0000","last_active":"2023-10-09T05:08:41.203678+0000","last_peered":"2023-10-09T05:08:41.203678+0000","last_clean":"2023-10-09T05:08:41.203678+0000","last_became_active":"2023-10-09T05:03:50.388186+0000","last_became_peered":"2023-10-09T05:03:50.388186+0000","last_unstale":"2023-10-09T05:08:41.203678+0000","last_undegraded":"2023-10-09T05:08:41.203678+0000","last_fullsized":"2023-10-09T05:08:41.203678+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":678,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:12:51.355127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"724'271","reported_seq":1102,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:42.785618+0000","last_change":"2023-10-09T05:08:41.142733+0000","last_active":"2023-10-09T05:08:42.785618+0000","last_peered":"2023-10-09T05:08:42.785618+0000","last_clean":"2023-10-09T05:08:42.785618+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:08:42.785618+0000","last_undegraded":"2023-10-09T05:08:42.785618+0000","last_fullsized":"2023-10-09T05:08:42.785618+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":271,"log_dups_size":0,"ondisk_log_size":271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038981900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1711,"num_read_kb":1450,"num_write":224,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"4.0","version":"0'0","reported_seq":229,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202779+0000","last_change":"2023-10-09T05:03:47.984765+0000","last_active":"2023-10-09T05:08:41.202779+0000","last_peered":"2023-10-09T05:08:41.202779+0000","last_clean":"2023-10-09T05:08:41.202779+0000","last_became_active":"2023-10-09T05:03:47.984383+0000","last_became_peered":"2023-10-09T05:03:47.984383+0000","last_unstale":"2023-10-09T05:08:41.202779+0000","last_undegraded":"2023-10-09T05:08:41.202779+0000","last_fullsized":"2023-10-09T05:08:41.202779+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:37:01.721883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"701'162","reported_seq":1147,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.202839+0000","last_change":"2023-10-09T05:07:56.029382+0000","last_active":"2023-10-09T05:08:41.202839+0000","last_peered":"2023-10-09T05:08:41.202839+0000","last_clean":"2023-10-09T05:08:41.202839+0000","last_became_active":"2023-10-09T05:07:53.144623+0000","last_became_peered":"2023-10-09T05:07:53.144623+0000","last_unstale":"2023-10-09T05:08:41.202839+0000","last_undegraded":"2023-10-09T05:08:41.202839+0000","last_fullsized":"2023-10-09T05:08:41.202839+0000","mapping_epoch":700,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":701,"parent":"0.0","parent_split_bits":3,"last_scrub":"701'162","last_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_deep_scrub":"701'162","last_deep_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.029274+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:26:37.229597+0000","scrub_duration":0.005003141,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"724'72","reported_seq":582,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:43.702913+0000","last_change":"2023-10-09T05:08:41.153613+0000","last_active":"2023-10-09T05:08:43.702913+0000","last_peered":"2023-10-09T05:08:43.702913+0000","last_clean":"2023-10-09T05:08:43.702913+0000","last_became_active":"2023-10-09T05:03:47.997391+0000","last_became_peered":"2023-10-09T05:03:47.997391+0000","last_unstale":"2023-10-09T05:08:43.702913+0000","last_undegraded":"2023-10-09T05:08:43.702913+0000","last_fullsized":"2023-10-09T05:08:43.702913+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:27:40.394703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028602599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":624,"num_read_kb":687,"num_write":48,"num_write_kb":89,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"8","length":"4"}]},{"pgid":"3.6","version":"68'1","reported_seq":159,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.142140+0000","last_change":"2023-10-09T05:03:24.566870+0000","last_active":"2023-10-09T05:08:41.142140+0000","last_peered":"2023-10-09T05:08:41.142140+0000","last_clean":"2023-10-09T05:08:41.142140+0000","last_became_active":"2023-10-09T05:03:24.566540+0000","last_became_peered":"2023-10-09T05:03:24.566540+0000","last_unstale":"2023-10-09T05:08:41.142140+0000","last_undegraded":"2023-10-09T05:08:41.142140+0000","last_fullsized":"2023-10-09T05:08:41.142140+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:39:12.185143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":195,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664738+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:08:37.664738+0000","last_peered":"2023-10-09T05:08:37.664738+0000","last_clean":"2023-10-09T05:08:37.664738+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:08:37.664738+0000","last_undegraded":"2023-10-09T05:08:37.664738+0000","last_fullsized":"2023-10-09T05:08:37.664738+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":185,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.666427+0000","last_change":"2023-10-09T05:03:24.412960+0000","last_active":"2023-10-09T05:08:37.666427+0000","last_peered":"2023-10-09T05:08:37.666427+0000","last_clean":"2023-10-09T05:08:37.666427+0000","last_became_active":"2023-10-09T05:03:24.412581+0000","last_became_peered":"2023-10-09T05:03:24.412581+0000","last_unstale":"2023-10-09T05:08:37.666427+0000","last_undegraded":"2023-10-09T05:08:37.666427+0000","last_fullsized":"2023-10-09T05:08:37.666427+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:17.661976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":140,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.142027+0000","last_change":"2023-10-09T05:02:33.839332+0000","last_active":"2023-10-09T05:08:41.142027+0000","last_peered":"2023-10-09T05:08:41.142027+0000","last_clean":"2023-10-09T05:08:41.142027+0000","last_became_active":"2023-10-09T05:02:33.838635+0000","last_became_peered":"2023-10-09T05:02:33.838635+0000","last_unstale":"2023-10-09T05:08:41.142027+0000","last_undegraded":"2023-10-09T05:08:41.142027+0000","last_fullsized":"2023-10-09T05:08:41.142027+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:39:06.759504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":863,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681266+0000","last_change":"2023-10-09T05:07:31.951921+0000","last_active":"2023-10-09T05:08:40.681266+0000","last_peered":"2023-10-09T05:08:40.681266+0000","last_clean":"2023-10-09T05:08:40.681266+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:08:40.681266+0000","last_undegraded":"2023-10-09T05:08:40.681266+0000","last_fullsized":"2023-10-09T05:08:40.681266+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.666203+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:08:37.666203+0000","last_peered":"2023-10-09T05:08:37.666203+0000","last_clean":"2023-10-09T05:08:37.666203+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:08:37.666203+0000","last_undegraded":"2023-10-09T05:08:37.666203+0000","last_fullsized":"2023-10-09T05:08:37.666203+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":803,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681268+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:08:40.681268+0000","last_peered":"2023-10-09T05:08:40.681268+0000","last_clean":"2023-10-09T05:08:40.681268+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:08:40.681268+0000","last_undegraded":"2023-10-09T05:08:40.681268+0000","last_fullsized":"2023-10-09T05:08:40.681268+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":78,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664994+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:08:37.664994+0000","last_peered":"2023-10-09T05:08:37.664994+0000","last_clean":"2023-10-09T05:08:37.664994+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:08:37.664994+0000","last_undegraded":"2023-10-09T05:08:37.664994+0000","last_fullsized":"2023-10-09T05:08:37.664994+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":346,"reported_epoch":724,"state":"active+clean","last_fresh":"2023-10-09T05:08:42.736638+0000","last_change":"2023-10-09T05:02:19.780582+0000","last_active":"2023-10-09T05:08:42.736638+0000","last_peered":"2023-10-09T05:08:42.736638+0000","last_clean":"2023-10-09T05:08:42.736638+0000","last_became_active":"2023-10-09T05:02:19.780197+0000","last_became_peered":"2023-10-09T05:02:19.780197+0000","last_unstale":"2023-10-09T05:08:42.736638+0000","last_undegraded":"2023-10-09T05:08:42.736638+0000","last_fullsized":"2023-10-09T05:08:42.736638+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:02:00.292221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":353,"reported_epoch":723,"state":"active+clean","last_fresh":"2023-10-09T05:08:41.204064+0000","last_change":"2023-10-09T05:03:36.639073+0000","last_active":"2023-10-09T05:08:41.204064+0000","last_peered":"2023-10-09T05:08:41.204064+0000","last_clean":"2023-10-09T05:08:41.204064+0000","last_became_active":"2023-10-09T05:03:36.638726+0000","last_became_peered":"2023-10-09T05:03:36.638726+0000","last_unstale":"2023-10-09T05:08:41.204064+0000","last_undegraded":"2023-10-09T05:08:41.204064+0000","last_fullsized":"2023-10-09T05:08:41.204064+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:27:58.211669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":102,"reported_epoch":722,"state":"active+clean","last_fresh":"2023-10-09T05:08:40.681178+0000","last_change":"2023-10-09T05:04:36.880386+0000","last_active":"2023-10-09T05:08:40.681178+0000","last_peered":"2023-10-09T05:08:40.681178+0000","last_clean":"2023-10-09T05:08:40.681178+0000","last_became_active":"2023-10-09T05:03:37.300954+0000","last_became_peered":"2023-10-09T05:03:37.300954+0000","last_unstale":"2023-10-09T05:08:40.681178+0000","last_undegraded":"2023-10-09T05:08:40.681178+0000","last_fullsized":"2023-10-09T05:08:40.681178+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:41:52.210997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":196,"reported_epoch":719,"state":"active+clean","last_fresh":"2023-10-09T05:08:37.664479+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:08:37.664479+0000","last_peered":"2023-10-09T05:08:37.664479+0000","last_clean":"2023-10-09T05:08:37.664479+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:08:37.664479+0000","last_undegraded":"2023-10-09T05:08:37.664479+0000","last_fullsized":"2023-10-09T05:08:37.664479+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":17710,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":381350,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":954734,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":1048681,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":4532,"num_read_kb":6847,"num_write":655,"num_write_kb":3411,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3219456,"data_stored":3166504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":750,"ondisk_log_size":750,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234736,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1588876,"kb_used_data":71428,"kb_used_omap":0,"kb_used_meta":1517440,"kb_avail":92156276,"statfs":{"total":95995035648,"available":94368026624,"internally_reserved":0,"allocated":73142272,"data_stored":71361888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1553858560},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":8.4380000000000006,"15min":3.371},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23599999999999999},"max":{"1min":0.74299999999999999,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.90300000000000002,"5min":8.8070000000000004,"15min":3.6389999999999998},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.7190000000000003,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":9.9890000000000008,"15min":4.1050000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26500000000000001},"max":{"1min":4.8730000000000002,"5min":322.31999999999999,"15min":322.31999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":9.907,"15min":4.0529999999999999},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":2.4409999999999998,"5min":322.14800000000002,"15min":322.14800000000002},"last":0.66000000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.8100000000000001,"15min":3.0329999999999999},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.6930000000000001,"5min":11.948,"15min":154.76400000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.8109999999999999,"15min":3.077},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.312},"max":{"1min":3.0310000000000001,"5min":11.968999999999999,"15min":154.80000000000001},"last":0.74099999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:08:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":15.032,"15min":6.2939999999999996},"min":{"1min":0.42499999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.181,"5min":349.863,"15min":349.863},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":14.420999999999999,"15min":6.1269999999999998},"min":{"1min":0.39900000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":4.3150000000000004,"5min":349.67500000000001,"15min":349.67500000000001},"last":0.69999999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:08:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.42299999999999999,"15min":0.39900000000000002},"min":{"1min":0.35399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80200000000000005,"5min":1.145,"15min":1.145},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.52400000000000002,"15min":0.48999999999999999},"min":{"1min":0.33000000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.81299999999999994,"5min":3.0609999999999999,"15min":3.0609999999999999},"last":0.56799999999999995}]},{"osd":5,"last update":"Mon Oct 9 05:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":1.3180000000000001,"15min":1.6319999999999999},"min":{"1min":0.34999999999999998,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":2.7189999999999999,"5min":74.915000000000006,"15min":131.84899999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.67000000000000004,"5min":1.3240000000000001,"15min":1.8180000000000001},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.26000000000000001},"max":{"1min":3.274,"5min":74.972999999999999,"15min":135.74199999999999},"last":0.67100000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:07:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.5,"15min":0.94399999999999995},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":0.88600000000000001,"5min":2.1829999999999998,"15min":57.168999999999997},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.54200000000000004,"15min":1.073},"min":{"1min":0.39500000000000002,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":57.183999999999997},"last":0.60199999999999998}]}]},{"osd":6,"up_from":46,"seq":197568496369,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1522276,"kb_used_data":7572,"kb_used_omap":0,"kb_used_meta":1514688,"kb_avail":92222876,"statfs":{"total":95995035648,"available":94436225024,"internally_reserved":0,"allocated":7753728,"data_stored":5970573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1551040512},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":5.5380000000000003,"5min":5.0940000000000003,"15min":3.383},"min":{"1min":0.252,"5min":0.218,"15min":0.218},"max":{"1min":122.07899999999999,"5min":122.07899999999999,"15min":321.346},"last":0.63500000000000001},{"interface":"front","average":{"1min":6.1070000000000002,"5min":5.2140000000000004,"15min":2.9169999999999998},"min":{"1min":0.26500000000000001,"5min":0.255,"15min":0.255},"max":{"1min":122.107,"5min":122.107,"15min":169.119},"last":0.5}]},{"osd":1,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":2.1429999999999998,"15min":2.153},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.2879999999999998,"5min":43.604999999999997,"15min":90.278999999999996},"last":0.625},{"interface":"front","average":{"1min":0.83499999999999996,"5min":2.274,"15min":2.1440000000000001},"min":{"1min":0.33500000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":3.5310000000000001,"5min":44.697000000000003,"15min":90.414000000000001},"last":0.48899999999999999}]},{"osd":2,"last update":"Mon Oct 9 05:07:48 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.8260000000000001,"15min":2.04},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25600000000000001},"max":{"1min":8.4800000000000004,"5min":11.032999999999999,"15min":133.535},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.6599999999999999,"15min":2.0230000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30599999999999999},"max":{"1min":3.6579999999999999,"5min":8.5920000000000005,"15min":133.51900000000001},"last":0.65100000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":6.1749999999999998,"5min":4.6959999999999997,"15min":3.472},"min":{"1min":0.38900000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":129.499,"5min":129.499,"15min":147.43600000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":6.976,"5min":4.9690000000000003,"15min":3.5680000000000001},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":130.994,"5min":130.994,"15min":147.41999999999999},"last":0.67600000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.88400000000000001,"15min":0.78400000000000003},"min":{"1min":0.33200000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.2280000000000002,"5min":18.186,"15min":18.186},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.077,"5min":1.073,"15min":1.0269999999999999},"min":{"1min":0.374,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.7710000000000008,"5min":17.597000000000001,"15min":17.597000000000001},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:08:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.67800000000000005,"15min":0.85399999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.4350000000000001,"5min":7.9539999999999997,"15min":25.934000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.70399999999999996,"15min":0.84199999999999997},"min":{"1min":0.29799999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":4.1909999999999998,"5min":7.2220000000000004,"15min":16.789000000000001},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:07:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.74299999999999999,"15min":0.80500000000000005},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.30399999999999999},"max":{"1min":2.3879999999999999,"5min":7.5039999999999996,"15min":7.5039999999999996},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.79800000000000004,"15min":0.90600000000000003},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.315},"max":{"1min":3.4009999999999998,"5min":8.3399999999999999,"15min":20.041},"last":0.41399999999999998}]}]},{"osd":1,"up_from":118,"seq":506806141436,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1427052,"kb_used_data":10592,"kb_used_omap":4,"kb_used_meta":1416443,"kb_avail":92318100,"statfs":{"total":95995035648,"available":94533734400,"internally_reserved":0,"allocated":10846208,"data_stored":6989465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1450438542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":5.9059999999999997,"5min":5.7039999999999997,"15min":2.5880000000000001},"min":{"1min":0.23200000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":42.058999999999997,"5min":100.247,"15min":100.247},"last":0.379},{"interface":"front","average":{"1min":31.283000000000001,"5min":11.256,"15min":4.4020000000000001},"min":{"1min":0.318,"5min":0.247,"15min":0.214},"max":{"1min":554.62,"5min":554.62,"15min":554.62},"last":0.40200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:07:52 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":2.0569999999999999,"15min":2.7589999999999999},"min":{"1min":0.23799999999999999,"5min":0.23499999999999999,"15min":0.21099999999999999},"max":{"1min":10.186,"5min":12.460000000000001,"15min":263.089},"last":0.495},{"interface":"front","average":{"1min":28.771999999999998,"5min":8.0389999999999997,"15min":4.7350000000000003},"min":{"1min":0.26000000000000001,"5min":0.252,"15min":0.222},"max":{"1min":554.31200000000001,"5min":554.31200000000001,"15min":554.31200000000001},"last":0.57099999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:08:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":8.1850000000000005,"15min":3.915},"min":{"1min":0.27700000000000002,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":2.3010000000000002,"5min":235.43600000000001,"15min":235.43600000000001},"last":0.434},{"interface":"front","average":{"1min":0.92100000000000004,"5min":12.523,"15min":5.1559999999999997},"min":{"1min":0.32300000000000001,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":4.2119999999999997,"5min":554.84400000000005,"15min":554.84400000000005},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Oct 9 05:08:36 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":6.8120000000000003,"15min":3.1200000000000001},"min":{"1min":0.41099999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.101,"5min":554.44000000000005,"15min":554.44000000000005},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.804,"15min":1.2330000000000001},"min":{"1min":0.44400000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.92000000000000004,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.305,"5min":7.766,"15min":3.3719999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":554.54700000000003,"5min":554.54700000000003,"15min":554.54700000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":5.1310000000000002,"5min":2.6259999999999999,"15min":1.744},"min":{"1min":0.39800000000000002,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":80.390000000000001,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":30.236999999999998,"5min":7.7699999999999996,"15min":3.387},"min":{"1min":0.40400000000000003,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":554.36500000000001,"5min":554.36500000000001,"15min":554.36500000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":5.0069999999999997,"5min":2.5939999999999999,"15min":1.7350000000000001},"min":{"1min":0.41999999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":78.278000000000006,"5min":78.278000000000006,"15min":78.278000000000006},"last":0.59799999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.968,"15min":1.5900000000000001},"min":{"1min":0.48299999999999998,"5min":0.41399999999999998,"15min":0.373},"max":{"1min":6.4850000000000003,"5min":18.899999999999999,"15min":19.664999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.589,"5min":1.6240000000000001,"15min":1.397},"min":{"1min":0.47299999999999998,"5min":0.39900000000000002,"15min":0.378},"max":{"1min":7.0099999999999998,"5min":11.903,"15min":11.903},"last":0.61299999999999999}]}]},{"osd":0,"up_from":91,"seq":390842024519,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1248352,"kb_used_data":20756,"kb_used_omap":5,"kb_used_meta":1227578,"kb_avail":92496800,"statfs":{"total":95995035648,"available":94716723200,"internally_reserved":0,"allocated":21254144,"data_stored":18150911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1257040506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:08:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":3.9220000000000002,"15min":2.6549999999999998},"min":{"1min":0.22600000000000001,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":17.704999999999998,"5min":53.176000000000002,"15min":155.411},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.3090000000000002,"5min":4.3460000000000001,"15min":2.7200000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":17.581,"5min":53.372999999999998,"15min":154.75},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:08:35 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":2.387,"15min":2.0939999999999999},"min":{"1min":0.27100000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.5580000000000001,"5min":20.010999999999999,"15min":53.719999999999999},"last":0.379},{"interface":"front","average":{"1min":0.64300000000000002,"5min":3.1819999999999999,"15min":2.3340000000000001},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.484,"5min":20.239000000000001,"15min":53.363999999999997},"last":0.46600000000000003}]},{"osd":3,"last update":"Mon Oct 9 05:08:04 2023","interfaces":[{"interface":"back","average":{"1min":14.112,"5min":10.515000000000001,"15min":5.5140000000000002},"min":{"1min":0.32100000000000001,"5min":0.30199999999999999,"15min":0.26300000000000001},"max":{"1min":267.63799999999998,"5min":290.89100000000002,"15min":290.89100000000002},"last":0.501},{"interface":"front","average":{"1min":13.535,"5min":10.983000000000001,"15min":5.7149999999999999},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":267.66199999999998,"5min":290.73899999999998,"15min":290.73899999999998},"last":0.42299999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:08:31 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":2.379,"15min":2.0270000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.5569999999999999,"5min":57.360999999999997,"15min":57.360999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":2.194,"15min":1.754},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.84599999999999997,"5min":57.338000000000001,"15min":57.338000000000001},"last":0.55200000000000005}]},{"osd":5,"last update":"Mon Oct 9 05:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":3.9119999999999999,"15min":1.9930000000000001},"min":{"1min":0.35799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.9450000000000003,"5min":106.922,"15min":106.922},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.2400000000000002,"15min":1.8340000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.34899999999999998},"max":{"1min":8.0749999999999993,"5min":106.771,"15min":106.771},"last":0.53300000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":4.6280000000000001,"15min":2.3929999999999998},"min":{"1min":0.38500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.9889999999999999,"5min":112.208,"15min":112.208},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.1120000000000001,"5min":3.9870000000000001,"15min":2.4140000000000001},"min":{"1min":0.375,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":8.0440000000000005,"5min":110.46899999999999,"15min":110.46899999999999},"last":0.64200000000000002}]},{"osd":7,"last update":"Mon Oct 9 05:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":2.044,"15min":1.538},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.39400000000000002},"max":{"1min":8.0079999999999991,"5min":13.787000000000001,"15min":21.658000000000001},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.097,"5min":1.8500000000000001,"15min":1.5109999999999999},"min":{"1min":0.45400000000000001,"5min":0.39900000000000002,"15min":0.38200000000000001},"max":{"1min":7.5419999999999998,"5min":11.842000000000001,"15min":21.68},"last":0.58499999999999996}]}]},{"osd":2,"up_from":0,"seq":949187772798,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":142,"seq":609885356492,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1027004,"kb_used_data":9460,"kb_used_omap":2,"kb_used_meta":1017533,"kb_avail":92718148,"statfs":{"total":95995035648,"available":94943383552,"internally_reserved":0,"allocated":9687040,"data_stored":6463767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1041953845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":4.6500000000000004,"15min":2.0499999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":1.595,"5min":108.631,"15min":108.631},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.81999999999999995,"5min":4.9509999999999996,"15min":2.2709999999999999},"min":{"1min":0.31,"5min":0.247,"15min":0.247},"max":{"1min":6.4020000000000001,"5min":110.012,"15min":110.012},"last":0.28699999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":2.472,"15min":1.516},"min":{"1min":0.27500000000000002,"5min":0.223,"15min":0.221},"max":{"1min":6.1219999999999999,"5min":15.218,"15min":15.218},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.21,"15min":1.419},"min":{"1min":0.29599999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.9509999999999996,"5min":17.544,"15min":26.692},"last":0.433}]},{"osd":2,"last update":"Mon Oct 9 05:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":1.5509999999999999,"15min":2.391},"min":{"1min":0.30499999999999999,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":3.0089999999999999,"5min":11.996,"15min":212.065},"last":0.499},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.756,"15min":2.464},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":5.7549999999999999,"5min":12.987,"15min":211.98400000000001},"last":0.32800000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:08:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.5740000000000001,"15min":1.671},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0409999999999999,"5min":11.193,"15min":11.193},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.3580000000000001,"15min":1.3839999999999999},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.91600000000000004,"5min":62.683999999999997,"15min":62.683999999999997},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":3.0110000000000001,"15min":1.619},"min":{"1min":0.36599999999999999,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":6.9939999999999998,"5min":75.478999999999999,"15min":75.478999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.97599999999999998,"5min":4.0499999999999998,"15min":2.1560000000000001},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.33600000000000002},"max":{"1min":3.9689999999999999,"5min":75.459000000000003,"15min":75.459000000000003},"last":0.46600000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":3.5070000000000001,"15min":1.889},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34799999999999998},"max":{"1min":3.6560000000000001,"5min":79.093999999999994,"15min":79.093999999999994},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.512,"5min":3.5379999999999998,"15min":2.0289999999999999},"min":{"1min":0.44700000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.0060000000000002,"5min":78.555999999999997,"15min":78.555999999999997},"last":0.54200000000000004}]},{"osd":7,"last update":"Mon Oct 9 05:08:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":2.4830000000000001,"15min":1.669},"min":{"1min":0.42999999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.5640000000000001,"5min":63.017000000000003,"15min":63.017000000000003},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.871,"5min":2.351,"15min":1.4079999999999999},"min":{"1min":0.38900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.8119999999999998,"5min":62.808,"15min":62.808},"last":0.59399999999999997}]}]},{"osd":4,"up_from":626,"seq":2688649527360,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":637200,"kb_used_data":232008,"kb_used_omap":22,"kb_used_meta":405161,"kb_avail":93107952,"statfs":{"total":95995035648,"available":95342542848,"internally_reserved":0,"allocated":237576192,"data_stored":235784041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":414885709},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":3.976,"15min":2.8519999999999999},"min":{"1min":0.24099999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.90800000000000003,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.327,"15min":3.556},"min":{"1min":0.254,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.2689999999999999,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.54300000000000004}]},{"osd":1,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":4.8940000000000001,"15min":3.0499999999999998},"min":{"1min":0.27900000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.0129999999999999,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":4.6390000000000002,"15min":2.8130000000000002},"min":{"1min":0.40999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.157,"5min":182.41,"15min":182.41},"last":0.44900000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":1.6870000000000001,"15min":1.655},"min":{"1min":0.35599999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.028,"5min":11.051,"15min":11.051},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.698,"15min":1.6899999999999999},"min":{"1min":0.45800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.97999999999999998,"5min":11.036,"15min":11.036},"last":0.48799999999999999}]},{"osd":3,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":7.4640000000000004,"15min":3.2709999999999999},"min":{"1min":0.35899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":1.8540000000000001,"5min":270.767,"15min":270.767},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.63800000000000001,"5min":7.5060000000000002,"15min":3.7309999999999999},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.0960000000000001,"5min":270.274,"15min":270.274},"last":0.52500000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":2.137,"15min":0.96899999999999997},"min":{"1min":0.33900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.4049999999999998,"5min":134.82900000000001,"15min":134.82900000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":2.161,"15min":1.0189999999999999},"min":{"1min":0.317,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.75600000000000001,"5min":134.60400000000001,"15min":134.60400000000001},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":2.282,"15min":1.202},"min":{"1min":0.38600000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72299999999999998,"5min":139.77799999999999,"15min":139.77799999999999},"last":0.503},{"interface":"front","average":{"1min":0.57199999999999995,"5min":2.3199999999999998,"15min":1.075},"min":{"1min":0.34000000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0609999999999999,"5min":140.78200000000001,"15min":140.78200000000001},"last":0.41699999999999998}]},{"osd":7,"last update":"Mon Oct 9 05:08:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53000000000000003,"15min":0.52700000000000002},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.752,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.58199999999999996,"15min":0.50800000000000001},"min":{"1min":0.377,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.109,"5min":3.109,"15min":3.109},"last":0.439}]}]},{"osd":5,"up_from":38,"seq":163208758013,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1807044,"kb_used_data":317820,"kb_used_omap":0,"kb_used_meta":1489216,"kb_avail":91938108,"statfs":{"total":95995035648,"available":94144622592,"internally_reserved":0,"allocated":325447680,"data_stored":323676941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1524957184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":6.9800000000000004,"15min":3.181},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.91600000000000004,"5min":212.72800000000001,"15min":212.72800000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.66800000000000004,"5min":7.2160000000000002,"15min":3.4079999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":4.1699999999999999,"5min":212.53999999999999,"15min":212.53999999999999},"last":0.45700000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:08:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7830000000000004,"5min":3.4239999999999999,"15min":3.016},"min":{"1min":0.29499999999999998,"5min":0.252,"15min":0.23200000000000001},"max":{"1min":77.268000000000001,"5min":113.977,"15min":262.26900000000001},"last":0.435},{"interface":"front","average":{"1min":4.7080000000000002,"5min":3.6259999999999999,"15min":3.0640000000000001},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":77.287999999999997,"5min":112.947,"15min":262.291},"last":0.97699999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.407,"15min":1.9239999999999999},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":3.0030000000000001,"5min":10.382999999999999,"15min":106.238},"last":0.83399999999999996},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.843,"15min":2.1269999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":8.9239999999999995,"5min":8.9239999999999995,"15min":104.31699999999999},"last":1.1100000000000001}]},{"osd":3,"last update":"Mon Oct 9 05:08:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":6.9809999999999999,"15min":3.7810000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9409999999999998,"5min":125.824,"15min":125.824},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.48,"5min":6.7469999999999999,"15min":3.798},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.4299999999999997,"5min":125.761,"15min":125.761},"last":0.70299999999999996}]},{"osd":4,"last update":"Mon Oct 9 05:08:33 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.433,"15min":0.438},"min":{"1min":0.28699999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.83499999999999996,"5min":1.99,"15min":1.99},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.47199999999999998,"15min":0.46400000000000002},"min":{"1min":0.26400000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.95699999999999996,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.58799999999999997}]},{"osd":6,"last update":"Mon Oct 9 05:08:42 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.49099999999999999,"15min":0.628},"min":{"1min":0.33900000000000002,"5min":0.26600000000000001,"15min":0.23300000000000001},"max":{"1min":0.80200000000000005,"5min":4.3360000000000003,"15min":11.093},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.497,"15min":0.63},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.78400000000000003,"5min":3.0590000000000002,"15min":5.9329999999999998},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:08:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.52300000000000002,"15min":0.75600000000000001},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.26700000000000002},"max":{"1min":2.1019999999999999,"5min":2.1019999999999999,"15min":8.7720000000000002},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.54600000000000004,"15min":0.80300000000000005},"min":{"1min":0.35699999999999998,"5min":0.31,"15min":0.29499999999999998},"max":{"1min":1.845,"5min":2.0489999999999999,"15min":11.791},"last":0.66800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":51,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1060864,"data_stored":1048646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2101248,"data_stored":2097168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":10361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":10240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:08:47.484 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:08:47.484 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-09T05:08:47.484 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [5, 7, 3, 6, 4, 0, 1] 2023-10-09T05:08:47.485 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-09T05:08:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:47 smithi177 ceph-mon[195797]: pgmap v2288: 83 pgs: 83 active+clean; 2.4 MiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 510 KiB/s rd, 524 KiB/s wr, 233 op/s 2023-10-09T05:08:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:47 smithi177 ceph-mon[195797]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:47 smithi177 ceph-mon[195797]: osdmap e725: 8 total, 8 up, 7 in 2023-10-09T05:08:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:47 smithi177 ceph-mon[195797]: failed to encode map e725 with expected crc 2023-10-09T05:08:47.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:47 smithi177 ceph-mon[195797]: failed to encode map e725 with expected crc 2023-10-09T05:08:48.417 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:08:48.417 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:08:48.417 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:08:48.418 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:08:48.418 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:08:48.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:48 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2292163121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:48.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:48 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2292163121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:48.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:48 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2292163121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-09T05:08:49.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:49 smithi029 ceph-mon[216903]: pgmap v2290: 83 pgs: 83 active+clean; 2.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 657 KiB/s rd, 619 KiB/s wr, 274 op/s 2023-10-09T05:08:49.463 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:49 smithi029 ceph-mon[216903]: 1.8 scrub starts 2023-10-09T05:08:49.463 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:49 smithi029 ceph-mon[216903]: 1.8 scrub ok 2023-10-09T05:08:49.463 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:49 smithi029 ceph-mon[223858]: pgmap v2290: 83 pgs: 83 active+clean; 2.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 657 KiB/s rd, 619 KiB/s wr, 274 op/s 2023-10-09T05:08:49.464 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:49 smithi029 ceph-mon[223858]: 1.8 scrub starts 2023-10-09T05:08:49.464 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:49 smithi029 ceph-mon[223858]: 1.8 scrub ok 2023-10-09T05:08:49.465 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-09T05:08:49.465 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:49.499 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:49.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:49 smithi177 ceph-mon[195797]: pgmap v2290: 83 pgs: 83 active+clean; 2.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 657 KiB/s rd, 619 KiB/s wr, 274 op/s 2023-10-09T05:08:49.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:49 smithi177 ceph-mon[195797]: 1.8 scrub starts 2023-10-09T05:08:49.776 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:49 smithi177 ceph-mon[195797]: 1.8 scrub ok 2023-10-09T05:08:51.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:51 smithi029 ceph-mon[216903]: pgmap v2291: 83 pgs: 83 active+clean; 4.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 370 op/s 2023-10-09T05:08:51.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:51 smithi029 ceph-mon[216903]: osdmap e726: 8 total, 8 up, 7 in 2023-10-09T05:08:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:51 smithi029 ceph-mon[223858]: pgmap v2291: 83 pgs: 83 active+clean; 4.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 370 op/s 2023-10-09T05:08:51.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:51 smithi029 ceph-mon[223858]: osdmap e726: 8 total, 8 up, 7 in 2023-10-09T05:08:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:51 smithi177 ceph-mon[195797]: pgmap v2291: 83 pgs: 83 active+clean; 4.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 370 op/s 2023-10-09T05:08:51.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:51 smithi177 ceph-mon[195797]: osdmap e726: 8 total, 8 up, 7 in 2023-10-09T05:08:51.625 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:52.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[216903]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[216903]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[216903]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[216903]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[223858]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[223858]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[223858]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:52 smithi029 ceph-mon[223858]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:52 smithi177 ceph-mon[195797]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:52 smithi177 ceph-mon[195797]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:52 smithi177 ceph-mon[195797]: failed to encode map e726 with expected crc 2023-10-09T05:08:52.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:52 smithi177 ceph-mon[195797]: failed to encode map e726 with expected crc 2023-10-09T05:08:53.159 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:08:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:08:52] "GET /metrics HTTP/1.1" 200 34289 "" "Prometheus/2.43.0" 2023-10-09T05:08:53.169 INFO:teuthology.orchestra.run.smithi029.stderr:noscrub is unset 2023-10-09T05:08:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:53 smithi029 ceph-mon[223858]: pgmap v2293: 83 pgs: 83 active+clean; 4.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 795 KiB/s rd, 1.1 MiB/s wr, 206 op/s 2023-10-09T05:08:53.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:53 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4077081297' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:08:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:53 smithi029 ceph-mon[216903]: pgmap v2293: 83 pgs: 83 active+clean; 4.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 795 KiB/s rd, 1.1 MiB/s wr, 206 op/s 2023-10-09T05:08:53.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:53 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4077081297' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:08:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:53 smithi177 ceph-mon[195797]: pgmap v2293: 83 pgs: 83 active+clean; 4.4 MiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 795 KiB/s rd, 1.1 MiB/s wr, 206 op/s 2023-10-09T05:08:53.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:53 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4077081297' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-09T05:08:54.182 INFO:teuthology.orchestra.run.smithi029.stderr:marked out osd.7. 2023-10-09T05:08:54.208 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:53 smithi029 conmon[216878]: 2023-10-09T05:08:53.970+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e727 definitely_dead 0 2023-10-09T05:08:54.208 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[216903]: failed to encode map e727 with expected crc 2023-10-09T05:08:54.208 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4077081297' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[216903]: osdmap e727: 8 total, 8 up, 7 in 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[216903]: failed to encode map e727 with expected crc 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3863229440' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[216903]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[223858]: failed to encode map e727 with expected crc 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4077081297' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[223858]: osdmap e727: 8 total, 8 up, 7 in 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[223858]: failed to encode map e727 with expected crc 2023-10-09T05:08:54.209 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3863229440' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:08:54.210 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:54 smithi029 ceph-mon[223858]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:08:54.239 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-09T05:08:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:54 smithi177 ceph-mon[195797]: failed to encode map e727 with expected crc 2023-10-09T05:08:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4077081297' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-09T05:08:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:54 smithi177 ceph-mon[195797]: osdmap e727: 8 total, 8 up, 7 in 2023-10-09T05:08:54.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:54 smithi177 ceph-mon[195797]: failed to encode map e727 with expected crc 2023-10-09T05:08:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3863229440' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-09T05:08:54.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:54 smithi177 ceph-mon[195797]: Client client.admin marked osd.7 out, while it was still marked up 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91912 14.b 14:d0cac1ad:::benchmark_data_smithi029_550670_object14301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:41.732359+0000", 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 433.013898993, 2023-10-09T05:08:54.749 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.863427890000001, 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91912 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732359+0000", 2023-10-09T05:08:54.750 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732359+0000", 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732358+0000", 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.751 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732364+0000", 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7499999999999997e-06 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732376+0000", 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1871999999999999e-05 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.732384+0000", 2023-10-09T05:08:54.752 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.289e-06 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.769573+0000", 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037189978999999998 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.780750+0000", 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011176317 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.753 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.932536+0000", 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.151786217 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.936798+0000", 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0042621489999999998 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.413913+0000", 2023-10-09T05:08:54.754 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.47711435200000002 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.541782+0000", 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.127869381 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.563926+0000", 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.022143889999999999 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.755 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.655023+0000", 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.091097062000000006 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.655040+0000", 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6758000000000001e-05 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421648+0000", 2023-10-09T05:08:54.756 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76660828000000003 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421661+0000", 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2758999999999999e-05 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.859547+0000", 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.437885865999998 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.757 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.859564+0000", 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7229999999999999e-05 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595770+0000", 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73620618999999998 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.758 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595787+0000", 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6833999999999998e-05 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.759 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91922 14.b 14:dc20e122:::benchmark_data_smithi029_550670_object14311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:41.955425+0000", 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 432.79083323600003, 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.640405913999999, 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91922 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.760 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955425+0000", 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955425+0000", 2023-10-09T05:08:54.761 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955423+0000", 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.762 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955433+0000", 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0509000000000001e-05 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955450+0000", 2023-10-09T05:08:54.763 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7051999999999999e-05 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.955462+0000", 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1929e-05 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.764 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.971473+0000", 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016010832999999999 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.982740+0000", 2023-10-09T05:08:54.765 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011266905000000001 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.037740+0000", 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.054999795999999997 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.766 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.043790+0000", 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0060503049999999997 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656039+0000", 2023-10-09T05:08:54.767 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.61224903200000003 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656054+0000", 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4387e-05 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.768 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421695+0000", 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76564138299999995 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421704+0000", 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6920000000000008e-06 2023-10-09T05:08:54.769 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865416+0000", 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.44371245 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.770 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865428+0000", 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1542999999999999e-05 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595820+0000", 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73039237700000004 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.771 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595831+0000", 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0611999999999999e-05 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.772 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91924 14.b 14:d1ba9b3e:::benchmark_data_smithi029_550670_object14313:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:41.980177+0000", 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 432.76608049599997, 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.615808210000001, 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.773 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91924 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.774 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980177+0000", 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980177+0000", 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.775 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980175+0000", 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.776 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.980184+0000", 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3799999999999994e-06 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.982798+0000", 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026139610000000001 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.777 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:41.982823+0000", 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5154999999999999e-05 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.778 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.175554+0000", 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.19273112000000001 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.181103+0000", 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0055490849999999996 2023-10-09T05:08:54.779 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.247397+0000", 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.066294343000000006 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.780 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.253507+0000", 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061097640000000002 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656152+0000", 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.40264439899999999 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656156+0000", 2023-10-09T05:08:54.781 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.6589999999999998e-06 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421835+0000", 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76567873099999995 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421848+0000", 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3365e-05 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.782 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865794+0000", 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443945398 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865800+0000", 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4579999999999998e-06 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595976+0000", 2023-10-09T05:08:54.783 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.730175558 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595986+0000", 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8530000000000004e-06 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.784 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91937 14.b 14:d29ea32b:::benchmark_data_smithi029_550670_object14326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.105162+0000", 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 432.641095404, 2023-10-09T05:08:54.785 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.490785804000001, 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91937 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.786 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105162+0000", 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.787 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105162+0000", 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105161+0000", 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.788 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105168+0000", 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8450000000000003e-06 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.789 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105183+0000", 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4808e-05 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.105192+0000", 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9930000000000001e-06 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.790 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.182056+0000", 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.076864049000000004 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.190862+0000", 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088061900000000002 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334875+0000", 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.144012217 2023-10-09T05:08:54.791 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.342436+0000", 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0075609090000000002 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656130+0000", 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.313694588 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.792 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656137+0000", 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.319e-06 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421791+0000", 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76565495100000003 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421799+0000", 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5320000000000001e-06 2023-10-09T05:08:54.793 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865761+0000", 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443961875999999 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865770+0000", 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9390000000000003e-06 2023-10-09T05:08:54.794 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595938+0000", 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73016817700000003 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.795 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595948+0000", 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0283e-05 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.796 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91941 14.b 14:d4fdc961:::benchmark_data_smithi029_550670_object14330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.136326+0000", 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 432.60993197200003, 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.459581669999999, 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.797 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91941 2023-10-09T05:08:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136326+0000", 2023-10-09T05:08:54.798 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136326+0000", 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.799 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136324+0000", 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136334+0000", 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.268e-06 2023-10-09T05:08:54.800 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136352+0000", 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8000999999999999e-05 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.801 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.136362+0000", 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0259e-05 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.211276+0000", 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.074914003000000007 2023-10-09T05:08:54.802 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.216717+0000", 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054411390000000002 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.292465+0000", 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.075747729999999999 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.803 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.297780+0000", 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0053155199999999998 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656110+0000", 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.358329862 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656116+0000", 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6370000000000004e-06 2023-10-09T05:08:54.804 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421759+0000", 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76564343599999996 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421767+0000", 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5839999999999997e-06 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.805 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865731+0000", 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443964449999999 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865738+0000", 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7830000000000001e-06 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.806 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595897+0000", 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73015950100000004 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595908+0000", 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0091999999999999e-05 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.807 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91961 14.b 14:db9a68b4:::benchmark_data_smithi029_550670_object14350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.334243+0000", 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 432.41201534499999, 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 18.261626548999999, 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91961 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.808 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334243+0000", 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334243+0000", 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.809 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334241+0000", 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334249+0000", 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1519999999999992e-06 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334264+0000", 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5427999999999999e-05 2023-10-09T05:08:54.810 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.334273+0000", 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3599999999999996e-06 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.351183+0000", 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016909699 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.811 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.360166+0000", 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0089832769999999996 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.620803+0000", 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.260637008 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.624479+0000", 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003676548 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.812 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656082+0000", 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.031603054999999998 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.656092+0000", 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7399999999999999e-06 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.813 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421728+0000", 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76563604299999999 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.421736+0000", 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9410000000000007e-06 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865697+0000", 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 16.443961297000001 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.814 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865705+0000", 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8460000000000008e-06 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595859+0000", 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73015364100000002 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.815 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.595869+0000", 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0155e-05 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91966 14.16 14:696ae628:::benchmark_data_smithi029_550670_object14355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:42.368550+0000", 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 432.37770835399999, 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 10.744465222000001, 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.816 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91966 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368550+0000", 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.817 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368550+0000", 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368548+0000", 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368556+0000", 2023-10-09T05:08:54.818 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.8619999999999992e-06 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368573+0000", 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6878999999999999e-05 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.819 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.368583+0000", 2023-10-09T05:08:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7140000000000005e-06 2023-10-09T05:08:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.820 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.369429+0000", 2023-10-09T05:08:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00084586399999999999 2023-10-09T05:08:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.821 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.373983+0000", 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045541640000000003 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.389294+0000", 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.015310486 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.822 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.391535+0000", 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0022415410000000001 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.557489+0000", 2023-10-09T05:08:54.823 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.165954305 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.560884+0000", 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033946269999999999 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.824 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 3,6", 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.586880+0000", 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025995541 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.613945+0000", 2023-10-09T05:08:54.825 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027065378000000001 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:42.613962+0000", 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7087000000000001e-05 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.419798+0000", 2023-10-09T05:08:54.826 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.80583640899999998 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:43.419816+0000", 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7589999999999999e-05 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:52.682897+0000", 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2630812010000003 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.827 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:52.688813+0000", 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059156670000000003 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:53.110634+0000", 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.42182064200000002 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:53.113015+0000", 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023811959999999999 2023-10-09T05:08:54.828 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:91974 14.b 14.b620bdab (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:01:51.596975+0000", 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 423.14928264399998, 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.000508494, 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.829 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 91974 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596975+0000", 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596975+0000", 2023-10-09T05:08:54.830 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596974+0000", 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.596983+0000", 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7499999999999998e-06 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.831 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.597000+0000", 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6419e-05 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:51.597028+0000", 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8773999999999999e-05 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865819+0000", 2023-10-09T05:08:54.832 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2687909620000006 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:01:59.865830+0000", 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0088999999999999e-05 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.833 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.597476+0000", 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.73164620300000005 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:02:00.597484+0000", 2023-10-09T05:08:54.834 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0169999999999996e-06 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:146786 14.1b 14:dac989b4:::benchmark_data_smithi029_550670_object69175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e678)", 2023-10-09T05:08:54.835 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:22.615752+0000", 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 92.130505841000002, 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0399380220000001, 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:08:54.836 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 146786 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615752+0000", 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.837 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615752+0000", 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.838 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615751+0000", 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615756+0000", 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0939999999999998e-06 2023-10-09T05:08:54.839 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615767+0000", 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0797e-05 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615774+0000", 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1720000000000001e-06 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.840 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615828+0000", 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3465999999999999e-05 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:22.615838+0000", 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0723000000000001e-05 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.644989+0000", 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.029150902 2023-10-09T05:08:54.841 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.645128+0000", 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00013848800000000001 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.645501+0000", 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00037291100000000002 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.842 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.647069+0000", 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015686070000000001 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.653392+0000", 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0063226360000000004 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.655584+0000", 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021923400000000001 2023-10-09T05:08:54.843 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.655639+0000", 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4795e-05 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:23.655690+0000", 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0948000000000001e-05 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.844 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.37851.0:23 2.0 2:1d803aee:::rbd_header.93db2e1031d1:head [call lock.lock in=8b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e710)", 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:20.133621+0000", 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 34.612637087000003, 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.108304277, 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.845 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.37851", 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2131895455", 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 23 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.846 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133621+0000", 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133621+0000", 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.847 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133620+0000", 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.848 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133624+0000", 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6200000000000001e-06 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133630+0000", 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6599999999999998e-06 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.849 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133635+0000", 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2370000000000003e-06 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.850 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133670+0000", 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.5046999999999998e-05 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133720+0000", 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0203e-05 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.133838+0000", 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000118019 2023-10-09T05:08:54.851 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.134177+0000", 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00033852099999999999 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241738+0000", 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.107561002 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.852 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241832+0000", 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4666000000000005e-05 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241884+0000", 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1443000000000002e-05 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241925+0000", 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1449000000000002e-05 2023-10-09T05:08:54.853 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop_reply(client.37851.0:23 2.0 e710/596 ondisk, result = 0)", 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:20.134336+0000", 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 34.611922292000003, 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.10740400999999999, 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.854 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.134336+0000", 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.134336+0000", 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.855 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.134334+0000", 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.134339+0000", 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.5050000000000001e-06 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.856 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241685+0000", 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.107345977 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241693+0000", 2023-10-09T05:08:54.857 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2050000000000002e-06 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241728+0000", 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4944000000000003e-05 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.858 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241734+0000", 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6869999999999999e-06 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:20.241740+0000", 2023-10-09T05:08:54.859 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8279999999999999e-06 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.47759.0:22 2.6 2:739d4f67:::rbd_header.93f3a3e3d455:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e712)", 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:29.622282+0000", 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 25.123975999999999, 2023-10-09T05:08:54.860 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.038360997000000001, 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.47759", 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2015051291", 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 22 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622282+0000", 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.861 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622282+0000", 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622280+0000", 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.862 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622291+0000", 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1065e-05 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622307+0000", 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5999e-05 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622317+0000", 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0298e-05 2023-10-09T05:08:54.863 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622367+0000", 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9428000000000002e-05 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.622485+0000", 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000117816 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:08:54.864 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.659587+0000", 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.037102740000000002 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.660201+0000", 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00061342199999999997 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.660549+0000", 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00034857399999999999 2023-10-09T05:08:54.865 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.660579+0000", 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9723999999999999e-05 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.660616+0000", 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.6959999999999998e-05 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.866 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:29.660643+0000", 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.6889999999999998e-05 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.37953.0:26 2.6 2:71c0fee8:::rbd_header.baa749f0b7b6:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e714)", 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:31.635691+0000", 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 23.110567122999999, 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.093729873000000005, 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.867 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.37953", 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/62719925", 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 26 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.635691+0000", 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.868 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.635691+0000", 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.635689+0000", 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.635698+0000", 2023-10-09T05:08:54.869 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.2900000000000002e-06 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.727973+0000", 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.092275134999999994 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.727992+0000", 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9164000000000002e-05 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.870 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.728123+0000", 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000131533 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.728237+0000", 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000113291 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:08:54.871 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.728596+0000", 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035901299999999999 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.729164+0000", 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00056877999999999996 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.729308+0000", 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014327200000000001 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.872 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.729340+0000", 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2678999999999997e-05 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.729383+0000", 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.3127999999999998e-05 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:31.729421+0000", 2023-10-09T05:08:54.873 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7132000000000003e-05 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.47849.0:32 2.6 2:739d4f67:::rbd_header.93f3a3e3d455:head [watch unwatch cookie 140175116162752] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e715)", 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:32.649441+0000", 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 22.096817064, 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027711290000000002, 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.874 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.47849", 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/111680537", 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 32 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649441+0000", 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.875 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649441+0000", 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649440+0000", 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649443+0000", 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9849999999999998e-06 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.876 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649450+0000", 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.2790000000000001e-06 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649457+0000", 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1800000000000001e-06 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649676+0000", 2023-10-09T05:08:54.877 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021977499999999999 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649726+0000", 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9200999999999999e-05 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649824+0000", 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8128000000000006e-05 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.878 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.650144+0000", 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032031399999999998 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.651170+0000", 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001026154 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.652112+0000", 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00094209800000000005 2023-10-09T05:08:54.879 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.652143+0000", 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0689999999999999e-05 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.652212+0000", 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8926999999999999e-05 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.880 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.47849.0:33 2.6 2:739d4f67:::rbd_header.93f3a3e3d455:head [notify-ack] snapc 0=[] ondisk+read+known_if_redirected+full_try+supports_pool_eio e715)", 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:32.649738+0000", 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 22.09652011, 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026838330000000001, 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.47849", 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/111680537", 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 33 2023-10-09T05:08:54.881 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649738+0000", 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649738+0000", 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.882 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649737+0000", 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649743+0000", 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8040000000000004e-06 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649755+0000", 2023-10-09T05:08:54.883 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1165000000000001e-05 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.649763+0000", 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1589999999999993e-06 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.650066+0000", 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00030303099999999999 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.884 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.650128+0000", 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1917999999999999e-05 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.652260+0000", 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021323599999999998 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.652337+0000", 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7287999999999996e-05 2023-10-09T05:08:54.885 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:32.652422+0000", 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4314999999999995e-05 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.38013.0:92 2.6 2:67cae2ed:::rbd_mirroring:head [call rbd.mirror_image_remove in=22b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e715)", 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:33.241449+0000", 2023-10-09T05:08:54.886 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 21.504808799999999, 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0045135069999999999, 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.38013", 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/3384660344", 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 92 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241449+0000", 2023-10-09T05:08:54.887 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241449+0000", 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241448+0000", 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241454+0000", 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3589999999999999e-06 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241465+0000", 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.163e-05 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241472+0000", 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9600000000000003e-06 2023-10-09T05:08:54.889 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.241507+0000", 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.4245000000000002e-05 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.244947+0000", 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0034405310000000001 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.890 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.245963+0000", 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001015534 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.38061.0:29 2.4 2:279d933a:::rbd_object_map.94adceca3b20:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:33.672383+0000", 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 21.073875036, 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0026545819999999999, 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.891 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.38061", 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/83726564", 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 29 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672383+0000", 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.892 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672383+0000", 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672382+0000", 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672386+0000", 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2209999999999999e-06 2023-10-09T05:08:54.893 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672396+0000", 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8989999999999996e-06 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672401+0000", 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8149999999999996e-06 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.894 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672438+0000", 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7144000000000002e-05 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672517+0000", 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8708999999999995e-05 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672777+0000", 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00025986599999999998 2023-10-09T05:08:54.895 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.673239+0000", 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00046215900000000002 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.673388+0000", 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014867600000000001 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.896 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.674896+0000", 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001508613 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.674948+0000", 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.1993000000000002e-05 2023-10-09T05:08:54.897 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675037+0000", 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9137999999999999e-05 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.898 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.38061.0:30 2.4 2:279d933a:::rbd_object_map.94adceca3b20:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:33.672412+0000", 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 21.073845725999998, 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0036151209999999998, 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.38061", 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/83726564", 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 30 2023-10-09T05:08:54.899 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672412+0000", 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672412+0000", 2023-10-09T05:08:54.900 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672412+0000", 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672415+0000", 2023-10-09T05:08:54.901 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0960000000000001e-06 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672423+0000", 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7430000000000002e-06 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.902 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672428+0000", 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5840000000000002e-06 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.673040+0000", 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00061188400000000004 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:54.903 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.673134+0000", 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.3549999999999997e-05 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675077+0000", 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0019436379999999999 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675136+0000", 2023-10-09T05:08:54.904 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8857999999999998e-05 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675322+0000", 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018629399999999999 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675779+0000", 2023-10-09T05:08:54.905 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045698899999999998 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675903+0000", 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012403699999999999 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675933+0000", 2023-10-09T05:08:54.906 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9728999999999999e-05 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675967+0000", 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3559000000000002e-05 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.676027+0000", 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.0606000000000002e-05 2023-10-09T05:08:54.907 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.38061.0:31 2.4 2:279d933a:::rbd_object_map.94adceca3b20:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e715)", 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:33.672442+0000", 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 21.073815524, 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0046734339999999997, 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.908 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.38061", 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/83726564", 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 31 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672442+0000", 2023-10-09T05:08:54.909 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672442+0000", 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672442+0000", 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:08:54.910 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672445+0000", 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3629999999999998e-06 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672452+0000", 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.6800000000000004e-06 2023-10-09T05:08:54.911 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.672458+0000", 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6840000000000001e-06 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.673171+0000", 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00071328899999999998 2023-10-09T05:08:54.912 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.673230+0000", 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.9289000000000001e-05 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675608+0000", 2023-10-09T05:08:54.913 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023775049999999998 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.675692+0000", 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4408999999999998e-05 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.914 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.676058+0000", 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036622199999999998 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.676112+0000", 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3117e-05 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.676436+0000", 2023-10-09T05:08:54.915 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00032433800000000002 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.677028+0000", 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00059196100000000001 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.916 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.677053+0000", 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5071000000000001e-05 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.677057+0000", 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2880000000000003e-06 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.677095+0000", 2023-10-09T05:08:54.917 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7583000000000003e-05 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:33.677116+0000", 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1041000000000001e-05 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.918 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.38115.0:23 2.6 2:6e419213:::rbd_header.bb6d68990365:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e717)", 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:35.653886+0000", 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 19.092372178000002, 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003180474, 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.38115", 2023-10-09T05:08:54.919 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2709829702", 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 23 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653886+0000", 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.920 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653886+0000", 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653884+0000", 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.921 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653894+0000", 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0424999999999999e-05 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653912+0000", 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7754999999999999e-05 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.922 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653921+0000", 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6859999999999995e-06 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.653963+0000", 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2441000000000002e-05 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.923 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.654064+0000", 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100348 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 4,6", 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.654349+0000", 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028570400000000001 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:08:54.924 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.654805+0000", 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00045590499999999999 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.655787+0000", 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00098161899999999998 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:08:54.925 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.656969+0000", 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0011825029999999999 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657026+0000", 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6424000000000001e-05 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:08:54.926 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657066+0000", 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0429999999999997e-05 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:08:54.927 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:08:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: pgmap v2295: 83 pgs: 83 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 670 KiB/s rd, 1.0 MiB/s wr, 191 op/s 2023-10-09T05:08:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3863229440' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: osdmap e728: 8 total, 8 up, 6 in 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:55 smithi177 ceph-mon[195797]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: pgmap v2295: 83 pgs: 83 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 670 KiB/s rd, 1.0 MiB/s wr, 191 op/s 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3863229440' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: osdmap e728: 8 total, 8 up, 6 in 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[216903]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: pgmap v2295: 83 pgs: 83 active+clean; 4.6 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 670 KiB/s rd, 1.0 MiB/s wr, 191 op/s 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3863229440' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: osdmap e728: 8 total, 8 up, 6 in 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:55.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:55 smithi029 ceph-mon[223858]: failed to encode map e728 with expected crc 2023-10-09T05:08:56.079 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-09T05:08:56.124 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:56.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: osdmap e729: 8 total, 8 up, 6 in 2023-10-09T05:08:56.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[216903]: osdmap e730: 8 total, 8 up, 6 in 2023-10-09T05:08:56.336 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: osdmap e729: 8 total, 8 up, 6 in 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:56 smithi029 ceph-mon[223858]: osdmap e730: 8 total, 8 up, 6 in 2023-10-09T05:08:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: osdmap e729: 8 total, 8 up, 6 in 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: failed to encode map e729 with expected crc 2023-10-09T05:08:56.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:56 smithi177 ceph-mon[195797]: osdmap e730: 8 total, 8 up, 6 in 2023-10-09T05:08:57.396 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:08:57.456 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: pgmap v2298: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2023-10-09T05:08:57.456 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[216903]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: pgmap v2298: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.457 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.458 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.458 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.458 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:57 smithi029 ceph-mon[223858]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: pgmap v2298: 83 pgs: 83 active+clean; 4.6 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: failed to encode map e730 with expected crc 2023-10-09T05:08:57.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:57 smithi177 ceph-mon[195797]: failed to encode map e730 with expected crc 2023-10-09T05:08:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: osdmap e731: 8 total, 8 up, 6 in 2023-10-09T05:08:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[216903]: osdmap e732: 8 total, 8 up, 6 in 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: osdmap e731: 8 total, 8 up, 6 in 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:58 smithi029 ceph-mon[223858]: osdmap e732: 8 total, 8 up, 6 in 2023-10-09T05:08:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: osdmap e731: 8 total, 8 up, 6 in 2023-10-09T05:08:58.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: failed to encode map e731 with expected crc 2023-10-09T05:08:58.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:58 smithi177 ceph-mon[195797]: osdmap e732: 8 total, 8 up, 6 in 2023-10-09T05:08:59.218 INFO:teuthology.orchestra.run.smithi029.stderr:nodeep-scrub is unset 2023-10-09T05:08:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: pgmap v2301: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 17 B/s, 2 objects/s recovering 2023-10-09T05:08:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/892493231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:59.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:08:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:59.608 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:08:59.608 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:08:59.608 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T05:08:59.608 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:08:59.632 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: pgmap v2301: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 17 B/s, 2 objects/s recovering 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/892493231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:59.633 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: pgmap v2301: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 17 B/s, 2 objects/s recovering 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: failed to encode map e732 with expected crc 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/892493231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-09T05:08:59.634 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:08:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:09:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T05:09:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:09:00.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: osdmap e733: 8 total, 8 up, 6 in 2023-10-09T05:09:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:00 smithi177 ceph-mon[195797]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: osdmap e733: 8 total, 8 up, 6 in 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.551 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[216903]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: osdmap e733: 8 total, 8 up, 6 in 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: failed to encode map e733 with expected crc 2023-10-09T05:09:00.552 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:00 smithi029 ceph-mon[223858]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.037 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T05:09:01.037 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-09T05:09:01.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-09T05:09:01.208 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-09T05:09:01.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[223858]: pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 1.1 MiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T05:09:01.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[223858]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[223858]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[223858]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[216903]: pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 1.1 MiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T05:09:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[216903]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[216903]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:01 smithi029 ceph-mon[216903]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.431 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:01 smithi177 ceph-mon[195797]: pgmap v2304: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 1.1 MiB/s, 2 keys/s, 14 objects/s recovering 2023-10-09T05:09:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:01 smithi177 ceph-mon[195797]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:01 smithi177 ceph-mon[195797]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:01 smithi177 ceph-mon[195797]: failed to encode map e733 with expected crc 2023-10-09T05:09:01.904 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:03.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:09:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:09:02] "GET /metrics HTTP/1.1" 200 34300 "" "Prometheus/2.43.0" 2023-10-09T05:09:03.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:03 smithi029 ceph-mon[216903]: pgmap v2305: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 768 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T05:09:03.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:03 smithi029 ceph-mon[223858]: pgmap v2305: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 768 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T05:09:03.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:03 smithi177 ceph-mon[195797]: pgmap v2305: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 768 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T05:09:03.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:03 smithi029 conmon[216878]: 2023-10-09T05:09:03.694+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e733 definitely_dead 0 2023-10-09T05:09:04.234 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.2. 2023-10-09T05:09:04.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3903956825' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T05:09:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3903956825' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T05:09:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3903956825' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-09T05:09:05.299 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-09T05:09:05.299 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-09T05:09:05.299 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-09T05:09:05.300 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-09T05:09:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: pgmap v2306: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 656 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T05:09:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3903956825' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T05:09:05.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: osdmap e734: 8 total, 8 up, 7 in 2023-10-09T05:09:05.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: pgmap v2306: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 656 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T05:09:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3903956825' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T05:09:05.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: osdmap e734: 8 total, 8 up, 7 in 2023-10-09T05:09:05.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:05 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.471 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:09:05.472 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:09:05.472 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:09:05.472 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:09:05.472 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:09:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: pgmap v2306: 83 pgs: 83 active+clean; 4.6 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 656 KiB/s, 1 keys/s, 8 objects/s recovering 2023-10-09T05:09:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3903956825' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-09T05:09:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: osdmap e734: 8 total, 8 up, 7 in 2023-10-09T05:09:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:05.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:05 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-09T05:09:06.517 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: osdmap e735: 8 total, 8 up, 7 in 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:09:06.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:06 smithi177 ceph-mon[195797]: osdmap e736: 8 total, 8 up, 7 in 2023-10-09T05:09:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: osdmap e735: 8 total, 8 up, 7 in 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[216903]: osdmap e736: 8 total, 8 up, 7 in 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e734 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: osdmap e735: 8 total, 8 up, 7 in 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: failed to encode map e735 with expected crc 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:09:06.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:06 smithi029 ceph-mon[223858]: osdmap e736: 8 total, 8 up, 7 in 2023-10-09T05:09:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[216903]: pgmap v2309: 83 pgs: 83 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:09:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[216903]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[216903]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[216903]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[223858]: pgmap v2309: 83 pgs: 83 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:09:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[223858]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[223858]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:07 smithi029 ceph-mon[223858]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:07 smithi177 ceph-mon[195797]: pgmap v2309: 83 pgs: 83 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail 2023-10-09T05:09:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:07 smithi177 ceph-mon[195797]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:07 smithi177 ceph-mon[195797]: failed to encode map e736 with expected crc 2023-10-09T05:09:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:07 smithi177 ceph-mon[195797]: failed to encode map e736 with expected crc 2023-10-09T05:09:08.181 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:08.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[216903]: failed to encode map e736 with expected crc 2023-10-09T05:09:08.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[216903]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[216903]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[216903]: osdmap e737: 8 total, 8 up, 7 in 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[216903]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[216903]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[223858]: failed to encode map e736 with expected crc 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[223858]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[223858]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[223858]: osdmap e737: 8 total, 8 up, 7 in 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[223858]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.451 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:08 smithi029 ceph-mon[223858]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:08 smithi177 ceph-mon[195797]: failed to encode map e736 with expected crc 2023-10-09T05:09:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:08 smithi177 ceph-mon[195797]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:08 smithi177 ceph-mon[195797]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:08 smithi177 ceph-mon[195797]: osdmap e737: 8 total, 8 up, 7 in 2023-10-09T05:09:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:08 smithi177 ceph-mon[195797]: failed to encode map e737 with expected crc 2023-10-09T05:09:08.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:08 smithi177 ceph-mon[195797]: failed to encode map e737 with expected crc 2023-10-09T05:09:09.279 INFO:teuthology.orchestra.run.smithi029.stderr:marked in osd.7. 2023-10-09T05:09:09.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: pgmap v2312: 83 pgs: 4 peering, 79 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 64 KiB/s, 2 objects/s recovering 2023-10-09T05:09:09.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: osdmap e738: 8 total, 8 up, 7 in 2023-10-09T05:09:09.306 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3435919934' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 conmon[216878]: 2023-10-09T05:09:09.098+0000 7f28c3c1b700 -1 mon.a@0(leader).osd e738 definitely_dead 0 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: pgmap v2312: 83 pgs: 4 peering, 79 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 64 KiB/s, 2 objects/s recovering 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.307 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: osdmap e738: 8 total, 8 up, 7 in 2023-10-09T05:09:09.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3435919934' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:09:09.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:09:09.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:09.308 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:09 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: pgmap v2312: 83 pgs: 4 peering, 79 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 64 KiB/s, 2 objects/s recovering 2023-10-09T05:09:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: osdmap e738: 8 total, 8 up, 7 in 2023-10-09T05:09:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: failed to encode map e738 with expected crc 2023-10-09T05:09:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3435919934' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:09:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-09T05:09:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:09.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:09 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:09:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: osdmap e739: 8 total, 8 up, 8 in 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:10.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:10 smithi177 ceph-mon[195797]: osdmap e740: 8 total, 8 up, 8 in 2023-10-09T05:09:10.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:09:10.600 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: osdmap e739: 8 total, 8 up, 8 in 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.601 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[223858]: osdmap e740: 8 total, 8 up, 8 in 2023-10-09T05:09:10.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: osdmap e739: 8 total, 8 up, 8 in 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.603 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: failed to encode map e739 with expected crc 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:10.604 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:10 smithi029 ceph-mon[216903]: osdmap e740: 8 total, 8 up, 8 in 2023-10-09T05:09:10.620 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-09T05:09:10.620 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-10-09T05:09:10.621 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-10-09T05:09:11.072 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:09:11.072 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:09:11.072 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:09:11.072 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93420 14.2 14:43ae63e0:::benchmark_data_smithi029_550670_object15809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:24.754998+0000", 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 346.31416698599998, 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.9690727130000001, 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93420 2023-10-09T05:09:11.073 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.754998+0000", 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.754998+0000", 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.074 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.754996+0000", 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755004+0000", 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.5299999999999999e-06 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.075 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755023+0000", 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9181000000000002e-05 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755031+0000", 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0649999999999999e-06 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755088+0000", 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7707999999999997e-05 2023-10-09T05:09:11.076 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:24.755100+0000", 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1593000000000001e-05 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.335001+0000", 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.579900517 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.077 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:25.335020+0000", 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9627e-05 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.645432+0000", 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3104114999999998 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:11.078 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.650356+0000", 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049244140000000002 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.662698+0000", 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012341431 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.664260+0000", 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001562934 2023-10-09T05:09:11.079 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.667511+0000", 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032506119999999999 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.687962+0000", 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.020450585 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.080 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.688145+0000", 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018303899999999999 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.719106+0000", 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030961059999999999 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.719995+0000", 2023-10-09T05:09:11.081 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00088945599999999995 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:28.724070+0000", 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0040750789999999997 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_repop(client.36948.0:93755 14.14 e641/638 14:2dfdf15f:::benchmark_data_smithi029_550670_object16144:head v 641'5490, mlcod=594'5440)", 2023-10-09T05:09:11.082 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.249417+0000", 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 319.81974805499999, 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.634247083, 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249417+0000", 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.083 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249417+0000", 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249415+0000", 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249427+0000", 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1328000000000001e-05 2023-10-09T05:09:11.084 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249442+0000", 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5837000000000001e-05 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.249452+0000", 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2529999999999993e-06 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.085 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.215500+0000", 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9660483769999999 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.215533+0000", 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2830000000000002e-05 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.883635+0000", 2023-10-09T05:09:11.086 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.66810195699999997 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.883664+0000", 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8776e-05 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93770 14.4 14:211bc49a:::benchmark_data_smithi029_550670_object16159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:09:11.087 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.428219+0000", 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 319.64094555499997, 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7114670619999997, 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93770 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.088 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428219+0000", 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428219+0000", 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.089 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428217+0000", 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428228+0000", 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0953e-05 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428256+0000", 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8501e-05 2023-10-09T05:09:11.090 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.428270+0000", 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4003e-05 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.900063+0000", 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4717922569999997 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.091 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.903238+0000", 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0031751729999999999 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.946608+0000", 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.043369759000000001 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.949983+0000", 2023-10-09T05:09:11.092 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033750970000000001 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.099731+0000", 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.149748467 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.102098+0000", 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023662840000000002 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.093 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.107041+0000", 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0049436259999999996 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.121280+0000", 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014238971 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.130592+0000", 2023-10-09T05:09:11.094 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0093116850000000001 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.131401+0000", 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000808722 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.132729+0000", 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013287170000000001 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.095 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.139686+0000", 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0069568520000000003 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93774 14.4 14:279fca4c:::benchmark_data_smithi029_550670_object16163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.461282+0000", 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 319.60788211400001, 2023-10-09T05:09:11.096 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7526535790000004, 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93774 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461282+0000", 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.097 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461282+0000", 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461280+0000", 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.098 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461290+0000", 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8290000000000003e-06 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461305+0000", 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4718e-05 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.099 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461320+0000", 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4907e-05 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.950440+0000", 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.4891200979999999 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.954595+0000", 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041548949999999996 2023-10-09T05:09:11.100 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.016951+0000", 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.062356379000000003 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.021414+0000", 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044629769999999999 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.101 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.158317+0000", 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.13690313800000001 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.160690+0000", 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023728540000000002 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:09:11.102 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.165538+0000", 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004848102 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.195961+0000", 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.030422825000000001 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.196603+0000", 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00064218499999999998 2023-10-09T05:09:11.103 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.198402+0000", 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017984419999999999 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.201683+0000", 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0032815539999999999 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.104 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.213936+0000", 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012252798000000001 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93775 14.f 14:f35a5edc:::benchmark_data_smithi029_550670_object16164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 319.60783335799999, 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1113186439999998, 2023-10-09T05:09:11.105 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93775 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.106 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461331+0000", 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.107 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461334+0000", 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.0879999999999999e-06 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461340+0000", 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5409999999999997e-06 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461346+0000", 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2780000000000001e-06 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.108 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461427+0000", 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1403000000000006e-05 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.461439+0000", 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2146e-05 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.109 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.855663+0000", 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.394224124 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.859257+0000", 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003593564 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.863603+0000", 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043460169999999998 2023-10-09T05:09:11.110 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.868058+0000", 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044545899999999996 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:55.868427+0000", 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00036936199999999998 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.111 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.564654+0000", 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6962266349999999 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.566002+0000", 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013480359999999999 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.572650+0000", 2023-10-09T05:09:11.112 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0066483480000000001 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:93781 14.4 14:2296b9e3:::benchmark_data_smithi029_550670_object16170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:03:51.589249+0000", 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 319.47991564799997, 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6379777039999999, 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.113 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 93781 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589249+0000", 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.114 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589249+0000", 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589247+0000", 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.115 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589256+0000", 2023-10-09T05:09:11.116 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7790000000000001e-06 2023-10-09T05:09:11.116 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.116 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589276+0000", 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9729999999999999e-05 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:51.589287+0000", 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0974e-05 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.117 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.955385+0000", 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.3660981110000003 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:56.959367+0000", 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039820539999999996 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.012478+0000", 2023-10-09T05:09:11.118 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.053111053999999998 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.016463+0000", 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0039855580000000002 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.180920+0000", 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.16445663399999999 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.119 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.183328+0000", 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024079969999999998 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 6,7", 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.188382+0000", 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050540079999999996 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.215773+0000", 2023-10-09T05:09:11.120 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.027390687 2023-10-09T05:09:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.121 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.216155+0000", 2023-10-09T05:09:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000382643 2023-10-09T05:09:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.122 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.217042+0000", 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00088696100000000002 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.218823+0000", 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00178072 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:03:57.227227+0000", 2023-10-09T05:09:11.123 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0084035540000000006 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108133 14.f 14:f3f59ff2:::benchmark_data_smithi029_550670_object30522:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.218787+0000", 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 234.850377847, 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.60949035699999998, 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.124 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108133 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218787+0000", 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.125 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218787+0000", 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218785+0000", 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.126 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218794+0000", 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6090000000000001e-06 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218807+0000", 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3541e-05 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.218817+0000", 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4590000000000004e-06 2023-10-09T05:09:11.127 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.219556+0000", 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00073897100000000005 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.223282+0000", 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037261769999999998 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.128 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.237906+0000", 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014624513 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.241433+0000", 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0035265940000000001 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.708381+0000", 2023-10-09T05:09:11.129 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.46694833699999999 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.711125+0000", 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002743639 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.715990+0000", 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048649770000000004 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.130 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.760270+0000", 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.044279639000000003 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.819221+0000", 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.058951642999999998 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.131 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.820560+0000", 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0013389369999999999 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.822350+0000", 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017902459999999999 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.828277+0000", 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059267640000000002 2023-10-09T05:09:11.132 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:108156 14.f 14:f4fd88a0:::benchmark_data_smithi029_550670_object30545:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e662)", 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:16.307420+0000", 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 234.76174496600001, 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.626578886, 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.133 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 108156 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307420+0000", 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.134 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307420+0000", 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307418+0000", 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307425+0000", 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.1570000000000003e-06 2023-10-09T05:09:11.135 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307438+0000", 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3005e-05 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.307447+0000", 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9069999999999996e-06 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.136 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.571211+0000", 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.26376351199999998 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for missing object", 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.573982+0000", 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027715790000000001 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.614003+0000", 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.040020695000000002 2023-10-09T05:09:11.137 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.617403+0000", 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0033996040000000001 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.873892+0000", 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.25648922499999999 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.138 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.877668+0000", 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0037761040000000002 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.883641+0000", 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0059727790000000001 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.139 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.895657+0000", 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012015836 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.895925+0000", 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026844899999999998 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.921679+0000", 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.025754174000000001 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.140 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.926718+0000", 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050388509999999996 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:16.933999+0000", 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072804990000000002 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.141 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:143877 14.1f 14:fc4be378:::benchmark_data_smithi029_550670_object66266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.683485+0000", 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.38567987499999, 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.921677933, 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 143877 2023-10-09T05:09:11.142 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683485+0000", 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683485+0000", 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.143 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683484+0000", 2023-10-09T05:09:11.144 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683489+0000", 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2460000000000003e-06 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.145 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683500+0000", 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1365e-05 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683506+0000", 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6409999999999999e-06 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683574+0000", 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7756999999999995e-05 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.146 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.683589+0000", 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4820999999999999e-05 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.596500+0000", 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.91291160000000005 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.147 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.596641+0000", 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014110199999999999 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.596946+0000", 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00030477199999999997 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.599779+0000", 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0028325640000000001 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.148 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602289+0000", 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0025105570000000001 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605071+0000", 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027813999999999998 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.149 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605121+0000", 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9975000000000002e-05 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605163+0000", 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2144000000000002e-05 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.150 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:143926 14.1f 14:fab57f18:::benchmark_data_smithi029_550670_object66315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.750456+0000", 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.31870856800001, 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.85484137900000001, 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 143926 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.151 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750456+0000", 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750456+0000", 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750455+0000", 2023-10-09T05:09:11.152 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750461+0000", 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.674e-06 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750478+0000", 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6211000000000001e-05 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.153 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750486+0000", 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4800000000000001e-06 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750544+0000", 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7738000000000002e-05 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.750558+0000", 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4418e-05 2023-10-09T05:09:11.154 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.598513+0000", 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.847954549 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.598594+0000", 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1077999999999999e-05 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.155 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.598750+0000", 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00015622999999999999 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602264+0000", 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00351425 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602306+0000", 2023-10-09T05:09:11.156 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.1863999999999998e-05 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605211+0000", 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029045249999999998 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605260+0000", 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.9271999999999999e-05 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.157 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605297+0000", 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7301000000000001e-05 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:143950 14.1f 14:fe5a45e5:::benchmark_data_smithi029_550670_object66339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.782774+0000", 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.28639065900001, 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.82266406199999997, 2023-10-09T05:09:11.158 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 143950 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782774+0000", 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.159 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782774+0000", 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782772+0000", 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.160 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782780+0000", 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.6299999999999998e-06 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782802+0000", 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2357999999999998e-05 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782811+0000", 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.5269999999999998e-06 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.161 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782880+0000", 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9418999999999994e-05 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.782891+0000", 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0784e-05 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.162 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.599809+0000", 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.81691770200000002 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.599912+0000", 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000103492 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.600073+0000", 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000160836 2023-10-09T05:09:11.163 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602268+0000", 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002194545 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602323+0000", 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5671999999999998e-05 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.164 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605359+0000", 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003035692 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605398+0000", 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8803000000000001e-05 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605438+0000", 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0181e-05 2023-10-09T05:09:11.165 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144006 14.1f 14:fa4040ce:::benchmark_data_smithi029_550670_object66395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.847699+0000", 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.22146539000001, 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76497105399999998, 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.166 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144006 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847699+0000", 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847699+0000", 2023-10-09T05:09:11.167 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847698+0000", 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847703+0000", 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0359999999999997e-06 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.168 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847715+0000", 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1471000000000001e-05 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847723+0000", 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1189999999999997e-06 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.169 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847782+0000", 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.8823999999999998e-05 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.847792+0000", 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0142999999999999e-05 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.601121+0000", 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75332884099999997 2023-10-09T05:09:11.170 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.601222+0000", 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100993 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.601370+0000", 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000147925 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.171 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605178+0000", 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.003808423 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605497+0000", 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031930499999999998 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612584+0000", 2023-10-09T05:09:11.172 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070867300000000003 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612631+0000", 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7379000000000002e-05 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612670+0000", 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8878000000000003e-05 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.173 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144010 14.1f 14:fe2fd9b5:::benchmark_data_smithi029_550670_object66399:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.850177+0000", 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.218987285, 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.76264098599999997, 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.174 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144010 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850177+0000", 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.175 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850177+0000", 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850176+0000", 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850181+0000", 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8500000000000002e-06 2023-10-09T05:09:11.176 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850193+0000", 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.187e-05 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850201+0000", 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1950000000000003e-06 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.177 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850267+0000", 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5468000000000004e-05 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.850280+0000", 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3149000000000001e-05 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602336+0000", 2023-10-09T05:09:11.178 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.75205588400000001 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602404+0000", 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8464e-05 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.602582+0000", 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017736 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.179 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605311+0000", 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0027294580000000001 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612714+0000", 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0074027729999999996 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612743+0000", 2023-10-09T05:09:11.180 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.9048e-05 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612786+0000", 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2973999999999999e-05 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612818+0000", 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.2325e-05 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.181 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144024 14.1f 14:fadb29ce:::benchmark_data_smithi029_550670_object66413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.871028+0000", 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.19813648900001, 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.74200370100000002, 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.182 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144024 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871028+0000", 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871028+0000", 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.183 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871026+0000", 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871035+0000", 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4179999999999997e-06 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.184 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871055+0000", 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0072e-05 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871066+0000", 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1045e-05 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871131+0000", 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.5434999999999999e-05 2023-10-09T05:09:11.185 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.871139+0000", 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.1179999999999992e-06 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.603529+0000", 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.732389226 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.186 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.603616+0000", 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.7508000000000003e-05 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.603784+0000", 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016797000000000001 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605462+0000", 2023-10-09T05:09:11.187 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001678262 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612858+0000", 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0073957570000000002 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612951+0000", 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2804000000000001e-05 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.188 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612992+0000", 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0672e-05 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613032+0000", 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.0250999999999998e-05 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.189 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144041 14.1f 14:fef3938f:::benchmark_data_smithi029_550670_object66430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.895941+0000", 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.17322401, 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.722092917, 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144041 2023-10-09T05:09:11.190 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.895941+0000", 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.895941+0000", 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.191 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.895939+0000", 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.895946+0000", 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.0690000000000004e-06 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.895959+0000", 2023-10-09T05:09:11.192 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2911000000000001e-05 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.895967+0000", 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.3629999999999994e-06 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.896042+0000", 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.4748999999999999e-05 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.193 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.896053+0000", 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0868e-05 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605527+0000", 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.709474245 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605628+0000", 2023-10-09T05:09:11.194 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000100975 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.605817+0000", 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018843000000000001 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612889+0000", 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0070725720000000001 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.195 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613065+0000", 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00017529199999999999 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.617933+0000", 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0048683600000000004 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.617985+0000", 2023-10-09T05:09:11.196 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.2216000000000003e-05 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618034+0000", 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.8418999999999998e-05 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144075 14.1f 14:fc116245:::benchmark_data_smithi029_550670_object66464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.197 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.950171+0000", 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.118993101, 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.66796179099999997, 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144075 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.198 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950171+0000", 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950171+0000", 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950170+0000", 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.199 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950176+0000", 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.6879999999999996e-06 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950188+0000", 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1951e-05 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.200 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950196+0000", 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7729999999999999e-06 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950257+0000", 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.1583999999999999e-05 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.950267+0000", 2023-10-09T05:09:11.201 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8020000000000003e-06 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.607166+0000", 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.65689900300000004 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.607264+0000", 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.8382999999999996e-05 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.202 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.607449+0000", 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000184267 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.612919+0000", 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0054698949999999998 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613092+0000", 2023-10-09T05:09:11.203 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000173838 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618044+0000", 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.004951609 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618095+0000", 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.0853000000000003e-05 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.204 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618133+0000", 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8350999999999999e-05 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144101 14.1f 14:ff8dc99f:::benchmark_data_smithi029_550670_object66490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:17.992975+0000", 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.07618953799999, 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.62529671600000003, 2023-10-09T05:09:11.205 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144101 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.992975+0000", 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.206 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.992975+0000", 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.992974+0000", 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.207 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.992980+0000", 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.7239999999999997e-06 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.992992+0000", 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2551000000000001e-05 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.993000+0000", 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8970000000000008e-06 2023-10-09T05:09:11.208 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.993087+0000", 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.6321000000000002e-05 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:17.993097+0000", 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0396e-05 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.209 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.610913+0000", 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.61781590200000003 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.611038+0000", 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00012548999999999999 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.611294+0000", 2023-10-09T05:09:11.210 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000255412 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613123+0000", 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.001828713 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618140+0000", 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0050173659999999997 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.211 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618187+0000", 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7194999999999998e-05 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618234+0000", 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.7271000000000002e-05 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618272+0000", 2023-10-09T05:09:11.212 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.7382999999999998e-05 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.36948.0:144117 14.1f 14:fd76f15f:::benchmark_data_smithi029_550670_object66506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e676)", 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:18.018035+0000", 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 113.051129382, 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.61123632800000005, 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.213 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.36948", 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 144117 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018035+0000", 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.214 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018035+0000", 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018034+0000", 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018041+0000", 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8290000000000002e-06 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.215 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018054+0000", 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3004e-05 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018061+0000", 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9289999999999998e-06 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.216 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018130+0000", 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8584999999999995e-05 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for peered", 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.018142+0000", 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1711e-05 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613144+0000", 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.59500260400000005 2023-10-09T05:09:11.217 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613256+0000", 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000111351 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 1,7", 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.613469+0000", 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00021342400000000001 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.617924+0000", 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0044553040000000002 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.618145+0000", 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000220731 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.629191+0000", 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011045345 2023-10-09T05:09:11.219 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.629233+0000", 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4.2030000000000002e-05 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:18.629272+0000", 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.8973000000000001e-05 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.220 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.47741.0:89 2.1 2:9773d2d8:::rbd_data.93db2e1031d1.000000000000004e:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e711)", 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:21.179618+0000", 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 49.889546238000001, 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0023830689999999998, 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.47741", 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/4152484448", 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 89 2023-10-09T05:09:11.221 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179618+0000", 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179618+0000", 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.222 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179617+0000", 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179626+0000", 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.55e-06 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.223 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179651+0000", 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4508e-05 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179664+0000", 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2887e-05 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179733+0000", 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9455000000000005e-05 2023-10-09T05:09:11.224 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.179741+0000", 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.7370000000000006e-06 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.181917+0000", 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0021757410000000001 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.225 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:21.182001+0000", 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.4926999999999995e-05 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "osd_op(client.38115.0:25 2.1 2:9bcfb8b9:::rbd_object_map.bb6d68990365.0000000000000009:head [writefull 0~35] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e717)", 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:08:35.657477+0000", 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 35.411687637, 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0043983039999999996, 2023-10-09T05:09:11.226 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client_info": { 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client": "client.38115", 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "client_addr": "172.21.15.29:0/2709829702", 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "tid": 25 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657477+0000", 2023-10-09T05:09:11.227 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657477+0000", 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657475+0000", 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.228 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657485+0000", 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4369999999999996e-06 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657504+0000", 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9278000000000001e-05 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657513+0000", 2023-10-09T05:09:11.229 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9029999999999993e-06 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657580+0000", 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.7645999999999996e-05 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op must wait for map", 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.657590+0000", 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.5200000000000003e-06 2023-10-09T05:09:11.230 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.659089+0000", 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0014988289999999999 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.659251+0000", 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00016193300000000001 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.231 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "waiting for subops from 0,1", 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.659400+0000", 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00014957600000000001 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "op_commit", 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.659834+0000", 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00043332899999999998 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.232 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.660270+0000", 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000436057 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.661816+0000", 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00154667 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "commit_sent", 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.661850+0000", 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3646999999999998e-05 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:11.233 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:08:35.661875+0000", 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5298000000000001e-05 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:11.234 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:09:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: pgmap v2315: 83 pgs: 5 peering, 78 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 65 KiB/s, 2 keys/s, 8 objects/s recovering 2023-10-09T05:09:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.529 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.530 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:11 smithi177 ceph-mon[195797]: osdmap e741: 8 total, 8 up, 8 in 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: pgmap v2315: 83 pgs: 5 peering, 78 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 65 KiB/s, 2 keys/s, 8 objects/s recovering 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[216903]: osdmap e741: 8 total, 8 up, 8 in 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: pgmap v2315: 83 pgs: 5 peering, 78 active+clean; 4.6 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 65 KiB/s, 2 keys/s, 8 objects/s recovering 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: failed to encode map e740 with expected crc 2023-10-09T05:09:11.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:11 smithi029 ceph-mon[223858]: osdmap e741: 8 total, 8 up, 8 in 2023-10-09T05:09:12.139 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-09T05:09:12.181 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:12.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[216903]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[216903]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[216903]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[216903]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[216903]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[223858]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[223858]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[223858]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[223858]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:12 smithi029 ceph-mon[223858]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.504 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:12 smithi177 ceph-mon[195797]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:12 smithi177 ceph-mon[195797]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:12 smithi177 ceph-mon[195797]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:12 smithi177 ceph-mon[195797]: failed to encode map e741 with expected crc 2023-10-09T05:09:12.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:12 smithi177 ceph-mon[195797]: failed to encode map e741 with expected crc 2023-10-09T05:09:13.084 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:09:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:09:12] "GET /metrics HTTP/1.1" 200 34300 "" "Prometheus/2.43.0" 2023-10-09T05:09:13.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: pgmap v2318: 83 pgs: 5 peering, 78 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 1.2 KiB/s, 2 keys/s, 5 objects/s recovering 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: osdmap e742: 8 total, 8 up, 8 in 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[216903]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: pgmap v2318: 83 pgs: 5 peering, 78 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 1.2 KiB/s, 2 keys/s, 5 objects/s recovering 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: osdmap e742: 8 total, 8 up, 8 in 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.423 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:13 smithi029 ceph-mon[223858]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: pgmap v2318: 83 pgs: 5 peering, 78 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 1.2 KiB/s, 2 keys/s, 5 objects/s recovering 2023-10-09T05:09:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: osdmap e742: 8 total, 8 up, 8 in 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:13.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:13 smithi177 ceph-mon[195797]: failed to encode map e742 with expected crc 2023-10-09T05:09:14.516 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:14.516 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:14.516 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:09:14.516 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:14.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.539 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: osdmap e743: 8 total, 8 up, 8 in 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.540 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:09:14.541 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: osdmap e743: 8 total, 8 up, 8 in 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: osdmap e743: 8 total, 8 up, 8 in 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: failed to encode map e743 with expected crc 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:14.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:14.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:09:14.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:09:14.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:15.218 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-09T05:09:15.568 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:15.587 INFO:teuthology.orchestra.run.smithi029.stderr:set pool 1 pgp_num to 11 2023-10-09T05:09:15.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:15 smithi029 ceph-mon[216903]: pgmap v2321: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 1.1 MiB/s, 10 objects/s recovering 2023-10-09T05:09:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2291645950' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:09:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:15 smithi029 ceph-mon[223858]: pgmap v2321: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 1.1 MiB/s, 10 objects/s recovering 2023-10-09T05:09:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2291645950' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:09:15.941 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:15 smithi177 ceph-mon[195797]: pgmap v2321: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 1.1 MiB/s, 10 objects/s recovering 2023-10-09T05:09:15.942 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2291645950' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-09T05:09:16.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-09T05:09:16.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:16 smithi177 ceph-mon[195797]: failed to encode map e744 with expected crc 2023-10-09T05:09:16.911 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:16 smithi177 ceph-mon[195797]: failed to encode map e744 with expected crc 2023-10-09T05:09:16.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2291645950' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T05:09:16.912 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:16 smithi177 ceph-mon[195797]: osdmap e744: 8 total, 8 up, 8 in 2023-10-09T05:09:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[216903]: failed to encode map e744 with expected crc 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[216903]: failed to encode map e744 with expected crc 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2291645950' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[216903]: osdmap e744: 8 total, 8 up, 8 in 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[223858]: failed to encode map e744 with expected crc 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[223858]: failed to encode map e744 with expected crc 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2291645950' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-10-09T05:09:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:16 smithi029 ceph-mon[223858]: osdmap e744: 8 total, 8 up, 8 in 2023-10-09T05:09:16.988 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:17.137 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:17.137 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:17.137 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:09:17.138 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:09:17.138 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:17 smithi029 ceph-mon[216903]: pgmap v2323: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 946 KiB/s, 2 keys/s, 12 objects/s recovering 2023-10-09T05:09:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:17 smithi029 ceph-mon[223858]: pgmap v2323: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 946 KiB/s, 2 keys/s, 12 objects/s recovering 2023-10-09T05:09:17.924 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:17 smithi177 ceph-mon[195797]: pgmap v2323: 83 pgs: 3 peering, 80 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 946 KiB/s, 2 keys/s, 12 objects/s recovering 2023-10-09T05:09:17.942 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-09T05:09:18.293 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:18.831 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a: {} 2023-10-09T05:09:18.841 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b: {} 2023-10-09T05:09:18.851 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c: {} 2023-10-09T05:09:18.852 INFO:teuthology.orchestra.run.smithi029.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:09:18.852 INFO:teuthology.orchestra.run.smithi029.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:09:18.852 INFO:teuthology.orchestra.run.smithi029.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-09T05:09:19.636 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:19 smithi177 ceph-mon[195797]: pgmap v2324: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 737 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T05:09:19.720 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:19 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1551833465' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:09:19.800 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-09T05:09:19.847 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:19.847 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:09:19.847 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:09:19.847 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:09:19.847 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.847 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82816 14.17 14:ecc022e2:::benchmark_data_smithi029_550670_object5205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:29.898851+0000", 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 589.94605104599998, 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94699071300000004, 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82816 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.848 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898851+0000", 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898851+0000", 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.849 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898849+0000", 2023-10-09T05:09:19.850 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:19.850 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898861+0000", 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2354999999999999e-05 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.851 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898877+0000", 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6138999999999999e-05 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898884+0000", 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8419999999999999e-06 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.852 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.898923+0000", 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8943000000000003e-05 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.853 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.899039+0000", 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011547100000000001 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,4", 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.899172+0000", 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000133614 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.900871+0000", 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016987599999999999 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.901300+0000", 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00042900700000000002 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.855 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.940487+0000", 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.039187249 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:29.940499+0000", 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.181e-05 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.845828+0000", 2023-10-09T05:09:19.856 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.905328778 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.845841+0000", 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3348e-05 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.857 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82827 14.7 14.50ff8567 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:30.306383+0000", 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 589.53851842400002, 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.539547675, 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82827 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.858 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306383+0000", 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306383+0000", 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.859 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306382+0000", 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306387+0000", 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7219999999999999e-06 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306398+0000", 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0701e-05 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.860 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306402+0000", 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4499999999999997e-06 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306464+0000", 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1679000000000004e-05 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.861 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.306477+0000", 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2880999999999999e-05 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.845922+0000", 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.53944571900000005 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.845931+0000", 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5269999999999998e-06 2023-10-09T05:09:19.862 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:82830 14.17 14.fa83cf97 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e545)", 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T04:59:30.406584+0000", 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 589.43831794799996, 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43928909500000002, 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.863 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82830 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406584+0000", 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406584+0000", 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.864 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406582+0000", 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406589+0000", 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7549999999999997e-06 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.865 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406602+0000", 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3324e-05 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406608+0000", 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.857e-06 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.866 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406678+0000", 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9889999999999997e-05 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.406686+0000", 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.729e-06 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.845864+0000", 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43917808400000002 2023-10-09T05:09:19.867 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T04:59:30.845873+0000", 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9309999999999997e-06 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120291 14.7 14:e31d2563:::benchmark_data_smithi029_550670_object42680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.229923+0000", 2023-10-09T05:09:19.868 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.61497861199999, 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.95677183899999996, 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120291 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.869 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229923+0000", 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229923+0000", 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229922+0000", 2023-10-09T05:09:19.870 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229927+0000", 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0409999999999997e-06 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229938+0000", 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1691999999999999e-05 2023-10-09T05:09:19.871 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229944+0000", 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3199999999999999e-06 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.229996+0000", 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1887e-05 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.872 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.230004+0000", 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3720000000000005e-06 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.179573+0000", 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94956869899999996 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.873 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.179659+0000", 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6637000000000004e-05 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.179893+0000", 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00023365699999999999 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.874 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186375+0000", 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064817269999999996 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186384+0000", 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1700000000000003e-06 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.875 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186594+0000", 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021032200000000001 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186653+0000", 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9030999999999998e-05 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.876 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186695+0000", 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.159e-05 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120298 14.7 14:e0386fc5:::benchmark_data_smithi029_550670_object42687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.259341+0000", 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.585560383, 2023-10-09T05:09:19.877 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92758556400000003, 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120298 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.878 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259341+0000", 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259341+0000", 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.879 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259339+0000", 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259351+0000", 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1313e-05 2023-10-09T05:09:19.880 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259381+0000", 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0258999999999999e-05 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259397+0000", 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5719999999999999e-05 2023-10-09T05:09:19.881 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259527+0000", 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013093999999999999 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.259544+0000", 2023-10-09T05:09:19.882 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6212999999999999e-05 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.182106+0000", 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92256274000000005 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.883 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.182210+0000", 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000103923 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.182508+0000", 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00029720300000000001 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.884 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186753+0000", 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042454499999999996 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186763+0000", 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0417999999999999e-05 2023-10-09T05:09:19.885 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186821+0000", 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7534000000000002e-05 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186887+0000", 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6099999999999994e-05 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.886 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186927+0000", 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9740999999999998e-05 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120364 14.7 14:e00ccc3f:::benchmark_data_smithi029_550670_object42753:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.538250+0000", 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.30665208100001, 2023-10-09T05:09:19.887 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.65308594099999995, 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120364 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.888 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538250+0000", 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538250+0000", 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538249+0000", 2023-10-09T05:09:19.889 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538252+0000", 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5679999999999998e-06 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538257+0000", 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4179999999999996e-06 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.890 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538260+0000", 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7499999999999999e-06 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538304+0000", 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3823999999999999e-05 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.538312+0000", 2023-10-09T05:09:19.891 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.1009999999999992e-06 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.184230+0000", 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.64591790500000001 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.184328+0000", 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8498999999999993e-05 2023-10-09T05:09:19.892 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.893 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.893 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.893 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.184566+0000", 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000237245 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186960+0000", 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023942299999999998 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.894 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191231+0000", 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0042709990000000001 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191256+0000", 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.495e-05 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191292+0000", 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6114e-05 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.895 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191335+0000", 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3665999999999998e-05 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120368 14.7 14:e40bc8a1:::benchmark_data_smithi029_550670_object42757:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.548202+0000", 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.29669919899999, 2023-10-09T05:09:19.896 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.64506941799999995, 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120368 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548202+0000", 2023-10-09T05:09:19.897 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548202+0000", 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548201+0000", 2023-10-09T05:09:19.898 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548206+0000", 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8949999999999997e-06 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548218+0000", 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1909000000000001e-05 2023-10-09T05:09:19.899 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548223+0000", 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8520000000000003e-06 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548274+0000", 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0797999999999997e-05 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.900 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.548284+0000", 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.859e-06 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.186992+0000", 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.638708041 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.187103+0000", 2023-10-09T05:09:19.901 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011106400000000001 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.187443+0000", 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034014199999999998 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191391+0000", 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039477339999999996 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.902 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191446+0000", 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5723000000000002e-05 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193205+0000", 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017591359999999999 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.903 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193235+0000", 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9833e-05 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193272+0000", 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6594999999999998e-05 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.904 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120370 14.7 14:e2773394:::benchmark_data_smithi029_550670_object42759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.554439+0000", 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.29046220699999, 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.63893957499999998, 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.905 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120370 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554439+0000", 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.906 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554439+0000", 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554439+0000", 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.907 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554443+0000", 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.4950000000000002e-06 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554455+0000", 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1874e-05 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.908 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554460+0000", 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0980000000000001e-06 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554498+0000", 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8155999999999999e-05 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.909 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.554504+0000", 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9660000000000001e-06 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.189036+0000", 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.63453170800000003 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.910 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.189156+0000", 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119811 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.189407+0000", 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025110900000000001 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.911 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191492+0000", 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020851110000000002 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193280+0000", 2023-10-09T05:09:19.912 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017879770000000001 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193322+0000", 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2243000000000002e-05 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.913 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193351+0000", 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9036000000000001e-05 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193379+0000", 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7715000000000001e-05 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.914 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120405 14.7 14:e0639125:::benchmark_data_smithi029_550670_object42794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.685492+0000", 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.15941001900001, 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.51295157800000002, 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.915 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120405 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685492+0000", 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.916 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685492+0000", 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685489+0000", 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.917 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685501+0000", 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1496e-05 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685533+0000", 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2406000000000001e-05 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685547+0000", 2023-10-09T05:09:19.918 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3287e-05 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685675+0000", 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012856 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.685697+0000", 2023-10-09T05:09:19.919 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1979000000000001e-05 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191527+0000", 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.50582975900000005 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191648+0000", 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012063800000000001 2023-10-09T05:09:19.920 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.191898+0000", 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000250344 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198060+0000", 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061615469999999999 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.921 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198089+0000", 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9611000000000002e-05 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198353+0000", 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026373899999999998 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.922 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198403+0000", 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.973e-05 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198443+0000", 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0584999999999999e-05 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.923 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:120420 14.7 14:e7e17410:::benchmark_data_smithi029_550670_object42809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e667)", 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:07.762423+0000", 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 192.08247894600001, 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43665837200000002, 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 120420 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.924 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762423+0000", 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762423+0000", 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762421+0000", 2023-10-09T05:09:19.925 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762427+0000", 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7150000000000003e-06 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762441+0000", 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3691e-05 2023-10-09T05:09:19.926 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762447+0000", 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2149999999999998e-06 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762522+0000", 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4527e-05 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.927 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:07.762530+0000", 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0139999999999998e-06 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193413+0000", 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.43088293599999999 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.928 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193483+0000", 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0469000000000006e-05 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,4", 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.193665+0000", 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00018201 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198116+0000", 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0044511999999999998 2023-10-09T05:09:19.929 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.198457+0000", 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000340945 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.199025+0000", 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000567557 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.930 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.199055+0000", 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0074999999999999e-05 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:08.199081+0000", 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6225e-05 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.931 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:123729 14.10 14:0c33f5a9:::benchmark_data_smithi029_550670_object46118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e669)", 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:19.203513+0000", 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 180.64138899299999, 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.34084924599999999, 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 123729 2023-10-09T05:09:19.932 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203513+0000", 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203513+0000", 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.933 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203510+0000", 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203523+0000", 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2483000000000001e-05 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203556+0000", 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2839000000000002e-05 2023-10-09T05:09:19.934 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.937 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.937 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.937 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203574+0000", 2023-10-09T05:09:19.937 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8359000000000002e-05 2023-10-09T05:09:19.937 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203702+0000", 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012833000000000001 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.203864+0000", 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00016176499999999999 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.938 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.204214+0000", 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00034985600000000001 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.205911+0000", 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00169724 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.939 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.207110+0000", 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011987980000000001 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.543651+0000", 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.336541325 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.543691+0000", 2023-10-09T05:09:19.940 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9277999999999999e-05 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:19.544362+0000", 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00067120899999999998 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:130168 14.a 14:5767d628:::benchmark_data_smithi029_550670_object52557:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e670)", 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:42.113258+0000", 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 157.73164371499999, 2023-10-09T05:09:19.941 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32682491299999999, 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 130168 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113258+0000", 2023-10-09T05:09:19.942 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113258+0000", 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113248+0000", 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999905 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.943 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113267+0000", 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.908e-05 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113293+0000", 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5176000000000001e-05 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.944 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113305+0000", 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2425999999999999e-05 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113441+0000", 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013575300000000001 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.945 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.113669+0000", 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00022808600000000001 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.114004+0000", 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00033529799999999999 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.115675+0000", 2023-10-09T05:09:19.946 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0016711009999999999 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.351322+0000", 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.235647154 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.439912+0000", 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.088589917000000004 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.947 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.440003+0000", 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.1150999999999995e-05 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.440083+0000", 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9354e-05 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.948 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:130171 14.1a 14:5f60af5a:::benchmark_data_smithi029_550670_object52560:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e670)", 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:42.120059+0000", 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 157.72484284699999, 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32013489699999997, 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 130171 2023-10-09T05:09:19.949 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.120059+0000", 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.120059+0000", 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.950 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.120058+0000", 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.120061+0000", 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.508e-06 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.951 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.246665+0000", 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.12660432199999999 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.246695+0000", 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9952999999999999e-05 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.246760+0000", 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4695000000000003e-05 2023-10-09T05:09:19.952 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.246878+0000", 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000117757 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3", 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.247200+0000", 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032288500000000002 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.953 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.249124+0000", 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019233220000000001 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.353705+0000", 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.104580784 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.440132+0000", 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.086427322000000001 2023-10-09T05:09:19.954 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.440167+0000", 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4656999999999997e-05 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:42.440194+0000", 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.709e-05 2023-10-09T05:09:19.955 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:131604 14.20 14:0447c0db:::benchmark_data_smithi029_550670_object53993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e670)", 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:46.752922+0000", 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 153.09197968199999, 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.31943586899999998, 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.956 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131604 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.752922+0000", 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.957 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.752922+0000", 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.752920+0000", 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.752927+0000", 2023-10-09T05:09:19.958 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7360000000000004e-06 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.752946+0000", 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9210000000000001e-05 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.752954+0000", 2023-10-09T05:09:19.959 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7300000000000005e-06 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.753014+0000", 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9852000000000002e-05 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.754909+0000", 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018945920000000001 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.960 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.051066+0000", 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29615761699999998 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.051134+0000", 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7377000000000002e-05 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:19.961 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.051379+0000", 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024580799999999997 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.052569+0000", 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011893819999999999 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.053033+0000", 2023-10-09T05:09:19.962 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00046406300000000002 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.071816+0000", 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018783557999999999 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.071877+0000", 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1101999999999999e-05 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.963 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.072358+0000", 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00048033800000000002 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132496 14.10 14:0d4896fa:::benchmark_data_smithi029_550670_object54885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.671268+0000", 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.17363322200001, 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33496790399999998, 2023-10-09T05:09:19.964 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132496 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671268+0000", 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.965 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671268+0000", 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671268+0000", 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.966 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671271+0000", 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0599999999999999e-06 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671289+0000", 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7858000000000001e-05 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671293+0000", 2023-10-09T05:09:19.967 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3080000000000001e-06 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671341+0000", 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7840999999999998e-05 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671450+0000", 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000109345 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.968 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.671790+0000", 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000339107 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.673606+0000", 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001816713 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.674811+0000", 2023-10-09T05:09:19.969 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012049389999999999 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.005407+0000", 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33059557000000001 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.005458+0000", 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0816000000000003e-05 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.970 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.006236+0000", 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000778713 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132516 14.10 14:0dd78d91:::benchmark_data_smithi029_550670_object54905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.706814+0000", 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.138087926, 2023-10-09T05:09:19.971 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33493864099999998, 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132516 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706814+0000", 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.972 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706814+0000", 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706813+0000", 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.973 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706816+0000", 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8059999999999999e-06 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706822+0000", 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4219999999999997e-06 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706826+0000", 2023-10-09T05:09:19.974 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7799999999999998e-06 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706877+0000", 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0386999999999997e-05 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.707023+0000", 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014648600000000001 2023-10-09T05:09:19.975 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.707398+0000", 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00037482200000000002 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.709213+0000", 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001814764 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.976 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.711100+0000", 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018869329999999999 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.041016+0000", 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.32991647600000001 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.041077+0000", 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1058999999999999e-05 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.977 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.041752+0000", 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00067512599999999996 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132517 14.0 14:03332c35:::benchmark_data_smithi029_550670_object54906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.706872+0000", 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 150.13803010500001, 2023-10-09T05:09:19.978 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.33184270399999999, 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132517 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706872+0000", 2023-10-09T05:09:19.979 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706872+0000", 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706871+0000", 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.980 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706874+0000", 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5899999999999999e-06 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706884+0000", 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5789999999999993e-06 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.981 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706890+0000", 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7089999999999999e-06 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.706921+0000", 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1275999999999997e-05 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.707021+0000", 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9844000000000002e-05 2023-10-09T05:09:19.982 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.707272+0000", 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00025171299999999998 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708520+0000", 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001247857 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.983 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708785+0000", 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026446 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.038650+0000", 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.329865452 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.038687+0000", 2023-10-09T05:09:19.984 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7183e-05 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.038714+0000", 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6755999999999999e-05 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.985 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.47741.0:82 2.3 2:cd5b9934:::rbd_data.93db2e1031d1.0000000000000046:head [sparse-read 0~4194304 [fadvise_nocache]] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e711)", 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:08:21.153223+0000", 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 58.691678948000003, 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039073160000000001, 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.47741", 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/4152484448", 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 82 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.986 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153223+0000", 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153223+0000", 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.987 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153222+0000", 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153227+0000", 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4909999999999996e-06 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153242+0000", 2023-10-09T05:09:19.988 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.491e-05 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153250+0000", 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3989999999999998e-06 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153358+0000", 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000108785 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.989 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.153365+0000", 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2729999999999999e-06 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.157027+0000", 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0036620749999999999 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.990 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:21.157130+0000", 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010328 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.47783.0:51 2.2 2:5da54907:::rbd_object_map.baa749f0b7b6.0000000000000005:head [writefull 0~35] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e713)", 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:08:30.633966+0000", 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 49.210935135, 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0043173059999999999, 2023-10-09T05:09:19.991 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.47783", 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/3492449525", 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 51 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.633966+0000", 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.992 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.633966+0000", 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.633965+0000", 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.993 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.633974+0000", 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6610000000000006e-06 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.633991+0000", 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7051e-05 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.633998+0000", 2023-10-09T05:09:19.994 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2989999999999999e-06 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.634063+0000", 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4561999999999996e-05 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op must wait for map", 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.634072+0000", 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.9039999999999998e-06 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.995 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.636923+0000", 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028516710000000001 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.637140+0000", 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00021690300000000001 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,6", 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.637269+0000", 2023-10-09T05:09:19.996 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012904100000000001 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.637681+0000", 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000412101 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.637984+0000", 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030266800000000002 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:19.997 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.638207+0000", 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000222926 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.638240+0000", 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3259999999999997e-05 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:30.638284+0000", 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3578000000000002e-05 2023-10-09T05:09:19.998 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.25226.0:113 2.2 2:4e99cc3e:::rbd_mirror_snapshot_schedule:head [omap-get-vals] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e702)", 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:08:59.106640+0000", 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 20.738261136999999, 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000532742, 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:19.999 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.25226", 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/41816888", 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 113 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.106640+0000", 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:20.000 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.106640+0000", 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.106639+0000", 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.106651+0000", 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2113000000000001e-05 2023-10-09T05:09:20.001 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.106672+0000", 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1518999999999998e-05 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.106683+0000", 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0813e-05 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.002 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.107097+0000", 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00041427599999999999 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:08:59.107173+0000", 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5798999999999997e-05 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:20.003 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:20.004 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:20.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:19 smithi029 ceph-mon[216903]: pgmap v2324: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 737 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T05:09:20.004 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:19 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1551833465' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:09:20.004 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:19 smithi029 ceph-mon[223858]: pgmap v2324: 83 pgs: 2 peering, 81 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 737 KiB/s, 1 keys/s, 9 objects/s recovering 2023-10-09T05:09:20.005 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:19 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1551833465' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-09T05:09:20.164 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:20.418 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-09T05:09:20.772 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:20.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:20.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:20.814 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:21.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:21 smithi029 ceph-mon[216903]: pgmap v2325: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 577 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T05:09:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:21 smithi029 ceph-mon[223858]: pgmap v2325: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 577 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T05:09:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:21 smithi177 ceph-mon[195797]: pgmap v2325: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 577 KiB/s, 1 keys/s, 7 objects/s recovering 2023-10-09T05:09:22.348 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:22.348 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:22.348 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:09:22.348 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:22.614 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.0 to 1 (10000) 2023-10-09T05:09:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:22 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4072604791' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:22 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:22.864 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:09:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:09:22] "GET /metrics HTTP/1.1" 200 34316 "" "Prometheus/2.43.0" 2023-10-09T05:09:22.864 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:22 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4072604791' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:22.864 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:22 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:22.865 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:22 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4072604791' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:22.865 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:22 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:22.915 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-09T05:09:23.260 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:23.561 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-09T05:09:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:23 smithi177 ceph-mon[195797]: pgmap v2326: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 38 KiB/s, 1 keys/s, 3 objects/s recovering 2023-10-09T05:09:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:23 smithi177 ceph-mon[195797]: failed to encode map e745 with expected crc 2023-10-09T05:09:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:23 smithi177 ceph-mon[195797]: failed to encode map e745 with expected crc 2023-10-09T05:09:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:23 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:09:23.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:23 smithi177 ceph-mon[195797]: osdmap e745: 8 total, 8 up, 8 in 2023-10-09T05:09:23.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[216903]: pgmap v2326: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 38 KiB/s, 1 keys/s, 3 objects/s recovering 2023-10-09T05:09:23.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[216903]: failed to encode map e745 with expected crc 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[216903]: failed to encode map e745 with expected crc 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[216903]: osdmap e745: 8 total, 8 up, 8 in 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[223858]: pgmap v2326: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 38 KiB/s, 1 keys/s, 3 objects/s recovering 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[223858]: failed to encode map e745 with expected crc 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[223858]: failed to encode map e745 with expected crc 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:09:23.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:23 smithi029 ceph-mon[223858]: osdmap e745: 8 total, 8 up, 8 in 2023-10-09T05:09:23.938 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:24.865 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:24.865 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:24.865 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:09:24.865 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:09:24.865 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:25.557 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-09T05:09:25.906 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:25 smithi029 ceph-mon[216903]: pgmap v2328: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:25 smithi029 ceph-mon[223858]: pgmap v2328: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:25 smithi177 ceph-mon[195797]: pgmap v2328: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:26.628 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-09T05:09:26.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:26 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3324769221' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:26.880 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:26 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3324769221' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:26.880 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:26 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3324769221' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:91970 14.4 14:21827fbb:::benchmark_data_smithi029_550670_object14359:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e576)", 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:43.424517+0000", 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 464.044813207, 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6851607230000001, 2023-10-09T05:09:27.472 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 91970 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424517+0000", 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424517+0000", 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424516+0000", 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424523+0000", 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4070000000000004e-06 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424535+0000", 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1990999999999999e-05 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.424543+0000", 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7840000000000004e-06 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.439331+0000", 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.014787692999999999 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.439342+0000", 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0967e-05 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.477 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:50.107237+0000", 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.6678946870000004 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:50.109678+0000", 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0024416030000000001 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.478 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e576,539 14:33039425:::benchmark_data_smithi029_550670_object2899:head,512'5047)", 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 463.03082056800002, 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0750518820000003, 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438510+0000", 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438513+0000", 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7309999999999999e-06 2023-10-09T05:09:27.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438521+0000", 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2729999999999997e-06 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438527+0000", 2023-10-09T05:09:27.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5690000000000001e-06 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513501+0000", 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0749741339999996 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513511+0000", 2023-10-09T05:09:27.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7140000000000005e-06 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513562+0000", 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0744000000000001e-05 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e577,539 14:32986eef:::benchmark_data_smithi029_550670_object2913:head,512'5048)", 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 463.03055523199998, 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0749084590000004, 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438775+0000", 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438778+0000", 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9220000000000001e-06 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438799+0000", 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0863999999999999e-05 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:44.438804+0000", 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.553e-06 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513642+0000", 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0748375110000001 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513648+0000", 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4370000000000004e-06 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513684+0000", 2023-10-09T05:09:27.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5435000000000002e-05 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e578,539 14:31dc1b0e:::benchmark_data_smithi029_550670_object2955:head,514'5049)", 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:45.494277+0000", 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 461.97505401000001, 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0194656899999996, 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:27.492 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494277+0000", 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494277+0000", 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494276+0000", 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494283+0000", 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3590000000000002e-06 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494299+0000", 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5611e-05 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:45.494307+0000", 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.3650000000000004e-06 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513707+0000", 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0193997130000003 2023-10-09T05:09:27.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513712+0000", 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7350000000000001e-06 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513742+0000", 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9881000000000001e-05 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e579,539 14:3a70dd65:::benchmark_data_smithi029_550670_object2985:head,523'5050)", 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:46.524601+0000", 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 460.94472946899998, 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.989196808, 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524601+0000", 2023-10-09T05:09:27.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524601+0000", 2023-10-09T05:09:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524600+0000", 2023-10-09T05:09:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524610+0000", 2023-10-09T05:09:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7880000000000001e-06 2023-10-09T05:09:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524623+0000", 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2352e-05 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:46.524645+0000", 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2943e-05 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.503 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513763+0000", 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9891175649999999 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513770+0000", 2023-10-09T05:09:27.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8299999999999998e-06 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513798+0000", 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8070000000000001e-05 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGRecoveryDelete(14.c e580,539 14:35c61ae4:::benchmark_data_smithi029_550670_object2997:head,523'5051)", 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:48.499945+0000", 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 458.96938579900001, 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0139077599999999, 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.506 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499945+0000", 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499945+0000", 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499944+0000", 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499953+0000", 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2609999999999999e-06 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499967+0000", 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4497e-05 2023-10-09T05:09:27.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:48.499976+0000", 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4300000000000006e-06 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513821+0000", 2023-10-09T05:09:27.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.01384556 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513829+0000", 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.25e-06 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:51.513853+0000", 2023-10-09T05:09:27.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3873e-05 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92055 14.1c 14:3e5947b2:::benchmark_data_smithi029_550670_object14444:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473836+0000", 2023-10-09T05:09:27.512 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99549487899998, 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7604524719999999, 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92055 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.513 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473836+0000", 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473836+0000", 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473834+0000", 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473844+0000", 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6069999999999997e-06 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473859+0000", 2023-10-09T05:09:27.516 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5299e-05 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473867+0000", 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7039999999999994e-06 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.517 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473922+0000", 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5556000000000001e-05 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473930+0000", 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0180000000000001e-06 2023-10-09T05:09:27.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.202764+0000", 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7288333829999996 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.206360+0000", 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003595844 2023-10-09T05:09:27.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.230795+0000", 2023-10-09T05:09:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024435351000000001 2023-10-09T05:09:27.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.234288+0000", 2023-10-09T05:09:27.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003493229 2023-10-09T05:09:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.523 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92069 14.c 14:365687b7:::benchmark_data_smithi029_550670_object14458:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99545156599999, 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6586282880000001, 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92069 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.524 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:09:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473879+0000", 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473882+0000", 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1420000000000002e-06 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473893+0000", 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1647999999999999e-05 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473897+0000", 2023-10-09T05:09:27.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6090000000000001e-06 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473947+0000", 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9599999999999999e-05 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473955+0000", 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0809999999999994e-06 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.078473+0000", 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6045187260000002 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.080872+0000", 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0023986160000000001 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.130299+0000", 2023-10-09T05:09:27.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.049426781000000003 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.132507+0000", 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022084539999999999 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92089 14.1c 14:3c0d3de8:::benchmark_data_smithi029_550670_object14478:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.530 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99541923999999, 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1017356180000002, 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92089 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473911+0000", 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473914+0000", 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3110000000000001e-06 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473921+0000", 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5810000000000002e-06 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473929+0000", 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.5390000000000007e-06 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473954+0000", 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4389999999999999e-05 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473958+0000", 2023-10-09T05:09:27.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0969999999999999e-06 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.206952+0000", 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7329936870000004 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.209989+0000", 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00303732 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.243365+0000", 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.033375984999999997 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.246468+0000", 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031034909999999999 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.543790+0000", 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29732161899999998 2023-10-09T05:09:27.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.547548+0000", 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037581609999999999 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.554854+0000", 2023-10-09T05:09:27.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073057 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.560593+0000", 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057395099999999998 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.564110+0000", 2023-10-09T05:09:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035166619999999998 2023-10-09T05:09:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.565379+0000", 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00126932 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.567433+0000", 2023-10-09T05:09:27.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020534860000000002 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.575647+0000", 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082141650000000007 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.541 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92094 14.c 14:37d5fb26:::benchmark_data_smithi029_550670_object14483:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473941+0000", 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.995390026, 2023-10-09T05:09:27.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9777295480000001, 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92094 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.543 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473941+0000", 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473941+0000", 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473940+0000", 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.544 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473943+0000", 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.481e-06 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473949+0000", 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.964e-06 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.545 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473953+0000", 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5360000000000003e-06 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473980+0000", 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6869e-05 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473986+0000", 2023-10-09T05:09:27.546 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1099999999999999e-06 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.101391+0000", 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6274046120000003 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.104333+0000", 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002942269 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.547 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.144237+0000", 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.039903921000000002 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.147577+0000", 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033400230000000001 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.377144+0000", 2023-10-09T05:09:27.548 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.22956679399999999 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.379900+0000", 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027565179999999999 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.385303+0000", 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054023279999999996 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.549 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.441018+0000", 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.055715573999999997 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.442356+0000", 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0013380620000000001 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.443438+0000", 2023-10-09T05:09:27.550 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010813909999999999 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.445000+0000", 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015619189999999999 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.451670+0000", 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0066705430000000001 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.551 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92107 14.1c 14:3ef3cb94:::benchmark_data_smithi029_550670_object14496:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99537322600003, 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3570599840000002, 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92107 2023-10-09T05:09:27.552 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.553 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473957+0000", 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.554 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473960+0000", 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.818e-06 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473966+0000", 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1890000000000004e-06 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.555 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473970+0000", 2023-10-09T05:09:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.827e-06 2023-10-09T05:09:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.556 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473985+0000", 2023-10-09T05:09:27.557 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4782e-05 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473988+0000", 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1109999999999999e-06 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.246758+0000", 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7727702189999999 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.558 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.249559+0000", 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002801347 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.266902+0000", 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017342453000000001 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.559 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.269999+0000", 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030973429999999998 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.635209+0000", 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.36521015099999998 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.560 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.637768+0000", 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002558383 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.643047+0000", 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052789719999999998 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.673755+0000", 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030708092999999999 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.561 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.674062+0000", 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00030731299999999998 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.823732+0000", 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14966990299999999 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.562 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.825231+0000", 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014994979999999999 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.831017+0000", 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005785887 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.563 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92161 14.c 14:309adb2f:::benchmark_data_smithi029_550670_object14550:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473974+0000", 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99535683400001, 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9877311280000001, 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92161 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.564 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473974+0000", 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473974+0000", 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473973+0000", 2023-10-09T05:09:27.565 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473976+0000", 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2730000000000001e-06 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473982+0000", 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8470000000000001e-06 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.566 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473985+0000", 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9009999999999996e-06 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474021+0000", 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5642999999999997e-05 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.567 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474031+0000", 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0321e-05 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.104642+0000", 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6306100810000004 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.108462+0000", 2023-10-09T05:09:27.568 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038201300000000001 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.160022+0000", 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.051560828000000003 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.163297+0000", 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032748669999999999 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.569 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.403246+0000", 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.23994837599999999 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.406013+0000", 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0027670709999999999 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.570 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.411554+0000", 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055412739999999997 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.441330+0000", 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029776054 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.452572+0000", 2023-10-09T05:09:27.571 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.011241660000000001 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.453779+0000", 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001206801 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.455288+0000", 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001509203 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.572 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.461705+0000", 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064171209999999996 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92165 14.c 14:3175512b:::benchmark_data_smithi029_550670_object14554:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.573 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.473998+0000", 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99533298199998, 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9992911180000004, 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92165 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.574 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473998+0000", 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473998+0000", 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.473997+0000", 2023-10-09T05:09:27.575 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474000+0000", 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3920000000000001e-06 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474006+0000", 2023-10-09T05:09:27.576 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9970000000000004e-06 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474010+0000", 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8369999999999999e-06 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474046+0000", 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6656000000000003e-05 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.577 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474049+0000", 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.207e-06 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.163697+0000", 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6896473920000004 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.166772+0000", 2023-10-09T05:09:27.578 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030751329999999999 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.183820+0000", 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017048447000000001 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.187210+0000", 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033900940000000002 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.579 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.428328+0000", 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.241117529 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.431210+0000", 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028819100000000001 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.580 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.437276+0000", 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0060659659999999999 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.452887+0000", 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015610939000000001 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.462618+0000", 2023-10-09T05:09:27.581 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097308080000000005 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.463787+0000", 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001168955 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.465742+0000", 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001955156 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.582 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.473289+0000", 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0075470160000000001 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92174 14.c 14:376305e3:::benchmark_data_smithi029_550670_object14563:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.995306357, 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0689501889999997, 2023-10-09T05:09:27.583 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92174 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.584 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474024+0000", 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.585 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474026+0000", 2023-10-09T05:09:27.586 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3549999999999999e-06 2023-10-09T05:09:27.586 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.586 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.586 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.586 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474032+0000", 2023-10-09T05:09:27.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8200000000000002e-06 2023-10-09T05:09:27.587 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.587 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.587 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.587 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474036+0000", 2023-10-09T05:09:27.587 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4240000000000002e-06 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474069+0000", 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3745000000000003e-05 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.588 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474075+0000", 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6489999999999996e-06 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.167560+0000", 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6934853460000001 2023-10-09T05:09:27.589 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.170840+0000", 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032801850000000001 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.590 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.198844+0000", 2023-10-09T05:09:27.591 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028004037999999998 2023-10-09T05:09:27.591 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.591 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.591 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.591 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.202080+0000", 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0032359670000000002 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.503990+0000", 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30190954800000003 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.506993+0000", 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030031099999999998 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.592 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.512309+0000", 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0053163180000000004 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.534225+0000", 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.021915537999999998 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.593 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.535117+0000", 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000892 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.535405+0000", 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028789000000000001 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.536826+0000", 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014215880000000001 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.594 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.542974+0000", 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0061479940000000004 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92180 14.1c 14:3ca86941:::benchmark_data_smithi029_550670_object14569:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.995280294, 2023-10-09T05:09:27.595 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3662740299999996, 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92180 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.596 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474050+0000", 2023-10-09T05:09:27.597 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474053+0000", 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.768e-06 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474059+0000", 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4579999999999998e-06 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.598 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474065+0000", 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6949999999999997e-06 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474084+0000", 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9241000000000001e-05 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474087+0000", 2023-10-09T05:09:27.599 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0939999999999999e-06 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.249878+0000", 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7757908350000005 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.252725+0000", 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028465470000000001 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.600 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.281495+0000", 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028770463999999999 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.284552+0000", 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030568209999999999 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.601 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.660798+0000", 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37624634699999998 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.663753+0000", 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002954852 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.602 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.669646+0000", 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058927800000000002 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.699397+0000", 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029751418000000002 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.831873+0000", 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.132475605 2023-10-09T05:09:27.603 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.833045+0000", 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001172137 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.834456+0000", 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014105350000000001 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.604 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.840324+0000", 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058687949999999996 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92182 14.1c 14:3fd3171a:::benchmark_data_smithi029_550670_object14571:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.995258605, 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3752296690000003, 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.605 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92182 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.606 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474072+0000", 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474074+0000", 2023-10-09T05:09:27.607 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2910000000000002e-06 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474079+0000", 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4280000000000004e-06 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474083+0000", 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6009999999999999e-06 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.608 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474100+0000", 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7336000000000001e-05 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474103+0000", 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.853e-06 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.253164+0000", 2023-10-09T05:09:27.609 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7790607170000001 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.256107+0000", 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029431840000000002 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.295790+0000", 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.039682679999999998 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.610 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.298786+0000", 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029965109999999999 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.687348+0000", 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.38856141199999999 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.690035+0000", 2023-10-09T05:09:27.611 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026874889999999999 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.695938+0000", 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0059025290000000001 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.722603+0000", 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026665728999999999 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.612 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.841224+0000", 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.118620266 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.842115+0000", 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00089085200000000003 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.843556+0000", 2023-10-09T05:09:27.613 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014413410000000001 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.849302+0000", 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057457899999999998 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92208 14.1c 14:3948b646:::benchmark_data_smithi029_550670_object14597:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.614 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99524228600001, 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3847670389999998, 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92208 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.615 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474088+0000", 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.616 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474094+0000", 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8320000000000002e-06 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474100+0000", 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.5479999999999998e-06 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474104+0000", 2023-10-09T05:09:27.617 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.6770000000000001e-06 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474124+0000", 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0072e-05 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474127+0000", 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.869e-06 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.618 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.299950+0000", 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8258230119999999 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.302809+0000", 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0028591889999999998 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.619 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.320588+0000", 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017778321999999999 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.323603+0000", 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030157880000000001 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.711636+0000", 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.388033138 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.620 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.714179+0000", 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002542248 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.719284+0000", 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051056419999999996 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.746816+0000", 2023-10-09T05:09:27.621 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.027531282000000001 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.850167+0000", 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.103351006 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.851177+0000", 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010101140000000001 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.622 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.852656+0000", 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001479314 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.858855+0000", 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006199296 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.623 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92211 14.1c 14:3ce01d9d:::benchmark_data_smithi029_550670_object14600:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474112+0000", 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99521880100002, 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4040441320000001, 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92211 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.624 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474112+0000", 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474112+0000", 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474111+0000", 2023-10-09T05:09:27.625 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474114+0000", 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2230000000000001e-06 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474119+0000", 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2970000000000003e-06 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.626 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474122+0000", 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4419999999999998e-06 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474142+0000", 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9797e-05 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:09:27.627 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9299999999999999e-06 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.303237+0000", 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8290917640000002 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.306169+0000", 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029325200000000001 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.628 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.349452+0000", 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.043282893000000003 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.352483+0000", 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003030516 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.629 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.759382+0000", 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40689936700000001 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.761976+0000", 2023-10-09T05:09:27.630 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025939309999999998 2023-10-09T05:09:27.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.631 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.631 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.631 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.767278+0000", 2023-10-09T05:09:27.631 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005301516 2023-10-09T05:09:27.631 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.632 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.632 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.798076+0000", 2023-10-09T05:09:27.632 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.030798012999999999 2023-10-09T05:09:27.632 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.632 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.633 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.869441+0000", 2023-10-09T05:09:27.633 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.071365108999999996 2023-10-09T05:09:27.633 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.633 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.633 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.870336+0000", 2023-10-09T05:09:27.634 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00089563100000000003 2023-10-09T05:09:27.634 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.634 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.634 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.634 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.871911+0000", 2023-10-09T05:09:27.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015744979999999999 2023-10-09T05:09:27.635 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.635 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.635 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.635 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.878156+0000", 2023-10-09T05:09:27.635 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0062450029999999998 2023-10-09T05:09:27.636 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.636 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.636 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.636 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.636 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.636 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92219 14.1c 14:3b1c3ae5:::benchmark_data_smithi029_550670_object14608:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474128+0000", 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99520285800003, 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.394299921, 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.637 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92219 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474128+0000", 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.638 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474128+0000", 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.639 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474127+0000", 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474129+0000", 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0320000000000002e-06 2023-10-09T05:09:27.640 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474136+0000", 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1340000000000001e-06 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474139+0000", 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.332e-06 2023-10-09T05:09:27.641 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474168+0000", 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9552000000000001e-05 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.642 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474172+0000", 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7910000000000002e-06 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.307293+0000", 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8331210049999997 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.310098+0000", 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002804993 2023-10-09T05:09:27.643 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.334589+0000", 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024490607000000001 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.337720+0000", 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031307420000000002 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.644 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.735191+0000", 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39747148799999998 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.737741+0000", 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025500929999999998 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.742871+0000", 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0051294360000000002 2023-10-09T05:09:27.645 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.770928+0000", 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.028057289999999999 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.859794+0000", 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.088865880999999994 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.646 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.860794+0000", 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001000623 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.862284+0000", 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0014899030000000001 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.868428+0000", 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006143328 2023-10-09T05:09:27.647 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92256 14.c 14:30b496db:::benchmark_data_smithi029_550670_object14645:head [delete] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e618)", 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 383.99518575000002, 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1024558200000003, 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:27.648 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92256 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:09:27.649 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474145+0000", 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474147+0000", 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3259999999999998e-06 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.650 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474152+0000", 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.3909999999999997e-06 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474156+0000", 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.0709999999999996e-06 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474173+0000", 2023-10-09T05:09:27.651 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.628e-05 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:03.474176+0000", 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9330000000000001e-06 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.202517+0000", 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7283419310000001 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.652 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.205529+0000", 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0030117899999999999 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.234712+0000", 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029183121999999999 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for rw locks", 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.238172+0000", 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003459419 2023-10-09T05:09:27.653 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.543400+0000", 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30522812399999999 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.547146+0000", 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003746217 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.654 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,2", 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.554862+0000", 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0077156530000000003 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.560664+0000", 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058017729999999997 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.561083+0000", 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00041973100000000001 2023-10-09T05:09:27.655 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.565647+0000", 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045636569999999996 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.567799+0000", 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021523919999999999 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:27.656 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.576601+0000", 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0088013180000000007 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:27.657 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:27.659 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-09T05:09:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: pgmap v2329: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3324769221' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:09:27.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: osdmap e746: 8 total, 8 up, 8 in 2023-10-09T05:09:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.778 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:27 smithi177 ceph-mon[195797]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: pgmap v2329: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:27.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3324769221' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:09:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: osdmap e746: 8 total, 8 up, 8 in 2023-10-09T05:09:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[216903]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: pgmap v2329: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3324769221' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: osdmap e746: 8 total, 8 up, 8 in 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: failed to encode map e746 with expected crc 2023-10-09T05:09:27.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:27 smithi029 ceph-mon[223858]: failed to encode map e746 with expected crc 2023-10-09T05:09:28.029 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:28.149 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-09T05:09:28.493 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: osdmap e747: 8 total, 8 up, 8 in 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: 1.2 scrub starts 2023-10-09T05:09:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: 1.2 scrub ok 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: osdmap e747: 8 total, 8 up, 8 in 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: failed to encode map e747 with expected crc 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: 1.2 scrub starts 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: 1.2 scrub ok 2023-10-09T05:09:28.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: failed to encode map e747 with expected crc 2023-10-09T05:09:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: osdmap e747: 8 total, 8 up, 8 in 2023-10-09T05:09:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: failed to encode map e747 with expected crc 2023-10-09T05:09:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: failed to encode map e747 with expected crc 2023-10-09T05:09:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: failed to encode map e747 with expected crc 2023-10-09T05:09:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: 1.2 scrub starts 2023-10-09T05:09:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: 1.2 scrub ok 2023-10-09T05:09:29.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:29.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:29 smithi177 ceph-mon[195797]: pgmap v2332: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:29.760 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:29 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:29.849 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:29 smithi029 ceph-mon[216903]: pgmap v2332: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:29.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:29 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:29.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:29 smithi029 ceph-mon[223858]: pgmap v2332: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:29.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:29 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:09:29.959 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:29.959 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:29.960 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:09:29.960 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:30.639 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.1 to 1 (10000) 2023-10-09T05:09:30.672 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-09T05:09:30.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:30 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2193270339' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:30.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:30 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:30 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2193270339' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:30.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:30 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:30 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2193270339' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:31.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:30 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:31.036 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:31.533 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-09T05:09:31.898 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[216903]: pgmap v2333: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:31.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[216903]: failed to encode map e748 with expected crc 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[216903]: osdmap e748: 8 total, 8 up, 8 in 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[216903]: failed to encode map e748 with expected crc 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[216903]: failed to encode map e748 with expected crc 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[223858]: pgmap v2333: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[223858]: failed to encode map e748 with expected crc 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[223858]: osdmap e748: 8 total, 8 up, 8 in 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[223858]: failed to encode map e748 with expected crc 2023-10-09T05:09:31.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:31 smithi029 ceph-mon[223858]: failed to encode map e748 with expected crc 2023-10-09T05:09:32.026 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:31 smithi177 ceph-mon[195797]: pgmap v2333: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:31 smithi177 ceph-mon[195797]: failed to encode map e748 with expected crc 2023-10-09T05:09:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:31 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:09:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:31 smithi177 ceph-mon[195797]: osdmap e748: 8 total, 8 up, 8 in 2023-10-09T05:09:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:31 smithi177 ceph-mon[195797]: failed to encode map e748 with expected crc 2023-10-09T05:09:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:31 smithi177 ceph-mon[195797]: failed to encode map e748 with expected crc 2023-10-09T05:09:32.519 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:32.520 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:32.520 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:09:32.520 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:09:32.520 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:33.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:09:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:09:32] "GET /metrics HTTP/1.1" 200 34297 "" "Prometheus/2.43.0" 2023-10-09T05:09:33.239 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-09T05:09:33.585 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:33 smithi029 ceph-mon[216903]: pgmap v2335: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 10 op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:09:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:33 smithi029 ceph-mon[216903]: 1.0 scrub starts 2023-10-09T05:09:33.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:33 smithi029 ceph-mon[216903]: 1.0 scrub ok 2023-10-09T05:09:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:33 smithi029 ceph-mon[223858]: pgmap v2335: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 10 op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:09:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:33 smithi029 ceph-mon[223858]: 1.0 scrub starts 2023-10-09T05:09:33.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:33 smithi029 ceph-mon[223858]: 1.0 scrub ok 2023-10-09T05:09:33.975 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:33 smithi177 ceph-mon[195797]: pgmap v2335: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 14 KiB/s rd, 36 KiB/s wr, 10 op/s; 79 KiB/s, 0 objects/s recovering 2023-10-09T05:09:33.975 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:33 smithi177 ceph-mon[195797]: 1.0 scrub starts 2023-10-09T05:09:33.975 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:33 smithi177 ceph-mon[195797]: 1.0 scrub ok 2023-10-09T05:09:34.648 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-09T05:09:34.823 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:34 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1341500174' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:34.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:34 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1341500174' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:34.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:34 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1341500174' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-09T05:09:35.052 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:35.052 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:91970 14.4 14:21827fbb:::benchmark_data_smithi029_550670_object14359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:01:42.408999+0000", 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 472.64204079000001, 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0219353019999999, 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:35.053 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 91970 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.408999+0000", 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.054 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.408999+0000", 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.408998+0000", 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409006+0000", 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4769999999999997e-06 2023-10-09T05:09:35.055 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409022+0000", 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6721000000000001e-05 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409029+0000", 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9090000000000003e-06 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.056 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.409089+0000", 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.9553999999999999e-05 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.410296+0000", 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001207414 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.057 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.416006+0000", 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057100390000000001 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.416141+0000", 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00013511600000000001 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 1,3,6", 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.416305+0000", 2023-10-09T05:09:35.058 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000163812 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.422708+0000", 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0064030770000000001 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.461435+0000", 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.038726869999999997 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.059 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:42.461448+0000", 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3370000000000001e-05 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.430921+0000", 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.96947220599999995 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.060 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:01:43.430935+0000", 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4063000000000001e-05 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:91985 14.2 14:410dd76c:::benchmark_data_smithi029_550670_object14374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:01.825669+0000", 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 453.22537113999999, 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1116379600000004, 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.061 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 91985 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825669+0000", 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.062 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825669+0000", 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825666+0000", 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999971 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.063 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825683+0000", 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6543999999999999e-05 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825706+0000", 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2974999999999999e-05 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:01.825717+0000", 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1087000000000001e-05 2023-10-09T05:09:35.064 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.801677+0000", 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9759603759999997 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.822265+0000", 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020587215999999998 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.065 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.882023+0000", 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.059758517999999997 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.887238+0000", 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0052146980000000002 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:09:35.066 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.896986+0000", 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0097482279999999994 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.909274+0000", 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012287836999999999 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.911026+0000", 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0017521800000000001 2023-10-09T05:09:35.067 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.912977+0000", 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001950818 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.915190+0000", 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022132549999999999 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.068 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.918685+0000", 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00349512 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:05.937307+0000", 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.018621746000000002 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.069 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 593/586 [PushReplyOp(14:4529595e:::benchmark_data_smithi029_550670_object74277:head)])", 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:06.566413+0000", 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 448.48462722699998, 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.060305104999999998, 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566413+0000", 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.070 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566413+0000", 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566411+0000", 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.071 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566424+0000", 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2454e-05 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566441+0000", 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6787e-05 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.566451+0000", 2023-10-09T05:09:35.072 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0146000000000001e-05 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.581924+0000", 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015473169 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.626718+0000", 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.044794187999999999 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.073 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:443b08e2:::benchmark_data_smithi029_550670_object74358:head)])", 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:06.754907+0000", 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 448.29613293900002, 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.064498344999999999, 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.074 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754907+0000", 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754907+0000", 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754906+0000", 2023-10-09T05:09:35.075 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754918+0000", 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2072000000000001e-05 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754936+0000", 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7836999999999999e-05 2023-10-09T05:09:35.076 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.754946+0000", 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0248000000000001e-05 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.797183+0000", 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.042237441000000001 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.077 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:06.819406+0000", 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.022222301999999999 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:44043727:::benchmark_data_smithi029_550670_object74467:head)])", 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:07.009229+0000", 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 448.04181144799998, 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.067423306000000002, 2023-10-09T05:09:35.078 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009229+0000", 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009229+0000", 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.079 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009228+0000", 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009237+0000", 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2399999999999996e-06 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.080 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009253+0000", 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6350000000000001e-05 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.009261+0000", 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.4080000000000001e-06 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.033341+0000", 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.024080958 2023-10-09T05:09:35.081 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:07.076652+0000", 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.043310596999999999 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:46efb29e:::benchmark_data_smithi029_550670_object74990:head)])", 2023-10-09T05:09:35.082 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:08.028376+0000", 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 447.02266389599998, 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.069276355999999997, 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028376+0000", 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.083 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028376+0000", 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028375+0000", 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028382+0000", 2023-10-09T05:09:35.084 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8260000000000004e-06 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028398+0000", 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5622999999999999e-05 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.028404+0000", 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4189999999999999e-06 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.085 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.053474+0000", 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.025070012999999999 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:08.097653+0000", 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.044178336999999998 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.086 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92031 14.2 14:400f9103:::benchmark_data_smithi029_550670_object14420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:10.530240+0000", 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 444.52080027800002, 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.14341484199999999, 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:35.087 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92031 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530240+0000", 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530240+0000", 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.088 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530238+0000", 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530248+0000", 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0087e-05 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.089 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530267+0000", 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9137999999999999e-05 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.530275+0000", 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0879999999999995e-06 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.535696+0000", 2023-10-09T05:09:35.090 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054208019999999997 2023-10-09T05:09:35.091 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.091 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.091 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.552710+0000", 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.017013654 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.609958+0000", 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.057248649999999998 2023-10-09T05:09:35.093 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.612646+0000", 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0026877989999999998 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.617288+0000", 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0046419479999999999 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.094 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.653851+0000", 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.036563307000000003 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.655902+0000", 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020504949999999998 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:35.095 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.656580+0000", 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00067814399999999999 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.658429+0000", 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001848992 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.660996+0000", 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025669460000000001 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.096 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.673655+0000", 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012658849 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.2 594/586 [PushReplyOp(14:4589460d:::benchmark_data_smithi029_550670_object76508:head)])", 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:02:10.570016+0000", 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 444.48102452799998, 2023-10-09T05:09:35.097 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.081763137, 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570016+0000", 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570016+0000", 2023-10-09T05:09:35.098 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570014+0000", 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570025+0000", 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1127e-05 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.099 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570044+0000", 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8454e-05 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.570053+0000", 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8780000000000004e-06 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.100 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.622346+0000", 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.052293555999999998 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:02:10.651779+0000", 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.029432494 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.101 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:92277 14.4 14:2748781f:::benchmark_data_smithi029_550670_object14666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:09.402221+0000", 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 385.64881891700003, 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.070626093000000001, 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 92277 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.102 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402221+0000", 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402221+0000", 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.103 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402219+0000", 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402235+0000", 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5835999999999999e-05 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402258+0000", 2023-10-09T05:09:35.104 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3258e-05 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402267+0000", 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4150000000000005e-06 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.402335+0000", 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.7275999999999998e-05 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.105 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.403203+0000", 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00086857000000000004 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.454287+0000", 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.051083739000000003 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.106 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.454400+0000", 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000113029 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.454574+0000", 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00017359400000000001 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.456547+0000", 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019731409999999999 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.107 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.462441+0000", 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058945339999999999 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.463065+0000", 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062352499999999999 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.108 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.471014+0000", 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079486799999999996 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.471090+0000", 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.5967e-05 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:09.472847+0000", 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001757855 2023-10-09T05:09:35.109 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93464 14.4 14:267b05b2:::benchmark_data_smithi029_550670_object15853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e630)", 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:31.775033+0000", 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 363.27600738899997, 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.086900612000000002, 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:35.110 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93464 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775033+0000", 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.111 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775033+0000", 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775032+0000", 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775038+0000", 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2709999999999998e-06 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.112 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775053+0000", 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4953000000000001e-05 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775059+0000", 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.789e-06 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.113 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.775129+0000", 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0958999999999996e-05 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.776109+0000", 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097991599999999995 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.855584+0000", 2023-10-09T05:09:35.114 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.079474808999999993 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.855678+0000", 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.4109000000000003e-05 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3,6", 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.855787+0000", 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010880200000000001 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.115 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.858366+0000", 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0025790800000000001 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.859247+0000", 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000880618 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.859494+0000", 2023-10-09T05:09:35.116 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00024704600000000001 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.860324+0000", 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083051700000000002 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.860366+0000", 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1770999999999997e-05 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.117 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:31.861933+0000", 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0015672850000000001 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93769 14.1f 14.e529d53f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e641)", 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:51.428253+0000", 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 343.62278688599997, 2023-10-09T05:09:35.118 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.582461071, 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93769 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.119 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428253+0000", 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428253+0000", 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428252+0000", 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.120 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428259+0000", 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0709999999999997e-06 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428273+0000", 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3718e-05 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.121 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428280+0000", 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3209999999999999e-06 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428351+0000", 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1095999999999998e-05 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:51.428364+0000", 2023-10-09T05:09:35.122 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2873e-05 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:52.010701+0000", 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.58233696199999996 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.123 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:52.010714+0000", 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3111e-05 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36948.0:97931 14.1f e652/642 14:fb9dafde:::benchmark_data_smithi029_550670_object20320:head v 652'5879, mlcod=0'0)", 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:21.081459+0000", 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.96958134400001, 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.13545954499999999, 2023-10-09T05:09:35.124 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081459+0000", 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081459+0000", 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.125 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081458+0000", 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081471+0000", 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.33e-05 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.126 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081491+0000", 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0157e-05 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081499+0000", 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.0099999999999995e-06 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081587+0000", 2023-10-09T05:09:35.127 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7734999999999999e-05 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.081621+0000", 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.4561999999999999e-05 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.216891+0000", 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.135269841 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.128 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.216918+0000", 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7217e-05 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36948.0:97940 14.f e652/641 14:f48c4e8b:::benchmark_data_smithi029_550670_object20329:head v 652'5807, mlcod=0'0)", 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:21.120358+0000", 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.93068201400001, 2023-10-09T05:09:35.129 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.096529511999999998, 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120358+0000", 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120358+0000", 2023-10-09T05:09:35.130 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120357+0000", 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120369+0000", 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2026e-05 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120384+0000", 2023-10-09T05:09:35.131 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.5478e-05 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120391+0000", 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2709999999999998e-06 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120436+0000", 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5090999999999999e-05 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.132 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.120469+0000", 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3311e-05 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.216839+0000", 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.096369837999999999 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.216888+0000", 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.8717000000000001e-05 2023-10-09T05:09:35.133 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36948.0:97943 14.5 e652/644 14:a00492e9:::benchmark_data_smithi029_550670_object20332:head v 652'5635, mlcod=651'5633)", 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:21.139909+0000", 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 313.91113136199999, 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.076978374000000002, 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.134 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139909+0000", 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139909+0000", 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139908+0000", 2023-10-09T05:09:35.135 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139917+0000", 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.5729999999999997e-06 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139935+0000", 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7597000000000001e-05 2023-10-09T05:09:35.136 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139942+0000", 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.1219999999999997e-06 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.139993+0000", 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1082999999999999e-05 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.137 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.140020+0000", 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6400000000000001e-05 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.216856+0000", 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.076836370000000001 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:21.216887+0000", 2023-10-09T05:09:35.138 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1313000000000003e-05 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36948.0:133013 14.4 e671/639 14:26df984e:::benchmark_data_smithi029_550670_object55402:head v 671'6704, mlcod=446'3440)", 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:51.233661+0000", 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 163.817379434, 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0096438849999999996, 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.139 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233661+0000", 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233661+0000", 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.140 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233658+0000", 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999976 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233674+0000", 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6059e-05 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.141 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233707+0000", 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.3030000000000001e-05 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233722+0000", 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4949e-05 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.142 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233830+0000", 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00010721399999999999 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.233855+0000", 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5267000000000001e-05 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.243280+0000", 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0094250619999999997 2023-10-09T05:09:35.143 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:51.243305+0000", 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.4677000000000001e-05 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36948.0:146570 14.15 e677/649 14:ae22bcbc:::benchmark_data_smithi029_550670_object68959:head v 677'7097, mlcod=677'7094)", 2023-10-09T05:09:35.144 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:07:22.185474+0000", 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 132.86556599799999, 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076972459999999996, 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185474+0000", 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.145 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185474+0000", 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185473+0000", 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.146 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185481+0000", 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.8860000000000004e-06 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185493+0000", 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2179e-05 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185500+0000", 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.511e-06 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.147 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.189280+0000", 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037801200000000001 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.189297+0000", 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6943e-05 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.148 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.193140+0000", 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003842874 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.193171+0000", 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1677000000000002e-05 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.149 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_repop(client.36948.0:146576 14.5 e677/644 14:a734d7c5:::benchmark_data_smithi029_550670_object68965:head v 677'7135, mlcod=677'7132)", 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:07:22.185506+0000", 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 132.86553404099999, 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0076285340000000002, 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185506+0000", 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.150 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185506+0000", 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185506+0000", 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.151 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185510+0000", 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9539999999999998e-06 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185515+0000", 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.412e-06 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.185519+0000", 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2969999999999999e-06 2023-10-09T05:09:35.152 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.186876+0000", 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001357271 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.186895+0000", 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8995e-05 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.153 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.193089+0000", 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.006194069 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:07:22.193135+0000", 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5781000000000001e-05 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.154 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(2.0 741/740 [PushOp(2:0de5d4e6:::rbd_data.bcd5b41e9b33.000000000000:head, version: 724'109, data_included: [0~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(2:0de5d4e6:::rbd_data.bcd5b41e9b33.000000000000:head@724'109, size: 4194304, copy_subset: [0~4194304], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:1048576, data_complete:false, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:09:11.595505+0000", 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 23.455535511000001, 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0020950309999999998, 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.595505+0000", 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.155 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.595505+0000", 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.595504+0000", 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.156 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.596956+0000", 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00145188 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.596975+0000", 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.9485000000000001e-05 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.597002+0000", 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6571e-05 2023-10-09T05:09:35.157 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.597063+0000", 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1396999999999995e-05 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.597069+0000", 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8459999999999996e-06 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.158 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.597600+0000", 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000530683 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPush(2.0 741/740 [PushOp(2:0de5d4e6:::rbd_data.bcd5b41e9b33.000000000000:head, version: 724'109, data_included: [3145728~1048576], data_size: 1048576, omap_header_size: 0, omap_entries_size: 0, attrset_size: 0, recovery_info: ObjectRecoveryInfo(2:0de5d4e6:::rbd_data.bcd5b41e9b33.000000000000:head@724'109, size: 4194304, copy_subset: [0~4194304], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(!first, data_recovered_to:3145728, data_complete:false, omap_recovered_to:, omap_complete:true, error:false))])", 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:09:11.605697+0000", 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 23.445343387000001, 2023-10-09T05:09:35.159 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002190226, 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.605697+0000", 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.605697+0000", 2023-10-09T05:09:35.160 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.605696+0000", 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.607254+0000", 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001557744 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.161 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.607271+0000", 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.6660999999999999e-05 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.607296+0000", 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5065000000000001e-05 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.607353+0000", 2023-10-09T05:09:35.162 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7627999999999998e-05 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.607362+0000", 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6130000000000004e-06 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:11.607887+0000", 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00052515699999999995 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.163 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDScrubReserve(1.0 REQUEST e748)", 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:09:32.822402+0000", 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 2.2286386230000002, 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014881700000000001, 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "started", 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.164 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822402+0000", 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822402+0000", 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822400+0000", 2023-10-09T05:09:35.165 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822409+0000", 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.0960000000000007e-06 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822432+0000", 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.3020999999999999e-05 2023-10-09T05:09:35.166 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822441+0000", 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8939999999999999e-06 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822492+0000", 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1051999999999998e-05 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.167 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822515+0000", 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.2880000000000001e-05 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:09:32.822550+0000", 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.5290999999999998e-05 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:35.168 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:35.169 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:35.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-09T05:09:35.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-09T05:09:35.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[223858]: pgmap v2336: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 9 op/s; 64 KiB/s, 0 objects/s recovering 2023-10-09T05:09:35.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[223858]: failed to encode map e749 with expected crc 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1341500174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[223858]: osdmap e749: 8 total, 8 up, 8 in 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[223858]: failed to encode map e749 with expected crc 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[223858]: failed to encode map e749 with expected crc 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[216903]: pgmap v2336: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 9 op/s; 64 KiB/s, 0 objects/s recovering 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[216903]: failed to encode map e749 with expected crc 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1341500174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[216903]: osdmap e749: 8 total, 8 up, 8 in 2023-10-09T05:09:35.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[216903]: failed to encode map e749 with expected crc 2023-10-09T05:09:35.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:35 smithi029 ceph-mon[216903]: failed to encode map e749 with expected crc 2023-10-09T05:09:35.961 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:35 smithi177 ceph-mon[195797]: pgmap v2336: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 9 op/s; 64 KiB/s, 0 objects/s recovering 2023-10-09T05:09:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:35 smithi177 ceph-mon[195797]: failed to encode map e749 with expected crc 2023-10-09T05:09:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:35 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1341500174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-09T05:09:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:35 smithi177 ceph-mon[195797]: osdmap e749: 8 total, 8 up, 8 in 2023-10-09T05:09:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:35 smithi177 ceph-mon[195797]: failed to encode map e749 with expected crc 2023-10-09T05:09:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:35 smithi177 ceph-mon[195797]: failed to encode map e749 with expected crc 2023-10-09T05:09:36.443 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:36.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:36 smithi029 ceph-mon[216903]: failed to encode map e749 with expected crc 2023-10-09T05:09:36.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:36 smithi029 ceph-mon[223858]: failed to encode map e749 with expected crc 2023-10-09T05:09:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:36 smithi177 ceph-mon[195797]: failed to encode map e749 with expected crc 2023-10-09T05:09:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:37 smithi029 ceph-mon[216903]: pgmap v2338: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:37 smithi029 ceph-mon[223858]: pgmap v2338: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:37 smithi177 ceph-mon[195797]: pgmap v2338: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:39.253 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:09:39.253 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:09:39.253 INFO:teuthology.orchestra.run.smithi029.stdout: "num_ops": 0 2023-10-09T05:09:39.253 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:09:39.656 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.3 to 1 (10000) 2023-10-09T05:09:39.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:39 smithi029 ceph-mon[216903]: pgmap v2339: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:39 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2854250272' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:39.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:39 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:39 smithi029 ceph-mon[223858]: pgmap v2339: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:39 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2854250272' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:39.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:39 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:39 smithi177 ceph-mon[195797]: pgmap v2339: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:39 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2854250272' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:39 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:40.412 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-09T05:09:40.808 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-09T05:09:40.880 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:40.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[216903]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[216903]: osdmap e750: 8 total, 8 up, 8 in 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[216903]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[216903]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[216903]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[223858]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[223858]: osdmap e750: 8 total, 8 up, 8 in 2023-10-09T05:09:40.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[223858]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[223858]: failed to encode map e750 with expected crc 2023-10-09T05:09:40.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:40 smithi029 ceph-mon[223858]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:40 smithi177 ceph-mon[195797]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:40 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:09:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:40 smithi177 ceph-mon[195797]: osdmap e750: 8 total, 8 up, 8 in 2023-10-09T05:09:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:40 smithi177 ceph-mon[195797]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:40 smithi177 ceph-mon[195797]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:40 smithi177 ceph-mon[195797]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:41 smithi029 ceph-mon[216903]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:41 smithi029 ceph-mon[216903]: pgmap v2341: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:41 smithi029 ceph-mon[223858]: failed to encode map e750 with expected crc 2023-10-09T05:09:41.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:41 smithi029 ceph-mon[223858]: pgmap v2341: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:41 smithi177 ceph-mon[195797]: failed to encode map e750 with expected crc 2023-10-09T05:09:42.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:41 smithi177 ceph-mon[195797]: pgmap v2341: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:42.210 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:42.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:09:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:09:42] "GET /metrics HTTP/1.1" 200 34297 "" "Prometheus/2.43.0" 2023-10-09T05:09:43.652 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:09:43.652 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [], 2023-10-09T05:09:43.652 INFO:teuthology.orchestra.run.smithi029.stdout: "complaint_time": 30, 2023-10-09T05:09:43.652 INFO:teuthology.orchestra.run.smithi029.stdout: "num_blocked_ops": 0 2023-10-09T05:09:43.652 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:09:43.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:43 smithi029 ceph-mon[223858]: pgmap v2342: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:43.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:43 smithi029 ceph-mon[216903]: pgmap v2342: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:43.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:43 smithi177 ceph-mon[195797]: pgmap v2342: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:44.624 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-09T05:09:45.002 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:45.670 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-09T05:09:45.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:45 smithi029 ceph-mon[216903]: pgmap v2343: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:45.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:45 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1380267392' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:45 smithi029 ceph-mon[223858]: pgmap v2343: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:45 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1380267392' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:45 smithi177 ceph-mon[195797]: pgmap v2343: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:45 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1380267392' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-09T05:09:46.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[216903]: failed to encode map e751 with expected crc 2023-10-09T05:09:46.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1380267392' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:09:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[216903]: osdmap e751: 8 total, 8 up, 8 in 2023-10-09T05:09:46.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[216903]: failed to encode map e751 with expected crc 2023-10-09T05:09:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[223858]: failed to encode map e751 with expected crc 2023-10-09T05:09:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1380267392' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:09:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[223858]: osdmap e751: 8 total, 8 up, 8 in 2023-10-09T05:09:46.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:46 smithi029 ceph-mon[223858]: failed to encode map e751 with expected crc 2023-10-09T05:09:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:46 smithi177 ceph-mon[195797]: failed to encode map e751 with expected crc 2023-10-09T05:09:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:46 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1380267392' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-09T05:09:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:46 smithi177 ceph-mon[195797]: osdmap e751: 8 total, 8 up, 8 in 2023-10-09T05:09:47.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:46 smithi177 ceph-mon[195797]: failed to encode map e751 with expected crc 2023-10-09T05:09:47.156 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-09T05:09:47.465 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2023-10-09T05:09:47.465 INFO:teuthology.orchestra.run.smithi029.stdout: "size": 20, 2023-10-09T05:09:47.465 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 600, 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "ops": [ 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.23 e658/615 cost 1049576)", 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:04:45.481529+0000", 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 301.98238148600001, 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010627640000000001, 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.466 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.481529+0000", 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.481529+0000", 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.467 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.481528+0000", 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.481549+0000", 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1192999999999999e-05 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.481571+0000", 2023-10-09T05:09:47.468 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2478999999999999e-05 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.481582+0000", 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0326000000000001e-05 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.482078+0000", 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000496081 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.469 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:04:45.492157+0000", 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010079124 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.1b e661/628 cost 1049576)", 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:12.613903+0000", 2023-10-09T05:09:47.470 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 274.85000817600002, 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011554205, 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.613903+0000", 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.471 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.613903+0000", 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.613901+0000", 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.472 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.613920+0000", 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8927999999999999e-05 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.613946+0000", 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.5522999999999999e-05 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.613957+0000", 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1194e-05 2023-10-09T05:09:47.473 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.615751+0000", 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0017943939999999999 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:12.625457+0000", 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0097056339999999994 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.474 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.6 e664/628 cost 1049576)", 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:34.940986+0000", 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 252.52292457799999, 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012153291, 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.940986+0000", 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.475 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.940986+0000", 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.940985+0000", 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.476 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.941002+0000", 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6923e-05 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.941029+0000", 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7784999999999999e-05 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.941039+0000", 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.4809999999999995e-06 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.477 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.941118+0000", 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.8953000000000001e-05 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:34.953140+0000", 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012021707 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.478 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.6 e664/628 cost 1049576)", 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:35.728131+0000", 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 251.73577946899999, 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.014092027999999999, 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.728131+0000", 2023-10-09T05:09:47.479 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.728131+0000", 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.728130+0000", 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.480 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.728149+0000", 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9267e-05 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.728174+0000", 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.4630999999999998e-05 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.728185+0000", 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1596999999999999e-05 2023-10-09T05:09:47.481 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.731105+0000", 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.002919736 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:35.742223+0000", 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011118193 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.482 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.f e664/641 cost 1049576)", 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:37.376104+0000", 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 250.08780710600001, 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011851204000000001, 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.483 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.376104+0000", 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.376104+0000", 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.376102+0000", 2023-10-09T05:09:47.484 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.376121+0000", 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8595000000000001e-05 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.376143+0000", 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2663e-05 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.485 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.376155+0000", 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1768e-05 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.379081+0000", 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0029258330000000001 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:37.387955+0000", 2023-10-09T05:09:47.486 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0088738740000000003 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.4 e665/639 cost 1049576)", 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:44.183492+0000", 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 243.28041835600001, 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016628536999999999, 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.487 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183492+0000", 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183492+0000", 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.488 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183491+0000", 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183507+0000", 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5897e-05 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183528+0000", 2023-10-09T05:09:47.489 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1192999999999999e-05 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183538+0000", 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.577e-06 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.183618+0000", 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9913999999999994e-05 2023-10-09T05:09:47.490 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:44.200121+0000", 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.016503205 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.b e665/628 cost 1049576)", 2023-10-09T05:09:47.491 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:47.733791+0000", 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 239.73011966199999, 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011506384999999999, 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.733791+0000", 2023-10-09T05:09:47.492 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.733791+0000", 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.733790+0000", 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.493 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.733802+0000", 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1211e-05 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.733812+0000", 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0409e-05 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.494 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.733819+0000", 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.9500000000000004e-06 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.745032+0000", 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011213589 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:47.745298+0000", 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00026508499999999997 2023-10-09T05:09:47.495 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.1f e665/642 cost 1049576)", 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:05:48.263014+0000", 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 239.200897247, 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011343784000000001, 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.496 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.263014+0000", 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.263014+0000", 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.497 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.263013+0000", 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.263025+0000", 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2853999999999999e-05 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.263041+0000", 2023-10-09T05:09:47.498 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5981000000000002e-05 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.263049+0000", 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 7.9119999999999998e-06 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.274078+0000", 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011028993000000001 2023-10-09T05:09:47.499 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:05:48.274357+0000", 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00027907000000000002 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.1b e666/628 cost 1049576)", 2023-10-09T05:09:47.500 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:05.826544+0000", 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 221.63736729199999, 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010655744999999999, 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.826544+0000", 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.501 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.826544+0000", 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.826542+0000", 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.502 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.826555+0000", 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.2702e-05 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.826565+0000", 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.009e-05 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.826571+0000", 2023-10-09T05:09:47.503 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.4309999999999999e-06 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.828073+0000", 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0015013400000000001 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:05.837199+0000", 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0091265559999999992 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.504 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.4 e668/639 cost 1049576)", 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:09.660979+0000", 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 217.80293203400001, 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010164939, 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.505 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.660979+0000", 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.660979+0000", 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.660977+0000", 2023-10-09T05:09:47.506 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.660996+0000", 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8332e-05 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.661024+0000", 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.8518e-05 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.507 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.661035+0000", 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1094000000000001e-05 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.668656+0000", 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0076207050000000002 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:09.671144+0000", 2023-10-09T05:09:47.508 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0024876360000000001 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.18 e668/603 cost 1049576)", 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:12.261102+0000", 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 215.202808428, 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013003083, 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.509 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.261102+0000", 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.261102+0000", 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.510 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.261101+0000", 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.261119+0000", 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.8488999999999999e-05 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.511 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.261142+0000", 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2535999999999999e-05 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.261153+0000", 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.1590000000000001e-05 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.267981+0000", 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0068279070000000002 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.512 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:12.274106+0000", 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0061241000000000004 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.16 e668/628 cost 1049576)", 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:17.086357+0000", 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 210.37755339099999, 2023-10-09T05:09:47.513 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012770769, 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.086357+0000", 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.514 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.086357+0000", 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.086356+0000", 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.086388+0000", 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.1650999999999999e-05 2023-10-09T05:09:47.515 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.086411+0000", 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3289e-05 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.086421+0000", 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0298e-05 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.516 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.088114+0000", 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0016930789999999999 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:17.099128+0000", 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011014006999999999 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.517 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.8 e670/603 cost 1049576)", 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:41.078264+0000", 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 186.38564643500001, 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011898409, 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.518 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.078264+0000", 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.078264+0000", 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.519 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.078263+0000", 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.078279+0000", 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.6194000000000001e-05 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.078302+0000", 2023-10-09T05:09:47.520 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2357e-05 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.078312+0000", 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0648e-05 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.079184+0000", 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00087206999999999996 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.521 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:41.090163+0000", 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.010978570999999999 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.b e671/628 cost 1049576)", 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:47.434034+0000", 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 180.029876363, 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013103907999999999, 2023-10-09T05:09:47.522 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.434034+0000", 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.434034+0000", 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.523 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.434033+0000", 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.434051+0000", 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7567999999999998e-05 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.524 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.434074+0000", 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.2908000000000002e-05 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.434084+0000", 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0182e-05 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.525 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.435286+0000", 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00120254 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:47.447138+0000", 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.011851907 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.526 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.1f e671/642 cost 1049576)", 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:06:49.563507+0000", 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 177.90040371399999, 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013636466, 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563507+0000", 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.527 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563507+0000", 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563506+0000", 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563524+0000", 2023-10-09T05:09:47.528 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.7785e-05 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563544+0000", 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.0472999999999999e-05 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563553+0000", 2023-10-09T05:09:47.529 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.9309999999999997e-06 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.563609+0000", 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 5.5924000000000002e-05 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:06:49.577144+0000", 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.013534594 2023-10-09T05:09:47.530 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.1f e674/642 cost 1049576)", 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:05.220870+0000", 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 162.24304059900001, 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0093323610000000008, 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.531 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.220870+0000", 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.220870+0000", 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.532 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.220869+0000", 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.220885+0000", 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.5554e-05 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.220906+0000", 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1537000000000001e-05 2023-10-09T05:09:47.533 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.220916+0000", 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.7419999999999992e-06 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.225083+0000", 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0041669350000000001 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.534 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:05.230203+0000", 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0051198349999999997 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPull(14.16 e674/628 cost 1049576)", 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:07:09.005900+0000", 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 158.45801117299999, 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.012886183000000001, 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.535 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "reached pg", 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.005900+0000", 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.005900+0000", 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.536 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.005899+0000", 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.005916+0000", 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.747e-05 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.537 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.005937+0000", 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1029000000000001e-05 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.005946+0000", 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 9.2949999999999999e-06 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.011558+0000", 2023-10-09T05:09:47.538 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0056114379999999998 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:07:09.018786+0000", 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.0072280779999999998 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(3.14 735/734 [PushOp(3:2e2fa760:::200.00000001:head, version: 82'22, data_included: [0~4090], data_size: 4090, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:2e2fa760:::200.00000001:head@82'22, size: 4090, copy_subset: [0~4090], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4090, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:09:47.539 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:09:05.244317+0000", 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 42.219593834999998, 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000424843, 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244317+0000", 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.540 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244317+0000", 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244316+0000", 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244344+0000", 2023-10-09T05:09:47.541 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.7869999999999999e-05 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244377+0000", 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 3.3643000000000003e-05 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244401+0000", 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.3926000000000001e-05 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.542 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244537+0000", 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000136031 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244559+0000", 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1736999999999999e-05 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.543 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.244742+0000", 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00018296999999999999 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(3.13 735/734 [PushOp(3:cd5a64a3:::100.00000000:head, version: 68'1, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 10, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:cd5a64a3:::100.00000000:head@68'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:09:05.246755+0000", 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 42.217155642999998, 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00031731700000000001, 2023-10-09T05:09:47.544 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246755+0000", 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246755+0000", 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.545 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246755+0000", 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246776+0000", 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1324000000000001e-05 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.546 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246795+0000", 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9511e-05 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246809+0000", 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.3662999999999999e-05 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.547 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246878+0000", 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 6.8623999999999999e-05 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.246892+0000", 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.4405e-05 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.548 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:05.247073+0000", 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.000180445 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "description": "MOSDPGPush(1.8 736/735 [PushOp(1:1b0e96ae:devicehealth::main.db-journal.0000000000000000:head, version: 22'116, data_included: [0~262688], data_size: 262688, omap_header_size: 0, omap_entries_size: 0, attrset_size: 9, recovery_info: ObjectRecoveryInfo(1:1b0e96ae:devicehealth::main.db-journal.0000000000000000:head@22'116, size: 262688, copy_subset: [0~262688], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:262688, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "initiated_at": "2023-10-09T05:09:07.199846+0000", 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "age": 40.264064388999998, 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00077223000000000005, 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "type_data": { 2023-10-09T05:09:47.549 INFO:teuthology.orchestra.run.smithi029.stdout: "flag_point": "started", 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "events": [ 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "initiated", 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.199846+0000", 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "throttled", 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.199846+0000", 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.550 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "header_read", 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.199846+0000", 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 4294967295.999999 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "all_read", 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.200203+0000", 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00035776800000000001 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "dispatched", 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.200225+0000", 2023-10-09T05:09:47.551 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 2.1262000000000001e-05 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "queued_for_pg", 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.200245+0000", 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.9806999999999999e-05 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "reached_pg", 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.200325+0000", 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 8.0939999999999994e-05 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.552 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "started", 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.200336+0000", 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 1.0285000000000001e-05 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: { 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: "event": "done", 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: "time": "2023-10-09T05:09:07.200619+0000", 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: "duration": 0.00028295099999999998 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: } 2023-10-09T05:09:47.553 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2023-10-09T05:09:47.554 INFO:teuthology.orchestra.run.smithi029.stdout:} 2023-10-09T05:09:47.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:47 smithi029 ceph-mon[216903]: pgmap v2345: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:47 smithi029 ceph-mon[223858]: pgmap v2345: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:47 smithi177 ceph-mon[195797]: pgmap v2345: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:48.429 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-09T05:09:48.521 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:48 smithi177 ceph-mon[195797]: failed to encode map e751 with expected crc 2023-10-09T05:09:48.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:48 smithi177 ceph-mon[195797]: failed to encode map e751 with expected crc 2023-10-09T05:09:48.788 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:49.028 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:48 smithi029 ceph-mon[216903]: failed to encode map e751 with expected crc 2023-10-09T05:09:49.028 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:48 smithi029 ceph-mon[216903]: failed to encode map e751 with expected crc 2023-10-09T05:09:49.028 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:48 smithi029 ceph-mon[223858]: failed to encode map e751 with expected crc 2023-10-09T05:09:49.029 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:48 smithi029 ceph-mon[223858]: failed to encode map e751 with expected crc 2023-10-09T05:09:50.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:49 smithi177 ceph-mon[195797]: pgmap v2346: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:50.129 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:49 smithi029 ceph-mon[216903]: pgmap v2346: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:50.129 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:49 smithi029 ceph-mon[223858]: pgmap v2346: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:50.362 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:50.362 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:50.362 INFO:teuthology.orchestra.run.smithi177.stdout: "num_ops": 0 2023-10-09T05:09:50.362 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:50.693 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.5 to 1 (10000) 2023-10-09T05:09:50.987 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:50 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2315468976' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:50.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:50 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2315468976' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:50.994 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:50 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2315468976' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:51.002 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-09T05:09:51.351 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:51.689 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-09T05:09:51.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[216903]: pgmap v2347: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:51.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[216903]: failed to encode map e752 with expected crc 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2315468976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[216903]: osdmap e752: 8 total, 8 up, 8 in 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[216903]: failed to encode map e752 with expected crc 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[223858]: pgmap v2347: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[223858]: failed to encode map e752 with expected crc 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2315468976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[223858]: osdmap e752: 8 total, 8 up, 8 in 2023-10-09T05:09:51.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:51 smithi029 ceph-mon[223858]: failed to encode map e752 with expected crc 2023-10-09T05:09:51.944 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:51 smithi177 ceph-mon[195797]: pgmap v2347: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:51.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:51 smithi177 ceph-mon[195797]: failed to encode map e752 with expected crc 2023-10-09T05:09:51.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:51 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2315468976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:09:51.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:51 smithi177 ceph-mon[195797]: osdmap e752: 8 total, 8 up, 8 in 2023-10-09T05:09:51.945 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:51 smithi177 ceph-mon[195797]: failed to encode map e752 with expected crc 2023-10-09T05:09:52.055 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:52.872 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:52.872 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [], 2023-10-09T05:09:52.873 INFO:teuthology.orchestra.run.smithi177.stdout: "complaint_time": 30, 2023-10-09T05:09:52.873 INFO:teuthology.orchestra.run.smithi177.stdout: "num_blocked_ops": 0 2023-10-09T05:09:52.873 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:52 smithi177 ceph-mon[195797]: failed to encode map e752 with expected crc 2023-10-09T05:09:53.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:52 smithi177 ceph-mon[195797]: failed to encode map e752 with expected crc 2023-10-09T05:09:53.171 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:09:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:09:52] "GET /metrics HTTP/1.1" 200 34300 "" "Prometheus/2.43.0" 2023-10-09T05:09:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:52 smithi029 ceph-mon[216903]: failed to encode map e752 with expected crc 2023-10-09T05:09:53.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:52 smithi029 ceph-mon[216903]: failed to encode map e752 with expected crc 2023-10-09T05:09:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:52 smithi029 ceph-mon[223858]: failed to encode map e752 with expected crc 2023-10-09T05:09:53.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:52 smithi029 ceph-mon[223858]: failed to encode map e752 with expected crc 2023-10-09T05:09:53.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-09T05:09:53.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:53 smithi177 ceph-mon[195797]: pgmap v2349: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:53.863 INFO:teuthology.orchestra.run.smithi177.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-09T05:09:54.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:53 smithi029 ceph-mon[216903]: pgmap v2349: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:54.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:53 smithi029 ceph-mon[223858]: pgmap v2349: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:54.708 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-09T05:09:55.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:54 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/805674536' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:55.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:54 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:55.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:54 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/805674536' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:55.035 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:54 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:55.035 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:54 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/805674536' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:55.035 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:54 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-09T05:09:55.382 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "size": 20, 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 600, 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "ops": [ 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93380 14.7 14.dfd5a7a7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.99881476199999, 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94592668400000002, 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.383 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93380 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.384 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381651+0000", 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381649+0000", 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381660+0000", 2023-10-09T05:09:55.385 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1054e-05 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381674+0000", 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.385e-05 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.386 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381683+0000", 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.7490000000000004e-06 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381756+0000", 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3750000000000004e-05 2023-10-09T05:09:55.387 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381767+0000", 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0208e-05 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327567+0000", 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94579980799999996 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.388 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327577+0000", 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.08e-05 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93400 14.1c 14:3dc013b3:::benchmark_data_smithi029_550670_object15789:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:09:55.389 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.99876208299997, 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3919103770000003, 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93400 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.390 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.391 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381703+0000", 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381706+0000", 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.8140000000000001e-06 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.392 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381711+0000", 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.575e-06 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381715+0000", 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.8240000000000001e-06 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381812+0000", 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.6555e-05 2023-10-09T05:09:55.393 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381822+0000", 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0302000000000001e-05 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.015600+0000", 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6337784260000001 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.394 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.017575+0000", 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019744390000000001 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.771430+0000", 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.75385499199999995 2023-10-09T05:09:55.395 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.773614+0000", 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0021838650000000001 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.396 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93401 14.7 14.decc63e7 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.99872660800003, 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94601877700000003, 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.397 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93401 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.398 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381739+0000", 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381738+0000", 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.399 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381741+0000", 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.6749999999999998e-06 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381747+0000", 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5799999999999999e-06 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381751+0000", 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6689999999999997e-06 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.400 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381798+0000", 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6303e-05 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381805+0000", 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0190000000000001e-06 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.401 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327746+0000", 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94594101799999997 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.327758+0000", 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1878999999999999e-05 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.402 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93405 14.17 14.5711f157 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.99870706000002, 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94466796099999994, 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.403 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93405 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:09:55.404 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381758+0000", 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381761+0000", 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.5280000000000002e-06 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.405 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381766+0000", 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9130000000000002e-06 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.381769+0000", 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.788e-06 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.406 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382138+0000", 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00036829199999999999 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382146+0000", 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.8200000000000003e-06 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.326413+0000", 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94426679400000002 2023-10-09T05:09:55.407 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:25.326426+0000", 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.3084999999999999e-05 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.408 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93407 14.5 14:a209006e:::benchmark_data_smithi029_550670_object15796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.998358096, 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2579248559999998, 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.409 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93407 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:09:55.410 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382107+0000", 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382114+0000", 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.2089999999999999e-06 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.411 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382124+0000", 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0268000000000001e-05 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382134+0000", 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.7899999999999994e-06 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.412 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382191+0000", 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6889999999999999e-05 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382201+0000", 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0359e-05 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.374861+0000", 2023-10-09T05:09:55.413 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.9926602410000003 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for missing object", 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:27.375679+0000", 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00081763400000000003 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.639295+0000", 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.263616246 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.414 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.640032+0000", 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00073696500000000002 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93418 14.1c 14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:09:55.415 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.99831529400001, 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.3997008879999999, 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93418 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:09:55.416 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382150+0000", 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.417 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382153+0000", 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0680000000000001e-06 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382160+0000", 2023-10-09T05:09:55.418 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9639999999999997e-06 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382165+0000", 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.4569999999999996e-06 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382198+0000", 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.2703999999999997e-05 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.419 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.382205+0000", 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8399999999999997e-06 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.017835+0000", 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.6356300770000001 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.420 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.025741+0000", 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0079060310000000009 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.751118+0000", 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72537742100000002 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.753397+0000", 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022792559999999999 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.421 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.757192+0000", 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0037947889999999998 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.760499+0000", 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033070700000000001 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.422 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.774353+0000", 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.013853202 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.775194+0000", 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00084157500000000001 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.776410+0000", 2023-10-09T05:09:55.423 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012155670000000001 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.781851+0000", 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0054413259999999998 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93419 14.15 14:a8313c41:::benchmark_data_smithi029_550670_object15808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e627)", 2023-10-09T05:09:55.424 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 390.97531961800001, 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2483047789999997, 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93419 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.425 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405146+0000", 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405145+0000", 2023-10-09T05:09:55.426 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405149+0000", 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6789999999999996e-06 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405159+0000", 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.8779999999999993e-06 2023-10-09T05:09:55.427 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405163+0000", 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.1359999999999999e-06 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405203+0000", 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9718999999999997e-05 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.428 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:24.405212+0000", 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2439999999999999e-06 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.625967+0000", 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.2207548089999998 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.429 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.629287+0000", 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003320042 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,3", 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.633107+0000", 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038200420000000001 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.430 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.641198+0000", 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0080906099999999998 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.646808+0000", 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0056098529999999997 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.431 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.647586+0000", 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000778562 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.648854+0000", 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001267617 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.653451+0000", 2023-10-09T05:09:55.432 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045966180000000002 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93432 14.c 14:374a58ab:::benchmark_data_smithi029_550670_object15821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 386.65560569399997, 2023-10-09T05:09:55.433 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.1048139219999999, 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93432 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.434 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724860+0000", 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724859+0000", 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:55.435 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724864+0000", 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1170000000000003e-06 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724876+0000", 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1911e-05 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.436 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.724881+0000", 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0379999999999999e-06 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.774671+0000", 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0497900250000001 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.783271+0000", 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.008600056 2023-10-09T05:09:55.437 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.806956+0000", 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.023684614999999999 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.809403+0000", 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002447467 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.438 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.813396+0000", 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0039930909999999998 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.816971+0000", 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00357516 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.439 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.822865+0000", 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0058936639999999998 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.823152+0000", 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028665299999999999 2023-10-09T05:09:55.440 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.824381+0000", 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012293689999999999 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.829674+0000", 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00529255 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.441 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93434 14.1c 14:38f7b3c2:::benchmark_data_smithi029_550670_object15823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e629)", 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 386.63929467299999, 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0950609299999998, 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.442 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93434 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741171+0000", 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.443 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741170+0000", 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741175+0000", 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.1479999999999997e-06 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.444 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741186+0000", 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1286e-05 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:28.741191+0000", 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5639999999999998e-06 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.782108+0000", 2023-10-09T05:09:55.445 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0409173940000001 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.790417+0000", 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0083089150000000001 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.810827+0000", 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.020409265999999999 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.446 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.812966+0000", 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.002139448 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.816862+0000", 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0038954900000000002 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.447 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.822572+0000", 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057100429999999997 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.823625+0000", 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010534870000000001 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.830012+0000", 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0063868370000000002 2023-10-09T05:09:55.448 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.831216+0000", 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0012044099999999999 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.836232+0000", 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.005015345 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.449 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.1c 629/627 [PushReplyOp(14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head)])", 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 385.35337358999999, 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71326082199999996, 2023-10-09T05:09:55.450 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027092+0000", 2023-10-09T05:09:55.451 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027091+0000", 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027097+0000", 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.5639999999999996e-06 2023-10-09T05:09:55.452 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027107+0000", 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.082e-05 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027113+0000", 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.8239999999999996e-06 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.453 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.735833+0000", 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.70872009899999999 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.740353+0000", 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0045193639999999997 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.454 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "MOSDPGPushReply(14.1c 629/627 [PushReplyOp(14:3cb46df6:::benchmark_data_smithi029_550670_object15807:head)])", 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 385.35317146900002, 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.72351207900000003, 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "reached pg", 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.455 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027294+0000", 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027293+0000", 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999995 2023-10-09T05:09:55.456 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027298+0000", 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.9649999999999999e-06 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027310+0000", 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.1353e-05 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.457 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.027316+0000", 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.2890000000000003e-06 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.740750+0000", 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.71343399900000004 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:30.750806+0000", 2023-10-09T05:09:55.458 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010056034 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93889 14.c 14:34db2e8f:::benchmark_data_smithi029_550670_object16278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 357.67331146999999, 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.485937375, 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.459 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93889 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:09:55.460 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707154+0000", 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707152+0000", 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.461 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707158+0000", 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.7729999999999996e-06 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707171+0000", 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2500000000000001e-05 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.462 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.707176+0000", 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.2159999999999997e-06 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806421+0000", 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.099245228000000005 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.463 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.814839+0000", 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0084180560000000001 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.035337+0000", 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.220497577 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.038518+0000", 2023-10-09T05:09:55.464 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0031815480000000002 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.044091+0000", 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0055728160000000004 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.181816+0000", 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.137724604 2023-10-09T05:09:55.465 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.182864+0000", 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010481100000000001 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.183975+0000", 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011108789999999999 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.466 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.185725+0000", 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00175004 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.193091+0000", 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0073666629999999999 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.467 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93893 14.1c 14:3ca6af9b:::benchmark_data_smithi029_550670_object16282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 357.641351886, 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.309352242, 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.468 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93893 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.469 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739113+0000", 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739112+0000", 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:55.470 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739121+0000", 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2280000000000004e-06 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739139+0000", 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.7754e-05 2023-10-09T05:09:55.471 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.739149+0000", 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0462000000000001e-05 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806087+0000", 2023-10-09T05:09:55.472 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.066937742999999994 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806189+0000", 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000102457 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.806269+0000", 2023-10-09T05:09:55.473 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9692999999999997e-05 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.814900+0000", 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0086310050000000006 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.474 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.830030+0000", 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015129956 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.047451+0000", 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.217420735 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.475 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.047488+0000", 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.7416999999999998e-05 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.048466+0000", 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097746399999999994 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.476 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:93904 14.c 14:33d8ac2a:::benchmark_data_smithi029_550670_object16293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e645)", 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 357.57752754699999, 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.39957331299999999, 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.477 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 93904 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802938+0000", 2023-10-09T05:09:55.478 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802936+0000", 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802945+0000", 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.4160000000000004e-06 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.479 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802956+0000", 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.117e-05 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.802962+0000", 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0020000000000003e-06 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.480 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.815216+0000", 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.012254335999999999 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:57.823445+0000", 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0082289100000000007 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.481 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.063574+0000", 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.240128587 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.066504+0000", 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0029302120000000002 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.482 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.177873+0000", 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.111368644 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.193885+0000", 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.016012229999999999 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.483 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.194152+0000", 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00026738700000000001 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.195130+0000", 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00097748699999999997 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.196716+0000", 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001586212 2023-10-09T05:09:55.484 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:03:58.202511+0000", 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0057953019999999996 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.485 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97141 14.15 14:ae3b35de:::benchmark_data_smithi029_550670_object19530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.28914486600002, 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.94747084699999995, 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97141 2023-10-09T05:09:55.486 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091321+0000", 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.487 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091319+0000", 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091327+0000", 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.9740000000000002e-06 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.488 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091342+0000", 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4747e-05 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091351+0000", 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2089999999999994e-06 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091416+0000", 2023-10-09T05:09:55.489 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4838999999999994e-05 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.091423+0000", 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8240000000000002e-06 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.018840+0000", 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.92741732099999996 2023-10-09T05:09:55.490 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.018983+0000", 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00014266600000000001 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.019109+0000", 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00012617699999999999 2023-10-09T05:09:55.491 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022437+0000", 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0033283459999999998 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022749+0000", 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00031135000000000001 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.492 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038718+0000", 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.015969552000000001 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038764+0000", 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.5419e-05 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.493 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.038791+0000", 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.7798e-05 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97180 14.15 14:ad3b1b35:::benchmark_data_smithi029_550670_object19569:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:09:55.494 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 339.04334602799997, 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.705525913, 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97180 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.495 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337119+0000", 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337118+0000", 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:55.496 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337124+0000", 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1090000000000003e-06 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337138+0000", 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.4467000000000001e-05 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.497 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337145+0000", 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0099999999999998e-06 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337214+0000", 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.8980999999999996e-05 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.337222+0000", 2023-10-09T05:09:55.498 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.7780000000000007e-06 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020444+0000", 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.68322159100000002 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020544+0000", 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000100602 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.499 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.020650+0000", 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000105921 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022505+0000", 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0018547959999999999 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.500 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022791+0000", 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00028577600000000001 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042584+0000", 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.019793139000000001 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042615+0000", 2023-10-09T05:09:55.501 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.1220000000000003e-05 2023-10-09T05:09:55.502 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.502 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.502 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.502 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.042645+0000", 2023-10-09T05:09:55.502 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0244999999999999e-05 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:97245 14.15 14:ae966d39:::benchmark_data_smithi029_550670_object19634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e649)", 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 338.73817408100001, 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.40766333999999999, 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.503 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 97245 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.504 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.505 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.505 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642291+0000", 2023-10-09T05:09:55.505 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.505 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.505 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.506 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.506 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642290+0000", 2023-10-09T05:09:55.506 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999981 2023-10-09T05:09:55.506 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.506 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642299+0000", 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9490000000000008e-06 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642312+0000", 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2159e-05 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.507 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642319+0000", 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.3329999999999999e-06 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642389+0000", 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.0358000000000006e-05 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.508 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for peered", 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:16.642399+0000", 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.2269999999999999e-06 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.509 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021495+0000", 2023-10-09T05:09:55.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.37909622700000001 2023-10-09T05:09:55.510 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.510 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.510 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.510 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021614+0000", 2023-10-09T05:09:55.510 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00011913600000000001 2023-10-09T05:09:55.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.511 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.511 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,7", 2023-10-09T05:09:55.511 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.021734+0000", 2023-10-09T05:09:55.511 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000119611 2023-10-09T05:09:55.511 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.512 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.512 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.022758+0000", 2023-10-09T05:09:55.512 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0010246599999999999 2023-10-09T05:09:55.512 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.512 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.023321+0000", 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00056311399999999998 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.513 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049876+0000", 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.026554853999999999 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049924+0000", 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.7633999999999998e-05 2023-10-09T05:09:55.514 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.515 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.515 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.515 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:04:17.049955+0000", 2023-10-09T05:09:55.515 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.0854e-05 2023-10-09T05:09:55.515 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.515 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.516 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.516 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.516 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.516 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:131600 14.1c 14:3ec2aceb:::benchmark_data_smithi029_550670_object53989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e670)", 2023-10-09T05:09:55.516 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:46.742939+0000", 2023-10-09T05:09:55.516 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 188.63752593300001, 2023-10-09T05:09:55.517 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29314816599999999, 2023-10-09T05:09:55.517 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.517 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.517 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.517 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.517 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 131600 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742939+0000", 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.518 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742939+0000", 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742938+0000", 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.9999986 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.519 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742947+0000", 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6570000000000003e-06 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742967+0000", 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 2.0922000000000001e-05 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.520 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.742976+0000", 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 8.6230000000000003e-06 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.743038+0000", 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.1655999999999994e-05 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.743721+0000", 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00068335399999999995 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.521 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.753880+0000", 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.010159385 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.753980+0000", 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.9647000000000003e-05 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.522 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.754306+0000", 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00032565200000000001 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.757336+0000", 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.003030516 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:46.757346+0000", 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0088000000000001e-05 2023-10-09T05:09:55.523 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.035468+0000", 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.278121758 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.035515+0000", 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.6464000000000001e-05 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.524 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:47.036088+0000", 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00057302700000000004 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132518 14.c 14:363cf21e:::benchmark_data_smithi029_550670_object54907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.708129+0000", 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 185.67233608000001, 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30830919800000001, 2023-10-09T05:09:55.525 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132518 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708129+0000", 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.526 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708129+0000", 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708128+0000", 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.527 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708135+0000", 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9009999999999997e-06 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708153+0000", 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.8155000000000002e-05 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708160+0000", 2023-10-09T05:09:55.528 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.4840000000000001e-06 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.708199+0000", 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 3.9623999999999999e-05 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.709034+0000", 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00083480299999999996 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.529 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.009495+0000", 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30046083800000001 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.009588+0000", 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 9.3477e-05 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.530 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.009884+0000", 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000295536 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.013458+0000", 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0035743480000000002 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.013468+0000", 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.0179e-05 2023-10-09T05:09:55.531 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.015745+0000", 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0022768760000000002 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.015816+0000", 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 7.08e-05 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.532 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016438+0000", 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00062235399999999998 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: "description": "osd_op(client.36948.0:132520 14.c 14:353a445f:::benchmark_data_smithi029_550670_object54909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e671)", 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: "initiated_at": "2023-10-09T05:06:49.713110+0000", 2023-10-09T05:09:55.533 INFO:teuthology.orchestra.run.smithi177.stdout: "age": 185.66735530400001, 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.30387030700000001, 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "type_data": { 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "client_info": { 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "client": "client.36948", 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "client_addr": "172.21.15.29:0/2739934079", 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: "tid": 132520 2023-10-09T05:09:55.534 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "events": [ 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "initiated", 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713110+0000", 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "throttled", 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713110+0000", 2023-10-09T05:09:55.535 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "header_read", 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713109+0000", 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4294967295.999999 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "all_read", 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713114+0000", 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.0300000000000001e-06 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.536 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "dispatched", 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713127+0000", 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 1.2754e-05 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "queued_for_pg", 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713132+0000", 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 5.412e-06 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.537 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713176+0000", 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 4.3383999999999997e-05 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for degraded object", 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:49.713822+0000", 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000646614 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "reached_pg", 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.012042+0000", 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.29822033100000001 2023-10-09T05:09:55.538 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "started", 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.012112+0000", 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.9851000000000006e-05 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "waiting for subops from 0,6", 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.012298+0000", 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.000185465 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.539 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "op_commit", 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.013478+0000", 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0011797400000000001 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.014564+0000", 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.001086683 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.540 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "sub_op_commit_rec", 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016498+0000", 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.0019336920000000001 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "commit_sent", 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016558+0000", 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 6.0248999999999999e-05 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: }, 2023-10-09T05:09:55.541 INFO:teuthology.orchestra.run.smithi177.stdout: { 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: "event": "done", 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: "time": "2023-10-09T05:06:50.016980+0000", 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: "duration": 0.00042221000000000001 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: } 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2023-10-09T05:09:55.542 INFO:teuthology.orchestra.run.smithi177.stdout:} 2023-10-09T05:09:55.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-09T05:09:56.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:55 smithi177 ceph-mon[195797]: pgmap v2350: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:55 smithi177 ceph-mon[195797]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:55 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:09:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:55 smithi177 ceph-mon[195797]: osdmap e753: 8 total, 8 up, 8 in 2023-10-09T05:09:56.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:55 smithi177 ceph-mon[195797]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.125 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:09:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[223858]: pgmap v2350: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[223858]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:09:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[223858]: osdmap e753: 8 total, 8 up, 8 in 2023-10-09T05:09:56.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[223858]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[216903]: pgmap v2350: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[216903]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-09T05:09:56.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[216903]: osdmap e753: 8 total, 8 up, 8 in 2023-10-09T05:09:56.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:55 smithi029 ceph-mon[216903]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.860 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:56 smithi029 ceph-mon[216903]: failed to encode map e753 with expected crc 2023-10-09T05:09:56.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:56 smithi029 ceph-mon[223858]: failed to encode map e753 with expected crc 2023-10-09T05:09:57.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:56 smithi177 ceph-mon[195797]: failed to encode map e753 with expected crc 2023-10-09T05:09:57.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:57 smithi029 ceph-mon[216903]: pgmap v2352: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:57.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:57 smithi029 ceph-mon[223858]: pgmap v2352: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:57 smithi177 ceph-mon[195797]: pgmap v2352: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:09:58.721 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.6 to 1 (10000) 2023-10-09T05:09:58.968 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:58 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1155114730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:09:58.968 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:58.969 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:58 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1155114730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:09:58.969 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:58 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1155114730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:09:59.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:09:59.717 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-09T05:10:00.001 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[216903]: pgmap v2353: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:00.001 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[216903]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[216903]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1155114730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[216903]: osdmap e754: 8 total, 8 up, 8 in 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[223858]: pgmap v2353: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[223858]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.002 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[223858]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1155114730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:10:00.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[223858]: osdmap e754: 8 total, 8 up, 8 in 2023-10-09T05:10:00.003 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:09:59 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:10:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:59 smithi177 ceph-mon[195797]: pgmap v2353: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:59 smithi177 ceph-mon[195797]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:59 smithi177 ceph-mon[195797]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1155114730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:10:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:59 smithi177 ceph-mon[195797]: osdmap e754: 8 total, 8 up, 8 in 2023-10-09T05:10:00.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:09:59 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-09T05:10:00.097 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: failed to encode map e755 with expected crc 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: osdmap e755: 8 total, 8 up, 8 in 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: failed to encode map e755 with expected crc 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: overall HEALTH_OK 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[216903]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: failed to encode map e754 with expected crc 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: failed to encode map e755 with expected crc 2023-10-09T05:10:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:10:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: osdmap e755: 8 total, 8 up, 8 in 2023-10-09T05:10:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: failed to encode map e755 with expected crc 2023-10-09T05:10:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: overall HEALTH_OK 2023-10-09T05:10:00.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:00 smithi029 ceph-mon[223858]: failed to encode map e754 with expected crc 2023-10-09T05:10:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: failed to encode map e754 with expected crc 2023-10-09T05:10:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: failed to encode map e755 with expected crc 2023-10-09T05:10:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-09T05:10:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: osdmap e755: 8 total, 8 up, 8 in 2023-10-09T05:10:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: failed to encode map e755 with expected crc 2023-10-09T05:10:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: overall HEALTH_OK 2023-10-09T05:10:01.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:00 smithi177 ceph-mon[195797]: failed to encode map e754 with expected crc 2023-10-09T05:10:01.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[216903]: pgmap v2356: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:01.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[216903]: failed to encode map e755 with expected crc 2023-10-09T05:10:01.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[216903]: failed to encode map e756 with expected crc 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[216903]: failed to encode map e756 with expected crc 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[216903]: osdmap e756: 8 total, 8 up, 8 in 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[223858]: pgmap v2356: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[223858]: failed to encode map e755 with expected crc 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[223858]: failed to encode map e756 with expected crc 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[223858]: failed to encode map e756 with expected crc 2023-10-09T05:10:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:01 smithi029 ceph-mon[223858]: osdmap e756: 8 total, 8 up, 8 in 2023-10-09T05:10:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:01 smithi177 ceph-mon[195797]: pgmap v2356: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:01 smithi177 ceph-mon[195797]: failed to encode map e755 with expected crc 2023-10-09T05:10:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:01 smithi177 ceph-mon[195797]: failed to encode map e756 with expected crc 2023-10-09T05:10:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:01 smithi177 ceph-mon[195797]: failed to encode map e756 with expected crc 2023-10-09T05:10:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:01 smithi177 ceph-mon[195797]: osdmap e756: 8 total, 8 up, 8 in 2023-10-09T05:10:02.741 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-09T05:10:02.993 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[216903]: failed to encode map e756 with expected crc 2023-10-09T05:10:02.993 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[216903]: failed to encode map e756 with expected crc 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1838336653' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:10:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:10:02] "GET /metrics HTTP/1.1" 200 34299 "" "Prometheus/2.43.0" 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[223858]: failed to encode map e756 with expected crc 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[223858]: failed to encode map e756 with expected crc 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1838336653' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:10:02.994 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:02 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:10:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:02 smithi177 ceph-mon[195797]: failed to encode map e756 with expected crc 2023-10-09T05:10:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:02 smithi177 ceph-mon[195797]: failed to encode map e756 with expected crc 2023-10-09T05:10:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:02 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1838336653' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:10:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:02 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-09T05:10:03.814 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-09T05:10:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:03 smithi177 ceph-mon[195797]: pgmap v2358: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:03 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:10:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:03 smithi177 ceph-mon[195797]: failed to encode map e757 with expected crc 2023-10-09T05:10:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:03 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:10:04.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:03 smithi177 ceph-mon[195797]: osdmap e757: 8 total, 8 up, 8 in 2023-10-09T05:10:04.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[216903]: pgmap v2358: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[216903]: failed to encode map e757 with expected crc 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[216903]: osdmap e757: 8 total, 8 up, 8 in 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[223858]: pgmap v2358: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[223858]: failed to encode map e757 with expected crc 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-09T05:10:04.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:03 smithi029 ceph-mon[223858]: osdmap e757: 8 total, 8 up, 8 in 2023-10-09T05:10:04.185 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:04.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:04 smithi029 ceph-mon[216903]: failed to encode map e757 with expected crc 2023-10-09T05:10:04.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:04 smithi029 ceph-mon[216903]: failed to encode map e757 with expected crc 2023-10-09T05:10:04.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:04 smithi029 ceph-mon[223858]: failed to encode map e757 with expected crc 2023-10-09T05:10:04.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:04 smithi029 ceph-mon[223858]: failed to encode map e757 with expected crc 2023-10-09T05:10:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:04 smithi177 ceph-mon[195797]: failed to encode map e757 with expected crc 2023-10-09T05:10:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:04 smithi177 ceph-mon[195797]: failed to encode map e757 with expected crc 2023-10-09T05:10:05.778 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:05 smithi029 ceph-mon[216903]: pgmap v2360: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:05.779 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:05 smithi029 ceph-mon[223858]: pgmap v2360: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:06.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:05 smithi177 ceph-mon[195797]: pgmap v2360: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:07.182 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.7 to 1 (10000) 2023-10-09T05:10:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[216903]: pgmap v2361: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1081189547' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:10:07.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T05:10:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[223858]: pgmap v2361: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1081189547' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:10:07.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:07 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T05:10:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:07 smithi177 ceph-mon[195797]: pgmap v2361: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1081189547' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:07.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:07 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:07 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-09T05:10:07.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:07 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T05:10:08.143 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-09T05:10:08.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[216903]: failed to encode map e758 with expected crc 2023-10-09T05:10:08.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[216903]: failed to encode map e758 with expected crc 2023-10-09T05:10:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:10:08.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[216903]: osdmap e758: 8 total, 8 up, 8 in 2023-10-09T05:10:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[223858]: failed to encode map e758 with expected crc 2023-10-09T05:10:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[223858]: failed to encode map e758 with expected crc 2023-10-09T05:10:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:10:08.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:08 smithi029 ceph-mon[223858]: osdmap e758: 8 total, 8 up, 8 in 2023-10-09T05:10:08.512 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:08 smithi177 ceph-mon[195797]: failed to encode map e758 with expected crc 2023-10-09T05:10:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:08 smithi177 ceph-mon[195797]: failed to encode map e758 with expected crc 2023-10-09T05:10:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:08 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:10:08.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:08 smithi177 ceph-mon[195797]: osdmap e758: 8 total, 8 up, 8 in 2023-10-09T05:10:09.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:09 smithi029 ceph-mon[216903]: pgmap v2363: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:09.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:09 smithi029 ceph-mon[223858]: pgmap v2363: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:09.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:09 smithi177 ceph-mon[195797]: pgmap v2363: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:11.196 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-09T05:10:11.491 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:11 smithi029 ceph-mon[216903]: pgmap v2364: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:11.492 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/4221311889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:11.492 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:11 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:11.492 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:11 smithi029 ceph-mon[223858]: pgmap v2364: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:11.492 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/4221311889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:11.492 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:11 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:11 smithi177 ceph-mon[195797]: pgmap v2364: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/4221311889' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:11.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:11 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-09T05:10:12.213 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-09T05:10:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:12 smithi177 ceph-mon[195797]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:12 smithi177 ceph-mon[195797]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:12 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:10:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:12 smithi177 ceph-mon[195797]: osdmap e759: 8 total, 8 up, 8 in 2023-10-09T05:10:12.528 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:12 smithi177 ceph-mon[195797]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.583 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[216903]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[216903]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:10:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[216903]: osdmap e759: 8 total, 8 up, 8 in 2023-10-09T05:10:12.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[216903]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[223858]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[223858]: failed to encode map e759 with expected crc 2023-10-09T05:10:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-09T05:10:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[223858]: osdmap e759: 8 total, 8 up, 8 in 2023-10-09T05:10:12.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:12 smithi029 ceph-mon[223858]: failed to encode map e759 with expected crc 2023-10-09T05:10:13.150 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:10:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:10:12] "GET /metrics HTTP/1.1" 200 34299 "" "Prometheus/2.43.0" 2023-10-09T05:10:13.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: pgmap v2366: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: Health check failed: Reduced data availability: 11 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: osdmap e760: 8 total, 8 up, 8 in 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[216903]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: pgmap v2366: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: Health check failed: Reduced data availability: 11 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: osdmap e760: 8 total, 8 up, 8 in 2023-10-09T05:10:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:13 smithi029 ceph-mon[223858]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: pgmap v2366: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: Health check failed: Reduced data availability: 11 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: osdmap e760: 8 total, 8 up, 8 in 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: failed to encode map e760 with expected crc 2023-10-09T05:10:13.527 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:13 smithi177 ceph-mon[195797]: failed to encode map e760 with expected crc 2023-10-09T05:10:14.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:14.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:14 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:10:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:14.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:14 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:10:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:15.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:14 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-09T05:10:15.641 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.2 to 1 (10000) 2023-10-09T05:10:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:15 smithi029 ceph-mon[216903]: pgmap v2368: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:15 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:15.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:15 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2881706383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:15 smithi029 ceph-mon[223858]: pgmap v2368: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:15 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:15.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:15 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2881706383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:15 smithi177 ceph-mon[195797]: pgmap v2368: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:15 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:16.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:15 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2881706383' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: failed to encode map e761 with expected crc 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2881706383' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: osdmap e761: 8 total, 8 up, 8 in 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: failed to encode map e761 with expected crc 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: osdmap e762: 8 total, 8 up, 8 in 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: failed to encode map e762 with expected crc 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[216903]: failed to encode map e762 with expected crc 2023-10-09T05:10:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: failed to encode map e761 with expected crc 2023-10-09T05:10:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2881706383' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:10:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: osdmap e761: 8 total, 8 up, 8 in 2023-10-09T05:10:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: failed to encode map e761 with expected crc 2023-10-09T05:10:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: osdmap e762: 8 total, 8 up, 8 in 2023-10-09T05:10:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: failed to encode map e762 with expected crc 2023-10-09T05:10:16.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:16 smithi029 ceph-mon[223858]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: failed to encode map e761 with expected crc 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2881706383' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: osdmap e761: 8 total, 8 up, 8 in 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: failed to encode map e761 with expected crc 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: osdmap e762: 8 total, 8 up, 8 in 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:16 smithi177 ceph-mon[195797]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.134 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-09T05:10:17.674 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: pgmap v2370: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: osdmap e763: 8 total, 8 up, 8 in 2023-10-09T05:10:17.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: failed to encode map e763 with expected crc 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: pgmap v2370: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: failed to encode map e762 with expected crc 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: osdmap e763: 8 total, 8 up, 8 in 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: failed to encode map e763 with expected crc 2023-10-09T05:10:17.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:17.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:17 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: pgmap v2370: 83 pgs: 11 peering, 72 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: failed to encode map e762 with expected crc 2023-10-09T05:10:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: failed to encode map e762 with expected crc 2023-10-09T05:10:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: failed to encode map e762 with expected crc 2023-10-09T05:10:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: failed to encode map e762 with expected crc 2023-10-09T05:10:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: osdmap e763: 8 total, 8 up, 8 in 2023-10-09T05:10:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: failed to encode map e763 with expected crc 2023-10-09T05:10:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:18.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:17 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:18.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[216903]: failed to encode map e763 with expected crc 2023-10-09T05:10:18.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[216903]: failed to encode map e763 with expected crc 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[216903]: osdmap e764: 8 total, 8 up, 8 in 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[216903]: failed to encode map e764 with expected crc 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[216903]: failed to encode map e764 with expected crc 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[223858]: failed to encode map e763 with expected crc 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[223858]: failed to encode map e763 with expected crc 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[223858]: osdmap e764: 8 total, 8 up, 8 in 2023-10-09T05:10:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[223858]: failed to encode map e764 with expected crc 2023-10-09T05:10:18.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:18 smithi029 ceph-mon[223858]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:18 smithi177 ceph-mon[195797]: failed to encode map e763 with expected crc 2023-10-09T05:10:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:18 smithi177 ceph-mon[195797]: failed to encode map e763 with expected crc 2023-10-09T05:10:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:18 smithi177 ceph-mon[195797]: osdmap e764: 8 total, 8 up, 8 in 2023-10-09T05:10:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:18 smithi177 ceph-mon[195797]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:18 smithi177 ceph-mon[195797]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: pgmap v2373: 83 pgs: 1 remapped+peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:19.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 11 pgs peering) 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: Cluster is now healthy 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: osdmap e765: 8 total, 8 up, 8 in 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: failed to encode map e765 with expected crc 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: pgmap v2373: 83 pgs: 1 remapped+peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: failed to encode map e764 with expected crc 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 11 pgs peering) 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: Cluster is now healthy 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: osdmap e765: 8 total, 8 up, 8 in 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: failed to encode map e765 with expected crc 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:19.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:19 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: pgmap v2373: 83 pgs: 1 remapped+peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: failed to encode map e764 with expected crc 2023-10-09T05:10:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: failed to encode map e764 with expected crc 2023-10-09T05:10:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: failed to encode map e764 with expected crc 2023-10-09T05:10:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 11 pgs peering) 2023-10-09T05:10:20.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: Cluster is now healthy 2023-10-09T05:10:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: osdmap e765: 8 total, 8 up, 8 in 2023-10-09T05:10:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:20.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:19 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:20.794 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/407816328' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:20.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: failed to encode map e765 with expected crc 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/407816328' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:20.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:20 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: failed to encode map e765 with expected crc 2023-10-09T05:10:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: failed to encode map e765 with expected crc 2023-10-09T05:10:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: failed to encode map e765 with expected crc 2023-10-09T05:10:21.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-09T05:10:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-09T05:10:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:21.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/407816328' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:21.029 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:20 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-09T05:10:21.655 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:21 smithi177 ceph-mon[195797]: pgmap v2376: 83 pgs: 1 peering, 1 remapped+peering, 81 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:21.655 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:21 smithi177 ceph-mon[195797]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.655 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:21 smithi177 ceph-mon[195797]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.655 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:21 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:10:21.656 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:21 smithi177 ceph-mon[195797]: osdmap e766: 8 total, 8 up, 8 in 2023-10-09T05:10:21.820 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-09T05:10:21.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[216903]: pgmap v2376: 83 pgs: 1 peering, 1 remapped+peering, 81 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:21.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[216903]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[216903]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[216903]: osdmap e766: 8 total, 8 up, 8 in 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[216903]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[223858]: pgmap v2376: 83 pgs: 1 peering, 1 remapped+peering, 81 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[223858]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[223858]: failed to encode map e766 with expected crc 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-09T05:10:21.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[223858]: osdmap e766: 8 total, 8 up, 8 in 2023-10-09T05:10:21.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:21 smithi029 ceph-mon[223858]: failed to encode map e766 with expected crc 2023-10-09T05:10:22.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:21 smithi177 ceph-mon[195797]: failed to encode map e766 with expected crc 2023-10-09T05:10:22.190 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:23.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:10:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:10:22] "GET /metrics HTTP/1.1" 200 34298 "" "Prometheus/2.43.0" 2023-10-09T05:10:23.876 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:23 smithi029 ceph-mon[216903]: pgmap v2378: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 116 B/s, 2 objects/s recovering 2023-10-09T05:10:23.877 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:23 smithi029 ceph-mon[223858]: pgmap v2378: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 116 B/s, 2 objects/s recovering 2023-10-09T05:10:24.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:23 smithi177 ceph-mon[195797]: pgmap v2378: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 116 B/s, 2 objects/s recovering 2023-10-09T05:10:24.668 INFO:teuthology.orchestra.run.smithi029.stderr:reweighted osd.4 to 1 (10000) 2023-10-09T05:10:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:24 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/935045203' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:24 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:24.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:24 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:24 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/935045203' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:24 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:24.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:24 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:24 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/935045203' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:24 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:25.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:24 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:25.663 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-09T05:10:25.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[216903]: pgmap v2379: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 92 B/s, 2 objects/s recovering 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[216903]: failed to encode map e767 with expected crc 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[216903]: failed to encode map e767 with expected crc 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[216903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[216903]: osdmap e767: 8 total, 8 up, 8 in 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[216903]: failed to encode map e767 with expected crc 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[223858]: pgmap v2379: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 92 B/s, 2 objects/s recovering 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[223858]: failed to encode map e767 with expected crc 2023-10-09T05:10:25.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[223858]: failed to encode map e767 with expected crc 2023-10-09T05:10:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[223858]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:10:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[223858]: osdmap e767: 8 total, 8 up, 8 in 2023-10-09T05:10:25.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:25 smithi029 ceph-mon[223858]: failed to encode map e767 with expected crc 2023-10-09T05:10:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:25 smithi177 ceph-mon[195797]: pgmap v2379: 83 pgs: 1 peering, 82 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 92 B/s, 2 objects/s recovering 2023-10-09T05:10:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:25 smithi177 ceph-mon[195797]: failed to encode map e767 with expected crc 2023-10-09T05:10:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:25 smithi177 ceph-mon[195797]: failed to encode map e767 with expected crc 2023-10-09T05:10:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:25 smithi177 ceph-mon[195797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:10:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:25 smithi177 ceph-mon[195797]: osdmap e767: 8 total, 8 up, 8 in 2023-10-09T05:10:26.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:25 smithi177 ceph-mon[195797]: failed to encode map e767 with expected crc 2023-10-09T05:10:26.031 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:26.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:26 smithi029 ceph-mon[216903]: failed to encode map e767 with expected crc 2023-10-09T05:10:26.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:26 smithi029 ceph-mon[223858]: failed to encode map e767 with expected crc 2023-10-09T05:10:27.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:26 smithi177 ceph-mon[195797]: failed to encode map e767 with expected crc 2023-10-09T05:10:27.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:27 smithi029 ceph-mon[216903]: pgmap v2381: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 81 B/s, 2 objects/s recovering 2023-10-09T05:10:27.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:27 smithi029 ceph-mon[223858]: pgmap v2381: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 81 B/s, 2 objects/s recovering 2023-10-09T05:10:28.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:27 smithi177 ceph-mon[195797]: pgmap v2381: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 81 B/s, 2 objects/s recovering 2023-10-09T05:10:28.683 INFO:teuthology.orchestra.run.smithi029.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-09T05:10:28.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:28 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/665096789' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:28.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:28 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:28 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/665096789' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:28.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:28 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:28 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/665096789' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-09T05:10:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:29.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:28 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:29.751 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-09T05:10:29.751 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-09T05:10:29.751 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-09T05:10:29.751 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-09T05:10:29.751 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-09T05:10:29.752 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: pgmap v2382: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 69 B/s, 1 objects/s recovering 2023-10-09T05:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/665096789' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: osdmap e768: 8 total, 8 up, 8 in 2023-10-09T05:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:29 smithi177 ceph-mon[195797]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.119 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: pgmap v2382: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 69 B/s, 1 objects/s recovering 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/665096789' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: osdmap e768: 8 total, 8 up, 8 in 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[223858]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: pgmap v2382: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 69 B/s, 1 objects/s recovering 2023-10-09T05:10:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/665096789' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-09T05:10:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: osdmap e768: 8 total, 8 up, 8 in 2023-10-09T05:10:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: failed to encode map e768 with expected crc 2023-10-09T05:10:30.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:29 smithi029 ceph-mon[216903]: failed to encode map e768 with expected crc 2023-10-09T05:10:31.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:31 smithi029 ceph-mon[216903]: pgmap v2384: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:31.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:31 smithi029 ceph-mon[223858]: pgmap v2384: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:32.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:31 smithi177 ceph-mon[195797]: pgmap v2384: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:32.269 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:10:32.269 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":768,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T05:10:28.669495+0000","last_up_change":"2023-10-09T05:03:23.292636+0000","last_in_change":"2023-10-09T05:09:09.258247+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"744","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"470","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"723","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":723,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:07:33.266053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:07:34.056234+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":761,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6803","nonce":709667403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6805","nonce":709667403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6809","nonce":709667403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6807","nonce":709667403}]},"public_addr":"172.21.15.29:6803/709667403","cluster_addr":"172.21.15.29:6805/709667403","heartbeat_back_addr":"172.21.15.29:6809/709667403","heartbeat_front_addr":"172.21.15.29:6807/709667403","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":114,"up_from":118,"up_thru":761,"down_at":115,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6811","nonce":1230632802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6813","nonce":1230632802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6817","nonce":1230632802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6815","nonce":1230632802}]},"public_addr":"172.21.15.29:6811/1230632802","cluster_addr":"172.21.15.29:6813/1230632802","heartbeat_back_addr":"172.21.15.29:6817/1230632802","heartbeat_front_addr":"172.21.15.29:6815/1230632802","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":135,"last_clean_end":200,"up_from":221,"up_thru":761,"down_at":201,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6819","nonce":2824881881}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6821","nonce":2824881881}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6825","nonce":2824881881}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6823","nonce":2824881881}]},"public_addr":"172.21.15.29:6819/2824881881","cluster_addr":"172.21.15.29:6821/2824881881","heartbeat_back_addr":"172.21.15.29:6825/2824881881","heartbeat_front_addr":"172.21.15.29:6823/2824881881","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":138,"up_from":142,"up_thru":761,"down_at":139,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6827","nonce":2977593922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6829","nonce":2977593922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6833","nonce":2977593922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6831","nonce":2977593922}]},"public_addr":"172.21.15.29:6827/2977593922","cluster_addr":"172.21.15.29:6829/2977593922","heartbeat_back_addr":"172.21.15.29:6833/2977593922","heartbeat_front_addr":"172.21.15.29:6831/2977593922","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":507,"last_clean_end":616,"up_from":626,"up_thru":764,"down_at":617,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6801","nonce":1622844329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6803","nonce":1622844329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6807","nonce":1622844329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6805","nonce":1622844329}]},"public_addr":"172.21.15.177:6801/1622844329","cluster_addr":"172.21.15.177:6803/1622844329","heartbeat_back_addr":"172.21.15.177:6807/1622844329","heartbeat_front_addr":"172.21.15.177:6805/1622844329","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":761,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":742,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":764,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":114},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":200},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":138},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":616},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/3682933143":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6801/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1432692814":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1371585021":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/3560371706":"2023-10-10T04:12:57.728727+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/3925658916":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.177:0/2501999039":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/1007503695":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/3506139723":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.177:6834/1319489652":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/4232895263":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/2321967128":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1787157051":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/1665676424":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1566968162":"2023-10-10T04:13:58.298189+0000","172.21.15.177:0/63096837":"2023-10-10T04:12:57.728727+0000","172.21.15.177:6835/1319489652":"2023-10-10T04:12:57.728727+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:10:32.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:32 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2308367689' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:10:32.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:32 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2308367689' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:10:33.032 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:32 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2308367689' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:10:33.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:10:32 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:10:32] "GET /metrics HTTP/1.1" 200 34286 "" "Prometheus/2.43.0" 2023-10-09T05:10:33.236 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-09T05:10:33.236 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-09T05:10:33.606 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:33.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:33 smithi029 ceph-mon[216903]: pgmap v2385: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:33.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:33 smithi029 ceph-mon[223858]: pgmap v2385: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:34.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:33 smithi177 ceph-mon[195797]: pgmap v2385: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:35.711 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:10:35.711 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":768,"fsid":"486585fa-6658-11ee-8db5-212e2dc638e7","created":"2023-10-09T04:00:46.050683+0000","modified":"2023-10-09T05:10:28.669495+0000","last_up_change":"2023-10-09T05:03:23.292636+0000","last_in_change":"2023-10-09T05:09:09.258247+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-09T04:03:57.603026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"744","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"470","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-09T04:06:27.633742+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"723","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":723,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-09T04:07:33.266053+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-09T04:07:34.056234+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a3e888dc-7381-4dc9-9e9e-ea492336810d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":761,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6803","nonce":709667403}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6805","nonce":709667403}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6809","nonce":709667403}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":709667403},{"type":"v1","addr":"172.21.15.29:6807","nonce":709667403}]},"public_addr":"172.21.15.29:6803/709667403","cluster_addr":"172.21.15.29:6805/709667403","heartbeat_back_addr":"172.21.15.29:6809/709667403","heartbeat_front_addr":"172.21.15.29:6807/709667403","state":["exists","up"]},{"osd":1,"uuid":"12e83e56-ffd0-4eb6-bb51-5612a7768792","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":114,"up_from":118,"up_thru":761,"down_at":115,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6811","nonce":1230632802}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6813","nonce":1230632802}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6817","nonce":1230632802}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1230632802},{"type":"v1","addr":"172.21.15.29:6815","nonce":1230632802}]},"public_addr":"172.21.15.29:6811/1230632802","cluster_addr":"172.21.15.29:6813/1230632802","heartbeat_back_addr":"172.21.15.29:6817/1230632802","heartbeat_front_addr":"172.21.15.29:6815/1230632802","state":["exists","up"]},{"osd":2,"uuid":"6620b1f9-fce6-4509-8c69-741afdb0332d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":135,"last_clean_end":200,"up_from":221,"up_thru":761,"down_at":201,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6819","nonce":2824881881}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6821","nonce":2824881881}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6825","nonce":2824881881}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2824881881},{"type":"v1","addr":"172.21.15.29:6823","nonce":2824881881}]},"public_addr":"172.21.15.29:6819/2824881881","cluster_addr":"172.21.15.29:6821/2824881881","heartbeat_back_addr":"172.21.15.29:6825/2824881881","heartbeat_front_addr":"172.21.15.29:6823/2824881881","state":["exists","up"]},{"osd":3,"uuid":"266c755e-25de-4dd5-a2a8-1b67b5fc8e3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":138,"up_from":142,"up_thru":761,"down_at":139,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6827","nonce":2977593922}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6829","nonce":2977593922}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6833","nonce":2977593922}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2977593922},{"type":"v1","addr":"172.21.15.29:6831","nonce":2977593922}]},"public_addr":"172.21.15.29:6827/2977593922","cluster_addr":"172.21.15.29:6829/2977593922","heartbeat_back_addr":"172.21.15.29:6833/2977593922","heartbeat_front_addr":"172.21.15.29:6831/2977593922","state":["exists","up"]},{"osd":4,"uuid":"4eed67c8-385c-41ab-9afc-a65ad85104d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":507,"last_clean_end":616,"up_from":626,"up_thru":764,"down_at":617,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6801","nonce":1622844329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6803","nonce":1622844329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6807","nonce":1622844329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":1622844329},{"type":"v1","addr":"172.21.15.177:6805","nonce":1622844329}]},"public_addr":"172.21.15.177:6801/1622844329","cluster_addr":"172.21.15.177:6803/1622844329","heartbeat_back_addr":"172.21.15.177:6807/1622844329","heartbeat_front_addr":"172.21.15.177:6805/1622844329","state":["exists","up"]},{"osd":5,"uuid":"d1c822a8-9ffa-4586-9773-007ad79e1b5f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":761,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6809","nonce":3350634946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6811","nonce":3350634946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6815","nonce":3350634946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3350634946},{"type":"v1","addr":"172.21.15.177:6813","nonce":3350634946}]},"public_addr":"172.21.15.177:6809/3350634946","cluster_addr":"172.21.15.177:6811/3350634946","heartbeat_back_addr":"172.21.15.177:6815/3350634946","heartbeat_front_addr":"172.21.15.177:6813/3350634946","state":["exists","up"]},{"osd":6,"uuid":"41a89acf-f32f-41bf-9eb6-5a6edeea9043","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":742,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6817","nonce":4114806886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6819","nonce":4114806886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6823","nonce":4114806886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":4114806886},{"type":"v1","addr":"172.21.15.177:6821","nonce":4114806886}]},"public_addr":"172.21.15.177:6817/4114806886","cluster_addr":"172.21.15.177:6819/4114806886","heartbeat_back_addr":"172.21.15.177:6823/4114806886","heartbeat_front_addr":"172.21.15.177:6821/4114806886","state":["exists","up"]},{"osd":7,"uuid":"cf54f5e9-1efd-44de-968c-59accfa44021","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":764,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6825","nonce":3009136577}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6827","nonce":3009136577}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6831","nonce":3009136577}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3009136577},{"type":"v1","addr":"172.21.15.177:6829","nonce":3009136577}]},"public_addr":"172.21.15.177:6825/3009136577","cluster_addr":"172.21.15.177:6827/3009136577","heartbeat_back_addr":"172.21.15.177:6831/3009136577","heartbeat_front_addr":"172.21.15.177:6829/3009136577","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:05.741316+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:30.471804+0000","dead_epoch":114},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:03:55.145154+0000","dead_epoch":200},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:20.542024+0000","dead_epoch":138},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:04:42.785328+0000","dead_epoch":616},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:05.368947+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:27.155695+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-10-09T04:05:48.945518+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/3682933143":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6801/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/1003732865":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1432692814":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1371585021":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/3560371706":"2023-10-10T04:12:57.728727+0000","172.21.15.29:6801/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/3925658916":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1647324901":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1481182":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/2363485910":"2023-10-10T04:01:43.065145+0000","172.21.15.177:0/2501999039":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/1007503695":"2023-10-10T04:13:58.298189+0000","172.21.15.29:0/1305222045":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/817958524":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/3506139723":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6800/2869772012":"2023-10-10T04:01:15.592153+0000","172.21.15.29:0/613592146":"2023-10-10T04:01:15.592153+0000","172.21.15.177:6834/1319489652":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/673424689":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/4232895263":"2023-10-10T04:13:58.298189+0000","172.21.15.29:6800/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:6800/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6800/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/2321967128":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/2548128137":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/4212129823":"2023-10-10T04:06:26.739366+0000","172.21.15.29:0/1787157051":"2023-10-10T04:11:25.861678+0000","172.21.15.177:0/1665676424":"2023-10-10T04:12:57.728727+0000","172.21.15.29:0/3335564331":"2023-10-10T04:06:26.739366+0000","172.21.15.29:6801/3407458368":"2023-10-10T04:11:25.861678+0000","172.21.15.29:0/40150084":"2023-10-10T04:01:15.592153+0000","172.21.15.29:6801/3451944530":"2023-10-10T04:01:43.065145+0000","172.21.15.29:0/1566968162":"2023-10-10T04:13:58.298189+0000","172.21.15.177:0/63096837":"2023-10-10T04:12:57.728727+0000","172.21.15.177:6835/1319489652":"2023-10-10T04:12:57.728727+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-09T05:10:35.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:35 smithi029 ceph-mon[216903]: pgmap v2386: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:35.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:35 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:35 smithi029 ceph-mon[223858]: pgmap v2386: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:35.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:35 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:35 smithi177 ceph-mon[195797]: pgmap v2386: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:36.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:35 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' 2023-10-09T05:10:36.742 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-09T05:10:36.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-09T05:10:36.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-09T05:10:36.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-09T05:10:36.744 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-09T05:10:36.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-09T05:10:36.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-09T05:10:36.745 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-09T05:10:36.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:36 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/816049659' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:10:36.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:36 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/816049659' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:10:37.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:36 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/816049659' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-09T05:10:37.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:37 smithi029 ceph-mon[216903]: pgmap v2387: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:37 smithi029 ceph-mon[223858]: pgmap v2387: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:38.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:37 smithi177 ceph-mon[195797]: pgmap v2387: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:38.261 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.311 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.409 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.546 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.547 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.548 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.548 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:38.672 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:39.664 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:39 smithi029 ceph-mon[216903]: pgmap v2388: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:39.665 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:39 smithi029 ceph-mon[223858]: pgmap v2388: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:40.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:39 smithi177 ceph-mon[195797]: pgmap v2388: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:41.861 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:41 smithi029 ceph-mon[216903]: pgmap v2389: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:41.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:41 smithi029 ceph-mon[223858]: pgmap v2389: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:42.030 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:41 smithi177 ceph-mon[195797]: pgmap v2389: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:43.087 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:10:42 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:10:42] "GET /metrics HTTP/1.1" 200 34286 "" "Prometheus/2.43.0" 2023-10-09T05:10:43.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:43 smithi029 ceph-mon[216903]: pgmap v2390: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:43.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:43 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:43.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:43 smithi029 ceph-mon[223858]: pgmap v2390: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:43.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:43 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:43 smithi177 ceph-mon[195797]: pgmap v2390: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:44.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:43 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:45.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:45 smithi029 ceph-mon[216903]: pgmap v2391: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:45.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:45 smithi029 ceph-mon[223858]: pgmap v2391: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:46.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:45 smithi177 ceph-mon[195797]: pgmap v2391: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:46.867 INFO:teuthology.orchestra.run.smithi029.stdout:390842024544 2023-10-09T05:10:46.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-09T05:10:47.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:47 smithi029 ceph-mon[216903]: pgmap v2392: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:47.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:47 smithi029 ceph-mon[223858]: pgmap v2392: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:48.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:47 smithi177 ceph-mon[195797]: pgmap v2392: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:50.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:49 smithi177 ceph-mon[195797]: pgmap v2393: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:50.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:49 smithi029 ceph-mon[216903]: pgmap v2393: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:50.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:49 smithi029 ceph-mon[223858]: pgmap v2393: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:50.825 INFO:teuthology.orchestra.run.smithi029.stdout:949187772824 2023-10-09T05:10:50.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-09T05:10:51.192 INFO:teuthology.orchestra.run.smithi029.stdout:231928234762 2023-10-09T05:10:51.192 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-09T05:10:51.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:51 smithi029 ceph-mon[216903]: pgmap v2394: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:51.738 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:51 smithi029 ceph-mon[223858]: pgmap v2394: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:52.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:51 smithi177 ceph-mon[195797]: pgmap v2394: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:53.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:10:52 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:10:52] "GET /metrics HTTP/1.1" 200 34286 "" "Prometheus/2.43.0" 2023-10-09T05:10:53.456 INFO:teuthology.orchestra.run.smithi029.stdout:609885356517 2023-10-09T05:10:53.457 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-09T05:10:53.567 INFO:teuthology.orchestra.run.smithi029.stdout:163208758039 2023-10-09T05:10:53.567 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-09T05:10:53.701 INFO:teuthology.orchestra.run.smithi029.stdout:506806141462 2023-10-09T05:10:53.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-09T05:10:53.861 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:53 smithi029 ceph-mon[216903]: pgmap v2395: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:53.861 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:53 smithi029 ceph-mon[223858]: pgmap v2395: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:53.909 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:53.944 INFO:teuthology.orchestra.run.smithi029.stdout:2688649527387 2023-10-09T05:10:53.944 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-09T05:10:54.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:53 smithi177 ceph-mon[195797]: pgmap v2395: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:54.995 INFO:teuthology.orchestra.run.smithi029.stdout:197568496395 2023-10-09T05:10:54.995 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-09T05:10:55.611 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:56.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:55 smithi177 ceph-mon[195797]: pgmap v2396: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:56.083 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:56.086 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:55 smithi029 ceph-mon[223858]: pgmap v2396: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:56.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:55 smithi029 ceph-mon[216903]: pgmap v2396: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:56.116 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:56.210 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:56.277 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:56.969 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:57.795 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:10:57.979 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:57 smithi029 ceph-mon[216903]: pgmap v2397: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:57.979 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:57 smithi029 ceph-mon[223858]: pgmap v2397: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:58.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:57 smithi177 ceph-mon[195797]: pgmap v2397: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:58.717 INFO:teuthology.orchestra.run.smithi029.stdout:390842024547 2023-10-09T05:10:59.031 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:58 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:59.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:58 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:59.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:58 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:10:59.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:59 smithi029 ceph-mon[216903]: pgmap v2398: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:59.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:10:59 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/984819359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:10:59.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:59 smithi029 ceph-mon[223858]: pgmap v2398: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:10:59.765 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:10:59 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/984819359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:11:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:59 smithi177 ceph-mon[195797]: pgmap v2398: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:00.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:10:59 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/984819359' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-09T05:11:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:00 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T05:11:00.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:00 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T05:11:01.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:00 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-10-09T05:11:01.362 INFO:teuthology.orchestra.run.smithi029.stdout:231928234764 2023-10-09T05:11:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: pgmap v2399: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T05:11:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: osdmap e769: 8 total, 8 up, 8 in 2023-10-09T05:11:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.028 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:01 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1887028693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:11:02.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: pgmap v2399: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:02.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T05:11:02.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.049 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: osdmap e769: 8 total, 8 up, 8 in 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1887028693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: pgmap v2399: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: osdmap e769: 8 total, 8 up, 8 in 2023-10-09T05:11:02.050 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: failed to encode map e769 with expected crc 2023-10-09T05:11:02.051 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:01 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1887028693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-09T05:11:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:02 smithi177 ceph-mon[195797]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:02 smithi177 ceph-mon[195797]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:02 smithi177 ceph-mon[195797]: osdmap e770: 8 total, 8 up, 8 in 2023-10-09T05:11:03.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:02 smithi177 ceph-mon[195797]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[216903]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[216903]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[216903]: osdmap e770: 8 total, 8 up, 8 in 2023-10-09T05:11:03.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[216903]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[223858]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[223858]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[223858]: osdmap e770: 8 total, 8 up, 8 in 2023-10-09T05:11:03.099 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:02 smithi029 ceph-mon[223858]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.099 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:02 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:11:02] "GET /metrics HTTP/1.1" 200 34285 "" "Prometheus/2.43.0" 2023-10-09T05:11:03.748 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024544 got 390842024547 for osd.0 2023-10-09T05:11:03.748 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:03.791 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234762 got 231928234764 for osd.7 2023-10-09T05:11:03.792 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:03.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:03 smithi029 ceph-mon[216903]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.916 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:03 smithi029 ceph-mon[216903]: pgmap v2402: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:03.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:03 smithi029 ceph-mon[223858]: failed to encode map e770 with expected crc 2023-10-09T05:11:03.916 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:03 smithi029 ceph-mon[223858]: pgmap v2402: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:03 smithi177 ceph-mon[195797]: failed to encode map e770 with expected crc 2023-10-09T05:11:04.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:03 smithi177 ceph-mon[195797]: pgmap v2402: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:04.542 INFO:teuthology.orchestra.run.smithi029.stdout:949187772827 2023-10-09T05:11:04.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:04 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2931479338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:11:04.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:04 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2931479338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:11:05.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:04 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2931479338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-09T05:11:06.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:05 smithi177 ceph-mon[195797]: pgmap v2403: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:06.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:05 smithi029 ceph-mon[216903]: pgmap v2403: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:06.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:05 smithi029 ceph-mon[223858]: pgmap v2403: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:06.739 INFO:tasks.cephadm.ceph_manager.ceph:need seq 949187772824 got 949187772827 for osd.2 2023-10-09T05:11:06.739 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:07.417 INFO:teuthology.orchestra.run.smithi029.stdout:2688649527390 2023-10-09T05:11:07.548 INFO:teuthology.orchestra.run.smithi029.stdout:163208758042 2023-10-09T05:11:07.852 INFO:teuthology.orchestra.run.smithi029.stdout:506806141465 2023-10-09T05:11:08.010 INFO:teuthology.orchestra.run.smithi029.stdout:609885356521 2023-10-09T05:11:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:07 smithi177 ceph-mon[195797]: pgmap v2404: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/3541385065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:11:08.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:07 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1657292565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:11:08.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:07 smithi029 ceph-mon[216903]: pgmap v2404: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:08.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/3541385065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:11:08.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:07 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1657292565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:11:08.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:07 smithi029 ceph-mon[223858]: pgmap v2404: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:08.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/3541385065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-09T05:11:08.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:07 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1657292565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-09T05:11:08.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1846252947' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:11:08.977 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:08 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/1634498791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:11:08.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1846252947' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:11:08.977 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:08 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/1634498791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:11:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1846252947' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-09T05:11:09.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:08 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/1634498791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-09T05:11:09.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:09 smithi029 ceph-mon[216903]: pgmap v2405: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:09.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:09 smithi029 ceph-mon[223858]: pgmap v2405: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:10.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:09 smithi177 ceph-mon[195797]: pgmap v2405: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:10.109 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2688649527387 got 2688649527390 for osd.4 2023-10-09T05:11:10.109 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:10.299 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208758039 got 163208758042 for osd.5 2023-10-09T05:11:10.299 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:10.441 INFO:tasks.cephadm.ceph_manager.ceph:need seq 609885356517 got 609885356521 for osd.3 2023-10-09T05:11:10.441 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:10.491 INFO:tasks.cephadm.ceph_manager.ceph:need seq 506806141462 got 506806141465 for osd.1 2023-10-09T05:11:10.491 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:10.836 INFO:teuthology.orchestra.run.smithi029.stdout:197568496399 2023-10-09T05:11:11.661 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496395 got 197568496399 for osd.6 2023-10-09T05:11:11.662 DEBUG:teuthology.parallel:result is None 2023-10-09T05:11:11.662 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-09T05:11:11.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:11:11.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:11 smithi029 ceph-mon[216903]: pgmap v2406: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:11.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:11 smithi029 ceph-mon[216903]: from='client.? 172.21.15.29:0/2336331923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:11:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:11 smithi029 ceph-mon[223858]: pgmap v2406: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:11.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:11 smithi029 ceph-mon[223858]: from='client.? 172.21.15.29:0/2336331923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:11:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:11 smithi177 ceph-mon[195797]: pgmap v2406: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:12.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:11 smithi177 ceph-mon[195797]: from='client.? 172.21.15.29:0/2336331923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-09T05:11:12.030 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:12.920 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:12 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:11:12] "GET /metrics HTTP/1.1" 200 34285 "" "Prometheus/2.43.0" 2023-10-09T05:11:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:13 smithi177 ceph-mon[195797]: pgmap v2407: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:14.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:13 smithi177 ceph-mon[195797]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:14.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:13 smithi029 ceph-mon[216903]: pgmap v2407: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:14.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:13 smithi029 ceph-mon[216903]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:13 smithi029 ceph-mon[223858]: pgmap v2407: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:14.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:13 smithi029 ceph-mon[223858]: from='mgr.25202 172.21.15.29:0/2188812110' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-09T05:11:14.184 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:11:14.185 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:11:15.180 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2408,"stamp":"2023-10-09T05:11:13.963843+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":5144514,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2803,"ondisk_log_size":2803,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9434500,"kb_used_data":63428,"kb_used_omap":42,"kb_used_meta":9370965,"kb_avail":740526716,"statfs":{"total":767960285184,"available":758299357184,"internally_reserved":0,"allocated":64950272,"data_stored":49376356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43557,"internal_metadata":9595868635},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003142"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":282,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701282+0000","last_change":"2023-10-09T05:09:05.237648+0000","last_active":"2023-10-09T05:11:00.701282+0000","last_peered":"2023-10-09T05:11:00.701282+0000","last_clean":"2023-10-09T05:11:00.701282+0000","last_became_active":"2023-10-09T05:09:05.237250+0000","last_became_peered":"2023-10-09T05:09:05.237250+0000","last_unstale":"2023-10-09T05:11:00.701282+0000","last_undegraded":"2023-10-09T05:11:00.701282+0000","last_fullsized":"2023-10-09T05:11:00.701282+0000","mapping_epoch":734,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":735,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:44.769366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":419,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699626+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:11:00.699626+0000","last_peered":"2023-10-09T05:11:00.699626+0000","last_clean":"2023-10-09T05:11:00.699626+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:11:00.699626+0000","last_undegraded":"2023-10-09T05:11:00.699626+0000","last_fullsized":"2023-10-09T05:11:00.699626+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":196,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694747+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:11:00.694747+0000","last_peered":"2023-10-09T05:11:00.694747+0000","last_clean":"2023-10-09T05:11:00.694747+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:11:00.694747+0000","last_undegraded":"2023-10-09T05:11:00.694747+0000","last_fullsized":"2023-10-09T05:11:00.694747+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982207+0000","last_change":"2023-10-09T05:09:27.665724+0000","last_active":"2023-10-09T05:11:00.982207+0000","last_peered":"2023-10-09T05:11:00.982207+0000","last_clean":"2023-10-09T05:11:00.982207+0000","last_became_active":"2023-10-09T05:09:27.665402+0000","last_became_peered":"2023-10-09T05:09:27.665402+0000","last_unstale":"2023-10-09T05:11:00.982207+0000","last_undegraded":"2023-10-09T05:11:00.982207+0000","last_fullsized":"2023-10-09T05:11:00.982207+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:56.519789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":401,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699101+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:11:00.699101+0000","last_peered":"2023-10-09T05:11:00.699101+0000","last_clean":"2023-10-09T05:11:00.699101+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:11:00.699101+0000","last_undegraded":"2023-10-09T05:11:00.699101+0000","last_fullsized":"2023-10-09T05:11:00.699101+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":296,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984072+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:11:00.984072+0000","last_peered":"2023-10-09T05:11:00.984072+0000","last_clean":"2023-10-09T05:11:00.984072+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:11:00.984072+0000","last_undegraded":"2023-10-09T05:11:00.984072+0000","last_fullsized":"2023-10-09T05:11:00.984072+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":630,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982309+0000","last_change":"2023-10-09T05:09:27.665287+0000","last_active":"2023-10-09T05:11:00.982309+0000","last_peered":"2023-10-09T05:11:00.982309+0000","last_clean":"2023-10-09T05:11:00.982309+0000","last_became_active":"2023-10-09T05:09:27.664944+0000","last_became_peered":"2023-10-09T05:09:27.664944+0000","last_unstale":"2023-10-09T05:11:00.982309+0000","last_undegraded":"2023-10-09T05:11:00.982309+0000","last_fullsized":"2023-10-09T05:11:00.982309+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:22.928578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.698999+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:11:00.698999+0000","last_peered":"2023-10-09T05:11:00.698999+0000","last_clean":"2023-10-09T05:11:00.698999+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:11:00.698999+0000","last_undegraded":"2023-10-09T05:11:00.698999+0000","last_fullsized":"2023-10-09T05:11:00.698999+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":177,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700417+0000","last_change":"2023-10-09T05:09:10.293342+0000","last_active":"2023-10-09T05:11:00.700417+0000","last_peered":"2023-10-09T05:11:00.700417+0000","last_clean":"2023-10-09T05:11:00.700417+0000","last_became_active":"2023-10-09T05:09:10.292951+0000","last_became_peered":"2023-10-09T05:09:10.292951+0000","last_unstale":"2023-10-09T05:11:00.700417+0000","last_undegraded":"2023-10-09T05:11:00.700417+0000","last_fullsized":"2023-10-09T05:11:00.700417+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:40.274186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":401,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699438+0000","last_change":"2023-10-09T05:09:10.280081+0000","last_active":"2023-10-09T05:11:00.699438+0000","last_peered":"2023-10-09T05:11:00.699438+0000","last_clean":"2023-10-09T05:11:00.699438+0000","last_became_active":"2023-10-09T05:09:10.279774+0000","last_became_peered":"2023-10-09T05:09:10.279774+0000","last_unstale":"2023-10-09T05:11:00.699438+0000","last_undegraded":"2023-10-09T05:11:00.699438+0000","last_fullsized":"2023-10-09T05:11:00.699438+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:57.567705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":461,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.706057+0000","last_change":"2023-10-09T05:10:12.210156+0000","last_active":"2023-10-09T05:11:00.706057+0000","last_peered":"2023-10-09T05:11:00.706057+0000","last_clean":"2023-10-09T05:11:00.706057+0000","last_became_active":"2023-10-09T05:10:12.209818+0000","last_became_peered":"2023-10-09T05:10:12.209818+0000","last_unstale":"2023-10-09T05:11:00.706057+0000","last_undegraded":"2023-10-09T05:11:00.706057+0000","last_fullsized":"2023-10-09T05:11:00.706057+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:07:46.991464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1080,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":205,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694661+0000","last_change":"2023-10-09T05:09:05.239952+0000","last_active":"2023-10-09T05:11:00.694661+0000","last_peered":"2023-10-09T05:11:00.694661+0000","last_clean":"2023-10-09T05:11:00.694661+0000","last_became_active":"2023-10-09T05:09:05.239675+0000","last_became_peered":"2023-10-09T05:09:05.239675+0000","last_unstale":"2023-10-09T05:11:00.694661+0000","last_undegraded":"2023-10-09T05:11:00.694661+0000","last_fullsized":"2023-10-09T05:11:00.694661+0000","mapping_epoch":734,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":735,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:28.664644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":285,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701231+0000","last_change":"2023-10-09T05:09:10.313568+0000","last_active":"2023-10-09T05:11:00.701231+0000","last_peered":"2023-10-09T05:11:00.701231+0000","last_clean":"2023-10-09T05:11:00.701231+0000","last_became_active":"2023-10-09T05:09:10.288379+0000","last_became_peered":"2023-10-09T05:09:10.288379+0000","last_unstale":"2023-10-09T05:11:00.701231+0000","last_undegraded":"2023-10-09T05:11:00.701231+0000","last_fullsized":"2023-10-09T05:11:00.701231+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:54.959372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":240,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701088+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:11:00.701088+0000","last_peered":"2023-10-09T05:11:00.701088+0000","last_clean":"2023-10-09T05:11:00.701088+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:11:00.701088+0000","last_undegraded":"2023-10-09T05:11:00.701088+0000","last_fullsized":"2023-10-09T05:11:00.701088+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":272,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705800+0000","last_change":"2023-10-09T05:10:12.208114+0000","last_active":"2023-10-09T05:11:00.705800+0000","last_peered":"2023-10-09T05:11:00.705800+0000","last_clean":"2023-10-09T05:11:00.705800+0000","last_became_active":"2023-10-09T05:10:12.207776+0000","last_became_peered":"2023-10-09T05:10:12.207776+0000","last_unstale":"2023-10-09T05:11:00.705800+0000","last_undegraded":"2023-10-09T05:11:00.705800+0000","last_fullsized":"2023-10-09T05:11:00.705800+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:06.947241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":383,"reported_epoch":770,"state":"active+clean","last_fresh":"2023-10-09T05:11:01.701158+0000","last_change":"2023-10-09T05:09:10.274048+0000","last_active":"2023-10-09T05:11:01.701158+0000","last_peered":"2023-10-09T05:11:01.701158+0000","last_clean":"2023-10-09T05:11:01.701158+0000","last_became_active":"2023-10-09T05:09:10.273740+0000","last_became_peered":"2023-10-09T05:09:10.273740+0000","last_unstale":"2023-10-09T05:11:01.701158+0000","last_undegraded":"2023-10-09T05:11:01.701158+0000","last_fullsized":"2023-10-09T05:11:01.701158+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:34.021710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":289,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.981997+0000","last_change":"2023-10-09T05:09:27.665248+0000","last_active":"2023-10-09T05:11:00.981997+0000","last_peered":"2023-10-09T05:11:00.981997+0000","last_clean":"2023-10-09T05:11:00.981997+0000","last_became_active":"2023-10-09T05:09:27.664873+0000","last_became_peered":"2023-10-09T05:09:27.664873+0000","last_unstale":"2023-10-09T05:11:00.981997+0000","last_undegraded":"2023-10-09T05:11:00.981997+0000","last_fullsized":"2023-10-09T05:11:00.981997+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:17.717559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700324+0000","last_change":"2023-10-09T05:09:10.295199+0000","last_active":"2023-10-09T05:11:00.700324+0000","last_peered":"2023-10-09T05:11:00.700324+0000","last_clean":"2023-10-09T05:11:00.700324+0000","last_became_active":"2023-10-09T05:09:10.292927+0000","last_became_peered":"2023-10-09T05:09:10.292927+0000","last_unstale":"2023-10-09T05:11:00.700324+0000","last_undegraded":"2023-10-09T05:11:00.700324+0000","last_fullsized":"2023-10-09T05:11:00.700324+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:36.159060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":213,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705497+0000","last_change":"2023-10-09T05:10:12.211033+0000","last_active":"2023-10-09T05:11:00.705497+0000","last_peered":"2023-10-09T05:11:00.705497+0000","last_clean":"2023-10-09T05:11:00.705497+0000","last_became_active":"2023-10-09T05:10:12.210688+0000","last_became_peered":"2023-10-09T05:10:12.210688+0000","last_unstale":"2023-10-09T05:11:00.705497+0000","last_undegraded":"2023-10-09T05:11:00.705497+0000","last_fullsized":"2023-10-09T05:11:00.705497+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:45.399387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":432,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982184+0000","last_change":"2023-10-09T05:09:27.665200+0000","last_active":"2023-10-09T05:11:00.982184+0000","last_peered":"2023-10-09T05:11:00.982184+0000","last_clean":"2023-10-09T05:11:00.982184+0000","last_became_active":"2023-10-09T05:09:27.664838+0000","last_became_peered":"2023-10-09T05:09:27.664838+0000","last_unstale":"2023-10-09T05:11:00.982184+0000","last_undegraded":"2023-10-09T05:11:00.982184+0000","last_fullsized":"2023-10-09T05:11:00.982184+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:06.260353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":404,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983755+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:11:00.983755+0000","last_peered":"2023-10-09T05:11:00.983755+0000","last_clean":"2023-10-09T05:11:00.983755+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:11:00.983755+0000","last_undegraded":"2023-10-09T05:11:00.983755+0000","last_fullsized":"2023-10-09T05:11:00.983755+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":303,"reported_epoch":770,"state":"active+clean","last_fresh":"2023-10-09T05:11:01.842506+0000","last_change":"2023-10-09T05:11:01.842506+0000","last_active":"2023-10-09T05:11:01.842506+0000","last_peered":"2023-10-09T05:11:01.842506+0000","last_clean":"2023-10-09T05:11:01.842506+0000","last_became_active":"2023-10-09T05:11:01.842277+0000","last_became_peered":"2023-10-09T05:11:01.842277+0000","last_unstale":"2023-10-09T05:11:01.842506+0000","last_undegraded":"2023-10-09T05:11:01.842506+0000","last_fullsized":"2023-10-09T05:11:01.842506+0000","mapping_epoch":769,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":770,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:54.146465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":307,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.706104+0000","last_change":"2023-10-09T05:10:12.209123+0000","last_active":"2023-10-09T05:11:00.706104+0000","last_peered":"2023-10-09T05:11:00.706104+0000","last_clean":"2023-10-09T05:11:00.706104+0000","last_became_active":"2023-10-09T05:10:12.208711+0000","last_became_peered":"2023-10-09T05:10:12.208711+0000","last_unstale":"2023-10-09T05:11:00.706104+0000","last_undegraded":"2023-10-09T05:11:00.706104+0000","last_fullsized":"2023-10-09T05:11:00.706104+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:13.123169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":177,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700106+0000","last_change":"2023-10-09T05:09:10.293964+0000","last_active":"2023-10-09T05:11:00.700106+0000","last_peered":"2023-10-09T05:11:00.700106+0000","last_clean":"2023-10-09T05:11:00.700106+0000","last_became_active":"2023-10-09T05:09:10.293585+0000","last_became_peered":"2023-10-09T05:09:10.293585+0000","last_unstale":"2023-10-09T05:11:00.700106+0000","last_undegraded":"2023-10-09T05:11:00.700106+0000","last_fullsized":"2023-10-09T05:11:00.700106+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:53.738992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":141,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.706013+0000","last_change":"2023-10-09T05:10:12.207492+0000","last_active":"2023-10-09T05:11:00.706013+0000","last_peered":"2023-10-09T05:11:00.706013+0000","last_clean":"2023-10-09T05:11:00.706013+0000","last_became_active":"2023-10-09T05:10:12.207143+0000","last_became_peered":"2023-10-09T05:10:12.207143+0000","last_unstale":"2023-10-09T05:11:00.706013+0000","last_undegraded":"2023-10-09T05:11:00.706013+0000","last_fullsized":"2023-10-09T05:11:00.706013+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:00.546236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":256,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700811+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:11:00.700811+0000","last_peered":"2023-10-09T05:11:00.700811+0000","last_clean":"2023-10-09T05:11:00.700811+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:11:00.700811+0000","last_undegraded":"2023-10-09T05:11:00.700811+0000","last_fullsized":"2023-10-09T05:11:00.700811+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":159,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705892+0000","last_change":"2023-10-09T05:10:12.211670+0000","last_active":"2023-10-09T05:11:00.705892+0000","last_peered":"2023-10-09T05:11:00.705892+0000","last_clean":"2023-10-09T05:11:00.705892+0000","last_became_active":"2023-10-09T05:10:12.211350+0000","last_became_peered":"2023-10-09T05:10:12.211350+0000","last_unstale":"2023-10-09T05:11:00.705892+0000","last_undegraded":"2023-10-09T05:11:00.705892+0000","last_fullsized":"2023-10-09T05:11:00.705892+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:30.615058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":336,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983883+0000","last_change":"2023-10-09T05:09:10.282347+0000","last_active":"2023-10-09T05:11:00.983883+0000","last_peered":"2023-10-09T05:11:00.983883+0000","last_clean":"2023-10-09T05:11:00.983883+0000","last_became_active":"2023-10-09T05:09:10.281988+0000","last_became_peered":"2023-10-09T05:09:10.281988+0000","last_unstale":"2023-10-09T05:11:00.983883+0000","last_undegraded":"2023-10-09T05:11:00.983883+0000","last_fullsized":"2023-10-09T05:11:00.983883+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:32.577269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":829,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700523+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:11:00.700523+0000","last_peered":"2023-10-09T05:11:00.700523+0000","last_clean":"2023-10-09T05:11:00.700523+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:11:00.700523+0000","last_undegraded":"2023-10-09T05:11:00.700523+0000","last_fullsized":"2023-10-09T05:11:00.700523+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982199+0000","last_change":"2023-10-09T05:09:27.625297+0000","last_active":"2023-10-09T05:11:00.982199+0000","last_peered":"2023-10-09T05:11:00.982199+0000","last_clean":"2023-10-09T05:11:00.982199+0000","last_became_active":"2023-10-09T05:09:27.624891+0000","last_became_peered":"2023-10-09T05:09:27.624891+0000","last_unstale":"2023-10-09T05:11:00.982199+0000","last_undegraded":"2023-10-09T05:11:00.982199+0000","last_fullsized":"2023-10-09T05:11:00.982199+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:30.267688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1247,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705621+0000","last_change":"2023-10-09T05:10:19.195136+0000","last_active":"2023-10-09T05:11:00.705621+0000","last_peered":"2023-10-09T05:11:00.705621+0000","last_clean":"2023-10-09T05:11:00.705621+0000","last_became_active":"2023-10-09T05:10:19.194649+0000","last_became_peered":"2023-10-09T05:10:19.194649+0000","last_unstale":"2023-10-09T05:11:00.705621+0000","last_undegraded":"2023-10-09T05:11:00.705621+0000","last_fullsized":"2023-10-09T05:11:00.705621+0000","mapping_epoch":764,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":765,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:54.997107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:54.997107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:38.221308+0000","scrub_duration":0.0021928569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":376,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699561+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:11:00.699561+0000","last_peered":"2023-10-09T05:11:00.699561+0000","last_clean":"2023-10-09T05:11:00.699561+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:11:00.699561+0000","last_undegraded":"2023-10-09T05:11:00.699561+0000","last_fullsized":"2023-10-09T05:11:00.699561+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700914+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:11:00.700914+0000","last_peered":"2023-10-09T05:11:00.700914+0000","last_clean":"2023-10-09T05:11:00.700914+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:11:00.700914+0000","last_undegraded":"2023-10-09T05:11:00.700914+0000","last_fullsized":"2023-10-09T05:11:00.700914+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1232,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699944+0000","last_change":"2023-10-09T05:09:08.254212+0000","last_active":"2023-10-09T05:11:00.699944+0000","last_peered":"2023-10-09T05:11:00.699944+0000","last_clean":"2023-10-09T05:11:00.699944+0000","last_became_active":"2023-10-09T05:09:08.253872+0000","last_became_peered":"2023-10-09T05:09:08.253872+0000","last_unstale":"2023-10-09T05:11:00.699944+0000","last_undegraded":"2023-10-09T05:11:00.699944+0000","last_fullsized":"2023-10-09T05:11:00.699944+0000","mapping_epoch":737,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":738,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:48.616814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:48.616814+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:23.216761+0000","scrub_duration":0.0021341469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":313,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983787+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:11:00.983787+0000","last_peered":"2023-10-09T05:11:00.983787+0000","last_clean":"2023-10-09T05:11:00.983787+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:11:00.983787+0000","last_undegraded":"2023-10-09T05:11:00.983787+0000","last_fullsized":"2023-10-09T05:11:00.983787+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":138,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701294+0000","last_change":"2023-10-09T05:10:16.182226+0000","last_active":"2023-10-09T05:11:00.701294+0000","last_peered":"2023-10-09T05:11:00.701294+0000","last_clean":"2023-10-09T05:11:00.701294+0000","last_became_active":"2023-10-09T05:10:16.181907+0000","last_became_peered":"2023-10-09T05:10:16.181907+0000","last_unstale":"2023-10-09T05:11:00.701294+0000","last_undegraded":"2023-10-09T05:11:00.701294+0000","last_fullsized":"2023-10-09T05:11:00.701294+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:12.875304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701009+0000","last_change":"2023-10-09T05:09:10.283620+0000","last_active":"2023-10-09T05:11:00.701009+0000","last_peered":"2023-10-09T05:11:00.701009+0000","last_clean":"2023-10-09T05:11:00.701009+0000","last_became_active":"2023-10-09T05:09:10.283099+0000","last_became_peered":"2023-10-09T05:09:10.283099+0000","last_unstale":"2023-10-09T05:11:00.701009+0000","last_undegraded":"2023-10-09T05:11:00.701009+0000","last_fullsized":"2023-10-09T05:11:00.701009+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:03.690878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701282+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:11:00.701282+0000","last_peered":"2023-10-09T05:11:00.701282+0000","last_clean":"2023-10-09T05:11:00.701282+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:11:00.701282+0000","last_undegraded":"2023-10-09T05:11:00.701282+0000","last_fullsized":"2023-10-09T05:11:00.701282+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1243,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700206+0000","last_change":"2023-10-09T05:09:08.253464+0000","last_active":"2023-10-09T05:11:00.700206+0000","last_peered":"2023-10-09T05:11:00.700206+0000","last_clean":"2023-10-09T05:11:00.700206+0000","last_became_active":"2023-10-09T05:09:08.253102+0000","last_became_peered":"2023-10-09T05:09:08.253102+0000","last_unstale":"2023-10-09T05:11:00.700206+0000","last_undegraded":"2023-10-09T05:11:00.700206+0000","last_fullsized":"2023-10-09T05:11:00.700206+0000","mapping_epoch":737,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":738,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:05.173448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:05.173448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:37.921959+0000","scrub_duration":0.001419465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":309,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.698954+0000","last_change":"2023-10-09T05:09:10.273499+0000","last_active":"2023-10-09T05:11:00.698954+0000","last_peered":"2023-10-09T05:11:00.698954+0000","last_clean":"2023-10-09T05:11:00.698954+0000","last_became_active":"2023-10-09T05:09:10.273195+0000","last_became_peered":"2023-10-09T05:09:10.273195+0000","last_unstale":"2023-10-09T05:11:00.698954+0000","last_undegraded":"2023-10-09T05:11:00.698954+0000","last_fullsized":"2023-10-09T05:11:00.698954+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:07.522564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":199,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694862+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:11:00.694862+0000","last_peered":"2023-10-09T05:11:00.694862+0000","last_clean":"2023-10-09T05:11:00.694862+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:11:00.694862+0000","last_undegraded":"2023-10-09T05:11:00.694862+0000","last_fullsized":"2023-10-09T05:11:00.694862+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1443,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984097+0000","last_change":"2023-10-09T05:09:13.543189+0000","last_active":"2023-10-09T05:11:00.984097+0000","last_peered":"2023-10-09T05:11:00.984097+0000","last_clean":"2023-10-09T05:11:00.984097+0000","last_became_active":"2023-10-09T05:09:13.542757+0000","last_became_peered":"2023-10-09T05:09:13.542757+0000","last_unstale":"2023-10-09T05:11:00.984097+0000","last_undegraded":"2023-10-09T05:11:00.984097+0000","last_fullsized":"2023-10-09T05:11:00.984097+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:56.681273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.681273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:31.565781+0000","scrub_duration":0.001633844,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"724'129","reported_seq":578,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694376+0000","last_change":"2023-10-09T05:09:13.296378+0000","last_active":"2023-10-09T05:11:00.694376+0000","last_peered":"2023-10-09T05:11:00.694376+0000","last_clean":"2023-10-09T05:11:00.694376+0000","last_became_active":"2023-10-09T05:09:13.295966+0000","last_became_peered":"2023-10-09T05:09:13.295966+0000","last_unstale":"2023-10-09T05:11:00.694376+0000","last_undegraded":"2023-10-09T05:11:00.694376+0000","last_fullsized":"2023-10-09T05:11:00.694376+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:23.681451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047247800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":400,"num_read_kb":709,"num_write":158,"num_write_kb":271,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":157,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700431+0000","last_change":"2023-10-09T05:09:05.237586+0000","last_active":"2023-10-09T05:11:00.700431+0000","last_peered":"2023-10-09T05:11:00.700431+0000","last_clean":"2023-10-09T05:11:00.700431+0000","last_became_active":"2023-10-09T05:09:05.237334+0000","last_became_peered":"2023-10-09T05:09:05.237334+0000","last_unstale":"2023-10-09T05:11:00.700431+0000","last_undegraded":"2023-10-09T05:11:00.700431+0000","last_fullsized":"2023-10-09T05:11:00.700431+0000","mapping_epoch":734,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":735,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:55.395287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694621+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:11:00.694621+0000","last_peered":"2023-10-09T05:11:00.694621+0000","last_clean":"2023-10-09T05:11:00.694621+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:11:00.694621+0000","last_undegraded":"2023-10-09T05:11:00.694621+0000","last_fullsized":"2023-10-09T05:11:00.694621+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1232,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700005+0000","last_change":"2023-10-09T05:09:08.253434+0000","last_active":"2023-10-09T05:11:00.700005+0000","last_peered":"2023-10-09T05:11:00.700005+0000","last_clean":"2023-10-09T05:11:00.700005+0000","last_became_active":"2023-10-09T05:09:08.253156+0000","last_became_peered":"2023-10-09T05:09:08.253156+0000","last_unstale":"2023-10-09T05:11:00.700005+0000","last_undegraded":"2023-10-09T05:11:00.700005+0000","last_fullsized":"2023-10-09T05:11:00.700005+0000","mapping_epoch":737,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":738,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:59.084186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:59.084186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:07.255235+0000","scrub_duration":0.001940289,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"724'79","reported_seq":1343,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700254+0000","last_change":"2023-10-09T05:08:41.151543+0000","last_active":"2023-10-09T05:11:00.700254+0000","last_peered":"2023-10-09T05:11:00.700254+0000","last_clean":"2023-10-09T05:11:00.700254+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:11:00.700254+0000","last_undegraded":"2023-10-09T05:11:00.700254+0000","last_fullsized":"2023-10-09T05:11:00.700254+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095775700000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":730,"num_read_kb":1477,"num_write":79,"num_write_kb":462,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":235,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700775+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:11:00.700775+0000","last_peered":"2023-10-09T05:11:00.700775+0000","last_clean":"2023-10-09T05:11:00.700775+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:11:00.700775+0000","last_undegraded":"2023-10-09T05:11:00.700775+0000","last_fullsized":"2023-10-09T05:11:00.700775+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700109+0000","last_change":"2023-10-09T05:10:16.184121+0000","last_active":"2023-10-09T05:11:00.700109+0000","last_peered":"2023-10-09T05:11:00.700109+0000","last_clean":"2023-10-09T05:11:00.700109+0000","last_became_active":"2023-10-09T05:10:16.183830+0000","last_became_peered":"2023-10-09T05:10:16.183830+0000","last_unstale":"2023-10-09T05:11:00.700109+0000","last_undegraded":"2023-10-09T05:11:00.700109+0000","last_fullsized":"2023-10-09T05:11:00.700109+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:40.778454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1241,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705739+0000","last_change":"2023-10-09T05:10:12.208474+0000","last_active":"2023-10-09T05:11:00.705739+0000","last_peered":"2023-10-09T05:11:00.705739+0000","last_clean":"2023-10-09T05:11:00.705739+0000","last_became_active":"2023-10-09T05:10:12.207353+0000","last_became_peered":"2023-10-09T05:10:12.207353+0000","last_unstale":"2023-10-09T05:11:00.705739+0000","last_undegraded":"2023-10-09T05:11:00.705739+0000","last_fullsized":"2023-10-09T05:11:00.705739+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:09:27.811929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:09:27.811929+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:57.418226+0000","scrub_duration":0.001713099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"724'67","reported_seq":45,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701862+0000","last_change":"2023-10-09T05:10:16.190109+0000","last_active":"2023-10-09T05:11:00.701862+0000","last_peered":"2023-10-09T05:11:00.701862+0000","last_clean":"2023-10-09T05:11:00.701862+0000","last_became_active":"2023-10-09T05:10:16.179712+0000","last_became_peered":"2023-10-09T05:10:16.179712+0000","last_unstale":"2023-10-09T05:11:00.701862+0000","last_undegraded":"2023-10-09T05:11:00.701862+0000","last_fullsized":"2023-10-09T05:11:00.701862+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.591725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061087000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":328,"num_read_kb":1510,"num_write":56,"num_write_kb":1199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":195,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.693940+0000","last_change":"2023-10-09T05:10:16.178981+0000","last_active":"2023-10-09T05:11:00.693940+0000","last_peered":"2023-10-09T05:11:00.693940+0000","last_clean":"2023-10-09T05:11:00.693940+0000","last_became_active":"2023-10-09T05:10:16.178646+0000","last_became_peered":"2023-10-09T05:10:16.178646+0000","last_unstale":"2023-10-09T05:11:00.693940+0000","last_undegraded":"2023-10-09T05:11:00.693940+0000","last_fullsized":"2023-10-09T05:11:00.693940+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:08.144860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.702392+0000","last_change":"2023-10-09T05:10:01.572404+0000","last_active":"2023-10-09T05:11:00.702392+0000","last_peered":"2023-10-09T05:11:00.702392+0000","last_clean":"2023-10-09T05:11:00.702392+0000","last_became_active":"2023-10-09T05:10:01.571931+0000","last_became_peered":"2023-10-09T05:10:01.571931+0000","last_unstale":"2023-10-09T05:11:00.702392+0000","last_undegraded":"2023-10-09T05:11:00.702392+0000","last_fullsized":"2023-10-09T05:11:00.702392+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.770672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"724'174","reported_seq":892,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700134+0000","last_change":"2023-10-09T05:09:13.540906+0000","last_active":"2023-10-09T05:11:00.700134+0000","last_peered":"2023-10-09T05:11:00.700134+0000","last_clean":"2023-10-09T05:11:00.700134+0000","last_became_active":"2023-10-09T05:09:13.540372+0000","last_became_peered":"2023-10-09T05:09:13.540372+0000","last_unstale":"2023-10-09T05:11:00.700134+0000","last_undegraded":"2023-10-09T05:11:00.700134+0000","last_fullsized":"2023-10-09T05:11:00.700134+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:21.040614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040793699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":925,"num_read_kb":1854,"num_write":140,"num_write_kb":1122,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1400,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705422+0000","last_change":"2023-10-09T05:10:12.211701+0000","last_active":"2023-10-09T05:11:00.705422+0000","last_peered":"2023-10-09T05:11:00.705422+0000","last_clean":"2023-10-09T05:11:00.705422+0000","last_became_active":"2023-10-09T05:10:12.211352+0000","last_became_peered":"2023-10-09T05:10:12.211352+0000","last_unstale":"2023-10-09T05:11:00.705422+0000","last_undegraded":"2023-10-09T05:11:00.705422+0000","last_fullsized":"2023-10-09T05:11:00.705422+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:09:32.823787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:09:32.823787+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:08.791250+0000","scrub_duration":0.0015712409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":265,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699443+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:11:00.699443+0000","last_peered":"2023-10-09T05:11:00.699443+0000","last_clean":"2023-10-09T05:11:00.699443+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:11:00.699443+0000","last_undegraded":"2023-10-09T05:11:00.699443+0000","last_fullsized":"2023-10-09T05:11:00.699443+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":310,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983995+0000","last_change":"2023-10-09T05:09:10.286480+0000","last_active":"2023-10-09T05:11:00.983995+0000","last_peered":"2023-10-09T05:11:00.983995+0000","last_clean":"2023-10-09T05:11:00.983995+0000","last_became_active":"2023-10-09T05:09:10.286182+0000","last_became_peered":"2023-10-09T05:09:10.286182+0000","last_unstale":"2023-10-09T05:11:00.983995+0000","last_undegraded":"2023-10-09T05:11:00.983995+0000","last_fullsized":"2023-10-09T05:11:00.983995+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:32.066948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1231,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694347+0000","last_change":"2023-10-09T05:07:58.306714+0000","last_active":"2023-10-09T05:11:00.694347+0000","last_peered":"2023-10-09T05:11:00.694347+0000","last_clean":"2023-10-09T05:11:00.694347+0000","last_became_active":"2023-10-09T05:07:52.143205+0000","last_became_peered":"2023-10-09T05:07:52.143205+0000","last_unstale":"2023-10-09T05:11:00.694347+0000","last_undegraded":"2023-10-09T05:11:00.694347+0000","last_fullsized":"2023-10-09T05:11:00.694347+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:58.306600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:58.306600+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.900624+0000","scrub_duration":0.001694367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"724'130","reported_seq":865,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705576+0000","last_change":"2023-10-09T05:10:12.205801+0000","last_active":"2023-10-09T05:11:00.705576+0000","last_peered":"2023-10-09T05:11:00.705576+0000","last_clean":"2023-10-09T05:11:00.705576+0000","last_became_active":"2023-10-09T05:10:12.205265+0000","last_became_peered":"2023-10-09T05:10:12.205265+0000","last_unstale":"2023-10-09T05:11:00.705576+0000","last_undegraded":"2023-10-09T05:11:00.705576+0000","last_fullsized":"2023-10-09T05:11:00.705576+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:44.711106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042191800000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1129,"num_read_kb":2343,"num_write":84,"num_write_kb":6329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":87,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.981913+0000","last_change":"2023-10-09T05:09:27.665313+0000","last_active":"2023-10-09T05:11:00.981913+0000","last_peered":"2023-10-09T05:11:00.981913+0000","last_clean":"2023-10-09T05:11:00.981913+0000","last_became_active":"2023-10-09T05:09:27.665030+0000","last_became_peered":"2023-10-09T05:09:27.665030+0000","last_unstale":"2023-10-09T05:11:00.981913+0000","last_undegraded":"2023-10-09T05:11:00.981913+0000","last_fullsized":"2023-10-09T05:11:00.981913+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:17.430065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":251,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982287+0000","last_change":"2023-10-09T05:10:16.183686+0000","last_active":"2023-10-09T05:11:00.982287+0000","last_peered":"2023-10-09T05:11:00.982287+0000","last_clean":"2023-10-09T05:11:00.982287+0000","last_became_active":"2023-10-09T05:10:16.183379+0000","last_became_peered":"2023-10-09T05:10:16.183379+0000","last_unstale":"2023-10-09T05:11:00.982287+0000","last_undegraded":"2023-10-09T05:11:00.982287+0000","last_fullsized":"2023-10-09T05:11:00.982287+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:29.821498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1397,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984090+0000","last_change":"2023-10-09T05:07:53.688815+0000","last_active":"2023-10-09T05:11:00.984090+0000","last_peered":"2023-10-09T05:11:00.984090+0000","last_clean":"2023-10-09T05:11:00.984090+0000","last_became_active":"2023-10-09T05:07:52.149137+0000","last_became_peered":"2023-10-09T05:07:52.149137+0000","last_unstale":"2023-10-09T05:11:00.984090+0000","last_undegraded":"2023-10-09T05:11:00.984090+0000","last_fullsized":"2023-10-09T05:11:00.984090+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:53.688730+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:53.688730+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:38.094584+0000","scrub_duration":0.0019408190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"724'44","reported_seq":364,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705943+0000","last_change":"2023-10-09T05:10:12.209042+0000","last_active":"2023-10-09T05:11:00.705943+0000","last_peered":"2023-10-09T05:11:00.705943+0000","last_clean":"2023-10-09T05:11:00.705943+0000","last_became_active":"2023-10-09T05:10:12.208540+0000","last_became_peered":"2023-10-09T05:10:12.208540+0000","last_unstale":"2023-10-09T05:11:00.705943+0000","last_undegraded":"2023-10-09T05:11:00.705943+0000","last_fullsized":"2023-10-09T05:11:00.705943+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:53.401398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033500400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":146,"num_read_kb":1134,"num_write":53,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":206,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694287+0000","last_change":"2023-10-09T05:10:16.180481+0000","last_active":"2023-10-09T05:11:00.694287+0000","last_peered":"2023-10-09T05:11:00.694287+0000","last_clean":"2023-10-09T05:11:00.694287+0000","last_became_active":"2023-10-09T05:10:16.180125+0000","last_became_peered":"2023-10-09T05:10:16.180125+0000","last_unstale":"2023-10-09T05:11:00.694287+0000","last_undegraded":"2023-10-09T05:11:00.694287+0000","last_fullsized":"2023-10-09T05:11:00.694287+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:02.328432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":439,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982089+0000","last_change":"2023-10-09T05:09:27.631516+0000","last_active":"2023-10-09T05:11:00.982089+0000","last_peered":"2023-10-09T05:11:00.982089+0000","last_clean":"2023-10-09T05:11:00.982089+0000","last_became_active":"2023-10-09T05:09:27.631192+0000","last_became_peered":"2023-10-09T05:09:27.631192+0000","last_unstale":"2023-10-09T05:11:00.982089+0000","last_undegraded":"2023-10-09T05:11:00.982089+0000","last_fullsized":"2023-10-09T05:11:00.982089+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:29.313177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1247,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701183+0000","last_change":"2023-10-09T05:10:19.194964+0000","last_active":"2023-10-09T05:11:00.701183+0000","last_peered":"2023-10-09T05:11:00.701183+0000","last_clean":"2023-10-09T05:11:00.701183+0000","last_became_active":"2023-10-09T05:10:19.194513+0000","last_became_peered":"2023-10-09T05:10:19.194513+0000","last_unstale":"2023-10-09T05:11:00.701183+0000","last_undegraded":"2023-10-09T05:11:00.701183+0000","last_fullsized":"2023-10-09T05:11:00.701183+0000","mapping_epoch":764,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":765,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:00.730094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:00.730094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:03.239194+0000","scrub_duration":0.0018014909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":428,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700569+0000","last_change":"2023-10-09T05:09:10.292493+0000","last_active":"2023-10-09T05:11:00.700569+0000","last_peered":"2023-10-09T05:11:00.700569+0000","last_clean":"2023-10-09T05:11:00.700569+0000","last_became_active":"2023-10-09T05:09:10.281968+0000","last_became_peered":"2023-10-09T05:09:10.281968+0000","last_unstale":"2023-10-09T05:11:00.700569+0000","last_undegraded":"2023-10-09T05:11:00.700569+0000","last_fullsized":"2023-10-09T05:11:00.700569+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:30.407675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"724'277","reported_seq":1158,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694272+0000","last_change":"2023-10-09T05:08:41.142733+0000","last_active":"2023-10-09T05:11:00.694272+0000","last_peered":"2023-10-09T05:11:00.694272+0000","last_clean":"2023-10-09T05:11:00.694272+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:11:00.694272+0000","last_undegraded":"2023-10-09T05:11:00.694272+0000","last_fullsized":"2023-10-09T05:11:00.694272+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":277,"log_dups_size":0,"ondisk_log_size":277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038981900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1711,"num_read_kb":1450,"num_write":224,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":294,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.698894+0000","last_change":"2023-10-09T05:09:10.277609+0000","last_active":"2023-10-09T05:11:00.698894+0000","last_peered":"2023-10-09T05:11:00.698894+0000","last_clean":"2023-10-09T05:11:00.698894+0000","last_became_active":"2023-10-09T05:09:10.277198+0000","last_became_peered":"2023-10-09T05:09:10.277198+0000","last_unstale":"2023-10-09T05:11:00.698894+0000","last_undegraded":"2023-10-09T05:11:00.698894+0000","last_fullsized":"2023-10-09T05:11:00.698894+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:30.809377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"701'162","reported_seq":1236,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699866+0000","last_change":"2023-10-09T05:09:13.540971+0000","last_active":"2023-10-09T05:11:00.699866+0000","last_peered":"2023-10-09T05:11:00.699866+0000","last_clean":"2023-10-09T05:11:00.699866+0000","last_became_active":"2023-10-09T05:09:13.540622+0000","last_became_peered":"2023-10-09T05:09:13.540622+0000","last_unstale":"2023-10-09T05:11:00.699866+0000","last_undegraded":"2023-10-09T05:11:00.699866+0000","last_fullsized":"2023-10-09T05:11:00.699866+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":3,"last_scrub":"701'162","last_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_deep_scrub":"701'162","last_deep_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.029274+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:23.603690+0000","scrub_duration":0.005003141,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"724'76","reported_seq":732,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984128+0000","last_change":"2023-10-09T05:09:10.316912+0000","last_active":"2023-10-09T05:11:00.984128+0000","last_peered":"2023-10-09T05:11:00.984128+0000","last_clean":"2023-10-09T05:11:00.984128+0000","last_became_active":"2023-10-09T05:09:10.289167+0000","last_became_peered":"2023-10-09T05:09:10.289167+0000","last_unstale":"2023-10-09T05:11:00.984128+0000","last_undegraded":"2023-10-09T05:11:00.984128+0000","last_fullsized":"2023-10-09T05:11:00.984128+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":76,"log_dups_size":0,"ondisk_log_size":76,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:02.138333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028602599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":626,"num_read_kb":2735,"num_write":52,"num_write_kb":2137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":219,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982271+0000","last_change":"2023-10-09T05:09:27.625392+0000","last_active":"2023-10-09T05:11:00.982271+0000","last_peered":"2023-10-09T05:11:00.982271+0000","last_clean":"2023-10-09T05:11:00.982271+0000","last_became_active":"2023-10-09T05:09:27.624996+0000","last_became_peered":"2023-10-09T05:09:27.624996+0000","last_unstale":"2023-10-09T05:11:00.982271+0000","last_undegraded":"2023-10-09T05:11:00.982271+0000","last_fullsized":"2023-10-09T05:11:00.982271+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:41.100557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700749+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:11:00.700749+0000","last_peered":"2023-10-09T05:11:00.700749+0000","last_clean":"2023-10-09T05:11:00.700749+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:11:00.700749+0000","last_undegraded":"2023-10-09T05:11:00.700749+0000","last_fullsized":"2023-10-09T05:11:00.700749+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":119,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982106+0000","last_change":"2023-10-09T05:09:27.630251+0000","last_active":"2023-10-09T05:11:00.982106+0000","last_peered":"2023-10-09T05:11:00.982106+0000","last_clean":"2023-10-09T05:11:00.982106+0000","last_became_active":"2023-10-09T05:09:27.629931+0000","last_became_peered":"2023-10-09T05:09:27.629931+0000","last_unstale":"2023-10-09T05:11:00.982106+0000","last_undegraded":"2023-10-09T05:11:00.982106+0000","last_fullsized":"2023-10-09T05:11:00.982106+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:38.445400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":19,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.702145+0000","last_change":"2023-10-09T05:10:16.181492+0000","last_active":"2023-10-09T05:11:00.702145+0000","last_peered":"2023-10-09T05:11:00.702145+0000","last_clean":"2023-10-09T05:11:00.702145+0000","last_became_active":"2023-10-09T05:10:16.181199+0000","last_became_peered":"2023-10-09T05:10:16.181199+0000","last_unstale":"2023-10-09T05:11:00.702145+0000","last_undegraded":"2023-10-09T05:11:00.702145+0000","last_fullsized":"2023-10-09T05:11:00.702145+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:00.699814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":910,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700486+0000","last_change":"2023-10-09T05:07:31.951921+0000","last_active":"2023-10-09T05:11:00.700486+0000","last_peered":"2023-10-09T05:11:00.700486+0000","last_clean":"2023-10-09T05:11:00.700486+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:11:00.700486+0000","last_undegraded":"2023-10-09T05:11:00.700486+0000","last_fullsized":"2023-10-09T05:11:00.700486+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":128,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700844+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:11:00.700844+0000","last_peered":"2023-10-09T05:11:00.700844+0000","last_clean":"2023-10-09T05:11:00.700844+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:11:00.700844+0000","last_undegraded":"2023-10-09T05:11:00.700844+0000","last_fullsized":"2023-10-09T05:11:00.700844+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":850,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700523+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:11:00.700523+0000","last_peered":"2023-10-09T05:11:00.700523+0000","last_clean":"2023-10-09T05:11:00.700523+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:11:00.700523+0000","last_undegraded":"2023-10-09T05:11:00.700523+0000","last_fullsized":"2023-10-09T05:11:00.700523+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700978+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:11:00.700978+0000","last_peered":"2023-10-09T05:11:00.700978+0000","last_clean":"2023-10-09T05:11:00.700978+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:11:00.700978+0000","last_undegraded":"2023-10-09T05:11:00.700978+0000","last_fullsized":"2023-10-09T05:11:00.700978+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":402,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699696+0000","last_change":"2023-10-09T05:10:16.180178+0000","last_active":"2023-10-09T05:11:00.699696+0000","last_peered":"2023-10-09T05:11:00.699696+0000","last_clean":"2023-10-09T05:11:00.699696+0000","last_became_active":"2023-10-09T05:10:16.179788+0000","last_became_peered":"2023-10-09T05:10:16.179788+0000","last_unstale":"2023-10-09T05:11:00.699696+0000","last_undegraded":"2023-10-09T05:11:00.699696+0000","last_fullsized":"2023-10-09T05:11:00.699696+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:24.719962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":409,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699525+0000","last_change":"2023-10-09T05:10:16.178525+0000","last_active":"2023-10-09T05:11:00.699525+0000","last_peered":"2023-10-09T05:11:00.699525+0000","last_clean":"2023-10-09T05:11:00.699525+0000","last_became_active":"2023-10-09T05:10:16.178186+0000","last_became_peered":"2023-10-09T05:10:16.178186+0000","last_unstale":"2023-10-09T05:11:00.699525+0000","last_undegraded":"2023-10-09T05:11:00.699525+0000","last_fullsized":"2023-10-09T05:11:00.699525+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:52.559595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":630,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982394+0000","last_change":"2023-10-09T05:10:16.187939+0000","last_active":"2023-10-09T05:11:00.982394+0000","last_peered":"2023-10-09T05:11:00.982394+0000","last_clean":"2023-10-09T05:11:00.982394+0000","last_became_active":"2023-10-09T05:10:16.184331+0000","last_became_peered":"2023-10-09T05:10:16.184331+0000","last_unstale":"2023-10-09T05:11:00.982394+0000","last_undegraded":"2023-10-09T05:11:00.982394+0000","last_fullsized":"2023-10-09T05:11:00.982394+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:22:05.752749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":246,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700489+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:11:00.700489+0000","last_peered":"2023-10-09T05:11:00.700489+0000","last_clean":"2023-10-09T05:11:00.700489+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:11:00.700489+0000","last_undegraded":"2023-10-09T05:11:00.700489+0000","last_fullsized":"2023-10-09T05:11:00.700489+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":32162,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234767,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515024,"kb_used_data":9864,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":92230128,"statfs":{"total":95995035648,"available":94443651072,"internally_reserved":0,"allocated":10100736,"data_stored":8825608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":5.3520000000000003,"15min":3.3540000000000001},"min":{"1min":0.27700000000000002,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":0.93200000000000005,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":5.6829999999999998,"15min":3.5600000000000001},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.79700000000000004,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.35599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":8.1430000000000007,"15min":4.0419999999999998},"min":{"1min":0.33400000000000002,"5min":0.29599999999999999,"15min":0.26500000000000001},"max":{"1min":0.998,"5min":322.31999999999999,"15min":322.31999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":8.125,"15min":4.0190000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":0.93400000000000005,"5min":322.14800000000002,"15min":322.14800000000002},"last":0.58599999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.89300000000000002,"15min":2.1680000000000001},"min":{"1min":0.52700000000000002,"5min":0.34000000000000002,"15min":0.29399999999999998},"max":{"1min":5.4489999999999998,"5min":6.3470000000000004,"15min":154.76400000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.86099999999999999,"15min":2.1699999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":5.4800000000000004,"5min":6.3579999999999997,"15min":154.80000000000001},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":11.513999999999999,"15min":6.2089999999999996},"min":{"1min":0.42699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.85599999999999998,"5min":349.863,"15min":349.863},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":11.202999999999999,"15min":6.0309999999999997},"min":{"1min":0.41999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.84099999999999997,"5min":349.67500000000001,"15min":349.67500000000001},"last":0.48899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.48399999999999999,"15min":0.41799999999999998},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":1.0860000000000001,"5min":1.145,"15min":1.145},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.53000000000000003,"15min":0.5},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.018,"5min":3.0609999999999999,"15min":3.0609999999999999},"last":0.61099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.3440000000000001,"15min":1.159},"min":{"1min":0.35099999999999998,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":1.1379999999999999,"5min":74.915000000000006,"15min":74.915000000000006},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.365,"15min":1.2390000000000001},"min":{"1min":0.39100000000000001,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":0.97099999999999997,"5min":74.972999999999999,"15min":74.972999999999999},"last":0.65500000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.59599999999999997,"15min":0.85599999999999998},"min":{"1min":0.434,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":1.1120000000000001,"5min":2.1829999999999998,"15min":57.168999999999997},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.59299999999999997,"15min":0.95799999999999996},"min":{"1min":0.42299999999999999,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":1.0629999999999999,"5min":2.9969999999999999,"15min":57.183999999999997},"last":0.68500000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496400,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1524232,"kb_used_data":5440,"kb_used_omap":0,"kb_used_meta":1518784,"kb_avail":92220920,"statfs":{"total":95995035648,"available":94434222080,"internally_reserved":0,"allocated":5570560,"data_stored":4303497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1555234816},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.528,"15min":3.3599999999999999},"min":{"1min":0.36399999999999999,"5min":0.22600000000000001,"15min":0.218},"max":{"1min":0.67300000000000004,"5min":122.07899999999999,"15min":321.346},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":2.7069999999999999,"15min":2.8519999999999999},"min":{"1min":0.37,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":0.70599999999999996,"5min":122.107,"15min":169.119},"last":0.51100000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.573,"15min":2.1139999999999999},"min":{"1min":0.40500000000000003,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":0.69199999999999995,"5min":43.604999999999997,"15min":90.278999999999996},"last":0.433},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.5700000000000001,"15min":2.1629999999999998},"min":{"1min":0.32800000000000001,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":0.79100000000000004,"5min":44.697000000000003,"15min":90.414000000000001},"last":0.32800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.1359999999999999,"15min":1.28},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25600000000000001},"max":{"1min":0.755,"5min":8.4800000000000004,"15min":23.943000000000001},"last":0.627},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.071,"15min":1.2549999999999999},"min":{"1min":0.45100000000000001,"5min":0.32300000000000001,"15min":0.30599999999999999},"max":{"1min":0.72799999999999998,"5min":8.0039999999999996,"15min":25.978999999999999},"last":0.60399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":2.4089999999999998,"15min":2.9729999999999999},"min":{"1min":0.435,"5min":0.38900000000000001,"15min":0.26600000000000001},"max":{"1min":0.79600000000000004,"5min":129.499,"15min":147.43600000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":2.6400000000000001,"15min":3.0619999999999998},"min":{"1min":0.439,"5min":0.33800000000000002,"15min":0.28899999999999998},"max":{"1min":0.78000000000000003,"5min":130.994,"15min":147.41999999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.57399999999999995,"15min":0.75800000000000001},"min":{"1min":0.311,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.87,"5min":5.3769999999999998,"15min":18.186},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.78300000000000003,"15min":0.96599999999999997},"min":{"1min":0.35099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.85799999999999998,"5min":8.7710000000000008,"15min":17.597000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.68300000000000005,"15min":0.752},"min":{"1min":0.35799999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":7.9539999999999997,"15min":25.934000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.72299999999999998,"15min":0.69299999999999995},"min":{"1min":0.46300000000000002,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":0.79300000000000004,"5min":5.2990000000000004,"15min":16.789000000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.752,"15min":0.66300000000000003},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":0.83899999999999997,"5min":7.5039999999999996,"15min":7.5039999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.77300000000000002,"15min":0.76500000000000001},"min":{"1min":0.48399999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.76300000000000001,"5min":8.3399999999999999,"15min":20.041},"last":0.64700000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141467,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1415636,"kb_used_data":11468,"kb_used_omap":4,"kb_used_meta":1404155,"kb_avail":92329516,"statfs":{"total":95995035648,"available":94545424384,"internally_reserved":0,"allocated":11743232,"data_stored":8695236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1437855630},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":3.0049999999999999,"15min":2.7749999999999999},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.20699999999999999},"max":{"1min":0.63900000000000001,"5min":61.344999999999999,"15min":100.247},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":8.1449999999999996,"15min":4.585},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.74399999999999999,"5min":554.62,"15min":554.62},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":1.1160000000000001,"15min":1.486},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":0.64200000000000002,"5min":10.393000000000001,"15min":63.143999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":6.8959999999999999,"15min":3.4910000000000001},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":0.61499999999999999,"5min":554.31200000000001,"15min":554.31200000000001},"last":0.38}]},{"osd":3,"last update":"Mon Oct 9 05:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":2.8100000000000001,"15min":3.5819999999999999},"min":{"1min":0.34200000000000003,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":0.628,"5min":102.729,"15min":235.43600000000001},"last":0.318},{"interface":"front","average":{"1min":0.45800000000000002,"5min":7.4210000000000003,"15min":4.851},"min":{"1min":0.32700000000000001,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":0.66000000000000003,"5min":554.84400000000005,"15min":554.84400000000005},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 05:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":6.3200000000000003,"15min":3.0289999999999999},"min":{"1min":0.42899999999999999,"5min":0.35699999999999998,"15min":0.34699999999999998},"max":{"1min":0.748,"5min":554.44000000000005,"15min":554.44000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.2929999999999999,"15min":1.1930000000000001},"min":{"1min":0.38600000000000001,"5min":0.32600000000000001,"15min":0.31900000000000001},"max":{"1min":1.349,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":7.2839999999999998,"15min":3.1930000000000001},"min":{"1min":0.42799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.98499999999999999,"5min":554.54700000000003,"15min":554.54700000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.67500000000000004,"5min":2.0379999999999998,"15min":1.6080000000000001},"min":{"1min":0.435,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":1.448,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":7.3220000000000001,"15min":3.2719999999999998},"min":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":0.92600000000000005,"5min":554.36500000000001,"15min":554.36500000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.74099999999999999,"5min":2.1720000000000002,"15min":1.673},"min":{"1min":0.51600000000000001,"5min":0.41999999999999998,"15min":0.36199999999999999},"max":{"1min":1.486,"5min":78.278000000000006,"15min":78.278000000000006},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1.5009999999999999,"15min":1.498},"min":{"1min":0.44800000000000001,"5min":0.439,"15min":0.373},"max":{"1min":1.4059999999999999,"5min":18.899999999999999,"15min":19.664999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.153,"15min":1.252},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.39300000000000002},"max":{"1min":0.88500000000000001,"5min":10.345000000000001,"15min":11.903},"last":0.59999999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024550,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1238292,"kb_used_data":10700,"kb_used_omap":5,"kb_used_meta":1227578,"kb_avail":92506860,"statfs":{"total":95995035648,"available":94727024640,"internally_reserved":0,"allocated":10956800,"data_stored":8616541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1257040506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:10:22 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":2.2549999999999999,"15min":2.649},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":0.52800000000000002,"5min":19.042000000000002,"15min":155.411},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.374,"5min":2.464,"15min":2.6709999999999998},"min":{"1min":0.254,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.56000000000000005,"5min":17.594000000000001,"15min":154.75},"last":0.59299999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":1.4410000000000001,"15min":1.6659999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.21099999999999999},"max":{"1min":0.63700000000000001,"5min":20.010999999999999,"15min":26.556999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.40100000000000002,"5min":1.7829999999999999,"15min":1.899},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":0.621,"5min":15.444000000000001,"15min":38.192},"last":0.52300000000000002}]},{"osd":3,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":3.6680000000000001,"15min":5.3330000000000002},"min":{"1min":0.315,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":0.65700000000000003,"5min":267.63799999999998,"15min":290.89100000000002},"last":0.627},{"interface":"front","average":{"1min":0.41899999999999998,"5min":3.7109999999999999,"15min":5.4050000000000002},"min":{"1min":0.318,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":0.53100000000000003,"5min":267.66199999999998,"15min":290.73899999999998},"last":0.68100000000000005}]},{"osd":4,"last update":"Mon Oct 9 05:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.79,"15min":1.8660000000000001},"min":{"1min":0.51100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.92500000000000004,"5min":57.360999999999997,"15min":57.360999999999997},"last":0.755},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.71,"15min":1.6359999999999999},"min":{"1min":0.42699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.96799999999999997,"5min":57.338000000000001,"15min":57.338000000000001},"last":0.85699999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.012,"15min":1.9139999999999999},"min":{"1min":0.48899999999999999,"5min":0.35799999999999998,"15min":0.30099999999999999},"max":{"1min":0.83799999999999997,"5min":106.922,"15min":106.922},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.5299999999999998,"15min":1.8080000000000001},"min":{"1min":0.38600000000000001,"5min":0.379,"15min":0.34899999999999998},"max":{"1min":0.86499999999999999,"5min":106.771,"15min":106.771},"last":0.90800000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":3.7530000000000001,"15min":2.3530000000000002},"min":{"1min":0.40500000000000003,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":0.85199999999999998,"5min":112.208,"15min":112.208},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.62,"5min":3.2850000000000001,"15min":2.3740000000000001},"min":{"1min":0.438,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":0.79200000000000004,"5min":110.46899999999999,"15min":110.46899999999999},"last":0.82199999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3360000000000001,"15min":1.494},"min":{"1min":0.47799999999999998,"5min":0.40899999999999997,"15min":0.39400000000000002},"max":{"1min":0.80600000000000005,"5min":13.787000000000001,"15min":21.658000000000001},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.29,"15min":1.444},"min":{"1min":0.54900000000000004,"5min":0.39900000000000002,"15min":0.38800000000000001},"max":{"1min":0.88900000000000001,"5min":9.9770000000000003,"15min":21.68},"last":0.92600000000000005}]}]},{"osd":2,"up_from":221,"seq":949187772829,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":806424,"kb_used_data":7120,"kb_used_omap":7,"kb_used_meta":799288,"kb_avail":92938728,"statfs":{"total":95995035648,"available":95169257472,"internally_reserved":0,"allocated":7290880,"data_stored":4709400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":818470993},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":2.6499999999999999,"15min":2.6749999999999998},"min":{"1min":0.22900000000000001,"5min":0.20999999999999999,"15min":0.19600000000000001},"max":{"1min":0.61899999999999999,"5min":120.062,"15min":126.77500000000001},"last":0.496},{"interface":"front","average":{"1min":0.35699999999999998,"5min":2.1629999999999998,"15min":2.2989999999999999},"min":{"1min":0.24199999999999999,"5min":0.20899999999999999,"15min":0.20699999999999999},"max":{"1min":0.504,"5min":118.41,"15min":120.937},"last":0.52200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":2.673,"15min":2.8919999999999999},"min":{"1min":0.27700000000000002,"5min":0.26100000000000001,"15min":0.24199999999999999},"max":{"1min":0.64000000000000001,"5min":100.01900000000001,"15min":135.38499999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":3.0840000000000001,"15min":3.3879999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":0.63200000000000001,"5min":102.179,"15min":135.23400000000001},"last":0.57899999999999996}]},{"osd":3,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":3.895,"15min":3.8109999999999999},"min":{"1min":0.317,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":0.58299999999999996,"5min":109.67,"15min":109.67},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.41199999999999998,"5min":4.141,"15min":3.8380000000000001},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.55200000000000005,"5min":109.746,"15min":109.746},"last":0.621}]},{"osd":4,"last update":"Mon Oct 9 05:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.98799999999999999,"15min":2.0699999999999998},"min":{"1min":0.40600000000000003,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":0.85599999999999998,"5min":5.3259999999999996,"15min":15.202},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.86599999999999999,"15min":1.417},"min":{"1min":0.44,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.81999999999999995,"5min":5.8280000000000003,"15min":10.038},"last":0.90900000000000003}]},{"osd":5,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.4470000000000001,"15min":1.2689999999999999},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.32600000000000001},"max":{"1min":0.877,"5min":55.994,"15min":55.994},"last":0.875},{"interface":"front","average":{"1min":0.629,"5min":1.4199999999999999,"15min":1.615},"min":{"1min":0.47099999999999997,"5min":0.38700000000000001,"15min":0.315},"max":{"1min":0.78200000000000003,"5min":55.950000000000003,"15min":55.950000000000003},"last":1.0129999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.5940000000000001,"15min":1.7210000000000001},"min":{"1min":0.45600000000000002,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.80500000000000005,"5min":58.192999999999998,"15min":72.662000000000006},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.5049999999999999,"15min":1.536},"min":{"1min":0.48599999999999999,"5min":0.375,"15min":0.35099999999999998},"max":{"1min":0.86199999999999999,"5min":59.064999999999998,"15min":70.938000000000002},"last":1.034}]},{"osd":7,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.0529999999999999,"15min":1.3999999999999999},"min":{"1min":0.42999999999999999,"5min":0.41799999999999998,"15min":0.372},"max":{"1min":0.85799999999999998,"5min":5.2709999999999999,"15min":15.241},"last":1.052},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.96499999999999997,"15min":1.2090000000000001},"min":{"1min":0.51200000000000001,"5min":0.44800000000000001,"15min":0.373},"max":{"1min":0.84299999999999997,"5min":5.952,"15min":10.122999999999999},"last":1.079}]}]},{"osd":3,"up_from":142,"seq":609885356523,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1024924,"kb_used_data":7380,"kb_used_omap":2,"kb_used_meta":1017533,"kb_avail":92720228,"statfs":{"total":95995035648,"available":94945513472,"internally_reserved":0,"allocated":7557120,"data_stored":5032835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1041953845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":2.8700000000000001,"15min":2.0209999999999999},"min":{"1min":0.253,"5min":0.191,"15min":0.191},"max":{"1min":0.91500000000000004,"5min":62.863,"15min":108.631},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.45100000000000001,"5min":3.1139999999999999,"15min":2.2290000000000001},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.247},"max":{"1min":0.97499999999999998,"5min":62.948,"15min":110.012},"last":0.42299999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":1.5469999999999999,"15min":1.4319999999999999},"min":{"1min":0.29799999999999999,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":0.94599999999999995,"5min":13.374000000000001,"15min":15.218},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.3220000000000001,"15min":1.3859999999999999},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.016,"5min":14.164,"15min":26.692},"last":0.505}]},{"osd":2,"last update":"Mon Oct 9 05:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":1.419,"15min":1.869},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.249},"max":{"1min":0.58899999999999997,"5min":54.999000000000002,"15min":212.065},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":1.3799999999999999,"15min":1.9770000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":0.51700000000000002,"5min":54.963000000000001,"15min":211.98400000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.016,"15min":1.524},"min":{"1min":0.45600000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0289999999999999,"5min":4.75,"15min":11.193},"last":0.63},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.9710000000000001,"15min":1.3500000000000001},"min":{"1min":0.51100000000000001,"5min":0.34899999999999998,"15min":0.309},"max":{"1min":0.997,"5min":62.683999999999997,"15min":62.683999999999997},"last":0.77700000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.036,"15min":1.538},"min":{"1min":0.432,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":2.1850000000000001,"5min":75.478999999999999,"15min":75.478999999999999},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.024,"15min":2.0750000000000002},"min":{"1min":0.498,"5min":0.379,"15min":0.35099999999999998},"max":{"1min":1.1180000000000001,"5min":75.459000000000003,"15min":75.459000000000003},"last":0.80000000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":3.048,"15min":1.8220000000000001},"min":{"1min":0.46100000000000002,"5min":0.35999999999999999,"15min":0.34799999999999998},"max":{"1min":1.1779999999999999,"5min":79.093999999999994,"15min":79.093999999999994},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.68000000000000005,"5min":3.0369999999999999,"15min":1.919},"min":{"1min":0.47099999999999997,"5min":0.39500000000000002,"15min":0.34799999999999998},"max":{"1min":1.196,"5min":78.555999999999997,"15min":78.555999999999997},"last":0.81100000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.99,"15min":1.595},"min":{"1min":0.51000000000000001,"5min":0.41199999999999998,"15min":0.39400000000000002},"max":{"1min":2.1720000000000002,"5min":63.017000000000003,"15min":63.017000000000003},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.76000000000000001,"5min":1.9890000000000001,"15min":1.3620000000000001},"min":{"1min":0.498,"5min":0.38900000000000001,"15min":0.32100000000000001},"max":{"1min":2.1549999999999998,"5min":62.808,"15min":62.808},"last":0.872}]}]},{"osd":4,"up_from":626,"seq":2688649527391,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410636,"kb_used_data":5444,"kb_used_omap":22,"kb_used_meta":405161,"kb_avail":93334516,"statfs":{"total":95995035648,"available":95574544384,"internally_reserved":0,"allocated":5574656,"data_stored":4303554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":414885709},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":3.226,"15min":2.6160000000000001},"min":{"1min":0.35899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.65100000000000002,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":3.4049999999999998,"15min":3.2069999999999999},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.89700000000000002,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.61099999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":4.1950000000000003,"15min":2.8380000000000001},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":0.71199999999999997,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":4.2169999999999996,"15min":2.6339999999999999},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.70399999999999996,"5min":182.41,"15min":182.41},"last":0.59899999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.2290000000000001,"15min":1.512},"min":{"1min":0.34200000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.93300000000000005,"5min":7.8639999999999999,"15min":11.051},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.2150000000000001,"15min":1.546},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.94999999999999996,"5min":7.8810000000000002,"15min":11.036},"last":0.56499999999999995}]},{"osd":3,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.4180000000000001,"15min":3.1920000000000002},"min":{"1min":0.38900000000000001,"5min":0.34599999999999997,"15min":0.318},"max":{"1min":0.77600000000000002,"5min":109.53400000000001,"15min":270.767},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":3.4399999999999999,"15min":3.5640000000000001},"min":{"1min":0.33200000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.91900000000000004,"5min":109.84,"15min":270.274},"last":0.39200000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":2.161,"15min":0.97899999999999998},"min":{"1min":0.28699999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.873,"5min":134.82900000000001,"15min":134.82900000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":2.1819999999999999,"15min":1.0269999999999999},"min":{"1min":0.32800000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.91800000000000004,"5min":134.60400000000001,"15min":134.60400000000001},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":2.266,"15min":1.1839999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.996,"5min":139.77799999999999,"15min":139.77799999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":2.335,"15min":1.085},"min":{"1min":0.308,"5min":0.30199999999999999,"15min":0.29999999999999999},"max":{"1min":0.85299999999999998,"5min":140.78200000000001,"15min":140.78200000000001},"last":0.627}]},{"osd":7,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55200000000000005,"15min":0.53300000000000003},"min":{"1min":0.34399999999999997,"5min":0.255,"15min":0.255},"max":{"1min":0.84299999999999997,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.60399999999999998,"15min":0.52100000000000002},"min":{"1min":0.32800000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.89500000000000002,"5min":3.109,"15min":3.109},"last":0.45600000000000002}]}]},{"osd":5,"up_from":38,"seq":163208758044,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1499332,"kb_used_data":6012,"kb_used_omap":0,"kb_used_meta":1493312,"kb_avail":92245820,"statfs":{"total":95995035648,"available":94459719680,"internally_reserved":0,"allocated":6156288,"data_stored":4889685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1529151488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":4.2160000000000002,"15min":3.1230000000000002},"min":{"1min":0.33100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.80400000000000005,"5min":212.72800000000001,"15min":212.72800000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":4.2880000000000003,"15min":3.298},"min":{"1min":0.33500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.75900000000000001,"5min":212.53999999999999,"15min":212.53999999999999},"last":0.63800000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":3.1070000000000002,"15min":2.9470000000000001},"min":{"1min":0.32200000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.81499999999999995,"5min":113.977,"15min":262.26900000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.54000000000000004,"5min":3.1070000000000002,"15min":2.9740000000000002},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78600000000000003,"5min":112.947,"15min":262.291},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.0209999999999999,"15min":1.641},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.27000000000000002},"max":{"1min":0.82099999999999995,"5min":9.6440000000000001,"15min":106.238},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.0920000000000001,"15min":1.7869999999999999},"min":{"1min":0.40100000000000002,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":0.76000000000000001,"5min":8.9239999999999995,"15min":104.31699999999999},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":3.1819999999999999,"15min":3.6440000000000001},"min":{"1min":0.36099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.78900000000000003,"5min":121.40600000000001,"15min":125.824},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":3.3340000000000001,"15min":3.7120000000000002},"min":{"1min":0.442,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.75,"5min":121.53700000000001,"15min":125.761},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.48299999999999998,"15min":0.45000000000000001},"min":{"1min":0.32600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.755,"5min":1.99,"15min":1.99},"last":0.746},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.51500000000000001,"15min":0.47599999999999998},"min":{"1min":0.41199999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.73099999999999998,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.60599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54200000000000004,"15min":0.58599999999999997},"min":{"1min":0.38100000000000001,"5min":0.28499999999999998,"15min":0.23300000000000001},"max":{"1min":0.75800000000000001,"5min":4.3360000000000003,"15min":11.093},"last":0.625},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.53800000000000003,"15min":0.55000000000000004},"min":{"1min":0.38800000000000001,"5min":0.29899999999999999,"15min":0.25600000000000001},"max":{"1min":0.77200000000000002,"5min":3.0590000000000002,"15min":5.9329999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.53800000000000003,"15min":0.60799999999999998},"min":{"1min":0.439,"5min":0.30399999999999999,"15min":0.26700000000000002},"max":{"1min":0.71599999999999997,"5min":2.1019999999999999,"15min":8.3520000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58999999999999997,"15min":0.69499999999999995},"min":{"1min":0.42299999999999999,"5min":0.31,"15min":0.31},"max":{"1min":0.76900000000000002,"5min":2.0489999999999999,"15min":11.791},"last":0.71799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:11:15.188 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-09T05:11:15.561 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-09T05:11:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:15 smithi177 ceph-mon[195797]: pgmap v2408: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:16.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:15 smithi177 ceph-mon[195797]: from='client.38787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:16.037 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:15 smithi029 ceph-mon[216903]: pgmap v2408: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:16.037 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:15 smithi029 ceph-mon[216903]: from='client.38787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:16.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:15 smithi029 ceph-mon[223858]: pgmap v2408: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:16.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:15 smithi029 ceph-mon[223858]: from='client.38787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:17.499 INFO:teuthology.orchestra.run.smithi029.stdout: 2023-10-09T05:11:17.501 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2023-10-09T05:11:17.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:17 smithi029 ceph-mon[216903]: pgmap v2409: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:17.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:17 smithi029 ceph-mon[223858]: pgmap v2409: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:18.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:17 smithi177 ceph-mon[195797]: pgmap v2409: 83 pgs: 83 active+clean; 4.6 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail 2023-10-09T05:11:18.404 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":2409,"stamp":"2023-10-09T05:11:15.964392+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":5144514,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2803,"ondisk_log_size":2803,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9434500,"kb_used_data":63428,"kb_used_omap":42,"kb_used_meta":9370965,"kb_avail":740526716,"statfs":{"total":767960285184,"available":758299357184,"internally_reserved":0,"allocated":64950272,"data_stored":49376356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43557,"internal_metadata":9595868635},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003297"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":282,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701282+0000","last_change":"2023-10-09T05:09:05.237648+0000","last_active":"2023-10-09T05:11:00.701282+0000","last_peered":"2023-10-09T05:11:00.701282+0000","last_clean":"2023-10-09T05:11:00.701282+0000","last_became_active":"2023-10-09T05:09:05.237250+0000","last_became_peered":"2023-10-09T05:09:05.237250+0000","last_unstale":"2023-10-09T05:11:00.701282+0000","last_undegraded":"2023-10-09T05:11:00.701282+0000","last_fullsized":"2023-10-09T05:11:00.701282+0000","mapping_epoch":734,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":735,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:08:44.769366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":419,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699626+0000","last_change":"2023-10-09T05:03:24.411270+0000","last_active":"2023-10-09T05:11:00.699626+0000","last_peered":"2023-10-09T05:11:00.699626+0000","last_clean":"2023-10-09T05:11:00.699626+0000","last_became_active":"2023-10-09T05:03:24.410916+0000","last_became_peered":"2023-10-09T05:03:24.410916+0000","last_unstale":"2023-10-09T05:11:00.699626+0000","last_undegraded":"2023-10-09T05:11:00.699626+0000","last_fullsized":"2023-10-09T05:11:00.699626+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:31:23.859775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":196,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694747+0000","last_change":"2023-10-09T05:04:20.873552+0000","last_active":"2023-10-09T05:11:00.694747+0000","last_peered":"2023-10-09T05:11:00.694747+0000","last_clean":"2023-10-09T05:11:00.694747+0000","last_became_active":"2023-10-09T05:04:20.873252+0000","last_became_peered":"2023-10-09T05:04:20.873252+0000","last_unstale":"2023-10-09T05:11:00.694747+0000","last_undegraded":"2023-10-09T05:11:00.694747+0000","last_fullsized":"2023-10-09T05:11:00.694747+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:28:17.636474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982207+0000","last_change":"2023-10-09T05:09:27.665724+0000","last_active":"2023-10-09T05:11:00.982207+0000","last_peered":"2023-10-09T05:11:00.982207+0000","last_clean":"2023-10-09T05:11:00.982207+0000","last_became_active":"2023-10-09T05:09:27.665402+0000","last_became_peered":"2023-10-09T05:09:27.665402+0000","last_unstale":"2023-10-09T05:11:00.982207+0000","last_undegraded":"2023-10-09T05:11:00.982207+0000","last_fullsized":"2023-10-09T05:11:00.982207+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:56.519789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":401,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699101+0000","last_change":"2023-10-09T05:02:19.765632+0000","last_active":"2023-10-09T05:11:00.699101+0000","last_peered":"2023-10-09T05:11:00.699101+0000","last_clean":"2023-10-09T05:11:00.699101+0000","last_became_active":"2023-10-09T05:02:19.765108+0000","last_became_peered":"2023-10-09T05:02:19.765108+0000","last_unstale":"2023-10-09T05:11:00.699101+0000","last_undegraded":"2023-10-09T05:11:00.699101+0000","last_fullsized":"2023-10-09T05:11:00.699101+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:16:41.370978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":296,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984072+0000","last_change":"2023-10-09T05:02:32.058533+0000","last_active":"2023-10-09T05:11:00.984072+0000","last_peered":"2023-10-09T05:11:00.984072+0000","last_clean":"2023-10-09T05:11:00.984072+0000","last_became_active":"2023-10-09T05:02:32.058275+0000","last_became_peered":"2023-10-09T05:02:32.058275+0000","last_unstale":"2023-10-09T05:11:00.984072+0000","last_undegraded":"2023-10-09T05:11:00.984072+0000","last_fullsized":"2023-10-09T05:11:00.984072+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:42:11.210037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":630,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982309+0000","last_change":"2023-10-09T05:09:27.665287+0000","last_active":"2023-10-09T05:11:00.982309+0000","last_peered":"2023-10-09T05:11:00.982309+0000","last_clean":"2023-10-09T05:11:00.982309+0000","last_became_active":"2023-10-09T05:09:27.664944+0000","last_became_peered":"2023-10-09T05:09:27.664944+0000","last_unstale":"2023-10-09T05:11:00.982309+0000","last_undegraded":"2023-10-09T05:11:00.982309+0000","last_fullsized":"2023-10-09T05:11:00.982309+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:22.928578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.698999+0000","last_change":"2023-10-09T05:02:19.787887+0000","last_active":"2023-10-09T05:11:00.698999+0000","last_peered":"2023-10-09T05:11:00.698999+0000","last_clean":"2023-10-09T05:11:00.698999+0000","last_became_active":"2023-10-09T05:02:19.787425+0000","last_became_peered":"2023-10-09T05:02:19.787425+0000","last_unstale":"2023-10-09T05:11:00.698999+0000","last_undegraded":"2023-10-09T05:11:00.698999+0000","last_fullsized":"2023-10-09T05:11:00.698999+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:44:15.357258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":177,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700417+0000","last_change":"2023-10-09T05:09:10.293342+0000","last_active":"2023-10-09T05:11:00.700417+0000","last_peered":"2023-10-09T05:11:00.700417+0000","last_clean":"2023-10-09T05:11:00.700417+0000","last_became_active":"2023-10-09T05:09:10.292951+0000","last_became_peered":"2023-10-09T05:09:10.292951+0000","last_unstale":"2023-10-09T05:11:00.700417+0000","last_undegraded":"2023-10-09T05:11:00.700417+0000","last_fullsized":"2023-10-09T05:11:00.700417+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:42:40.274186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":401,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699438+0000","last_change":"2023-10-09T05:09:10.280081+0000","last_active":"2023-10-09T05:11:00.699438+0000","last_peered":"2023-10-09T05:11:00.699438+0000","last_clean":"2023-10-09T05:11:00.699438+0000","last_became_active":"2023-10-09T05:09:10.279774+0000","last_became_peered":"2023-10-09T05:09:10.279774+0000","last_unstale":"2023-10-09T05:11:00.699438+0000","last_undegraded":"2023-10-09T05:11:00.699438+0000","last_fullsized":"2023-10-09T05:11:00.699438+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:34:57.567705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":461,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.706057+0000","last_change":"2023-10-09T05:10:12.210156+0000","last_active":"2023-10-09T05:11:00.706057+0000","last_peered":"2023-10-09T05:11:00.706057+0000","last_clean":"2023-10-09T05:11:00.706057+0000","last_became_active":"2023-10-09T05:10:12.209818+0000","last_became_peered":"2023-10-09T05:10:12.209818+0000","last_unstale":"2023-10-09T05:11:00.706057+0000","last_undegraded":"2023-10-09T05:11:00.706057+0000","last_fullsized":"2023-10-09T05:11:00.706057+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:07:46.991464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1080,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":205,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694661+0000","last_change":"2023-10-09T05:09:05.239952+0000","last_active":"2023-10-09T05:11:00.694661+0000","last_peered":"2023-10-09T05:11:00.694661+0000","last_clean":"2023-10-09T05:11:00.694661+0000","last_became_active":"2023-10-09T05:09:05.239675+0000","last_became_peered":"2023-10-09T05:09:05.239675+0000","last_unstale":"2023-10-09T05:11:00.694661+0000","last_undegraded":"2023-10-09T05:11:00.694661+0000","last_fullsized":"2023-10-09T05:11:00.694661+0000","mapping_epoch":734,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":735,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:37:28.664644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":285,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701231+0000","last_change":"2023-10-09T05:09:10.313568+0000","last_active":"2023-10-09T05:11:00.701231+0000","last_peered":"2023-10-09T05:11:00.701231+0000","last_clean":"2023-10-09T05:11:00.701231+0000","last_became_active":"2023-10-09T05:09:10.288379+0000","last_became_peered":"2023-10-09T05:09:10.288379+0000","last_unstale":"2023-10-09T05:11:00.701231+0000","last_undegraded":"2023-10-09T05:11:00.701231+0000","last_fullsized":"2023-10-09T05:11:00.701231+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:55:54.959372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":240,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701088+0000","last_change":"2023-10-09T05:03:24.752619+0000","last_active":"2023-10-09T05:11:00.701088+0000","last_peered":"2023-10-09T05:11:00.701088+0000","last_clean":"2023-10-09T05:11:00.701088+0000","last_became_active":"2023-10-09T05:03:24.752297+0000","last_became_peered":"2023-10-09T05:03:24.752297+0000","last_unstale":"2023-10-09T05:11:00.701088+0000","last_undegraded":"2023-10-09T05:11:00.701088+0000","last_fullsized":"2023-10-09T05:11:00.701088+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:10:13.005828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":272,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705800+0000","last_change":"2023-10-09T05:10:12.208114+0000","last_active":"2023-10-09T05:11:00.705800+0000","last_peered":"2023-10-09T05:11:00.705800+0000","last_clean":"2023-10-09T05:11:00.705800+0000","last_became_active":"2023-10-09T05:10:12.207776+0000","last_became_peered":"2023-10-09T05:10:12.207776+0000","last_unstale":"2023-10-09T05:11:00.705800+0000","last_undegraded":"2023-10-09T05:11:00.705800+0000","last_fullsized":"2023-10-09T05:11:00.705800+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:16:06.947241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":383,"reported_epoch":770,"state":"active+clean","last_fresh":"2023-10-09T05:11:01.701158+0000","last_change":"2023-10-09T05:09:10.274048+0000","last_active":"2023-10-09T05:11:01.701158+0000","last_peered":"2023-10-09T05:11:01.701158+0000","last_clean":"2023-10-09T05:11:01.701158+0000","last_became_active":"2023-10-09T05:09:10.273740+0000","last_became_peered":"2023-10-09T05:09:10.273740+0000","last_unstale":"2023-10-09T05:11:01.701158+0000","last_undegraded":"2023-10-09T05:11:01.701158+0000","last_fullsized":"2023-10-09T05:11:01.701158+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:34.021710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":289,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.981997+0000","last_change":"2023-10-09T05:09:27.665248+0000","last_active":"2023-10-09T05:11:00.981997+0000","last_peered":"2023-10-09T05:11:00.981997+0000","last_clean":"2023-10-09T05:11:00.981997+0000","last_became_active":"2023-10-09T05:09:27.664873+0000","last_became_peered":"2023-10-09T05:09:27.664873+0000","last_unstale":"2023-10-09T05:11:00.981997+0000","last_undegraded":"2023-10-09T05:11:00.981997+0000","last_fullsized":"2023-10-09T05:11:00.981997+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:37:17.717559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700324+0000","last_change":"2023-10-09T05:09:10.295199+0000","last_active":"2023-10-09T05:11:00.700324+0000","last_peered":"2023-10-09T05:11:00.700324+0000","last_clean":"2023-10-09T05:11:00.700324+0000","last_became_active":"2023-10-09T05:09:10.292927+0000","last_became_peered":"2023-10-09T05:09:10.292927+0000","last_unstale":"2023-10-09T05:11:00.700324+0000","last_undegraded":"2023-10-09T05:11:00.700324+0000","last_fullsized":"2023-10-09T05:11:00.700324+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:10:36.159060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":213,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705497+0000","last_change":"2023-10-09T05:10:12.211033+0000","last_active":"2023-10-09T05:11:00.705497+0000","last_peered":"2023-10-09T05:11:00.705497+0000","last_clean":"2023-10-09T05:11:00.705497+0000","last_became_active":"2023-10-09T05:10:12.210688+0000","last_became_peered":"2023-10-09T05:10:12.210688+0000","last_unstale":"2023-10-09T05:11:00.705497+0000","last_undegraded":"2023-10-09T05:11:00.705497+0000","last_fullsized":"2023-10-09T05:11:00.705497+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:29:45.399387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":432,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982184+0000","last_change":"2023-10-09T05:09:27.665200+0000","last_active":"2023-10-09T05:11:00.982184+0000","last_peered":"2023-10-09T05:11:00.982184+0000","last_clean":"2023-10-09T05:11:00.982184+0000","last_became_active":"2023-10-09T05:09:27.664838+0000","last_became_peered":"2023-10-09T05:09:27.664838+0000","last_unstale":"2023-10-09T05:11:00.982184+0000","last_undegraded":"2023-10-09T05:11:00.982184+0000","last_fullsized":"2023-10-09T05:11:00.982184+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:21:06.260353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":404,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983755+0000","last_change":"2023-10-09T05:03:49.845262+0000","last_active":"2023-10-09T05:11:00.983755+0000","last_peered":"2023-10-09T05:11:00.983755+0000","last_clean":"2023-10-09T05:11:00.983755+0000","last_became_active":"2023-10-09T05:03:36.651750+0000","last_became_peered":"2023-10-09T05:03:36.651750+0000","last_unstale":"2023-10-09T05:11:00.983755+0000","last_undegraded":"2023-10-09T05:11:00.983755+0000","last_fullsized":"2023-10-09T05:11:00.983755+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":639,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:54:08.216801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":303,"reported_epoch":770,"state":"active+clean","last_fresh":"2023-10-09T05:11:01.842506+0000","last_change":"2023-10-09T05:11:01.842506+0000","last_active":"2023-10-09T05:11:01.842506+0000","last_peered":"2023-10-09T05:11:01.842506+0000","last_clean":"2023-10-09T05:11:01.842506+0000","last_became_active":"2023-10-09T05:11:01.842277+0000","last_became_peered":"2023-10-09T05:11:01.842277+0000","last_unstale":"2023-10-09T05:11:01.842506+0000","last_undegraded":"2023-10-09T05:11:01.842506+0000","last_fullsized":"2023-10-09T05:11:01.842506+0000","mapping_epoch":769,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":770,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:25:54.146465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":307,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.706104+0000","last_change":"2023-10-09T05:10:12.209123+0000","last_active":"2023-10-09T05:11:00.706104+0000","last_peered":"2023-10-09T05:11:00.706104+0000","last_clean":"2023-10-09T05:11:00.706104+0000","last_became_active":"2023-10-09T05:10:12.208711+0000","last_became_peered":"2023-10-09T05:10:12.208711+0000","last_unstale":"2023-10-09T05:11:00.706104+0000","last_undegraded":"2023-10-09T05:11:00.706104+0000","last_fullsized":"2023-10-09T05:11:00.706104+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:03:13.123169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":177,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700106+0000","last_change":"2023-10-09T05:09:10.293964+0000","last_active":"2023-10-09T05:11:00.700106+0000","last_peered":"2023-10-09T05:11:00.700106+0000","last_clean":"2023-10-09T05:11:00.700106+0000","last_became_active":"2023-10-09T05:09:10.293585+0000","last_became_peered":"2023-10-09T05:09:10.293585+0000","last_unstale":"2023-10-09T05:11:00.700106+0000","last_undegraded":"2023-10-09T05:11:00.700106+0000","last_fullsized":"2023-10-09T05:11:00.700106+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:18:53.738992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":141,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.706013+0000","last_change":"2023-10-09T05:10:12.207492+0000","last_active":"2023-10-09T05:11:00.706013+0000","last_peered":"2023-10-09T05:11:00.706013+0000","last_clean":"2023-10-09T05:11:00.706013+0000","last_became_active":"2023-10-09T05:10:12.207143+0000","last_became_peered":"2023-10-09T05:10:12.207143+0000","last_unstale":"2023-10-09T05:11:00.706013+0000","last_undegraded":"2023-10-09T05:11:00.706013+0000","last_fullsized":"2023-10-09T05:11:00.706013+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:55:00.546236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":256,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700811+0000","last_change":"2023-10-09T05:03:24.338875+0000","last_active":"2023-10-09T05:11:00.700811+0000","last_peered":"2023-10-09T05:11:00.700811+0000","last_clean":"2023-10-09T05:11:00.700811+0000","last_became_active":"2023-10-09T05:03:24.338190+0000","last_became_peered":"2023-10-09T05:03:24.338190+0000","last_unstale":"2023-10-09T05:11:00.700811+0000","last_undegraded":"2023-10-09T05:11:00.700811+0000","last_fullsized":"2023-10-09T05:11:00.700811+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:02:30.987518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":159,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705892+0000","last_change":"2023-10-09T05:10:12.211670+0000","last_active":"2023-10-09T05:11:00.705892+0000","last_peered":"2023-10-09T05:11:00.705892+0000","last_clean":"2023-10-09T05:11:00.705892+0000","last_became_active":"2023-10-09T05:10:12.211350+0000","last_became_peered":"2023-10-09T05:10:12.211350+0000","last_unstale":"2023-10-09T05:11:00.705892+0000","last_undegraded":"2023-10-09T05:11:00.705892+0000","last_fullsized":"2023-10-09T05:11:00.705892+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:37:30.615058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":336,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983883+0000","last_change":"2023-10-09T05:09:10.282347+0000","last_active":"2023-10-09T05:11:00.983883+0000","last_peered":"2023-10-09T05:11:00.983883+0000","last_clean":"2023-10-09T05:11:00.983883+0000","last_became_active":"2023-10-09T05:09:10.281988+0000","last_became_peered":"2023-10-09T05:09:10.281988+0000","last_unstale":"2023-10-09T05:11:00.983883+0000","last_undegraded":"2023-10-09T05:11:00.983883+0000","last_fullsized":"2023-10-09T05:11:00.983883+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:59:32.577269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":829,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700523+0000","last_change":"2023-10-09T04:58:07.897767+0000","last_active":"2023-10-09T05:11:00.700523+0000","last_peered":"2023-10-09T05:11:00.700523+0000","last_clean":"2023-10-09T05:11:00.700523+0000","last_became_active":"2023-10-09T04:58:07.890597+0000","last_became_peered":"2023-10-09T04:58:07.890597+0000","last_unstale":"2023-10-09T05:11:00.700523+0000","last_undegraded":"2023-10-09T05:11:00.700523+0000","last_fullsized":"2023-10-09T05:11:00.700523+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:03:45.005553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982199+0000","last_change":"2023-10-09T05:09:27.625297+0000","last_active":"2023-10-09T05:11:00.982199+0000","last_peered":"2023-10-09T05:11:00.982199+0000","last_clean":"2023-10-09T05:11:00.982199+0000","last_became_active":"2023-10-09T05:09:27.624891+0000","last_became_peered":"2023-10-09T05:09:27.624891+0000","last_unstale":"2023-10-09T05:11:00.982199+0000","last_undegraded":"2023-10-09T05:11:00.982199+0000","last_fullsized":"2023-10-09T05:11:00.982199+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:21:30.267688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"82'161","reported_seq":1247,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705621+0000","last_change":"2023-10-09T05:10:19.195136+0000","last_active":"2023-10-09T05:11:00.705621+0000","last_peered":"2023-10-09T05:11:00.705621+0000","last_clean":"2023-10-09T05:11:00.705621+0000","last_became_active":"2023-10-09T05:10:19.194649+0000","last_became_peered":"2023-10-09T05:10:19.194649+0000","last_unstale":"2023-10-09T05:11:00.705621+0000","last_undegraded":"2023-10-09T05:11:00.705621+0000","last_fullsized":"2023-10-09T05:11:00.705621+0000","mapping_epoch":764,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":765,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:54.997107+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:54.997107+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:24:38.221308+0000","scrub_duration":0.0021928569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":376,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699561+0000","last_change":"2023-10-09T05:03:24.414758+0000","last_active":"2023-10-09T05:11:00.699561+0000","last_peered":"2023-10-09T05:11:00.699561+0000","last_clean":"2023-10-09T05:11:00.699561+0000","last_became_active":"2023-10-09T05:03:24.414463+0000","last_became_peered":"2023-10-09T05:03:24.414463+0000","last_unstale":"2023-10-09T05:11:00.699561+0000","last_undegraded":"2023-10-09T05:11:00.699561+0000","last_fullsized":"2023-10-09T05:11:00.699561+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:51:30.512580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700914+0000","last_change":"2023-10-09T05:03:24.391158+0000","last_active":"2023-10-09T05:11:00.700914+0000","last_peered":"2023-10-09T05:11:00.700914+0000","last_clean":"2023-10-09T05:11:00.700914+0000","last_became_active":"2023-10-09T05:03:24.390895+0000","last_became_peered":"2023-10-09T05:03:24.390895+0000","last_unstale":"2023-10-09T05:11:00.700914+0000","last_undegraded":"2023-10-09T05:11:00.700914+0000","last_fullsized":"2023-10-09T05:11:00.700914+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:42:01.904763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"82'161","reported_seq":1232,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699944+0000","last_change":"2023-10-09T05:09:08.254212+0000","last_active":"2023-10-09T05:11:00.699944+0000","last_peered":"2023-10-09T05:11:00.699944+0000","last_clean":"2023-10-09T05:11:00.699944+0000","last_became_active":"2023-10-09T05:09:08.253872+0000","last_became_peered":"2023-10-09T05:09:08.253872+0000","last_unstale":"2023-10-09T05:11:00.699944+0000","last_undegraded":"2023-10-09T05:11:00.699944+0000","last_fullsized":"2023-10-09T05:11:00.699944+0000","mapping_epoch":737,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":738,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:48.616814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:48.616814+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:42:23.216761+0000","scrub_duration":0.0021341469999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":313,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983787+0000","last_change":"2023-10-09T05:03:24.385513+0000","last_active":"2023-10-09T05:11:00.983787+0000","last_peered":"2023-10-09T05:11:00.983787+0000","last_clean":"2023-10-09T05:11:00.983787+0000","last_became_active":"2023-10-09T05:03:24.385171+0000","last_became_peered":"2023-10-09T05:03:24.385171+0000","last_unstale":"2023-10-09T05:11:00.983787+0000","last_undegraded":"2023-10-09T05:11:00.983787+0000","last_fullsized":"2023-10-09T05:11:00.983787+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:29.658357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":138,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701294+0000","last_change":"2023-10-09T05:10:16.182226+0000","last_active":"2023-10-09T05:11:00.701294+0000","last_peered":"2023-10-09T05:11:00.701294+0000","last_clean":"2023-10-09T05:11:00.701294+0000","last_became_active":"2023-10-09T05:10:16.181907+0000","last_became_peered":"2023-10-09T05:10:16.181907+0000","last_unstale":"2023-10-09T05:11:00.701294+0000","last_undegraded":"2023-10-09T05:11:00.701294+0000","last_fullsized":"2023-10-09T05:11:00.701294+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:30:12.875304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701009+0000","last_change":"2023-10-09T05:09:10.283620+0000","last_active":"2023-10-09T05:11:00.701009+0000","last_peered":"2023-10-09T05:11:00.701009+0000","last_clean":"2023-10-09T05:11:00.701009+0000","last_became_active":"2023-10-09T05:09:10.283099+0000","last_became_peered":"2023-10-09T05:09:10.283099+0000","last_unstale":"2023-10-09T05:11:00.701009+0000","last_undegraded":"2023-10-09T05:11:00.701009+0000","last_fullsized":"2023-10-09T05:11:00.701009+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:01:03.690878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701282+0000","last_change":"2023-10-09T05:03:24.395429+0000","last_active":"2023-10-09T05:11:00.701282+0000","last_peered":"2023-10-09T05:11:00.701282+0000","last_clean":"2023-10-09T05:11:00.701282+0000","last_became_active":"2023-10-09T05:03:24.395115+0000","last_became_peered":"2023-10-09T05:03:24.395115+0000","last_unstale":"2023-10-09T05:11:00.701282+0000","last_undegraded":"2023-10-09T05:11:00.701282+0000","last_fullsized":"2023-10-09T05:11:00.701282+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:23:13.084184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"82'161","reported_seq":1243,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700206+0000","last_change":"2023-10-09T05:09:08.253464+0000","last_active":"2023-10-09T05:11:00.700206+0000","last_peered":"2023-10-09T05:11:00.700206+0000","last_clean":"2023-10-09T05:11:00.700206+0000","last_became_active":"2023-10-09T05:09:08.253102+0000","last_became_peered":"2023-10-09T05:09:08.253102+0000","last_unstale":"2023-10-09T05:11:00.700206+0000","last_undegraded":"2023-10-09T05:11:00.700206+0000","last_fullsized":"2023-10-09T05:11:00.700206+0000","mapping_epoch":737,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":738,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:05.173448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:05.173448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:01:37.921959+0000","scrub_duration":0.001419465,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":309,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.698954+0000","last_change":"2023-10-09T05:09:10.273499+0000","last_active":"2023-10-09T05:11:00.698954+0000","last_peered":"2023-10-09T05:11:00.698954+0000","last_clean":"2023-10-09T05:11:00.698954+0000","last_became_active":"2023-10-09T05:09:10.273195+0000","last_became_peered":"2023-10-09T05:09:10.273195+0000","last_unstale":"2023-10-09T05:11:00.698954+0000","last_undegraded":"2023-10-09T05:11:00.698954+0000","last_fullsized":"2023-10-09T05:11:00.698954+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:20:07.522564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":199,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694862+0000","last_change":"2023-10-09T05:03:25.371523+0000","last_active":"2023-10-09T05:11:00.694862+0000","last_peered":"2023-10-09T05:11:00.694862+0000","last_clean":"2023-10-09T05:11:00.694862+0000","last_became_active":"2023-10-09T05:03:25.371184+0000","last_became_peered":"2023-10-09T05:03:25.371184+0000","last_unstale":"2023-10-09T05:11:00.694862+0000","last_undegraded":"2023-10-09T05:11:00.694862+0000","last_fullsized":"2023-10-09T05:11:00.694862+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:37:57.356195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"82'161","reported_seq":1443,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984097+0000","last_change":"2023-10-09T05:09:13.543189+0000","last_active":"2023-10-09T05:11:00.984097+0000","last_peered":"2023-10-09T05:11:00.984097+0000","last_clean":"2023-10-09T05:11:00.984097+0000","last_became_active":"2023-10-09T05:09:13.542757+0000","last_became_peered":"2023-10-09T05:09:13.542757+0000","last_unstale":"2023-10-09T05:11:00.984097+0000","last_undegraded":"2023-10-09T05:11:00.984097+0000","last_fullsized":"2023-10-09T05:11:00.984097+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:56.681273+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.681273+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:37:31.565781+0000","scrub_duration":0.001633844,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"724'129","reported_seq":578,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694376+0000","last_change":"2023-10-09T05:09:13.296378+0000","last_active":"2023-10-09T05:11:00.694376+0000","last_peered":"2023-10-09T05:11:00.694376+0000","last_clean":"2023-10-09T05:11:00.694376+0000","last_became_active":"2023-10-09T05:09:13.295966+0000","last_became_peered":"2023-10-09T05:09:13.295966+0000","last_unstale":"2023-10-09T05:11:00.694376+0000","last_undegraded":"2023-10-09T05:11:00.694376+0000","last_fullsized":"2023-10-09T05:11:00.694376+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:11:23.681451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047247800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":400,"num_read_kb":709,"num_write":158,"num_write_kb":271,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":157,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700431+0000","last_change":"2023-10-09T05:09:05.237586+0000","last_active":"2023-10-09T05:11:00.700431+0000","last_peered":"2023-10-09T05:11:00.700431+0000","last_clean":"2023-10-09T05:11:00.700431+0000","last_became_active":"2023-10-09T05:09:05.237334+0000","last_became_peered":"2023-10-09T05:09:05.237334+0000","last_unstale":"2023-10-09T05:11:00.700431+0000","last_undegraded":"2023-10-09T05:11:00.700431+0000","last_fullsized":"2023-10-09T05:11:00.700431+0000","mapping_epoch":734,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":735,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:16:55.395287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":206,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694621+0000","last_change":"2023-10-09T05:04:20.887644+0000","last_active":"2023-10-09T05:11:00.694621+0000","last_peered":"2023-10-09T05:11:00.694621+0000","last_clean":"2023-10-09T05:11:00.694621+0000","last_became_active":"2023-10-09T05:04:20.887323+0000","last_became_peered":"2023-10-09T05:04:20.887323+0000","last_unstale":"2023-10-09T05:11:00.694621+0000","last_undegraded":"2023-10-09T05:11:00.694621+0000","last_fullsized":"2023-10-09T05:11:00.694621+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:26:20.650668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"82'161","reported_seq":1232,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700005+0000","last_change":"2023-10-09T05:09:08.253434+0000","last_active":"2023-10-09T05:11:00.700005+0000","last_peered":"2023-10-09T05:11:00.700005+0000","last_clean":"2023-10-09T05:11:00.700005+0000","last_became_active":"2023-10-09T05:09:08.253156+0000","last_became_peered":"2023-10-09T05:09:08.253156+0000","last_unstale":"2023-10-09T05:11:00.700005+0000","last_undegraded":"2023-10-09T05:11:00.700005+0000","last_fullsized":"2023-10-09T05:11:00.700005+0000","mapping_epoch":737,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":738,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:59.084186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:59.084186+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T17:02:07.255235+0000","scrub_duration":0.001940289,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"724'79","reported_seq":1343,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700254+0000","last_change":"2023-10-09T05:08:41.151543+0000","last_active":"2023-10-09T05:11:00.700254+0000","last_peered":"2023-10-09T05:11:00.700254+0000","last_clean":"2023-10-09T05:11:00.700254+0000","last_became_active":"2023-10-09T05:02:19.782967+0000","last_became_peered":"2023-10-09T05:02:19.782967+0000","last_unstale":"2023-10-09T05:11:00.700254+0000","last_undegraded":"2023-10-09T05:11:00.700254+0000","last_fullsized":"2023-10-09T05:11:00.700254+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:52.007007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00095775700000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":730,"num_read_kb":1477,"num_write":79,"num_write_kb":462,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":235,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700775+0000","last_change":"2023-10-09T05:03:24.412559+0000","last_active":"2023-10-09T05:11:00.700775+0000","last_peered":"2023-10-09T05:11:00.700775+0000","last_clean":"2023-10-09T05:11:00.700775+0000","last_became_active":"2023-10-09T05:03:24.412210+0000","last_became_peered":"2023-10-09T05:03:24.412210+0000","last_unstale":"2023-10-09T05:11:00.700775+0000","last_undegraded":"2023-10-09T05:11:00.700775+0000","last_fullsized":"2023-10-09T05:11:00.700775+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:26:23.565469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700109+0000","last_change":"2023-10-09T05:10:16.184121+0000","last_active":"2023-10-09T05:11:00.700109+0000","last_peered":"2023-10-09T05:11:00.700109+0000","last_clean":"2023-10-09T05:11:00.700109+0000","last_became_active":"2023-10-09T05:10:16.183830+0000","last_became_peered":"2023-10-09T05:10:16.183830+0000","last_unstale":"2023-10-09T05:11:00.700109+0000","last_undegraded":"2023-10-09T05:11:00.700109+0000","last_fullsized":"2023-10-09T05:11:00.700109+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:40.778454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"82'161","reported_seq":1241,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705739+0000","last_change":"2023-10-09T05:10:12.208474+0000","last_active":"2023-10-09T05:11:00.705739+0000","last_peered":"2023-10-09T05:11:00.705739+0000","last_clean":"2023-10-09T05:11:00.705739+0000","last_became_active":"2023-10-09T05:10:12.207353+0000","last_became_peered":"2023-10-09T05:10:12.207353+0000","last_unstale":"2023-10-09T05:11:00.705739+0000","last_undegraded":"2023-10-09T05:11:00.705739+0000","last_fullsized":"2023-10-09T05:11:00.705739+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":4,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:09:27.811929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:09:27.811929+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:51:57.418226+0000","scrub_duration":0.001713099,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"724'67","reported_seq":45,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701862+0000","last_change":"2023-10-09T05:10:16.190109+0000","last_active":"2023-10-09T05:11:00.701862+0000","last_peered":"2023-10-09T05:11:00.701862+0000","last_clean":"2023-10-09T05:11:00.701862+0000","last_became_active":"2023-10-09T05:10:16.179712+0000","last_became_peered":"2023-10-09T05:10:16.179712+0000","last_unstale":"2023-10-09T05:11:00.701862+0000","last_undegraded":"2023-10-09T05:11:00.701862+0000","last_fullsized":"2023-10-09T05:11:00.701862+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:14.591725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061087000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":328,"num_read_kb":1510,"num_write":56,"num_write_kb":1199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":195,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.693940+0000","last_change":"2023-10-09T05:10:16.178981+0000","last_active":"2023-10-09T05:11:00.693940+0000","last_peered":"2023-10-09T05:11:00.693940+0000","last_clean":"2023-10-09T05:11:00.693940+0000","last_became_active":"2023-10-09T05:10:16.178646+0000","last_became_peered":"2023-10-09T05:10:16.178646+0000","last_unstale":"2023-10-09T05:11:00.693940+0000","last_undegraded":"2023-10-09T05:11:00.693940+0000","last_fullsized":"2023-10-09T05:11:00.693940+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:32:08.144860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":25,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.702392+0000","last_change":"2023-10-09T05:10:01.572404+0000","last_active":"2023-10-09T05:11:00.702392+0000","last_peered":"2023-10-09T05:11:00.702392+0000","last_clean":"2023-10-09T05:11:00.702392+0000","last_became_active":"2023-10-09T05:10:01.571931+0000","last_became_peered":"2023-10-09T05:10:01.571931+0000","last_unstale":"2023-10-09T05:11:00.702392+0000","last_undegraded":"2023-10-09T05:11:00.702392+0000","last_fullsized":"2023-10-09T05:11:00.702392+0000","mapping_epoch":755,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":756,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:13:31.770672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"724'174","reported_seq":892,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700134+0000","last_change":"2023-10-09T05:09:13.540906+0000","last_active":"2023-10-09T05:11:00.700134+0000","last_peered":"2023-10-09T05:11:00.700134+0000","last_clean":"2023-10-09T05:11:00.700134+0000","last_became_active":"2023-10-09T05:09:13.540372+0000","last_became_peered":"2023-10-09T05:09:13.540372+0000","last_unstale":"2023-10-09T05:11:00.700134+0000","last_undegraded":"2023-10-09T05:11:00.700134+0000","last_fullsized":"2023-10-09T05:11:00.700134+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":174,"log_dups_size":0,"ondisk_log_size":174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:34:21.040614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040793699999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":925,"num_read_kb":1854,"num_write":140,"num_write_kb":1122,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":1400,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705422+0000","last_change":"2023-10-09T05:10:12.211701+0000","last_active":"2023-10-09T05:11:00.705422+0000","last_peered":"2023-10-09T05:11:00.705422+0000","last_clean":"2023-10-09T05:11:00.705422+0000","last_became_active":"2023-10-09T05:10:12.211352+0000","last_became_peered":"2023-10-09T05:10:12.211352+0000","last_unstale":"2023-10-09T05:11:00.705422+0000","last_undegraded":"2023-10-09T05:11:00.705422+0000","last_fullsized":"2023-10-09T05:11:00.705422+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:09:32.823787+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:09:32.823787+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T11:51:08.791250+0000","scrub_duration":0.0015712409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":265,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699443+0000","last_change":"2023-10-09T05:03:35.984925+0000","last_active":"2023-10-09T05:11:00.699443+0000","last_peered":"2023-10-09T05:11:00.699443+0000","last_clean":"2023-10-09T05:11:00.699443+0000","last_became_active":"2023-10-09T05:03:35.984502+0000","last_became_peered":"2023-10-09T05:03:35.984502+0000","last_unstale":"2023-10-09T05:11:00.699443+0000","last_undegraded":"2023-10-09T05:11:00.699443+0000","last_fullsized":"2023-10-09T05:11:00.699443+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:46:16.764427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":310,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.983995+0000","last_change":"2023-10-09T05:09:10.286480+0000","last_active":"2023-10-09T05:11:00.983995+0000","last_peered":"2023-10-09T05:11:00.983995+0000","last_clean":"2023-10-09T05:11:00.983995+0000","last_became_active":"2023-10-09T05:09:10.286182+0000","last_became_peered":"2023-10-09T05:09:10.286182+0000","last_unstale":"2023-10-09T05:11:00.983995+0000","last_undegraded":"2023-10-09T05:11:00.983995+0000","last_fullsized":"2023-10-09T05:11:00.983995+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:42:32.066948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"82'161","reported_seq":1231,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694347+0000","last_change":"2023-10-09T05:07:58.306714+0000","last_active":"2023-10-09T05:11:00.694347+0000","last_peered":"2023-10-09T05:11:00.694347+0000","last_clean":"2023-10-09T05:11:00.694347+0000","last_became_active":"2023-10-09T05:07:52.143205+0000","last_became_peered":"2023-10-09T05:07:52.143205+0000","last_unstale":"2023-10-09T05:11:00.694347+0000","last_undegraded":"2023-10-09T05:11:00.694347+0000","last_fullsized":"2023-10-09T05:11:00.694347+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:58.306600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:58.306600+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T13:10:05.900624+0000","scrub_duration":0.001694367,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"724'130","reported_seq":865,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705576+0000","last_change":"2023-10-09T05:10:12.205801+0000","last_active":"2023-10-09T05:11:00.705576+0000","last_peered":"2023-10-09T05:11:00.705576+0000","last_clean":"2023-10-09T05:11:00.705576+0000","last_became_active":"2023-10-09T05:10:12.205265+0000","last_became_peered":"2023-10-09T05:10:12.205265+0000","last_unstale":"2023-10-09T05:11:00.705576+0000","last_undegraded":"2023-10-09T05:11:00.705576+0000","last_fullsized":"2023-10-09T05:11:00.705576+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:31:44.711106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042191800000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1129,"num_read_kb":2343,"num_write":84,"num_write_kb":6329,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":87,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.981913+0000","last_change":"2023-10-09T05:09:27.665313+0000","last_active":"2023-10-09T05:11:00.981913+0000","last_peered":"2023-10-09T05:11:00.981913+0000","last_clean":"2023-10-09T05:11:00.981913+0000","last_became_active":"2023-10-09T05:09:27.665030+0000","last_became_peered":"2023-10-09T05:09:27.665030+0000","last_unstale":"2023-10-09T05:11:00.981913+0000","last_undegraded":"2023-10-09T05:11:00.981913+0000","last_fullsized":"2023-10-09T05:11:00.981913+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:26:17.430065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":251,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982287+0000","last_change":"2023-10-09T05:10:16.183686+0000","last_active":"2023-10-09T05:11:00.982287+0000","last_peered":"2023-10-09T05:11:00.982287+0000","last_clean":"2023-10-09T05:11:00.982287+0000","last_became_active":"2023-10-09T05:10:16.183379+0000","last_became_peered":"2023-10-09T05:10:16.183379+0000","last_unstale":"2023-10-09T05:11:00.982287+0000","last_undegraded":"2023-10-09T05:11:00.982287+0000","last_fullsized":"2023-10-09T05:11:00.982287+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:45:29.821498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"82'161","reported_seq":1397,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984090+0000","last_change":"2023-10-09T05:07:53.688815+0000","last_active":"2023-10-09T05:11:00.984090+0000","last_peered":"2023-10-09T05:11:00.984090+0000","last_clean":"2023-10-09T05:11:00.984090+0000","last_became_active":"2023-10-09T05:07:52.149137+0000","last_became_peered":"2023-10-09T05:07:52.149137+0000","last_unstale":"2023-10-09T05:11:00.984090+0000","last_undegraded":"2023-10-09T05:11:00.984090+0000","last_fullsized":"2023-10-09T05:11:00.984090+0000","mapping_epoch":699,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":700,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:07:53.688730+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:07:53.688730+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:54:38.094584+0000","scrub_duration":0.0019408190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"724'44","reported_seq":364,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.705943+0000","last_change":"2023-10-09T05:10:12.209042+0000","last_active":"2023-10-09T05:11:00.705943+0000","last_peered":"2023-10-09T05:11:00.705943+0000","last_clean":"2023-10-09T05:11:00.705943+0000","last_became_active":"2023-10-09T05:10:12.208540+0000","last_became_peered":"2023-10-09T05:10:12.208540+0000","last_unstale":"2023-10-09T05:11:00.705943+0000","last_undegraded":"2023-10-09T05:11:00.705943+0000","last_fullsized":"2023-10-09T05:11:00.705943+0000","mapping_epoch":759,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":760,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":44,"log_dups_size":0,"ondisk_log_size":44,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:44:53.401398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033500400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":146,"num_read_kb":1134,"num_write":53,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":206,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694287+0000","last_change":"2023-10-09T05:10:16.180481+0000","last_active":"2023-10-09T05:11:00.694287+0000","last_peered":"2023-10-09T05:11:00.694287+0000","last_clean":"2023-10-09T05:11:00.694287+0000","last_became_active":"2023-10-09T05:10:16.180125+0000","last_became_peered":"2023-10-09T05:10:16.180125+0000","last_unstale":"2023-10-09T05:11:00.694287+0000","last_undegraded":"2023-10-09T05:11:00.694287+0000","last_fullsized":"2023-10-09T05:11:00.694287+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:35:02.328432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":439,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982089+0000","last_change":"2023-10-09T05:09:27.631516+0000","last_active":"2023-10-09T05:11:00.982089+0000","last_peered":"2023-10-09T05:11:00.982089+0000","last_clean":"2023-10-09T05:11:00.982089+0000","last_became_active":"2023-10-09T05:09:27.631192+0000","last_became_peered":"2023-10-09T05:09:27.631192+0000","last_unstale":"2023-10-09T05:11:00.982089+0000","last_undegraded":"2023-10-09T05:11:00.982089+0000","last_fullsized":"2023-10-09T05:11:00.982089+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T10:56:29.313177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"82'161","reported_seq":1247,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.701183+0000","last_change":"2023-10-09T05:10:19.194964+0000","last_active":"2023-10-09T05:11:00.701183+0000","last_peered":"2023-10-09T05:11:00.701183+0000","last_clean":"2023-10-09T05:11:00.701183+0000","last_became_active":"2023-10-09T05:10:19.194513+0000","last_became_peered":"2023-10-09T05:10:19.194513+0000","last_unstale":"2023-10-09T05:11:00.701183+0000","last_undegraded":"2023-10-09T05:11:00.701183+0000","last_fullsized":"2023-10-09T05:11:00.701183+0000","mapping_epoch":764,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":765,"parent":"0.0","parent_split_bits":3,"last_scrub":"82'161","last_scrub_stamp":"2023-10-09T05:08:00.730094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:03:57.976027+0000","last_clean_scrub_stamp":"2023-10-09T05:08:00.730094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:33:03.239194+0000","scrub_duration":0.0018014909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":428,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700569+0000","last_change":"2023-10-09T05:09:10.292493+0000","last_active":"2023-10-09T05:11:00.700569+0000","last_peered":"2023-10-09T05:11:00.700569+0000","last_clean":"2023-10-09T05:11:00.700569+0000","last_became_active":"2023-10-09T05:09:10.281968+0000","last_became_peered":"2023-10-09T05:09:10.281968+0000","last_unstale":"2023-10-09T05:11:00.700569+0000","last_undegraded":"2023-10-09T05:11:00.700569+0000","last_fullsized":"2023-10-09T05:11:00.700569+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:34:30.407675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"724'277","reported_seq":1158,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.694272+0000","last_change":"2023-10-09T05:08:41.142733+0000","last_active":"2023-10-09T05:11:00.694272+0000","last_peered":"2023-10-09T05:11:00.694272+0000","last_clean":"2023-10-09T05:11:00.694272+0000","last_became_active":"2023-10-09T05:03:24.395849+0000","last_became_peered":"2023-10-09T05:03:24.395849+0000","last_unstale":"2023-10-09T05:11:00.694272+0000","last_undegraded":"2023-10-09T05:11:00.694272+0000","last_fullsized":"2023-10-09T05:11:00.694272+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":277,"log_dups_size":0,"ondisk_log_size":277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:03:17.212885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038981900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1711,"num_read_kb":1450,"num_write":224,"num_write_kb":114,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":294,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.698894+0000","last_change":"2023-10-09T05:09:10.277609+0000","last_active":"2023-10-09T05:11:00.698894+0000","last_peered":"2023-10-09T05:11:00.698894+0000","last_clean":"2023-10-09T05:11:00.698894+0000","last_became_active":"2023-10-09T05:09:10.277198+0000","last_became_peered":"2023-10-09T05:09:10.277198+0000","last_unstale":"2023-10-09T05:11:00.698894+0000","last_undegraded":"2023-10-09T05:11:00.698894+0000","last_fullsized":"2023-10-09T05:11:00.698894+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T09:11:30.809377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"701'162","reported_seq":1236,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699866+0000","last_change":"2023-10-09T05:09:13.540971+0000","last_active":"2023-10-09T05:11:00.699866+0000","last_peered":"2023-10-09T05:11:00.699866+0000","last_clean":"2023-10-09T05:11:00.699866+0000","last_became_active":"2023-10-09T05:09:13.540622+0000","last_became_peered":"2023-10-09T05:09:13.540622+0000","last_unstale":"2023-10-09T05:11:00.699866+0000","last_undegraded":"2023-10-09T05:11:00.699866+0000","last_fullsized":"2023-10-09T05:11:00.699866+0000","mapping_epoch":742,"log_start":"0'0","ondisk_log_start":"0'0","created":470,"last_epoch_clean":743,"parent":"0.0","parent_split_bits":3,"last_scrub":"701'162","last_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_deep_scrub":"701'162","last_deep_scrub_stamp":"2023-10-09T05:07:56.029274+0000","last_clean_scrub_stamp":"2023-10-09T05:07:56.029274+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:27:23.603690+0000","scrub_duration":0.005003141,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"724'76","reported_seq":732,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.984128+0000","last_change":"2023-10-09T05:09:10.316912+0000","last_active":"2023-10-09T05:11:00.984128+0000","last_peered":"2023-10-09T05:11:00.984128+0000","last_clean":"2023-10-09T05:11:00.984128+0000","last_became_active":"2023-10-09T05:09:10.289167+0000","last_became_peered":"2023-10-09T05:09:10.289167+0000","last_unstale":"2023-10-09T05:11:00.984128+0000","last_undegraded":"2023-10-09T05:11:00.984128+0000","last_fullsized":"2023-10-09T05:11:00.984128+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:06:27.890842+0000","last_clean_scrub_stamp":"2023-10-09T04:06:27.890842+0000","objects_scrubbed":0,"log_size":76,"log_dups_size":0,"ondisk_log_size":76,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T08:39:02.138333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028602599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":626,"num_read_kb":2735,"num_write":52,"num_write_kb":2137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":219,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982271+0000","last_change":"2023-10-09T05:09:27.625392+0000","last_active":"2023-10-09T05:11:00.982271+0000","last_peered":"2023-10-09T05:11:00.982271+0000","last_clean":"2023-10-09T05:11:00.982271+0000","last_became_active":"2023-10-09T05:09:27.624996+0000","last_became_peered":"2023-10-09T05:09:27.624996+0000","last_unstale":"2023-10-09T05:11:00.982271+0000","last_undegraded":"2023-10-09T05:11:00.982271+0000","last_fullsized":"2023-10-09T05:11:00.982271+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:57:41.100557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700749+0000","last_change":"2023-10-09T05:03:36.613157+0000","last_active":"2023-10-09T05:11:00.700749+0000","last_peered":"2023-10-09T05:11:00.700749+0000","last_clean":"2023-10-09T05:11:00.700749+0000","last_became_active":"2023-10-09T05:03:36.612822+0000","last_became_peered":"2023-10-09T05:03:36.612822+0000","last_unstale":"2023-10-09T05:11:00.700749+0000","last_undegraded":"2023-10-09T05:11:00.700749+0000","last_fullsized":"2023-10-09T05:11:00.700749+0000","mapping_epoch":631,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":632,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:22.593686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":119,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982106+0000","last_change":"2023-10-09T05:09:27.630251+0000","last_active":"2023-10-09T05:11:00.982106+0000","last_peered":"2023-10-09T05:11:00.982106+0000","last_clean":"2023-10-09T05:11:00.982106+0000","last_became_active":"2023-10-09T05:09:27.629931+0000","last_became_peered":"2023-10-09T05:09:27.629931+0000","last_unstale":"2023-10-09T05:11:00.982106+0000","last_undegraded":"2023-10-09T05:11:00.982106+0000","last_fullsized":"2023-10-09T05:11:00.982106+0000","mapping_epoch":746,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":747,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T12:04:38.445400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":19,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.702145+0000","last_change":"2023-10-09T05:10:16.181492+0000","last_active":"2023-10-09T05:11:00.702145+0000","last_peered":"2023-10-09T05:11:00.702145+0000","last_clean":"2023-10-09T05:11:00.702145+0000","last_became_active":"2023-10-09T05:10:16.181199+0000","last_became_peered":"2023-10-09T05:10:16.181199+0000","last_unstale":"2023-10-09T05:11:00.702145+0000","last_undegraded":"2023-10-09T05:11:00.702145+0000","last_fullsized":"2023-10-09T05:11:00.702145+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:28:00.699814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":910,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700486+0000","last_change":"2023-10-09T05:07:31.951921+0000","last_active":"2023-10-09T05:11:00.700486+0000","last_peered":"2023-10-09T05:11:00.700486+0000","last_clean":"2023-10-09T05:11:00.700486+0000","last_became_active":"2023-10-09T05:03:30.578422+0000","last_became_peered":"2023-10-09T05:03:30.578422+0000","last_unstale":"2023-10-09T05:11:00.700486+0000","last_undegraded":"2023-10-09T05:11:00.700486+0000","last_fullsized":"2023-10-09T05:11:00.700486+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":680,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:07:56.835248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":128,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700844+0000","last_change":"2023-10-09T05:04:20.901848+0000","last_active":"2023-10-09T05:11:00.700844+0000","last_peered":"2023-10-09T05:11:00.700844+0000","last_clean":"2023-10-09T05:11:00.700844+0000","last_became_active":"2023-10-09T05:04:20.901414+0000","last_became_peered":"2023-10-09T05:04:20.901414+0000","last_unstale":"2023-10-09T05:11:00.700844+0000","last_undegraded":"2023-10-09T05:11:00.700844+0000","last_fullsized":"2023-10-09T05:11:00.700844+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T06:48:52.783734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":850,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700523+0000","last_change":"2023-10-09T05:03:24.352607+0000","last_active":"2023-10-09T05:11:00.700523+0000","last_peered":"2023-10-09T05:11:00.700523+0000","last_clean":"2023-10-09T05:11:00.700523+0000","last_became_active":"2023-10-09T05:03:24.352207+0000","last_became_peered":"2023-10-09T05:03:24.352207+0000","last_unstale":"2023-10-09T05:11:00.700523+0000","last_undegraded":"2023-10-09T05:11:00.700523+0000","last_fullsized":"2023-10-09T05:11:00.700523+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:14:40.552387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700978+0000","last_change":"2023-10-09T05:04:20.902362+0000","last_active":"2023-10-09T05:11:00.700978+0000","last_peered":"2023-10-09T05:11:00.700978+0000","last_clean":"2023-10-09T05:11:00.700978+0000","last_became_active":"2023-10-09T05:04:20.902149+0000","last_became_peered":"2023-10-09T05:04:20.902149+0000","last_unstale":"2023-10-09T05:11:00.700978+0000","last_undegraded":"2023-10-09T05:11:00.700978+0000","last_fullsized":"2023-10-09T05:11:00.700978+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T07:05:54.984829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":402,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699696+0000","last_change":"2023-10-09T05:10:16.180178+0000","last_active":"2023-10-09T05:11:00.699696+0000","last_peered":"2023-10-09T05:11:00.699696+0000","last_clean":"2023-10-09T05:11:00.699696+0000","last_became_active":"2023-10-09T05:10:16.179788+0000","last_became_peered":"2023-10-09T05:10:16.179788+0000","last_unstale":"2023-10-09T05:11:00.699696+0000","last_undegraded":"2023-10-09T05:11:00.699696+0000","last_fullsized":"2023-10-09T05:11:00.699696+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T14:10:24.719962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":409,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.699525+0000","last_change":"2023-10-09T05:10:16.178525+0000","last_active":"2023-10-09T05:11:00.699525+0000","last_peered":"2023-10-09T05:11:00.699525+0000","last_clean":"2023-10-09T05:11:00.699525+0000","last_became_active":"2023-10-09T05:10:16.178186+0000","last_became_peered":"2023-10-09T05:10:16.178186+0000","last_unstale":"2023-10-09T05:11:00.699525+0000","last_undegraded":"2023-10-09T05:11:00.699525+0000","last_fullsized":"2023-10-09T05:11:00.699525+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T05:40:52.559595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":630,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.982394+0000","last_change":"2023-10-09T05:10:16.187939+0000","last_active":"2023-10-09T05:11:00.982394+0000","last_peered":"2023-10-09T05:11:00.982394+0000","last_clean":"2023-10-09T05:11:00.982394+0000","last_became_active":"2023-10-09T05:10:16.184331+0000","last_became_peered":"2023-10-09T05:10:16.184331+0000","last_unstale":"2023-10-09T05:11:00.982394+0000","last_undegraded":"2023-10-09T05:11:00.982394+0000","last_fullsized":"2023-10-09T05:11:00.982394+0000","mapping_epoch":761,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":762,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:33.613343+0000","last_clean_scrub_stamp":"2023-10-09T04:07:33.613343+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T04:22:05.752749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":6648,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":246,"reported_epoch":769,"state":"active+clean","last_fresh":"2023-10-09T05:11:00.700489+0000","last_change":"2023-10-09T05:03:24.353293+0000","last_active":"2023-10-09T05:11:00.700489+0000","last_peered":"2023-10-09T05:11:00.700489+0000","last_clean":"2023-10-09T05:11:00.700489+0000","last_became_active":"2023-10-09T05:03:24.352980+0000","last_became_peered":"2023-10-09T05:03:24.352980+0000","last_unstale":"2023-10-09T05:11:00.700489+0000","last_undegraded":"2023-10-09T05:11:00.700489+0000","last_fullsized":"2023-10-09T05:11:00.700489+0000","mapping_epoch":626,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":627,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-09T04:07:34.617315+0000","last_clean_scrub_stamp":"2023-10-09T04:07:34.617315+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-10T15:39:49.005729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":32162,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42374,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":918048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1772,"ondisk_log_size":1772,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234767,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515024,"kb_used_data":9864,"kb_used_omap":0,"kb_used_meta":1505152,"kb_avail":92230128,"statfs":{"total":95995035648,"available":94443651072,"internally_reserved":0,"allocated":10100736,"data_stored":8825608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1541275648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":5.3520000000000003,"15min":3.3540000000000001},"min":{"1min":0.27700000000000002,"5min":0.23899999999999999,"15min":0.23599999999999999},"max":{"1min":0.93200000000000005,"5min":221.91800000000001,"15min":221.91800000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":5.6829999999999998,"15min":3.5600000000000001},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.79700000000000004,"5min":221.87799999999999,"15min":221.87799999999999},"last":0.35599999999999998}]},{"osd":1,"last update":"Mon Oct 9 05:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":8.1430000000000007,"15min":4.0419999999999998},"min":{"1min":0.33400000000000002,"5min":0.29599999999999999,"15min":0.26500000000000001},"max":{"1min":0.998,"5min":322.31999999999999,"15min":322.31999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":8.125,"15min":4.0190000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.28399999999999997},"max":{"1min":0.93400000000000005,"5min":322.14800000000002,"15min":322.14800000000002},"last":0.58599999999999997}]},{"osd":2,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.89300000000000002,"15min":2.1680000000000001},"min":{"1min":0.52700000000000002,"5min":0.34000000000000002,"15min":0.29399999999999998},"max":{"1min":5.4489999999999998,"5min":6.3470000000000004,"15min":154.76400000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.86099999999999999,"15min":2.1699999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":5.4800000000000004,"5min":6.3579999999999997,"15min":154.80000000000001},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":11.513999999999999,"15min":6.2089999999999996},"min":{"1min":0.42699999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.85599999999999998,"5min":349.863,"15min":349.863},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":11.202999999999999,"15min":6.0309999999999997},"min":{"1min":0.41999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.84099999999999997,"5min":349.67500000000001,"15min":349.67500000000001},"last":0.48899999999999999}]},{"osd":4,"last update":"Mon Oct 9 05:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.48399999999999999,"15min":0.41799999999999998},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":1.0860000000000001,"5min":1.145,"15min":1.145},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.53000000000000003,"15min":0.5},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.018,"5min":3.0609999999999999,"15min":3.0609999999999999},"last":0.61099999999999999}]},{"osd":5,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.3440000000000001,"15min":1.159},"min":{"1min":0.35099999999999998,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":1.1379999999999999,"5min":74.915000000000006,"15min":74.915000000000006},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.365,"15min":1.2390000000000001},"min":{"1min":0.39100000000000001,"5min":0.28599999999999998,"15min":0.26000000000000001},"max":{"1min":0.97099999999999997,"5min":74.972999999999999,"15min":74.972999999999999},"last":0.65500000000000003}]},{"osd":6,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.59599999999999997,"15min":0.85599999999999998},"min":{"1min":0.434,"5min":0.308,"15min":0.29399999999999998},"max":{"1min":1.1120000000000001,"5min":2.1829999999999998,"15min":57.168999999999997},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.59299999999999997,"15min":0.95799999999999996},"min":{"1min":0.42299999999999999,"5min":0.32900000000000001,"15min":0.29999999999999999},"max":{"1min":1.0629999999999999,"5min":2.9969999999999999,"15min":57.183999999999997},"last":0.68500000000000005}]}]},{"osd":6,"up_from":46,"seq":197568496400,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1524232,"kb_used_data":5440,"kb_used_omap":0,"kb_used_meta":1518784,"kb_avail":92220920,"statfs":{"total":95995035648,"available":94434222080,"internally_reserved":0,"allocated":5570560,"data_stored":4303497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1555234816},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.528,"15min":3.3599999999999999},"min":{"1min":0.36399999999999999,"5min":0.22600000000000001,"15min":0.218},"max":{"1min":0.67300000000000004,"5min":122.07899999999999,"15min":321.346},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":2.7069999999999999,"15min":2.8519999999999999},"min":{"1min":0.37,"5min":0.26500000000000001,"15min":0.255},"max":{"1min":0.70599999999999996,"5min":122.107,"15min":169.119},"last":0.51100000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.573,"15min":2.1139999999999999},"min":{"1min":0.40500000000000003,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":0.69199999999999995,"5min":43.604999999999997,"15min":90.278999999999996},"last":0.433},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.5700000000000001,"15min":2.1629999999999998},"min":{"1min":0.32800000000000001,"5min":0.28899999999999998,"15min":0.251},"max":{"1min":0.79100000000000004,"5min":44.697000000000003,"15min":90.414000000000001},"last":0.32800000000000001}]},{"osd":2,"last update":"Mon Oct 9 05:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.1359999999999999,"15min":1.28},"min":{"1min":0.39300000000000002,"5min":0.25800000000000001,"15min":0.25600000000000001},"max":{"1min":0.755,"5min":8.4800000000000004,"15min":23.943000000000001},"last":0.627},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.071,"15min":1.2549999999999999},"min":{"1min":0.45100000000000001,"5min":0.32300000000000001,"15min":0.30599999999999999},"max":{"1min":0.72799999999999998,"5min":8.0039999999999996,"15min":25.978999999999999},"last":0.60399999999999998}]},{"osd":3,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":2.4089999999999998,"15min":2.9729999999999999},"min":{"1min":0.435,"5min":0.38900000000000001,"15min":0.26600000000000001},"max":{"1min":0.79600000000000004,"5min":129.499,"15min":147.43600000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":2.6400000000000001,"15min":3.0619999999999998},"min":{"1min":0.439,"5min":0.33800000000000002,"15min":0.28899999999999998},"max":{"1min":0.78000000000000003,"5min":130.994,"15min":147.41999999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.57399999999999995,"15min":0.75800000000000001},"min":{"1min":0.311,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.87,"5min":5.3769999999999998,"15min":18.186},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.78300000000000003,"15min":0.96599999999999997},"min":{"1min":0.35099999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.85799999999999998,"5min":8.7710000000000008,"15min":17.597000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Mon Oct 9 05:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.68300000000000005,"15min":0.752},"min":{"1min":0.35799999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":7.9539999999999997,"15min":25.934000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.72299999999999998,"15min":0.69299999999999995},"min":{"1min":0.46300000000000002,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":0.79300000000000004,"5min":5.2990000000000004,"15min":16.789000000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Mon Oct 9 05:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.752,"15min":0.66300000000000003},"min":{"1min":0.496,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":0.83899999999999997,"5min":7.5039999999999996,"15min":7.5039999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.622,"5min":0.77300000000000002,"15min":0.76500000000000001},"min":{"1min":0.48399999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.76300000000000001,"5min":8.3399999999999999,"15min":20.041},"last":0.64700000000000002}]}]},{"osd":1,"up_from":118,"seq":506806141467,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1415636,"kb_used_data":11468,"kb_used_omap":4,"kb_used_meta":1404155,"kb_avail":92329516,"statfs":{"total":95995035648,"available":94545424384,"internally_reserved":0,"allocated":11743232,"data_stored":8695236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1437855630},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":3.0049999999999999,"15min":2.7749999999999999},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.20699999999999999},"max":{"1min":0.63900000000000001,"5min":61.344999999999999,"15min":100.247},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":8.1449999999999996,"15min":4.585},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.74399999999999999,"5min":554.62,"15min":554.62},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Oct 9 05:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":1.1160000000000001,"15min":1.486},"min":{"1min":0.28100000000000003,"5min":0.23799999999999999,"15min":0.23499999999999999},"max":{"1min":0.64200000000000002,"5min":10.393000000000001,"15min":63.143999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":6.8959999999999999,"15min":3.4910000000000001},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.252},"max":{"1min":0.61499999999999999,"5min":554.31200000000001,"15min":554.31200000000001},"last":0.38}]},{"osd":3,"last update":"Mon Oct 9 05:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":2.8100000000000001,"15min":3.5819999999999999},"min":{"1min":0.34200000000000003,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":0.628,"5min":102.729,"15min":235.43600000000001},"last":0.318},{"interface":"front","average":{"1min":0.45800000000000002,"5min":7.4210000000000003,"15min":4.851},"min":{"1min":0.32700000000000001,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":0.66000000000000003,"5min":554.84400000000005,"15min":554.84400000000005},"last":0.433}]},{"osd":4,"last update":"Mon Oct 9 05:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":6.3200000000000003,"15min":3.0289999999999999},"min":{"1min":0.42899999999999999,"5min":0.35699999999999998,"15min":0.34699999999999998},"max":{"1min":0.748,"5min":554.44000000000005,"15min":554.44000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.2929999999999999,"15min":1.1930000000000001},"min":{"1min":0.38600000000000001,"5min":0.32600000000000001,"15min":0.31900000000000001},"max":{"1min":1.349,"5min":18.832999999999998,"15min":18.832999999999998},"last":0.47199999999999998}]},{"osd":5,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":7.2839999999999998,"15min":3.1930000000000001},"min":{"1min":0.42799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.98499999999999999,"5min":554.54700000000003,"15min":554.54700000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.67500000000000004,"5min":2.0379999999999998,"15min":1.6080000000000001},"min":{"1min":0.435,"5min":0.39000000000000001,"15min":0.35899999999999999},"max":{"1min":1.448,"5min":80.390000000000001,"15min":80.390000000000001},"last":0.496}]},{"osd":6,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":7.3220000000000001,"15min":3.2719999999999998},"min":{"1min":0.45100000000000001,"5min":0.36199999999999999,"15min":0.35699999999999998},"max":{"1min":0.92600000000000005,"5min":554.36500000000001,"15min":554.36500000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.74099999999999999,"5min":2.1720000000000002,"15min":1.673},"min":{"1min":0.51600000000000001,"5min":0.41999999999999998,"15min":0.36199999999999999},"max":{"1min":1.486,"5min":78.278000000000006,"15min":78.278000000000006},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":1.5009999999999999,"15min":1.498},"min":{"1min":0.44800000000000001,"5min":0.439,"15min":0.373},"max":{"1min":1.4059999999999999,"5min":18.899999999999999,"15min":19.664999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.153,"15min":1.252},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.39300000000000002},"max":{"1min":0.88500000000000001,"5min":10.345000000000001,"15min":11.903},"last":0.59999999999999998}]}]},{"osd":0,"up_from":91,"seq":390842024551,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1238292,"kb_used_data":10700,"kb_used_omap":5,"kb_used_meta":1227578,"kb_avail":92506860,"statfs":{"total":95995035648,"available":94727024640,"internally_reserved":0,"allocated":10956800,"data_stored":8616541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1257040506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Oct 9 05:10:22 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":2.2549999999999999,"15min":2.649},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.21299999999999999},"max":{"1min":0.52800000000000002,"5min":19.042000000000002,"15min":155.411},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.374,"5min":2.464,"15min":2.6709999999999998},"min":{"1min":0.254,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.56000000000000005,"5min":17.594000000000001,"15min":154.75},"last":0.48299999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":1.4410000000000001,"15min":1.6659999999999999},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.21099999999999999},"max":{"1min":0.63700000000000001,"5min":20.010999999999999,"15min":26.556999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.40100000000000002,"5min":1.7829999999999999,"15min":1.899},"min":{"1min":0.28100000000000003,"5min":0.24099999999999999,"15min":0.22800000000000001},"max":{"1min":0.621,"5min":15.444000000000001,"15min":38.192},"last":0.58999999999999997}]},{"osd":3,"last update":"Mon Oct 9 05:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":3.6680000000000001,"15min":5.3330000000000002},"min":{"1min":0.315,"5min":0.307,"15min":0.26300000000000001},"max":{"1min":0.65700000000000003,"5min":267.63799999999998,"15min":290.89100000000002},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":3.7109999999999999,"15min":5.4050000000000002},"min":{"1min":0.318,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":0.53100000000000003,"5min":267.66199999999998,"15min":290.73899999999998},"last":0.63100000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.79,"15min":1.8660000000000001},"min":{"1min":0.51100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.92500000000000004,"5min":57.360999999999997,"15min":57.360999999999997},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.71,"15min":1.6359999999999999},"min":{"1min":0.42699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.96799999999999997,"5min":57.338000000000001,"15min":57.338000000000001},"last":0.77100000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":3.012,"15min":1.9139999999999999},"min":{"1min":0.48899999999999999,"5min":0.35799999999999998,"15min":0.30099999999999999},"max":{"1min":0.83799999999999997,"5min":106.922,"15min":106.922},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.5299999999999998,"15min":1.8080000000000001},"min":{"1min":0.38600000000000001,"5min":0.379,"15min":0.34899999999999998},"max":{"1min":0.86499999999999999,"5min":106.771,"15min":106.771},"last":0.70999999999999996}]},{"osd":6,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":3.7530000000000001,"15min":2.3530000000000002},"min":{"1min":0.40500000000000003,"5min":0.376,"15min":0.33700000000000002},"max":{"1min":0.85199999999999998,"5min":112.208,"15min":112.208},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.62,"5min":3.2850000000000001,"15min":2.3740000000000001},"min":{"1min":0.438,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":0.79200000000000004,"5min":110.46899999999999,"15min":110.46899999999999},"last":0.752}]},{"osd":7,"last update":"Mon Oct 9 05:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3360000000000001,"15min":1.494},"min":{"1min":0.47799999999999998,"5min":0.40899999999999997,"15min":0.39400000000000002},"max":{"1min":0.80600000000000005,"5min":13.787000000000001,"15min":21.658000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.29,"15min":1.444},"min":{"1min":0.54900000000000004,"5min":0.39900000000000002,"15min":0.38800000000000001},"max":{"1min":0.88900000000000001,"5min":9.9770000000000003,"15min":21.68},"last":0.83799999999999997}]}]},{"osd":2,"up_from":221,"seq":949187772830,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":806424,"kb_used_data":7120,"kb_used_omap":7,"kb_used_meta":799288,"kb_avail":92938728,"statfs":{"total":95995035648,"available":95169257472,"internally_reserved":0,"allocated":7290880,"data_stored":4709400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":818470993},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":2.6499999999999999,"15min":2.6749999999999998},"min":{"1min":0.22900000000000001,"5min":0.20999999999999999,"15min":0.19600000000000001},"max":{"1min":0.61899999999999999,"5min":120.062,"15min":126.77500000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.35699999999999998,"5min":2.1629999999999998,"15min":2.2989999999999999},"min":{"1min":0.24199999999999999,"5min":0.20899999999999999,"15min":0.20699999999999999},"max":{"1min":0.504,"5min":118.41,"15min":120.937},"last":0.33200000000000002}]},{"osd":1,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":2.673,"15min":2.8919999999999999},"min":{"1min":0.27700000000000002,"5min":0.26100000000000001,"15min":0.24199999999999999},"max":{"1min":0.64000000000000001,"5min":100.01900000000001,"15min":135.38499999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":3.0840000000000001,"15min":3.3879999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22500000000000001},"max":{"1min":0.63200000000000001,"5min":102.179,"15min":135.23400000000001},"last":0.311}]},{"osd":3,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":3.895,"15min":3.8109999999999999},"min":{"1min":0.317,"5min":0.28399999999999997,"15min":0.254},"max":{"1min":0.58299999999999996,"5min":109.67,"15min":109.67},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":4.141,"15min":3.8380000000000001},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.55200000000000005,"5min":109.746,"15min":109.746},"last":0.438}]},{"osd":4,"last update":"Mon Oct 9 05:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.98799999999999999,"15min":2.0699999999999998},"min":{"1min":0.40600000000000003,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":0.85599999999999998,"5min":5.3259999999999996,"15min":15.202},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.86599999999999999,"15min":1.417},"min":{"1min":0.44,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.81999999999999995,"5min":5.8280000000000003,"15min":10.038},"last":0.64900000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.4470000000000001,"15min":1.2689999999999999},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.32600000000000001},"max":{"1min":0.877,"5min":55.994,"15min":55.994},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.629,"5min":1.4199999999999999,"15min":1.615},"min":{"1min":0.47099999999999997,"5min":0.38700000000000001,"15min":0.315},"max":{"1min":0.78200000000000003,"5min":55.950000000000003,"15min":55.950000000000003},"last":0.79100000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":1.5940000000000001,"15min":1.7210000000000001},"min":{"1min":0.45600000000000002,"5min":0.372,"15min":0.33300000000000002},"max":{"1min":0.80500000000000005,"5min":58.192999999999998,"15min":72.662000000000006},"last":0.754},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.5049999999999999,"15min":1.536},"min":{"1min":0.48599999999999999,"5min":0.375,"15min":0.35099999999999998},"max":{"1min":0.86199999999999999,"5min":59.064999999999998,"15min":70.938000000000002},"last":0.82499999999999996}]},{"osd":7,"last update":"Mon Oct 9 05:10:52 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.0529999999999999,"15min":1.3999999999999999},"min":{"1min":0.42999999999999999,"5min":0.41799999999999998,"15min":0.372},"max":{"1min":0.85799999999999998,"5min":5.2709999999999999,"15min":15.241},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.96499999999999997,"15min":1.2090000000000001},"min":{"1min":0.51200000000000001,"5min":0.44800000000000001,"15min":0.373},"max":{"1min":0.84299999999999997,"5min":5.952,"15min":10.122999999999999},"last":0.84999999999999998}]}]},{"osd":3,"up_from":142,"seq":609885356523,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1024924,"kb_used_data":7380,"kb_used_omap":2,"kb_used_meta":1017533,"kb_avail":92720228,"statfs":{"total":95995035648,"available":94945513472,"internally_reserved":0,"allocated":7557120,"data_stored":5032835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":1041953845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":2.8700000000000001,"15min":2.0209999999999999},"min":{"1min":0.253,"5min":0.191,"15min":0.191},"max":{"1min":0.91500000000000004,"5min":62.863,"15min":108.631},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.45100000000000001,"5min":3.1139999999999999,"15min":2.2290000000000001},"min":{"1min":0.31,"5min":0.28599999999999998,"15min":0.247},"max":{"1min":0.97499999999999998,"5min":62.948,"15min":110.012},"last":0.42299999999999999}]},{"osd":1,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":1.5469999999999999,"15min":1.4319999999999999},"min":{"1min":0.29799999999999999,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":0.94599999999999995,"5min":13.374000000000001,"15min":15.218},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.3220000000000001,"15min":1.3859999999999999},"min":{"1min":0.26300000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":1.016,"5min":14.164,"15min":26.692},"last":0.505}]},{"osd":2,"last update":"Mon Oct 9 05:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":1.419,"15min":1.869},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.249},"max":{"1min":0.58899999999999997,"5min":54.999000000000002,"15min":212.065},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":1.3799999999999999,"15min":1.9770000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":0.51700000000000002,"5min":54.963000000000001,"15min":211.98400000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Oct 9 05:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":1.016,"15min":1.524},"min":{"1min":0.45600000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.0289999999999999,"5min":4.75,"15min":11.193},"last":0.63},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.9710000000000001,"15min":1.3500000000000001},"min":{"1min":0.51100000000000001,"5min":0.34899999999999998,"15min":0.309},"max":{"1min":0.997,"5min":62.683999999999997,"15min":62.683999999999997},"last":0.77700000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.036,"15min":1.538},"min":{"1min":0.432,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":2.1850000000000001,"5min":75.478999999999999,"15min":75.478999999999999},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.66900000000000004,"5min":3.024,"15min":2.0750000000000002},"min":{"1min":0.498,"5min":0.379,"15min":0.35099999999999998},"max":{"1min":1.1180000000000001,"5min":75.459000000000003,"15min":75.459000000000003},"last":0.80000000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":3.048,"15min":1.8220000000000001},"min":{"1min":0.46100000000000002,"5min":0.35999999999999999,"15min":0.34799999999999998},"max":{"1min":1.1779999999999999,"5min":79.093999999999994,"15min":79.093999999999994},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.68000000000000005,"5min":3.0369999999999999,"15min":1.919},"min":{"1min":0.47099999999999997,"5min":0.39500000000000002,"15min":0.34799999999999998},"max":{"1min":1.196,"5min":78.555999999999997,"15min":78.555999999999997},"last":0.81100000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.99,"15min":1.595},"min":{"1min":0.51000000000000001,"5min":0.41199999999999998,"15min":0.39400000000000002},"max":{"1min":2.1720000000000002,"5min":63.017000000000003,"15min":63.017000000000003},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.76000000000000001,"5min":1.9890000000000001,"15min":1.3620000000000001},"min":{"1min":0.498,"5min":0.38900000000000001,"15min":0.32100000000000001},"max":{"1min":2.1549999999999998,"5min":62.808,"15min":62.808},"last":0.872}]}]},{"osd":4,"up_from":626,"seq":2688649527392,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410636,"kb_used_data":5444,"kb_used_omap":22,"kb_used_meta":405161,"kb_avail":93334516,"statfs":{"total":95995035648,"available":95574544384,"internally_reserved":0,"allocated":5574656,"data_stored":4303554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":414885709},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":3.226,"15min":2.6160000000000001},"min":{"1min":0.35899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.65100000000000002,"5min":130.34200000000001,"15min":130.34200000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":3.4049999999999998,"15min":3.2069999999999999},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.89700000000000002,"5min":130.49700000000001,"15min":130.49700000000001},"last":0.56999999999999995}]},{"osd":1,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":4.1950000000000003,"15min":2.8380000000000001},"min":{"1min":0.374,"5min":0.27500000000000002,"15min":0.27300000000000002},"max":{"1min":0.71199999999999997,"5min":182.32900000000001,"15min":182.32900000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":4.2169999999999996,"15min":2.6339999999999999},"min":{"1min":0.36599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.70399999999999996,"5min":182.41,"15min":182.41},"last":0.53000000000000003}]},{"osd":2,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.2290000000000001,"15min":1.512},"min":{"1min":0.34200000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.93300000000000005,"5min":7.8639999999999999,"15min":11.051},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.2150000000000001,"15min":1.546},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.94999999999999996,"5min":7.8810000000000002,"15min":11.036},"last":0.68400000000000005}]},{"osd":3,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":3.4180000000000001,"15min":3.1920000000000002},"min":{"1min":0.38900000000000001,"5min":0.34599999999999997,"15min":0.318},"max":{"1min":0.77600000000000002,"5min":109.53400000000001,"15min":270.767},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":3.4399999999999999,"15min":3.5640000000000001},"min":{"1min":0.33200000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.91900000000000004,"5min":109.84,"15min":270.274},"last":0.64800000000000002}]},{"osd":5,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":2.161,"15min":0.97899999999999998},"min":{"1min":0.28699999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.873,"5min":134.82900000000001,"15min":134.82900000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":2.1819999999999999,"15min":1.0269999999999999},"min":{"1min":0.32800000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.91800000000000004,"5min":134.60400000000001,"15min":134.60400000000001},"last":0.67000000000000004}]},{"osd":6,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":2.266,"15min":1.1839999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.996,"5min":139.77799999999999,"15min":139.77799999999999},"last":0.625},{"interface":"front","average":{"1min":0.54300000000000004,"5min":2.335,"15min":1.085},"min":{"1min":0.308,"5min":0.30199999999999999,"15min":0.29999999999999999},"max":{"1min":0.85299999999999998,"5min":140.78200000000001,"15min":140.78200000000001},"last":0.55400000000000005}]},{"osd":7,"last update":"Mon Oct 9 05:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55200000000000005,"15min":0.53300000000000003},"min":{"1min":0.34399999999999997,"5min":0.255,"15min":0.255},"max":{"1min":0.84299999999999997,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.60399999999999998,"15min":0.52100000000000002},"min":{"1min":0.32800000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.89500000000000002,"5min":3.109,"15min":3.109},"last":0.60799999999999998}]}]},{"osd":5,"up_from":38,"seq":163208758044,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1499332,"kb_used_data":6012,"kb_used_omap":0,"kb_used_meta":1493312,"kb_avail":92245820,"statfs":{"total":95995035648,"available":94459719680,"internally_reserved":0,"allocated":6156288,"data_stored":4889685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1529151488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Oct 9 05:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":4.2160000000000002,"15min":3.1230000000000002},"min":{"1min":0.33100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.80400000000000005,"5min":212.72800000000001,"15min":212.72800000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":4.2880000000000003,"15min":3.298},"min":{"1min":0.33500000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.75900000000000001,"5min":212.53999999999999,"15min":212.53999999999999},"last":0.63800000000000001}]},{"osd":1,"last update":"Mon Oct 9 05:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":3.1070000000000002,"15min":2.9470000000000001},"min":{"1min":0.32200000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.81499999999999995,"5min":113.977,"15min":262.26900000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.54000000000000004,"5min":3.1070000000000002,"15min":2.9740000000000002},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78600000000000003,"5min":112.947,"15min":262.291},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon Oct 9 05:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.0209999999999999,"15min":1.641},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.27000000000000002},"max":{"1min":0.82099999999999995,"5min":9.6440000000000001,"15min":106.238},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.0920000000000001,"15min":1.7869999999999999},"min":{"1min":0.40100000000000002,"5min":0.317,"15min":0.29999999999999999},"max":{"1min":0.76000000000000001,"5min":8.9239999999999995,"15min":104.31699999999999},"last":0.54600000000000004}]},{"osd":3,"last update":"Mon Oct 9 05:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":3.1819999999999999,"15min":3.6440000000000001},"min":{"1min":0.36099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.78900000000000003,"5min":121.40600000000001,"15min":125.824},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":3.3340000000000001,"15min":3.7120000000000002},"min":{"1min":0.442,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.75,"5min":121.53700000000001,"15min":125.761},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Oct 9 05:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.48299999999999998,"15min":0.45000000000000001},"min":{"1min":0.32600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.755,"5min":1.99,"15min":1.99},"last":0.746},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.51500000000000001,"15min":0.47599999999999998},"min":{"1min":0.41199999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.73099999999999998,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.60599999999999998}]},{"osd":6,"last update":"Mon Oct 9 05:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54200000000000004,"15min":0.58599999999999997},"min":{"1min":0.38100000000000001,"5min":0.28499999999999998,"15min":0.23300000000000001},"max":{"1min":0.75800000000000001,"5min":4.3360000000000003,"15min":11.093},"last":0.625},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.53800000000000003,"15min":0.55000000000000004},"min":{"1min":0.38800000000000001,"5min":0.29899999999999999,"15min":0.25600000000000001},"max":{"1min":0.77200000000000002,"5min":3.0590000000000002,"15min":5.9329999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Mon Oct 9 05:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.53800000000000003,"15min":0.60799999999999998},"min":{"1min":0.439,"5min":0.30399999999999999,"15min":0.26700000000000002},"max":{"1min":0.71599999999999997,"5min":2.1019999999999999,"15min":8.3520000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58999999999999997,"15min":0.69499999999999995},"min":{"1min":0.42299999999999999,"5min":0.31,"15min":0.31},"max":{"1min":0.76900000000000002,"5min":2.0489999999999999,"15min":11.791},"last":0.71799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22707,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-09T05:11:18.410 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-09T05:11:18.411 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T05:11:18.477 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=f27578678b6047de84add1aa4b1a49c2 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_a340fb5463aa3715e93e7472d5f60dfdcf184ab3/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-09T05:11:18.480 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-09T05:11:18.490 INFO:tasks.cephadm:Teardown begin 2023-10-09T05:11:18.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:11:18.547 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:11:18.615 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-09T05:11:18.616 DEBUG:teuthology.orchestra.run.smithi029:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:11:18.636 DEBUG:teuthology.orchestra.run.smithi177:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-09T05:11:18.655 INFO:tasks.cephadm:Stopping all daemons... 2023-10-09T05:11:18.655 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-09T05:11:18.656 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a 2023-10-09T05:11:18.992 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:18 smithi029 ceph-mon[223858]: from='client.38793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:18.993 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:18 smithi029 systemd[1]: Stopping Ceph mon.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:18.993 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:18 smithi029 ceph-mon[216903]: from='client.38793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:19.027 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:18 smithi177 ceph-mon[195797]: from='client.38793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-09T05:11:19.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:18 smithi029 conmon[216878]: 2023-10-09T05:11:18.991+0000 7f28c9426700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:19.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Oct 09 05:11:18 smithi029 conmon[216878]: 2023-10-09T05:11:18.991+0000 7f28c9426700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-09T05:11:19.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:19 smithi029 conmon[202582]: [09/Oct/2023:05:11:19] ENGINE Bus STOPPING 2023-10-09T05:11:19.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:19 smithi029 conmon[202582]: [09/Oct/2023:05:11:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-09T05:11:19.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:19 smithi029 conmon[202582]: [09/Oct/2023:05:11:19] ENGINE Bus STOPPED 2023-10-09T05:11:19.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:19 smithi029 conmon[202582]: [09/Oct/2023:05:11:19] ENGINE Bus STARTING 2023-10-09T05:11:19.423 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:19 smithi029 conmon[202582]: [09/Oct/2023:05:11:19] ENGINE Serving on http://:::9283 2023-10-09T05:11:19.423 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:19 smithi029 conmon[202582]: [09/Oct/2023:05:11:19] ENGINE Bus STARTED 2023-10-09T05:11:20.283 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-09T05:11:20.342 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:20.342 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-09T05:11:20.342 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-09T05:11:20.342 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.c 2023-10-09T05:11:20.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:20 smithi029 systemd[1]: Stopping Ceph mon.c for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:21.118 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:20 smithi029 conmon[223833]: 2023-10-09T05:11:20.742+0000 7f1ad040b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:21.119 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:20 smithi029 conmon[223833]: 2023-10-09T05:11:20.742+0000 7f1ad040b700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-09T05:11:21.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Oct 09 05:11:21 smithi029 bash[746027]: 7e2b5984a98662edae48f925c20edae8e844bd6a4e67395360eca375d54d773f 2023-10-09T05:11:22.088 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-09T05:11:22.133 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:22.133 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-09T05:11:22.133 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-09T05:11:22.133 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.b 2023-10-09T05:11:22.431 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:22 smithi177 systemd[1]: Stopping Ceph mon.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:22 smithi177 conmon[195773]: 2023-10-09T05:11:22.429+0000 7fe83f37c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:22.777 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:22 smithi177 conmon[195773]: 2023-10-09T05:11:22.429+0000 7fe83f37c700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-09T05:11:23.170 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:22 smithi029 conmon[202582]: ::ffff:172.21.15.177 - - [09/Oct/2023:05:11:22] "GET /metrics HTTP/1.1" 200 34284 "" "Prometheus/2.43.0" 2023-10-09T05:11:23.221 INFO:journalctl@ceph.mon.b.smithi177.stdout:Oct 09 05:11:22 smithi177 bash[451884]: 8b10005e656535485ab06d18167ec102424da0a6387a643fabbddcb1142c2299 2023-10-09T05:11:23.728 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-09T05:11:23.768 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:23.769 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-09T05:11:23.769 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-09T05:11:23.769 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y 2023-10-09T05:11:24.084 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:23 smithi029 systemd[1]: Stopping Ceph mgr.y for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:24.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:24 smithi029 conmon[202582]: 2023-10-09T05:11:24.083+0000 7f606c8da700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:24.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Oct 09 05:11:24 smithi029 conmon[202582]: 2023-10-09T05:11:24.083+0000 7f606c8da700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-09T05:11:25.140 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-09T05:11:25.179 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:25.180 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-09T05:11:25.180 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-09T05:11:25.181 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x 2023-10-09T05:11:25.211 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:11:24 smithi177 conmon[179305]: ts=2023-10-09T05:11:24.838Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:11:25.212 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:11:24 smithi177 conmon[179305]: ts=2023-10-09T05:11:24.838Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:11:25.212 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:11:24 smithi177 conmon[179305]: ts=2023-10-09T05:11:24.845Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:11:25.212 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:11:24 smithi177 conmon[179305]: ts=2023-10-09T05:11:24.845Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:11:25.527 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 05:11:25 smithi177 systemd[1]: Stopping Ceph mgr.x for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:26.027 INFO:journalctl@ceph.mgr.x.smithi177.stdout:Oct 09 05:11:25 smithi177 bash[452184]: c43f5ec5cd804743305f3be1f9163d3ed7cc0f086bd62c87104a59343cc9f20a 2023-10-09T05:11:26.272 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-09T05:11:26.352 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:26.352 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-09T05:11:26.352 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-09T05:11:26.353 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.0 2023-10-09T05:11:26.671 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 05:11:26 smithi029 systemd[1]: Stopping Ceph osd.0 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:27.170 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 05:11:26 smithi029 conmon[241447]: 2023-10-09T05:11:26.674+0000 7f122f489700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:27.171 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 05:11:26 smithi029 conmon[241447]: 2023-10-09T05:11:26.674+0000 7f122f489700 -1 osd.0 770 *** Got signal Terminated *** 2023-10-09T05:11:27.171 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 05:11:26 smithi029 conmon[241447]: 2023-10-09T05:11:26.674+0000 7f122f489700 -1 osd.0 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:11:32.670 INFO:journalctl@ceph.osd.0.smithi029.stdout:Oct 09 05:11:32 smithi029 bash[746627]: e385ec644294308898589b59f72e0463224682fc177c5c9a09f117e57fcd1b3f 2023-10-09T05:11:35.593 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-09T05:11:35.642 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:35.642 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-09T05:11:35.643 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-09T05:11:35.643 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.1 2023-10-09T05:11:35.915 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 05:11:35 smithi029 systemd[1]: Stopping Ceph osd.1 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:36.170 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 05:11:35 smithi029 conmon[255177]: 2023-10-09T05:11:35.914+0000 7f8ad9ffc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:36.171 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 05:11:35 smithi029 conmon[255177]: 2023-10-09T05:11:35.914+0000 7f8ad9ffc700 -1 osd.1 770 *** Got signal Terminated *** 2023-10-09T05:11:36.171 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 05:11:35 smithi029 conmon[255177]: 2023-10-09T05:11:35.914+0000 7f8ad9ffc700 -1 osd.1 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:11:42.170 INFO:journalctl@ceph.osd.1.smithi029.stdout:Oct 09 05:11:41 smithi029 bash[747213]: f8791635683fcef5dca76defb5eec05aac07d62920d3feffa1553d299f973932 2023-10-09T05:11:44.164 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-09T05:11:44.209 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:44.209 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-09T05:11:44.210 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-09T05:11:44.210 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2 2023-10-09T05:11:44.420 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 05:11:44 smithi029 systemd[1]: Stopping Ceph osd.2 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:44.920 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 05:11:44 smithi029 conmon[374274]: 2023-10-09T05:11:44.577+0000 7faae2280700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:44.921 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 05:11:44 smithi029 conmon[374274]: 2023-10-09T05:11:44.577+0000 7faae2280700 -1 osd.2 770 *** Got signal Terminated *** 2023-10-09T05:11:44.921 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 05:11:44 smithi029 conmon[374274]: 2023-10-09T05:11:44.577+0000 7faae2280700 -1 osd.2 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:11:50.671 INFO:journalctl@ceph.osd.2.smithi029.stdout:Oct 09 05:11:50 smithi029 bash[747794]: 0979e032489c1c475f793a6a77b5e4861c0b48f6882252a7efb628bfeca4f05a 2023-10-09T05:11:52.681 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-09T05:11:52.730 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:11:52.731 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-09T05:11:52.731 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-09T05:11:52.731 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.3 2023-10-09T05:11:52.777 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:52 smithi177 conmon[131831]: 2023-10-09T05:11:52.452+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:53.039 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:52 smithi029 systemd[1]: Stopping Ceph osd.3 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:11:53.039 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:52 smithi029 conmon[275254]: 2023-10-09T05:11:52.907+0000 7fee0a60f700 -1 osd.3 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:27.442771+0000 front 2023-10-09T05:11:27.442781+0000 (oldest deadline 2023-10-09T05:11:52.742972+0000) 2023-10-09T05:11:53.420 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:53 smithi029 conmon[275254]: 2023-10-09T05:11:53.038+0000 7fee1081e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:11:53.420 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:53 smithi029 conmon[275254]: 2023-10-09T05:11:53.038+0000 7fee1081e700 -1 osd.3 770 *** Got signal Terminated *** 2023-10-09T05:11:53.421 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:53 smithi029 conmon[275254]: 2023-10-09T05:11:53.038+0000 7fee1081e700 -1 osd.3 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:11:53.493 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:53 smithi177 conmon[140729]: 2023-10-09T05:11:53.213+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:11:53.777 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:53 smithi177 conmon[131831]: 2023-10-09T05:11:53.493+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:53.777 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:53 smithi177 conmon[410843]: 2023-10-09T05:11:53.715+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:11:54.170 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:53 smithi029 conmon[275254]: 2023-10-09T05:11:53.894+0000 7fee0a60f700 -1 osd.3 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:27.442771+0000 front 2023-10-09T05:11:27.442781+0000 (oldest deadline 2023-10-09T05:11:52.742972+0000) 2023-10-09T05:11:54.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:54 smithi177 conmon[140729]: 2023-10-09T05:11:54.186+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:11:55.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:54 smithi177 conmon[410843]: 2023-10-09T05:11:54.761+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:11:55.027 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:54 smithi177 conmon[131831]: 2023-10-09T05:11:54.537+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:55.028 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:11:54 smithi177 conmon[123039]: 2023-10-09T05:11:54.741+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:11:55.170 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:54 smithi029 conmon[275254]: 2023-10-09T05:11:54.909+0000 7fee0a60f700 -1 osd.3 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:27.442771+0000 front 2023-10-09T05:11:27.442781+0000 (oldest deadline 2023-10-09T05:11:52.742972+0000) 2023-10-09T05:11:55.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:55 smithi177 conmon[140729]: 2023-10-09T05:11:55.208+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:11:56.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:11:55 smithi177 conmon[123039]: 2023-10-09T05:11:55.763+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:11:56.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:55 smithi177 conmon[410843]: 2023-10-09T05:11:55.800+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:11:56.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:55 smithi177 conmon[131831]: 2023-10-09T05:11:55.577+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:56.171 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:55 smithi029 conmon[275254]: 2023-10-09T05:11:55.862+0000 7fee0a60f700 -1 osd.3 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:27.442771+0000 front 2023-10-09T05:11:27.442781+0000 (oldest deadline 2023-10-09T05:11:52.742972+0000) 2023-10-09T05:11:56.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:56 smithi177 conmon[140729]: 2023-10-09T05:11:56.255+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:11:57.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:56 smithi177 conmon[410843]: 2023-10-09T05:11:56.804+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:11:57.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:11:56 smithi177 conmon[123039]: 2023-10-09T05:11:56.754+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:11:57.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:56 smithi177 conmon[131831]: 2023-10-09T05:11:56.616+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:57.170 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:56 smithi029 conmon[275254]: 2023-10-09T05:11:56.831+0000 7fee0a60f700 -1 osd.3 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:27.442771+0000 front 2023-10-09T05:11:27.442781+0000 (oldest deadline 2023-10-09T05:11:52.742972+0000) 2023-10-09T05:11:57.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:57 smithi177 conmon[140729]: 2023-10-09T05:11:57.214+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:11:58.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:11:57 smithi177 conmon[123039]: 2023-10-09T05:11:57.755+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:11:58.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:57 smithi177 conmon[410843]: 2023-10-09T05:11:57.775+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:11:58.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:57 smithi177 conmon[131831]: 2023-10-09T05:11:57.622+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:58.170 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:57 smithi029 conmon[275254]: 2023-10-09T05:11:57.876+0000 7fee0a60f700 -1 osd.3 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:27.442771+0000 front 2023-10-09T05:11:27.442781+0000 (oldest deadline 2023-10-09T05:11:52.742972+0000) 2023-10-09T05:11:58.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:58 smithi177 conmon[140729]: 2023-10-09T05:11:58.172+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:11:59.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:11:58 smithi177 conmon[123039]: 2023-10-09T05:11:58.798+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:11:59.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:58 smithi177 conmon[410843]: 2023-10-09T05:11:58.768+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:11:59.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:58 smithi177 conmon[131831]: 2023-10-09T05:11:58.670+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:11:59.170 INFO:journalctl@ceph.osd.3.smithi029.stdout:Oct 09 05:11:58 smithi029 bash[748373]: 8400b9c194003b5256e185328544a5c5c0e460efa9d14817ca5b9e1b2efadfcd 2023-10-09T05:11:59.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:11:59 smithi177 conmon[140729]: 2023-10-09T05:11:59.175+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:00.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:11:59 smithi177 conmon[123039]: 2023-10-09T05:11:59.809+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:00.028 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:11:59 smithi177 conmon[410843]: 2023-10-09T05:11:59.798+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:00.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:11:59 smithi177 conmon[131831]: 2023-10-09T05:11:59.709+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:00.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:00 smithi177 conmon[140729]: 2023-10-09T05:12:00.175+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:01.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:00 smithi177 conmon[123039]: 2023-10-09T05:12:00.854+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:01.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:00 smithi177 conmon[410843]: 2023-10-09T05:12:00.837+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:01.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:00 smithi177 conmon[131831]: 2023-10-09T05:12:00.740+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:01.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:01 smithi177 conmon[140729]: 2023-10-09T05:12:01.224+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:02.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:01 smithi177 conmon[123039]: 2023-10-09T05:12:01.844+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:02.027 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:01 smithi177 conmon[410843]: 2023-10-09T05:12:01.849+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:02.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:01 smithi177 conmon[131831]: 2023-10-09T05:12:01.725+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:02.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:02 smithi177 conmon[140729]: 2023-10-09T05:12:02.244+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:03.027 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:02 smithi177 conmon[123039]: 2023-10-09T05:12:02.875+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:03.028 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:02 smithi177 conmon[410843]: 2023-10-09T05:12:02.863+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:03.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:02 smithi177 conmon[131831]: 2023-10-09T05:12:02.771+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:03.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:03 smithi177 conmon[140729]: 2023-10-09T05:12:03.285+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:04.271 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:03 smithi177 conmon[410843]: 2023-10-09T05:12:03.827+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:04.272 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:03 smithi177 conmon[123039]: 2023-10-09T05:12:03.885+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:04.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:03 smithi177 conmon[131831]: 2023-10-09T05:12:03.785+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:04.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:03 smithi177 conmon[131831]: 2023-10-09T05:12:03.785+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:04.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:04 smithi177 conmon[140729]: 2023-10-09T05:12:04.271+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:05.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:04 smithi177 conmon[410843]: 2023-10-09T05:12:04.872+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:05.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:04 smithi177 conmon[123039]: 2023-10-09T05:12:04.914+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:05.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:04 smithi177 conmon[123039]: 2023-10-09T05:12:04.914+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:05.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:04 smithi177 conmon[131831]: 2023-10-09T05:12:04.791+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:05.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:04 smithi177 conmon[131831]: 2023-10-09T05:12:04.791+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:05.599 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-09T05:12:05.647 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:12:05.648 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-09T05:12:05.648 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-09T05:12:05.648 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4 2023-10-09T05:12:05.680 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[140729]: 2023-10-09T05:12:05.307+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:05.680 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[140729]: 2023-10-09T05:12:05.307+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:06.027 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[131831]: 2023-10-09T05:12:05.840+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:06.029 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[131831]: 2023-10-09T05:12:05.840+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:06.030 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[123039]: 2023-10-09T05:12:05.888+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:06.030 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[123039]: 2023-10-09T05:12:05.888+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:06.030 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:05 smithi177 systemd[1]: Stopping Ceph osd.4 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:06.030 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[410843]: 2023-10-09T05:12:05.870+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:06.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[410843]: 2023-10-09T05:12:05.870+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.868939+0000 front 2023-10-09T05:11:39.868916+0000 (oldest deadline 2023-10-09T05:12:05.168841+0000) 2023-10-09T05:12:06.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[410843]: 2023-10-09T05:12:05.929+0000 7ff97b693700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:12:06.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[410843]: 2023-10-09T05:12:05.929+0000 7ff97b693700 -1 osd.4 770 *** Got signal Terminated *** 2023-10-09T05:12:06.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:05 smithi177 conmon[410843]: 2023-10-09T05:12:05.929+0000 7ff97b693700 -1 osd.4 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:12:06.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[140729]: 2023-10-09T05:12:06.354+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:06.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[140729]: 2023-10-09T05:12:06.354+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:07.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[131831]: 2023-10-09T05:12:06.862+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:07.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[131831]: 2023-10-09T05:12:06.862+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:07.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[410843]: 2023-10-09T05:12:06.915+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:07.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[410843]: 2023-10-09T05:12:06.915+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.868939+0000 front 2023-10-09T05:11:39.868916+0000 (oldest deadline 2023-10-09T05:12:05.168841+0000) 2023-10-09T05:12:07.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[123039]: 2023-10-09T05:12:06.934+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:07.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:06 smithi177 conmon[123039]: 2023-10-09T05:12:06.934+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:07.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[140729]: 2023-10-09T05:12:07.362+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:07.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[140729]: 2023-10-09T05:12:07.362+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:08.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[131831]: 2023-10-09T05:12:07.862+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:08.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[131831]: 2023-10-09T05:12:07.862+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:08.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[410843]: 2023-10-09T05:12:07.960+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:08.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[410843]: 2023-10-09T05:12:07.960+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.868939+0000 front 2023-10-09T05:11:39.868916+0000 (oldest deadline 2023-10-09T05:12:05.168841+0000) 2023-10-09T05:12:08.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[123039]: 2023-10-09T05:12:07.914+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:08.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:07 smithi177 conmon[123039]: 2023-10-09T05:12:07.914+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:08.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[140729]: 2023-10-09T05:12:08.409+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:08.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[140729]: 2023-10-09T05:12:08.409+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:09.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[131831]: 2023-10-09T05:12:08.898+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:09.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[131831]: 2023-10-09T05:12:08.898+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:09.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[410843]: 2023-10-09T05:12:08.973+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:09.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[410843]: 2023-10-09T05:12:08.973+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.868939+0000 front 2023-10-09T05:11:39.868916+0000 (oldest deadline 2023-10-09T05:12:05.168841+0000) 2023-10-09T05:12:09.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[123039]: 2023-10-09T05:12:08.957+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:09.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:08 smithi177 conmon[123039]: 2023-10-09T05:12:08.958+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:09.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[140729]: 2023-10-09T05:12:09.372+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:09.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[140729]: 2023-10-09T05:12:09.372+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:10.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[131831]: 2023-10-09T05:12:09.876+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:10.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[131831]: 2023-10-09T05:12:09.876+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:10.277 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[410843]: 2023-10-09T05:12:09.967+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:31.266849+0000 front 2023-10-09T05:11:31.266745+0000 (oldest deadline 2023-10-09T05:11:53.467317+0000) 2023-10-09T05:12:10.278 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[410843]: 2023-10-09T05:12:09.967+0000 7ff97548c700 -1 osd.4 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.868939+0000 front 2023-10-09T05:11:39.868916+0000 (oldest deadline 2023-10-09T05:12:05.168841+0000) 2023-10-09T05:12:10.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[123039]: 2023-10-09T05:12:09.910+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:10.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:09 smithi177 conmon[123039]: 2023-10-09T05:12:09.910+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:10.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[140729]: 2023-10-09T05:12:10.360+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:10.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[140729]: 2023-10-09T05:12:10.360+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:10.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[140729]: 2023-10-09T05:12:10.360+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:11.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[131831]: 2023-10-09T05:12:10.828+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:11.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[131831]: 2023-10-09T05:12:10.828+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:11.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[123039]: 2023-10-09T05:12:10.904+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:11.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:10 smithi177 conmon[123039]: 2023-10-09T05:12:10.904+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:11.595 INFO:journalctl@ceph.osd.4.smithi177.stdout:Oct 09 05:12:11 smithi177 bash[452491]: c60e52eb4c6c3745edfb13038b08fb6256d1b6865674e57326f534793cbb4af2 2023-10-09T05:12:11.596 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[140729]: 2023-10-09T05:12:11.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:11.596 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[140729]: 2023-10-09T05:12:11.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:11.596 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[140729]: 2023-10-09T05:12:11.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:11.881 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[131831]: 2023-10-09T05:12:11.808+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:11.881 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[131831]: 2023-10-09T05:12:11.808+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:12.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[123039]: 2023-10-09T05:12:11.880+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:12.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:11 smithi177 conmon[123039]: 2023-10-09T05:12:11.880+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:12.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[140729]: 2023-10-09T05:12:12.353+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:12.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[140729]: 2023-10-09T05:12:12.353+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:12.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[140729]: 2023-10-09T05:12:12.353+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:13.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[123039]: 2023-10-09T05:12:12.877+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:13.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[123039]: 2023-10-09T05:12:12.877+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:13.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[131831]: 2023-10-09T05:12:12.840+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:13.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[131831]: 2023-10-09T05:12:12.840+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:13.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:12 smithi177 conmon[131831]: 2023-10-09T05:12:12.840+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:13.739 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[140729]: 2023-10-09T05:12:13.305+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:13.739 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[140729]: 2023-10-09T05:12:13.305+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:13.739 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[140729]: 2023-10-09T05:12:13.305+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:13.744 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-09T05:12:13.825 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:12:13.825 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-09T05:12:13.825 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-09T05:12:13.825 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.5 2023-10-09T05:12:14.027 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[131831]: 2023-10-09T05:12:13.832+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:14.027 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[131831]: 2023-10-09T05:12:13.832+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:14.028 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[131831]: 2023-10-09T05:12:13.832+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:14.028 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[123039]: 2023-10-09T05:12:13.853+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:14.029 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:13 smithi177 conmon[123039]: 2023-10-09T05:12:13.853+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:14.029 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:13 smithi177 systemd[1]: Stopping Ceph osd.5 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:14.527 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[123039]: 2023-10-09T05:12:14.127+0000 7f634d511700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:12:14.563 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[123039]: 2023-10-09T05:12:14.127+0000 7f634d511700 -1 osd.5 770 *** Got signal Terminated *** 2023-10-09T05:12:14.563 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[123039]: 2023-10-09T05:12:14.127+0000 7f634d511700 -1 osd.5 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:12:14.564 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[140729]: 2023-10-09T05:12:14.290+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:14.564 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[140729]: 2023-10-09T05:12:14.290+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:14.564 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[140729]: 2023-10-09T05:12:14.290+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:15.251 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[123039]: 2023-10-09T05:12:14.847+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:15.251 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[123039]: 2023-10-09T05:12:14.847+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:15.251 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[123039]: 2023-10-09T05:12:14.847+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:49.526310+0000 front 2023-10-09T05:11:49.526254+0000 (oldest deadline 2023-10-09T05:12:14.226239+0000) 2023-10-09T05:12:15.252 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[131831]: 2023-10-09T05:12:14.820+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:15.252 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[131831]: 2023-10-09T05:12:14.820+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:15.252 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:14 smithi177 conmon[131831]: 2023-10-09T05:12:14.820+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:15.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[140729]: 2023-10-09T05:12:15.250+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:15.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[140729]: 2023-10-09T05:12:15.250+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:15.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[140729]: 2023-10-09T05:12:15.250+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:16.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[123039]: 2023-10-09T05:12:15.858+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:16.277 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[123039]: 2023-10-09T05:12:15.858+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:16.278 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[123039]: 2023-10-09T05:12:15.858+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:49.526310+0000 front 2023-10-09T05:11:49.526254+0000 (oldest deadline 2023-10-09T05:12:14.226239+0000) 2023-10-09T05:12:16.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[131831]: 2023-10-09T05:12:15.799+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:16.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[131831]: 2023-10-09T05:12:15.799+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:16.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:15 smithi177 conmon[131831]: 2023-10-09T05:12:15.799+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:16.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[140729]: 2023-10-09T05:12:16.277+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:16.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[140729]: 2023-10-09T05:12:16.277+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:16.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[140729]: 2023-10-09T05:12:16.277+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:17.261 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[123039]: 2023-10-09T05:12:16.870+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:17.261 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[123039]: 2023-10-09T05:12:16.870+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:17.261 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[123039]: 2023-10-09T05:12:16.870+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:49.526310+0000 front 2023-10-09T05:11:49.526254+0000 (oldest deadline 2023-10-09T05:12:14.226239+0000) 2023-10-09T05:12:17.262 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[131831]: 2023-10-09T05:12:16.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:17.262 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[131831]: 2023-10-09T05:12:16.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:17.262 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:16 smithi177 conmon[131831]: 2023-10-09T05:12:16.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:17.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[140729]: 2023-10-09T05:12:17.260+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:17.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[140729]: 2023-10-09T05:12:17.260+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:17.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[140729]: 2023-10-09T05:12:17.260+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:18.243 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[123039]: 2023-10-09T05:12:17.822+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:18.244 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[123039]: 2023-10-09T05:12:17.822+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:18.244 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[123039]: 2023-10-09T05:12:17.822+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:49.526310+0000 front 2023-10-09T05:11:49.526254+0000 (oldest deadline 2023-10-09T05:12:14.226239+0000) 2023-10-09T05:12:18.244 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[131831]: 2023-10-09T05:12:17.856+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:18.244 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[131831]: 2023-10-09T05:12:17.856+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:18.245 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:17 smithi177 conmon[131831]: 2023-10-09T05:12:17.856+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:18.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[140729]: 2023-10-09T05:12:18.243+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:18.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[140729]: 2023-10-09T05:12:18.243+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:18.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[140729]: 2023-10-09T05:12:18.243+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:19.271 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[123039]: 2023-10-09T05:12:18.830+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:30.922660+0000 front 2023-10-09T05:11:30.922675+0000 (oldest deadline 2023-10-09T05:11:54.422751+0000) 2023-10-09T05:12:19.271 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[123039]: 2023-10-09T05:12:18.830+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:39.525295+0000 front 2023-10-09T05:11:39.525432+0000 (oldest deadline 2023-10-09T05:12:04.825320+0000) 2023-10-09T05:12:19.271 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[123039]: 2023-10-09T05:12:18.830+0000 7f634ab11700 -1 osd.5 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:49.526310+0000 front 2023-10-09T05:11:49.526254+0000 (oldest deadline 2023-10-09T05:12:14.226239+0000) 2023-10-09T05:12:19.271 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[131831]: 2023-10-09T05:12:18.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:19.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[131831]: 2023-10-09T05:12:18.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:19.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:18 smithi177 conmon[131831]: 2023-10-09T05:12:18.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:19.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:19 smithi177 conmon[140729]: 2023-10-09T05:12:19.270+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:19.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:19 smithi177 conmon[140729]: 2023-10-09T05:12:19.270+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:19.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:19 smithi177 conmon[140729]: 2023-10-09T05:12:19.270+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:20.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:19 smithi177 conmon[131831]: 2023-10-09T05:12:19.865+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:20.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:19 smithi177 conmon[131831]: 2023-10-09T05:12:19.865+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:20.278 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:19 smithi177 conmon[131831]: 2023-10-09T05:12:19.865+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:20.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:20 smithi177 conmon[140729]: 2023-10-09T05:12:20.289+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:20.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:20 smithi177 conmon[140729]: 2023-10-09T05:12:20.289+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:20.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:20 smithi177 conmon[140729]: 2023-10-09T05:12:20.289+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:20.778 INFO:journalctl@ceph.osd.5.smithi177.stdout:Oct 09 05:12:20 smithi177 bash[453072]: 28d4ab6e8bfead5b0babe2cf7848b74f1d744020c5eef94a872d860a39a40efc 2023-10-09T05:12:21.249 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:20 smithi177 conmon[131831]: 2023-10-09T05:12:20.891+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:21.250 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:20 smithi177 conmon[131831]: 2023-10-09T05:12:20.891+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:21.250 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:20 smithi177 conmon[131831]: 2023-10-09T05:12:20.891+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:21.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[140729]: 2023-10-09T05:12:21.248+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:21.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[140729]: 2023-10-09T05:12:21.248+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:21.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[140729]: 2023-10-09T05:12:21.248+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:21.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[140729]: 2023-10-09T05:12:21.248+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:21.943 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-09T05:12:21.984 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:12:21.984 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-09T05:12:21.985 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-09T05:12:21.985 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.6 2023-10-09T05:12:22.233 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[131831]: 2023-10-09T05:12:21.887+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:22.233 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[131831]: 2023-10-09T05:12:21.887+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:22.233 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[131831]: 2023-10-09T05:12:21.887+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:22.234 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:21 smithi177 conmon[131831]: 2023-10-09T05:12:21.887+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.285141+0000 front 2023-10-09T05:11:56.285172+0000 (oldest deadline 2023-10-09T05:12:20.985040+0000) 2023-10-09T05:12:22.234 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 systemd[1]: Stopping Ceph osd.6 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:22.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[140729]: 2023-10-09T05:12:22.232+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:22.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[140729]: 2023-10-09T05:12:22.232+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:22.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[140729]: 2023-10-09T05:12:22.232+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:22.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[140729]: 2023-10-09T05:12:22.232+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:22.528 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.273+0000 7fb2c4075700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:12:22.528 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.273+0000 7fb2c4075700 -1 osd.6 770 *** Got signal Terminated *** 2023-10-09T05:12:22.528 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.273+0000 7fb2c4075700 -1 osd.6 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:12:23.259 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:23.260 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:23.260 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:23.260 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:22 smithi177 conmon[131831]: 2023-10-09T05:12:22.849+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.285141+0000 front 2023-10-09T05:11:56.285172+0000 (oldest deadline 2023-10-09T05:12:20.985040+0000) 2023-10-09T05:12:23.527 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[174387]: logger=cleanup t=2023-10-09T05:12:23.275114722Z level=info msg="Completed cleanup jobs" duration=91.008672ms 2023-10-09T05:12:23.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[140729]: 2023-10-09T05:12:23.259+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:23.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[140729]: 2023-10-09T05:12:23.259+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:23.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[140729]: 2023-10-09T05:12:23.259+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:23.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[140729]: 2023-10-09T05:12:23.259+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:24.220 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[131831]: 2023-10-09T05:12:23.897+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:24.220 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[131831]: 2023-10-09T05:12:23.897+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:24.220 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[131831]: 2023-10-09T05:12:23.897+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:24.220 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:23 smithi177 conmon[131831]: 2023-10-09T05:12:23.897+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.285141+0000 front 2023-10-09T05:11:56.285172+0000 (oldest deadline 2023-10-09T05:12:20.985040+0000) 2023-10-09T05:12:24.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[140729]: 2023-10-09T05:12:24.219+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:24.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[140729]: 2023-10-09T05:12:24.219+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:24.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[140729]: 2023-10-09T05:12:24.219+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:24.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[140729]: 2023-10-09T05:12:24.219+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:25.255 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[131831]: 2023-10-09T05:12:24.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:25.255 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[131831]: 2023-10-09T05:12:24.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:25.255 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[131831]: 2023-10-09T05:12:24.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:25.255 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[131831]: 2023-10-09T05:12:24.861+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.285141+0000 front 2023-10-09T05:11:56.285172+0000 (oldest deadline 2023-10-09T05:12:20.985040+0000) 2023-10-09T05:12:25.256 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[179305]: ts=2023-10-09T05:12:24.837Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:12:25.256 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[179305]: ts=2023-10-09T05:12:24.837Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:12:25.256 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[179305]: ts=2023-10-09T05:12:24.845Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:12:25.256 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:24 smithi177 conmon[179305]: ts=2023-10-09T05:12:24.846Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.29:8765: connect: connection refused" 2023-10-09T05:12:25.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[140729]: 2023-10-09T05:12:25.254+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:25.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[140729]: 2023-10-09T05:12:25.254+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:25.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[140729]: 2023-10-09T05:12:25.254+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:25.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[140729]: 2023-10-09T05:12:25.254+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:26.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[131831]: 2023-10-09T05:12:25.838+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:26.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[131831]: 2023-10-09T05:12:25.838+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:26.272 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[131831]: 2023-10-09T05:12:25.838+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:26.273 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:25 smithi177 conmon[131831]: 2023-10-09T05:12:25.838+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.285141+0000 front 2023-10-09T05:11:56.285172+0000 (oldest deadline 2023-10-09T05:12:20.985040+0000) 2023-10-09T05:12:26.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[140729]: 2023-10-09T05:12:26.271+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:26.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[140729]: 2023-10-09T05:12:26.271+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:26.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[140729]: 2023-10-09T05:12:26.271+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:26.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[140729]: 2023-10-09T05:12:26.271+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:27.254 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[131831]: 2023-10-09T05:12:26.831+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.481155+0000 front 2023-10-09T05:11:29.481328+0000 (oldest deadline 2023-10-09T05:11:52.381201+0000) 2023-10-09T05:12:27.254 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[131831]: 2023-10-09T05:12:26.831+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.582786+0000 front 2023-10-09T05:11:37.582664+0000 (oldest deadline 2023-10-09T05:12:03.482572+0000) 2023-10-09T05:12:27.254 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[131831]: 2023-10-09T05:12:26.831+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:46.383753+0000 front 2023-10-09T05:11:46.383741+0000 (oldest deadline 2023-10-09T05:12:12.784195+0000) 2023-10-09T05:12:27.254 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:26 smithi177 conmon[131831]: 2023-10-09T05:12:26.831+0000 7fb2bf671700 -1 osd.6 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.285141+0000 front 2023-10-09T05:11:56.285172+0000 (oldest deadline 2023-10-09T05:12:20.985040+0000) 2023-10-09T05:12:27.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:27 smithi177 conmon[140729]: 2023-10-09T05:12:27.253+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:27.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:27 smithi177 conmon[140729]: 2023-10-09T05:12:27.253+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:27.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:27 smithi177 conmon[140729]: 2023-10-09T05:12:27.253+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:27.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:27 smithi177 conmon[140729]: 2023-10-09T05:12:27.253+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:28.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Oct 09 05:12:27 smithi177 bash[453668]: 03ede6ceccc01559f758237f27a8154a7f3335cf45ae81babb9875d898bb12c2 2023-10-09T05:12:28.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:28 smithi177 conmon[140729]: 2023-10-09T05:12:28.291+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:28.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:28 smithi177 conmon[140729]: 2023-10-09T05:12:28.291+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:28.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:28 smithi177 conmon[140729]: 2023-10-09T05:12:28.291+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:28.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:28 smithi177 conmon[140729]: 2023-10-09T05:12:28.291+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:29.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:29 smithi177 conmon[140729]: 2023-10-09T05:12:29.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:29.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:29 smithi177 conmon[140729]: 2023-10-09T05:12:29.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:29.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:29 smithi177 conmon[140729]: 2023-10-09T05:12:29.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:29.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:29 smithi177 conmon[140729]: 2023-10-09T05:12:29.329+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:29.791 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-09T05:12:29.831 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:12:29.832 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-09T05:12:29.832 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-09T05:12:29.832 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.7 2023-10-09T05:12:30.176 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:29 smithi177 systemd[1]: Stopping Ceph osd.7 for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:30.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.175+0000 7fe2bdc25700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-09T05:12:30.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.175+0000 7fe2bdc25700 -1 osd.7 770 *** Got signal Terminated *** 2023-10-09T05:12:30.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.175+0000 7fe2bdc25700 -1 osd.7 770 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-09T05:12:30.527 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:30.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:30.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:30.528 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:30 smithi177 conmon[140729]: 2023-10-09T05:12:30.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:31.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:31 smithi177 conmon[140729]: 2023-10-09T05:12:31.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:31.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:31 smithi177 conmon[140729]: 2023-10-09T05:12:31.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:31.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:31 smithi177 conmon[140729]: 2023-10-09T05:12:31.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:31.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:31 smithi177 conmon[140729]: 2023-10-09T05:12:31.335+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:32.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:32 smithi177 conmon[140729]: 2023-10-09T05:12:32.368+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:32.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:32 smithi177 conmon[140729]: 2023-10-09T05:12:32.368+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:32.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:32 smithi177 conmon[140729]: 2023-10-09T05:12:32.368+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:32.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:32 smithi177 conmon[140729]: 2023-10-09T05:12:32.368+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:32.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:32 smithi177 conmon[140729]: 2023-10-09T05:12:32.368+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2023-10-09T05:12:09.923098+0000 front 2023-10-09T05:12:09.923122+0000 (oldest deadline 2023-10-09T05:12:32.223221+0000) 2023-10-09T05:12:33.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:33 smithi177 conmon[140729]: 2023-10-09T05:12:33.337+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:33.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:33 smithi177 conmon[140729]: 2023-10-09T05:12:33.337+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:33.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:33 smithi177 conmon[140729]: 2023-10-09T05:12:33.337+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:33.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:33 smithi177 conmon[140729]: 2023-10-09T05:12:33.337+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:33.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:33 smithi177 conmon[140729]: 2023-10-09T05:12:33.337+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2023-10-09T05:12:09.923098+0000 front 2023-10-09T05:12:09.923122+0000 (oldest deadline 2023-10-09T05:12:32.223221+0000) 2023-10-09T05:12:34.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:33 smithi029 conmon[200683]: ts=2023-10-09T05:12:33.897Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2023-10-09T05:12:34.171 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:33 smithi029 conmon[200683]: ts=2023-10-09T05:12:33.897Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi177.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.177:8443: connect: connection refused" 2023-10-09T05:12:34.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:34 smithi177 conmon[140729]: 2023-10-09T05:12:34.367+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2023-10-09T05:11:29.816702+0000 front 2023-10-09T05:11:29.816554+0000 (oldest deadline 2023-10-09T05:11:52.716715+0000) 2023-10-09T05:12:34.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:34 smithi177 conmon[140729]: 2023-10-09T05:12:34.367+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2023-10-09T05:11:37.318085+0000 front 2023-10-09T05:11:37.318230+0000 (oldest deadline 2023-10-09T05:12:04.918736+0000) 2023-10-09T05:12:34.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:34 smithi177 conmon[140729]: 2023-10-09T05:12:34.367+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2023-10-09T05:11:44.919359+0000 front 2023-10-09T05:11:44.919351+0000 (oldest deadline 2023-10-09T05:12:10.219214+0000) 2023-10-09T05:12:34.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:34 smithi177 conmon[140729]: 2023-10-09T05:12:34.367+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2023-10-09T05:11:56.020657+0000 front 2023-10-09T05:11:56.020686+0000 (oldest deadline 2023-10-09T05:12:20.720606+0000) 2023-10-09T05:12:34.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:34 smithi177 conmon[140729]: 2023-10-09T05:12:34.367+0000 7fe2bba26700 -1 osd.7 770 heartbeat_check: no reply from 172.21.15.177:6804 osd.4 since back 2023-10-09T05:12:09.923098+0000 front 2023-10-09T05:12:09.923122+0000 (oldest deadline 2023-10-09T05:12:32.223221+0000) 2023-10-09T05:12:36.027 INFO:journalctl@ceph.osd.7.smithi177.stdout:Oct 09 05:12:35 smithi177 bash[454251]: 0f3217cc8fa16e7f07ff99f3f76b04a44d9af3d62bf6ebad6906f182bc16d259 2023-10-09T05:12:37.565 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-09T05:12:37.606 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:12:37.606 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-09T05:12:37.607 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-09T05:12:37.607 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-486585fa-6658-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-09T05:12:37.915 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 systemd[1]: Stopping Ceph prometheus.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:38.277 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:12:38.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-10-09T05:12:38.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-10-09T05:12:38.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-09T05:12:38.278 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.915Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.916Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.920Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.920Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-10-09T05:12:38.279 INFO:journalctl@ceph.prometheus.a.smithi177.stdout:Oct 09 05:12:37 smithi177 conmon[179305]: ts=2023-10-09T05:12:37.920Z caller=main.go:1147 level=info msg="See you next time!" 2023-10-09T05:12:38.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-486585fa-6658-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-09T05:12:38.608 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-09T05:12:38.608 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-09T05:12:38.608 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T05:12:40.124 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:39 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:40.420 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:40 smithi029 conmon[200683]: ts=2023-10-09T05:12:40.124Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-10-09T05:12:40.421 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:40 smithi029 bash[749262]: 7ab788eff7bee282e96ae2441cd3b9015c7100c966a0a4742e28efdc23d9cd5d 2023-10-09T05:12:41.375 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:40 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-09T05:12:41.375 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Oct 09 05:12:40 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:12:41.376 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 05:12:41 smithi029 systemd[1]: Stopping Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:42.110 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 05:12:41 smithi029 bash[749545]: 2be0e618a0f3fdc1dbacb71fc95d571446c64f53d14083b20c4ca1954510e13a 2023-10-09T05:12:42.110 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 05:12:41 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:12:42.111 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 05:12:41 smithi029 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-09T05:12:42.111 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Oct 09 05:12:41 smithi029 systemd[1]: Stopped Ceph node-exporter.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:12:51.136 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-09T05:12:52.527 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:12:52 smithi177 systemd[1]: Stopping Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:53.318 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:12:53 smithi177 bash[455434]: dbe79334ca87f7b53849e7539d7a21ed2a796e24d5636db57bb29bade77a1c9f 2023-10-09T05:12:53.318 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:12:53 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:12:53.319 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:12:53 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-10-09T05:12:53.319 INFO:journalctl@ceph.grafana.a.smithi177.stdout:Oct 09 05:12:53 smithi177 systemd[1]: Stopped Ceph grafana.a for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:12:53.663 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 05:12:53 smithi177 systemd[1]: Stopping Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7... 2023-10-09T05:12:54.351 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 05:12:54 smithi177 bash[455718]: 2adf0c28b4540e065c2489e1d32ae43e882c4903b94c6c5bcc91f0c9478932ce 2023-10-09T05:12:54.352 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 05:12:54 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-09T05:12:54.352 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 05:12:54 smithi177 systemd[1]: ceph-486585fa-6658-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-09T05:12:54.352 INFO:journalctl@ceph.node-exporter.b.smithi177.stdout:Oct 09 05:12:54 smithi177 systemd[1]: Stopped Ceph node-exporter.b for 486585fa-6658-11ee-8db5-212e2dc638e7. 2023-10-09T05:12:59.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:12:59.572 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-09T05:12:59.602 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-09T05:12:59.604 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/remote/smithi029/crash 2023-10-09T05:12:59.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T05:12:59.709 INFO:teuthology.orchestra.run.smithi029.stderr:tar: /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T05:12:59.710 INFO:teuthology.orchestra.run.smithi029.stderr:tar: Error is not recoverable: exiting now 2023-10-09T05:12:59.711 DEBUG:teuthology.misc:Transferring archived files from smithi177:/var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/remote/smithi177/crash 2023-10-09T05:12:59.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/crash -- . 2023-10-09T05:12:59.771 INFO:teuthology.orchestra.run.smithi177.stderr:tar: /var/lib/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-09T05:12:59.771 INFO:teuthology.orchestra.run.smithi177.stderr:tar: Error is not recoverable: exiting now 2023-10-09T05:12:59.773 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-09T05:12:59.773 DEBUG:teuthology.orchestra.run.smithi029:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-09T05:12:59.805 INFO:teuthology.orchestra.run.smithi029.stderr:grep: /var/log/ceph/486585fa-6658-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-09T05:12:59.809 INFO:tasks.cephadm:Compressing logs... 2023-10-09T05:12:59.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:12:59.852 DEBUG:teuthology.orchestra.run.smithi177:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:12:59.881 INFO:teuthology.orchestra.run.smithi029.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T05:12:59.908 INFO:teuthology.orchestra.run.smithi177.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-09T05:29:52.020 INFO:tasks.cephadm:Archiving logs... 2023-10-09T05:29:52.045 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/remote/smithi029/log 2023-10-09T05:29:52.047 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T05:31:29.929 DEBUG:teuthology.misc:Transferring archived files from smithi177:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/remote/smithi177/log 2023-10-09T05:31:29.930 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-09T05:33:24.836 INFO:tasks.cephadm:Removing cluster... 2023-10-09T05:33:24.837 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 --force 2023-10-09T05:33:26.455 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 486585fa-6658-11ee-8db5-212e2dc638e7 --force 2023-10-09T05:33:28.117 INFO:tasks.cephadm:Removing cephadm ... 2023-10-09T05:33:28.117 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T05:33:28.139 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-09T05:33:28.160 INFO:tasks.cephadm:Teardown complete 2023-10-09T05:33:28.161 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-09T05:33:28.171 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-09T05:33:28.171 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T05:33:28.182 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-09T05:33:28.297 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-09T05:33:28.308 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-09T05:33:28.308 DEBUG:teuthology.orchestra.run.smithi029:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:33:28.311 DEBUG:teuthology.orchestra.run.smithi177:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-09T05:33:28.334 INFO:teuthology.orchestra.run.smithi177.stderr:bash: ntpq: command not found 2023-10-09T05:33:28.338 INFO:teuthology.orchestra.run.smithi029.stderr:bash: ntpq: command not found 2023-10-09T05:33:28.354 INFO:teuthology.orchestra.run.smithi177.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:33:28.354 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================== 2023-10-09T05:33:28.354 INFO:teuthology.orchestra.run.smithi177.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 15 -454us[ -454us] +/- 97ms 2023-10-09T05:33:28.355 INFO:teuthology.orchestra.run.smithi177.stdout:^* hv02.front.sepia.ceph.com 3 6 377 16 +204us[ +241us] +/- 62ms 2023-10-09T05:33:28.355 INFO:teuthology.orchestra.run.smithi177.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 16 +262us[ +262us] +/- 69ms 2023-10-09T05:33:28.355 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:33:28.357 INFO:teuthology.orchestra.run.smithi029.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-09T05:33:28.357 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================== 2023-10-09T05:33:28.358 INFO:teuthology.orchestra.run.smithi029.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 9 -494us[ -494us] +/- 97ms 2023-10-09T05:33:28.358 INFO:teuthology.orchestra.run.smithi029.stdout:^* hv02.front.sepia.ceph.com 3 6 377 29 +157us[ +169us] +/- 62ms 2023-10-09T05:33:28.358 INFO:teuthology.orchestra.run.smithi029.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 20 +239us[ +239us] +/- 69ms 2023-10-09T05:33:28.358 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-09T05:33:28.359 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-09T05:33:28.369 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-09T05:33:28.369 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-09T05:33:28.380 DEBUG:teuthology.orchestra.run.smithi029:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T05:33:28.545 DEBUG:teuthology.orchestra.run.smithi177:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-09T05:33:28.712 DEBUG:teuthology.orchestra.run.smithi029:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T05:33:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.748 DEBUG:teuthology.orchestra.run.smithi177:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-09T05:33:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:28.780 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-09T05:33:28.792 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-09T05:33:28.801 INFO:teuthology.task.internal:Duration was 6524.756295 seconds 2023-10-09T05:33:28.802 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-09T05:33:28.812 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-09T05:33:28.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T05:33:28.815 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-09T05:33:28.882 INFO:teuthology.orchestra.run.smithi029.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T05:33:28.886 INFO:teuthology.orchestra.run.smithi177.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-09T05:33:29.299 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-09T05:33:29.299 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi029.front.sepia.ceph.com 2023-10-09T05:33:29.299 DEBUG:teuthology.orchestra.run.smithi029:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T05:33:29.336 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi177.front.sepia.ceph.com 2023-10-09T05:33:29.336 DEBUG:teuthology.orchestra.run.smithi177:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-09T05:33:29.364 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-09T05:33:29.365 DEBUG:teuthology.orchestra.run.smithi029:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:33:29.379 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-09T05:33:29.565 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-09T05:33:29.565 DEBUG:teuthology.orchestra.run.smithi029:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T05:33:29.568 DEBUG:teuthology.orchestra.run.smithi177:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-09T05:33:29.985 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-09T05:33:29.996 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-09T05:33:29.997 DEBUG:teuthology.orchestra.run.smithi029:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T05:33:30.035 DEBUG:teuthology.orchestra.run.smithi177:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-09T05:33:30.102 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-09T05:33:30.116 DEBUG:teuthology.orchestra.run.smithi029:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:33:30.118 DEBUG:teuthology.orchestra.run.smithi177:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:33:30.145 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern = core 2023-10-09T05:33:30.155 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = core 2023-10-09T05:33:30.171 DEBUG:teuthology.orchestra.run.smithi029:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:33:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.224 DEBUG:teuthology.orchestra.run.smithi177:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-09T05:33:30.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.241 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-09T05:33:30.252 INFO:teuthology.task.internal:Transferring archived files... 2023-10-09T05:33:30.253 DEBUG:teuthology.misc:Transferring archived files from smithi029:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/remote/smithi029 2023-10-09T05:33:30.254 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T05:33:30.337 DEBUG:teuthology.misc:Transferring archived files from smithi177:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454/remote/smithi177 2023-10-09T05:33:30.337 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-09T05:33:30.387 INFO:teuthology.task.internal:Removing archive directory... 2023-10-09T05:33:30.388 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T05:33:30.390 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-09T05:33:30.433 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-09T05:33:30.445 INFO:teuthology.task.internal:Not uploading archives. 2023-10-09T05:33:30.445 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-09T05:33:30.457 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-09T05:33:30.457 DEBUG:teuthology.orchestra.run.smithi029:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T05:33:30.459 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-09T05:33:30.474 INFO:teuthology.orchestra.run.smithi029.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 9 05:33 /home/ubuntu/cephtest 2023-10-09T05:33:30.474 INFO:teuthology.orchestra.run.smithi029.stdout: 5636097 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 9 05:08 /home/ubuntu/cephtest/mnt.0 2023-10-09T05:33:30.474 INFO:teuthology.orchestra.run.smithi029.stdout: 5636098 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 9 05:08 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-09T05:33:30.475 INFO:teuthology.orchestra.run.smithi029.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-09T05:33:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-09T05:33:30.476 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi029 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T05:33:30.483 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-09T05:33:30.498 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-09T05:33:30.509 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi029 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-09T05:33:30.509 INFO:teuthology.nuke:Checking targets against current locks 2023-10-09T05:33:30.517 INFO:teuthology.orchestra.run.smithi177.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 9 05:33 /home/ubuntu/cephtest 2023-10-09T05:33:30.547 DEBUG:teuthology.nuke:shortname: smithi029 2023-10-09T05:33:30.548 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:33:30.572 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi029.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:39:03.451333', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T05:33:30.575 DEBUG:teuthology.nuke:shortname: smithi177 2023-10-09T05:33:30.576 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-09T05:33:30.601 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-10-07_14:23:11-upgrade:quincy-x-reef-distro-default-smithi/7417454', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-09 03:39:03.450024', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-09T05:33:30.633 INFO:teuthology.orchestra.console.smithi029:Power off 2023-10-09T05:33:30.633 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T05:33:30.647 INFO:teuthology.orchestra.console.smithi177:Power off 2023-10-09T05:33:30.648 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-09T05:33:30.656 DEBUG:teuthology.orchestra.console.smithi029:power off output: Chassis Power Control: Down/Off 2023-10-09T05:33:30.657 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:33:30.669 DEBUG:teuthology.orchestra.console.smithi177:power off output: Chassis Power Control: Down/Off 2023-10-09T05:33:30.670 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:33:30.680 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is on 2023-10-09T05:33:30.693 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is on 2023-10-09T05:33:34.682 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:33:34.694 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:33:34.796 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is on 2023-10-09T05:33:34.803 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is on 2023-10-09T05:33:38.802 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:33:38.810 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-09T05:33:38.912 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is off 2023-10-09T05:33:38.919 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is off 2023-10-09T05:33:39.014 INFO:teuthology.orchestra.console.smithi029:Power off completed 2023-10-09T05:33:39.021 INFO:teuthology.orchestra.console.smithi177:Power off completed 2023-10-09T05:33:39.125 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 6524.756294727325 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=f27578678b6047de84add1aa4b1a49c2 status: fail success: false 2023-10-09T05:33:39.126 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-09T05:33:39.206 INFO:teuthology.run:FAIL